Merge branch 'GP-1551_emteere_ReParseHeaderFiles_2' into Ghidra_10.1

(Closes #615)
This commit is contained in:
ghidra1 2021-12-09 11:11:14 -05:00
commit 1538b49d0b
11 changed files with 261 additions and 110 deletions

View File

@ -0,0 +1,97 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//
// Synchronize the category path name case from the first archive into the second
// archive.
//
//@category Data Types
import java.io.File;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.data.Category;
import ghidra.program.model.data.FileDataTypeManager;
import ghidra.util.InvalidNameException;
import ghidra.util.exception.DuplicateNameException;
public class SynchronizeGDTCategoryPaths extends GhidraScript {
private File firstFile;
private File secondFile;
private FileDataTypeManager firstArchive;
private FileDataTypeManager secondArchive;
@Override
protected void run() throws Exception {
firstFile = askFile("Select First GDT File", "Select 1st");
secondFile = askFile("Select Second GDT File", "Select 2nd");
try {
firstArchive = FileDataTypeManager.openFileArchive(firstFile, false);
secondArchive = FileDataTypeManager.openFileArchive(secondFile, true);
int transactionID = secondArchive.startTransaction("Synchronize Category Path Names");
Category firstCategory = firstArchive.getRootCategory();
Category secondCategory = secondArchive.getRootCategory();
synchronizeCategory(firstCategory, secondCategory);
secondArchive.endTransaction(transactionID, true);
}
finally {
if (firstArchive != null) {
firstArchive.close();
}
secondArchive.save();
if (secondArchive != null) {
secondArchive.close();
}
}
}
private void synchronizeCategory(Category firstCategory, Category secondCategory) {
Category[] firstCategories = firstCategory.getCategories();
for (Category categoryA : firstCategories) {
// loop through categories looking for a case agnostic path match
Category[] secondCategories = secondCategory.getCategories();
boolean foundIt = false;
for (Category categoryB : secondCategories) {
if (categoryA.getName().equalsIgnoreCase(categoryB.getName())) {
// if not the exact same name, rename it
if (!categoryA.getName().equals(categoryB.getName())) {
try {
println(
"Renamed " + categoryB.getName() + " to " + categoryA.getName());
categoryB.setName(categoryA.getName());
foundIt = true;
}
catch (DuplicateNameException | InvalidNameException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
else {
foundIt = true;
}
synchronizeCategory(categoryA, categoryB);
}
}
if (!foundIt) {
println("Couldn't find matching category for " + categoryA.getName());
}
}
}
}

View File

@ -22,7 +22,6 @@ import ghidra.app.util.cparser.CPP.PreProcessor.PPToken;
import ghidra.program.model.data.*;
import ghidra.program.util.AddressEvaluator;
import ghidra.util.Msg;
import ghidra.util.NumericUtilities;
/**
*
@ -113,7 +112,8 @@ public class DefineTable {
node = new Hashtable();
findTable.put(chObj, node);
findTable = node;
} else {
}
else {
findTable = node;
}
}
@ -271,14 +271,14 @@ public class DefineTable {
*/
private String macroSub(String image, int pos, ArrayList<String> sublist) {
int replaceCount = 0;
StringBuffer buf = new StringBuffer(image);
// don't replace an infinite number of times.
//HashMap<String,Integer> lastReplStrings = new HashMap<String,Integer>();
while (pos < buf.length() && replaceCount < 900000) {
String defName = getDefineAt(buf, pos);
if (shouldReplace(buf,defName,pos)) {
if (shouldReplace(buf, defName, pos)) {
// stop recursion on the same replacement string
// if (lastReplStrings.containsKey(defName)) {
// int lastpos = lastReplStrings.get(defName);
@ -296,13 +296,15 @@ public class DefineTable {
// is there a replacement string
if (newpos == -1) {
pos++;
} else {
}
else {
//System.err.println(" replace " + defName + " with " + buf.substring(pos,newpos));
//lastReplStrings.put(defName,pos + defName.length());
pos = newpos;
replaceCount++;
}
} else {
}
else {
pos++;
}
}
@ -312,36 +314,34 @@ public class DefineTable {
return buf.toString();
}
private boolean shouldReplace(StringBuffer buf, String defName, int pos) {
if (defName == null) {
return false;
}
//String nextRepl = "";
int currIndex = buf.indexOf(defName, pos);
if (currIndex < 0)
if (currIndex < 0) {
return false; // nothing to replace
}
// this match is not exact so skip it (borrowing from JavaCharacter)
if (currIndex > 0
&& (Character
.isJavaIdentifierStart(buf.charAt(currIndex - 1)) || Character
.isJavaIdentifierPart(buf.charAt(currIndex - 1)))) {
if (currIndex > 0 && (Character.isJavaIdentifierStart(buf.charAt(currIndex - 1)) ||
Character.isJavaIdentifierPart(buf.charAt(currIndex - 1)))) {
return false;
}
int afterIndex = currIndex + defName.length();
if (afterIndex < buf.length()
&& (Character.isJavaIdentifierStart(buf.charAt(afterIndex)) || Character
.isJavaIdentifierPart(buf.charAt(afterIndex)))) {
if (afterIndex < buf.length() && (Character.isJavaIdentifierStart(buf.charAt(afterIndex)) ||
Character.isJavaIdentifierPart(buf.charAt(afterIndex)))) {
return false;
}
//nextRepl = image.substring(0, currIndex); // shift to location
String replacementString = defs.get(defName).image; // get replacement text
if (replacementString.equals(defName))
if (replacementString.equals(defName)) {
return false; // no need to replace
}
// // check that macro argv arguments match
// Vector<PPToken> argv = getArgs(defName);
// if (argv != null && argv.size() > 0) {
@ -360,41 +360,40 @@ public class DefineTable {
// return false;
// }
// }
return true;
}
int replace(StringBuffer buf, String currKey, int fromIndex, ArrayList<String> sublist) {
String replacementString = null;
if (sublist == null) {
sublist = new ArrayList<String>();
}
//String nextRepl = "";
int currIndex = buf.indexOf(currKey, fromIndex);
if (currIndex < 0)
if (currIndex < 0) {
return -1; // nothing to replace
}
// this match is not exact so skip it (borrowing from JavaCharacter)
if (currIndex > 0
&& (Character
.isJavaIdentifierStart(buf.charAt(currIndex - 1)) || Character
.isJavaIdentifierPart(buf.charAt(currIndex - 1)))) {
if (currIndex > 0 && (Character.isJavaIdentifierStart(buf.charAt(currIndex - 1)) ||
Character.isJavaIdentifierPart(buf.charAt(currIndex - 1)))) {
return -1;
}
int afterIndex = currIndex + currKey.length();
if (afterIndex < buf.length()
&& (Character.isJavaIdentifierStart(buf.charAt(afterIndex)) || Character
.isJavaIdentifierPart(buf.charAt(afterIndex)))) {
if (afterIndex < buf.length() && (Character.isJavaIdentifierStart(buf.charAt(afterIndex)) ||
Character.isJavaIdentifierPart(buf.charAt(afterIndex)))) {
return -1;
}
//nextRepl = image.substring(0, currIndex); // shift to location
replacementString = defs.get(currKey).image; // get replacement text
if (replacementString.equals(currKey))
if (replacementString.equals(currKey)) {
return -1; // no need to replace
}
// if current def has args, take care of the replacement of them
Vector<PPToken> argv = getArgs(currKey);
int replacedSubpieceLen = currKey.length();
@ -406,8 +405,7 @@ public class DefineTable {
// need to scan carefully, and recursively
// there shouldn't be so many globals...
// could be screwed up by so many things
String parms = getParams(buf, currIndex + currKey.length(),
(char) 0);
String parms = getParams(buf, currIndex + currKey.length(), (char) 0);
int parmslen = parms.length();
if (parmslen < 2) {
@ -417,30 +415,30 @@ public class DefineTable {
if (!parms.startsWith("(") || !parms.endsWith(")")) {
return -1;
}
parms = parms.substring(1, parms.length() - 1);
replacementString = subParams(replacementString, currKey, parms, argv);
replacementString = joinPdPd(replacementString);
replacedSubpieceLen += parmslen;
}
// you may add an else if{} block to warn of malformed macros
// but the actual culprit may be the Define() non-terminal
// but the actual culprit may be the Define() non-terminal
//if (replString != null)
// nextRepl += replString;
sublist = new ArrayList<String>(sublist);
sublist.add(currKey);
String newReplString = macroSub(replacementString,0, sublist);
String newReplString = macroSub(replacementString, 0, sublist);
if (newReplString != null) {
replacementString = newReplString;
}
buf.replace(currIndex, currIndex+replacedSubpieceLen, replacementString);
buf.replace(currIndex, currIndex + replacedSubpieceLen, replacementString);
//nextRepl += image.substring(currIndex + currKey.length());
return currIndex+replacementString.length();
return currIndex + replacementString.length();
}
/**
* expand a define with arguments
*
@ -462,13 +460,10 @@ public class DefineTable {
}
pos += argValue.length() + 1;
if (index >= argv.size()) {
Msg.error(
this,
"Define parameter mismatch for macro " + defName
+ "(" + parms + ")" + " Expected "
+ argv.size() + " arguments. "
+ " badarg(" + index + ") " + argValue
+ " args processed : " + argsfound);
Msg.error(this,
"Define parameter mismatch for macro " + defName + "(" + parms + ")" +
" Expected " + argv.size() + " arguments. " + " badarg(" + index + ") " +
argValue + " args processed : " + argsfound);
return replString;
}
String curArgName = argv.elementAt(index).image;
@ -489,20 +484,16 @@ public class DefineTable {
// this match is not exact so skip it (borrowing from
// JavaCharacter)
if (curpos > 0
&& (Character.isJavaIdentifierStart(substString
.charAt(curpos - 1)) || Character
.isJavaIdentifierPart(substString
.charAt(curpos - 1)))) {
if (curpos > 0 &&
(Character.isJavaIdentifierStart(substString.charAt(curpos - 1)) ||
Character.isJavaIdentifierPart(substString.charAt(curpos - 1)))) {
continue;
}
int afterIndex = curpos + curArgName.length();
if (afterIndex < substString.length()
&& (Character.isJavaIdentifierStart(substString
.charAt(afterIndex)) || Character
.isJavaIdentifierPart(substString
.charAt(afterIndex)))) {
if (afterIndex < substString.length() &&
(Character.isJavaIdentifierStart(substString.charAt(afterIndex)) ||
Character.isJavaIdentifierPart(substString.charAt(afterIndex)))) {
continue;
}
@ -516,10 +507,10 @@ public class DefineTable {
}
beginPos.add(insertLoc, begin);
endPos.add(insertLoc,
new Integer(curpos + curArgName.length()));
endPos.add(insertLoc, new Integer(curpos + curArgName.length()));
subValue.add(insertLoc, argValue);
} while (curpos >= 0);
}
while (curpos >= 0);
}
StringBuffer buf = new StringBuffer();
@ -566,8 +557,9 @@ public class DefineTable {
}
if (!hitQuote && ch == ')') {
depth--;
if (depth == 0 && endChar == 0)
if (depth == 0 && endChar == 0) {
break;
}
// hit a paren above depth, back up
if (depth < 0) {
pos--;
@ -617,35 +609,37 @@ public class DefineTable {
inString = !inString;
}
quotePos--;
} while (quotePos > currIndex);
}
while (quotePos > currIndex);
int afterIndex = currIndex + 2;
while (currIndex > 0 && image.charAt(currIndex - 1) == ' ') {
currIndex--; // scan back for first non-blank before ##
}
while (afterIndex < image.length()
&& image.charAt(afterIndex) == ' ') {
while (afterIndex < image.length() && image.charAt(afterIndex) == ' ') {
afterIndex++; // scan back for first non-blank before ##
}
if (!inString) {
buf.replace(currIndex, afterIndex, "");
currIndex--;
} else {
}
else {
currIndex -= 2;
}
}
} while (currIndex > 0);
}
while (currIndex > 0);
image = buf.toString();
return image;
}
/**
* Given a data type manager, populate defines with constant values as Enums
*
*/
public void populateDefineEquates(DataTypeManager dtMgr) {
int transactionID = dtMgr.startTransaction("Add Equates");
@ -663,15 +657,20 @@ public class DefineTable {
String strValue = getValue(defName);
String strExpanded = expand(strValue, true);
strValue = strExpanded;
// strip off any casting/parentheses
strValue = stripCast(strValue);
long value = 0;
Long lvalue = getCValue(strValue);
if (lvalue == null) {
lvalue = AddressEvaluator.evaluateToLong(strValue);
try {
lvalue = AddressEvaluator.evaluateToLong(strValue);
}
catch (Exception exc) {
// ignore didn't parse well
}
if (lvalue == null) {
continue;
}
@ -710,9 +709,10 @@ public class DefineTable {
if (strValue.startsWith("0x")) {
start = 2;
radix = 16;
} else if (strValue.startsWith("0")) {
}
else if (strValue.startsWith("0")) {
start = 1;
radix = 8;
radix = 8;
}
if (strValue.endsWith("ul") || strValue.endsWith("ll")) {
strValue = strValue.substring(0, strValue.length() - 2);
@ -720,11 +720,11 @@ public class DefineTable {
else if (strValue.endsWith("l") || strValue.endsWith("u")) {
strValue = strValue.substring(0, strValue.length() - 1);
}
if (start != 0) {
strValue = strValue.substring(start);
}
return Long.parseLong(strValue, radix);
}
catch (RuntimeException e) {
@ -732,7 +732,7 @@ public class DefineTable {
}
return null;
}
/*
* create a category path based on a name, or the root category with no name
*/
@ -757,41 +757,42 @@ public class DefineTable {
}
return path.substring(slashpos + 1);
}
/*
* Strip off any casts
*/
private static String stripCast(String strValue) {
strValue = strValue.trim();
int pos = 0;
while (pos < strValue.length()) {
int procLen = 1;
int startPos = strValue.indexOf('(',pos);
int startPos = strValue.indexOf('(', pos);
if (startPos == -1) {
return strValue; // done, no more open parens
}
pos = startPos;
int endParen = strValue.indexOf(')', pos+1);
if (endParen != -1) {
String subStr = strValue.substring(pos+1, endParen);
if (subStr.length() > 0) {
int subPos = 0;
subStr = subStr.trim();
boolean isValid = Character.isJavaIdentifierStart(subStr.charAt(0));
while (isValid && subPos < subStr.length()) {
char ch = subStr.charAt(subPos++);
isValid |= Character.isJavaIdentifierPart(ch);
}
// if looks like a cast, throw it away
if (isValid) {
strValue = strValue.substring(0, pos) + strValue.substring(endParen+1);
procLen = 0;
}
int endParen = strValue.indexOf(')', pos + 1);
if (endParen != -1) {
String subStr = strValue.substring(pos + 1, endParen);
if (subStr.length() > 0) {
int subPos = 0;
subStr = subStr.trim();
boolean isValid = Character.isJavaIdentifierStart(subStr.charAt(0));
while (isValid && subPos < subStr.length()) {
char ch = subStr.charAt(subPos++);
isValid |= Character.isJavaIdentifierPart(ch);
}
// if looks like a cast, throw it away
if (isValid) {
strValue = strValue.substring(0, pos) + strValue.substring(endParen + 1);
procLen = 0;
}
} else {
return strValue; // no more end parens, just finish
}
}
else {
return strValue; // no more end parens, just finish
}
pos = pos + procLen;
}
return strValue;

View File

@ -2596,7 +2596,7 @@ Object CastExpression() : {
(
LOOKAHEAD("(" TypeName() ")")
(
"(" TypeName() ")" ( CastExpression() |
"(" TypeName() ")" ( obj = CastExpression() |
( "{" InitializerList() [ "," ] "}" ) )
)
|
@ -2633,13 +2633,12 @@ Object UnaryExpression() : {
|
<SIZEOF>
(
LOOKAHEAD(UnaryExpression() )
UnaryExpression()
|
"(" dt = TypeName() ")"
"(" (dt = TypeName() | <IDENTIFIER> | obj = ConstantExpression() ) ")"
{
if (dt != null) {
obj = new Integer(dt.getLength());
obj = Long.valueOf(dt.getLength());
} else if (obj != null && obj instanceof String) {
obj = Long.valueOf(((String) obj).length() - 1); // will include "" plus \0
}
}
)
@ -2690,7 +2689,7 @@ Object PrimaryExpression() : {
Object obj = null;
}
{
( <IDENTIFIER> |
( obj = <IDENTIFIER> |
obj = Constant() |
"(" obj = Expression() ")" )
{
@ -2729,11 +2728,11 @@ Object Constant() : {
}
if (sval.startsWith("0x") || sval.startsWith("0X")) {
BigInteger bigConst = new BigInteger(sval.substring(2), 16);
obj = new Long(bigConst.longValue());
obj = Long.valueOf(bigConst.longValue());
}
else {
BigInteger bigConst = new BigInteger(sval);
obj = new Long(bigConst.longValue());
obj = Long.valueOf(bigConst.longValue());
}
}
|
@ -2748,7 +2747,7 @@ Object Constant() : {
obj = new Character(t.image.charAt(0));
} else if (t.image.length() == 4) {
long cval = ((long)t.image.charAt(0) << 24) + ((long)t.image.charAt(1) << 16) + ((long)t.image.charAt(2) << 8) + ((long) t.image.charAt(3));
obj = new Long(cval);
obj = Long.valueOf(cval);
}
}
|

View File

@ -789,11 +789,28 @@ public class PreProcessor {
iFile = new File(parent + File.separator + filename);
if (iFile.exists())
return iFile;
// try just in this directory
File sameiFile = new File(parent + File.separator
+ (new File(filename)).getName());
if (sameiFile.exists())
return sameiFile;
// try all files in this directory doing to-lower on both input file and output file
// if match return it
File folder = new File(parent);
if (folder.isDirectory()) {
File[] listOfFiles = folder.listFiles();
if (listOfFiles != null) {
for (File file : listOfFiles) {
if (file.isFile() && filename.compareToIgnoreCase(file.getName()) == 0) {
return file;
}
}
}
}
if (possibleXsym) {
String child = "";
// look up parent chain, looking for a file that has an Xsym in it

View File

@ -429,5 +429,23 @@ public class CParserTest extends AbstractGenericTest {
assertTrue(a.isZeroLength());
assertTrue(a.getDataType() instanceof UnsignedLongDataType);
assertEquals(4, a.getElementLength());
dt = dtMgr.getDataType(new CategoryPath("/"), "sizeof_t");
assertTrue(dt instanceof Structure);
sdt = (Structure) dt;
DataTypeComponent cdt = sdt.getComponent(0);
assertTrue(cdt.getDataType() instanceof Array);
assertEquals("Array field defined with sizeof typedef", 128, cdt.getLength());
dt = dtMgr.getDataType(new CategoryPath("/"), "cpu_set_t");
assertTrue(dt instanceof Structure);
sdt = (Structure) dt;
cdt = sdt.getComponent(0);
assertTrue(cdt.getDataType() instanceof Array);
assertEquals("Array field defined with sizeof typedef", 128, cdt.getLength());
cdt = sdt.getComponent(1);
assertTrue(cdt.getDataType() instanceof Array);
assertEquals("Array field defined with sizeof typedef", 2084, cdt.getLength());
}
}

View File

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* EXCLUDE: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,6 +17,7 @@
/** Test parsing header file for CParser. Most of the file is just checked to make sure it gets through parsing.
** Some data types are checked. More checking of the parsed information would be beneficial at some point.
**/
/**
** use of long as an attribute
@ -229,6 +229,8 @@ __checkint(int val, int* err) {
if (val < (-2147483647-1) || val > 2147483647) {
*err |= OVR_ERR;
}
val = 8 * sizeof(val);
return (int32_t) val;
}
@ -339,6 +341,22 @@ typedef struct sigevent
} sigevent_t;
/*
* Complicated sizeof array size
*/
typedef struct
{
unsigned long int val[(1024 / (8 * sizeof (unsigned long int)))];
} sizeof_t;
typedef unsigned long int __cpu_mask;
typedef struct
{
__cpu_mask __bits[1024 / (8 * (int) sizeof (__cpu_mask))];
char szUrl[(2048 + 32 + sizeof("://"))] ;
} cpu_set_t;
void __mem_func (void *, char **, int ***, long (*) (size_t),
@ -389,7 +407,7 @@ struct sockaddr_in
struct in_addr sin_addr;
unsigned char sin_zero[sizeof (struct bob) -
unsigned char sin_zero[3 * sizeof (struct bob) -
(sizeof (unsigned short int)) -
sizeof (int) -
sizeof (struct bob)];

View File

@ -105,6 +105,7 @@ public class DataTypeArchiveTransformerPanel extends JPanel {
gbc.gridx = 3;
gbc.gridwidth = 1;
useOldFileIDCheckBox = new GCheckBox(" Use Old File ID");
useOldFileIDCheckBox.setSelected(true);
filePanel.add(useOldFileIDCheckBox, gbc);
}