GT-3235 Correct parsing of *.ord MS ordinal symbol map files

This commit is contained in:
ghidra1 2019-10-10 17:55:44 -04:00 committed by Ryan Kurtz
parent 8eef7ddc0a
commit cf32f48605
8 changed files with 286 additions and 148 deletions

View File

@ -35,6 +35,7 @@ data/parserprofiles/objc_mac_carbon.prf||GHIDRA||reviewed||END|
data/parserprofiles/vs12Local.prf||GHIDRA||||END|
data/pcodetest/EmuTesting.gdt||GHIDRA||||END|
data/stringngrams/StringModel.sng||GHIDRA||reviewed||END|
data/symbols/README.txt||GHIDRA||||END|
data/symbols/win32/kernel32.hints||GHIDRA||||END|
data/symbols/win32/mfc100.exports||GHIDRA||||END|
data/symbols/win32/mfc100u.exports||GHIDRA||||END|
@ -1186,3 +1187,4 @@ src/test.slow/resources/filterTestDirList.txt||GHIDRA||||END|
src/test.slow/resources/ghidra/app/plugin/core/datamgr/TestDataType.txt||GHIDRA||||END|
src/test.slow/resources/ghidra/app/script/GhidraScriptAsk.properties||GHIDRA||||END|
src/test/resources/defaultTools/TestCodeBrowser.tool||GHIDRA||||END|
src/test/resources/ghidra/app/util/opinion/test.ord||GHIDRA||||END|

View File

@ -0,0 +1,26 @@
The use of library symbol table information is currently limited to Windows
x86 (32-bit and 64-bit). Currently, the use of these library symbol files are
best suited for library names which incorporate an API version (e.g., mfc100.dll)
since the .exports and .ord files will utilize this same name for identification.
When the PE loader detects a library dependency (i.e., DLL) and the library is
not loaded, the subdirectories win32 or win64 will be searched for a corresponding
.exports or .ord file to provide ordinal-to-symbol name mappings. User generated
.exports and .ord files may also be stored/read from the user's 'symbols' resource
directory contained within the user's version-specific .ghidra directory (e.g.,
%HOMEPATH%/.ghidra/.ghidra-9.0/symbols/win64).
The .exports files can be generated from a loaded library program and can also
provide function stack purge and function comment information. The Ghidra script
'CreateExportFileForDLL' may be used to generate a .exports file for the current
program which will be stored within the user's 'symbols' resource directory
mentioned above.
Many library functions are referenced through the use of ordinals and may be
missing real symbol names. In such cases it may be desirable to rely on ordinal
to symbol name map .ord files which may be generated with the following command:
DUMPBIN /EXPORTS <DLL-FILEPATH>
The DUMPBIN utility is provided with Microsoft Visual Studio. The resulting output
should be stored within a .ord file using the DLL name (e.g., mfc100.ord).

View File

@ -29,14 +29,22 @@
import generic.jar.ResourceFile;
import ghidra.app.script.GhidraScript;
import ghidra.app.util.opinion.LibraryLookupTable;
import ghidra.util.Msg;
import ghidra.util.SystemUtilities;
public class CreateExportFileForDLL extends GhidraScript {
@Override
public void run() throws Exception {
if (currentProgram == null) {
Msg.error(this, "Script requires active program");
}
// push this .dll into the location of the system .exports files.
// must have write permissions.
ResourceFile file = LibraryLookupTable.createFile(currentProgram, false, true, monitor);
ResourceFile file = LibraryLookupTable.createFile(currentProgram, false,
SystemUtilities.isInDevelopmentMode(), monitor);
println("Created .exports file : " + file.getAbsolutePath());
}

View File

@ -139,7 +139,8 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
}
@Override
public String validateOptions(ByteProvider provider, LoadSpec loadSpec, List<Option> options, Program program) {
public String validateOptions(ByteProvider provider, LoadSpec loadSpec, List<Option> options,
Program program) {
if (options != null) {
for (Option option : options) {
@ -336,8 +337,8 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
Address imageBaseAddr = language.getAddressFactory().getDefaultAddressSpace().getAddress(
loadSpec.getDesiredImageBase());
Program program = createProgram(provider, programName, imageBaseAddr, getName(),
language, compilerSpec, consumer);
Program program = createProgram(provider, programName, imageBaseAddr, getName(), language,
compilerSpec, consumer);
int transactionID = program.startTransaction("importing");
boolean success = false;
@ -623,8 +624,8 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
*/
protected boolean importLibrary(String libName, DomainFolder libFolder, File libFile,
LoadSpec loadSpec, List<Option> options, MessageLog log, Object consumer,
Set<String> unprocessedLibs, List<Program> programList,
TaskMonitor monitor) throws CancelledException, IOException {
Set<String> unprocessedLibs, List<Program> programList, TaskMonitor monitor)
throws CancelledException, IOException {
if (!libFile.isFile()) {
return false;
@ -658,8 +659,7 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
protected boolean importLibrary(String libName, DomainFolder libFolder, File libFile,
ByteProvider provider, LoadSpec loadSpec, List<Option> options, MessageLog log,
Object consumer, Set<String> unprocessedLibs, List<Program> programList,
TaskMonitor monitor)
throws CancelledException, IOException {
TaskMonitor monitor) throws CancelledException, IOException {
Program lib = null;
int size = loadSpec.getLanguageCompilerSpec().getLanguageDescription().getSize();
@ -670,6 +670,8 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
return false;
}
if (!isLoadLibraries(options)) {
// TODO: LibraryLookupTable support currently assumes Windows for x86 (32 or 64 bit).
// Need to investigate adding support for other architectures
if (LibraryLookupTable.hasFileAndPathAndTimeStampMatch(libFile, size)) {
return true;// no need to really import it
}
@ -710,9 +712,11 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
* @param monitor the task monitor
* @param size the language size
* @param program the loaded library program
* @throws CancelledException thrown is task cancelled
*
*/
protected void createExportsFile(String libName, File libFile, MessageLog log,
TaskMonitor monitor, int size, Program program) {
TaskMonitor monitor, int size, Program program) throws CancelledException {
if (!LibraryLookupTable.libraryLookupTableFileExists(libName, size) ||
!LibraryLookupTable.hasFileAndPathAndTimeStampMatch(libFile, size)) {
@ -728,8 +732,7 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
}
}
protected LoadSpec getLoadSpec(LoadSpec loadSpec, ByteProvider provider)
throws IOException {
protected LoadSpec getLoadSpec(LoadSpec loadSpec, ByteProvider provider) throws IOException {
LanguageCompilerSpecPair pair = loadSpec.getLanguageCompilerSpec();
Collection<LoadSpec> loadSpecs = findSupportedLoadSpecs(provider);
if (loadSpecs != null) { // shouldn't be null, but protect against rogue loaders
@ -758,9 +761,8 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader
// Check based on the original program name, not on the name I gave this program
int size = program.getLanguage().getLanguageDescription().getSize();
LibrarySymbolTable symtab =
LibraryLookupTable.getSymbolTable(new File(program.getExecutablePath()).getName(),
size);
LibrarySymbolTable symtab = LibraryLookupTable.getSymbolTable(
new File(program.getExecutablePath()).getName(), size);
if (symtab == null) {
// now try based on the name given to the program
symtab = LibraryLookupTable.getSymbolTable(program.getName(), size);

View File

@ -25,9 +25,14 @@ import ghidra.framework.options.Options;
import ghidra.program.model.listing.Program;
import ghidra.util.Msg;
import ghidra.util.datastruct.FixedSizeHashMap;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
public class LibraryLookupTable {
static final String EXPORTS_FILE_EXTENSION = ".exports";
static final String ORDINAL_MAPPING_FILE_EXTENSION = ".ord";
private static final int MAX_CACHE_ITEMS = 10;
private static Map<String, LibrarySymbolTable> cacheMap =
@ -111,12 +116,12 @@ public class LibraryLookupTable {
}
public synchronized static ResourceFile createFile(Program program, boolean overwrite,
TaskMonitor monitor) throws IOException {
TaskMonitor monitor) throws IOException, CancelledException {
return createFile(program, overwrite, false, monitor);
}
public synchronized static ResourceFile createFile(Program program, boolean overwrite,
boolean inSystem, TaskMonitor monitor) throws IOException {
boolean inSystem, TaskMonitor monitor) throws IOException, CancelledException {
ResourceFile file = null;
int size = program.getLanguage().getLanguageDescription().getSize();
@ -152,22 +157,35 @@ public class LibraryLookupTable {
symTab.applyOrdinalFile(existingDefFile, false);
}
if (!monitor.isCancelled()) {
File f = file.getFile(true);
if (f == null) {
Msg.warn(LibraryLookupTable.class, "Can't write to installation directory");
}
else {
symTab.write(f, new File(program.getExecutablePath()), version);
}
monitor.checkCanceled();
File f = file.getFile(true);
if (f == null) {
Msg.warn(LibraryLookupTable.class, "Can't write to installation directory");
}
else {
symTab.write(f, new File(program.getExecutablePath()), version);
}
return file;
}
/**
* Get the symbol table associated with the DLL name.
*
* Get the symbol table associated with the DLL name. If not previously
* generated for the given dllName, it will be constructed from a .exports
* file found within the 'symbols' resource area. If a .exports file
* is not found a similarly named .ord file will be used if found. The
* .exports file is a Ghidra XML file formatted file, while the .ord file
* is produced with the Visual Studio DUMPBIN /EXPORTS command. The default
* resource area is located within the directory
* <pre>
* Ghidra/Features/Base/data/symbols/[win32|win64]
* </pre>
* Alternatively, a user specific resource directory may be used which
* is located at
* <pre>
* &lt;USER_HOME&gt;/.ghidra/&lt;.ghidraVersion&gt;/symbols/[win32|win64]
* </pre>
* The cacheMap is a static cache which always returns the same
* instance for a given DLL name.
*
@ -212,19 +230,19 @@ public class LibraryLookupTable {
}
synchronized static ResourceFile getExistingExportsFile(String dllName, int size) {
return getExistingExtensionedFile(dllName, ".exports", size);
return getExistingExtensionedFile(dllName, EXPORTS_FILE_EXTENSION, size);
}
synchronized static ResourceFile getNewExportsFile(String dllName, int size) {
return getNewExtensionedFile(dllName, ".exports", size);
return getNewExtensionedFile(dllName, EXPORTS_FILE_EXTENSION, size);
}
private static ResourceFile getNewSystemExportsFile(String name, int size) {
return getNewSystemExtensionedFile(name, ".exports", size);
return getNewSystemExtensionedFile(name, EXPORTS_FILE_EXTENSION, size);
}
synchronized static ResourceFile getExistingOrdinalFile(String dllName, int size) {
return getExistingExtensionedFile(dllName, ".ord", size);
return getExistingExtensionedFile(dllName, ORDINAL_MAPPING_FILE_EXTENSION, size);
}
synchronized static boolean hasFileAndPathAndTimeStampMatch(File libraryFile, int size) {

View File

@ -33,6 +33,7 @@ import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.model.scalar.Scalar;
import ghidra.program.model.symbol.*;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import ghidra.util.xml.GenericXMLOutputter;
import ghidra.util.xml.XmlUtilities;
@ -62,6 +63,11 @@ class LibrarySymbolTable {
private HashMap<Integer, LibraryExportedSymbol> ordMap = new HashMap<>();
private Set<String> forwards = new HashSet<>();
/**
* Construct an empty library symbol table
* @param tableName symbol table name (generally same as associated library name)
* @param size the architecture size of the DLL (e.g., 32 or 64).
*/
LibrarySymbolTable(String tableName, int size) {
this.tableName = tableName.toLowerCase();
this.size = size;
@ -69,11 +75,24 @@ class LibrarySymbolTable {
tempPurge = size <= 32 ? -1 : 0; // assume 0 purge for 64-bit
}
/**
* Construct library symbol table from an existing symbol exports file in XML format
* @param libraryFile existing symbol exports file
* @param size the architecture size of the DLL (e.g., 32 or 64).
* @throws IOException thrown if file IO error occurs
*/
LibrarySymbolTable(ResourceFile libraryFile, int size) throws IOException {
read(libraryFile, size);
}
LibrarySymbolTable(Program library, TaskMonitor monitor) {
/**
* Construct a library symbol table based upon a specified library in the
* form of a {@link Program} object.
* @param library library program
* @param monitor task monitor
* @throws CancelledException thrown if task cancelled
*/
LibrarySymbolTable(Program library, TaskMonitor monitor) throws CancelledException {
tableName = new File(library.getExecutablePath()).getName().toLowerCase();
size = library.getLanguage().getLanguageDescription().getSize();
@ -85,7 +104,8 @@ class LibrarySymbolTable {
// go through all the symbols looking for Ordinal_#
// get the number and name for the symbol
SymbolIterator iter = symTab.getSymbolIterator(SymbolUtilities.ORDINAL_PREFIX + "*", true);
while (iter.hasNext() && !monitor.isCancelled()) {
while (iter.hasNext()) {
monitor.checkCanceled();
Symbol sym = iter.next();
int ordinal = SymbolUtilities.getOrdinalValue(sym.getName());
if (ordinal == -1) {
@ -219,68 +239,99 @@ class LibrarySymbolTable {
});
}
/**
* Parse a ordinal exports file produced by Microsoft DUMPBIN /EXPORTS &lt;DLL&gt;
* It is expected to start parsing lines following the table header containing the 'ordinal' header.
* Each ordinal mapping line is expected to have the format, starting with ordinal number and
* ending with symbol name:
* &lt;ordinal&gt; &lt;other-column-data&gt; &lt;name&gt;
* The name column contains the symbol name followed by an optional demangled form. If the name starts with
* [NONAME] this will be stripped.
* @param ordinalExportsFile file path to ordinal mapping file produced by DUMPBIN /EXPORTS
* @param addMissingOrdinals if true new entries will be created for ordinal mappings
* not already existing within this symbol table, otherwise only those which already
* exist will be updated with a name if specified by mapping file.
*/
public void applyOrdinalFile(ResourceFile ordinalExportsFile, boolean addMissingOrdinals) {
try {
InputStreamReader ir = new InputStreamReader(ordinalExportsFile.getInputStream());
BufferedReader in = new BufferedReader(ir);
try (BufferedReader in =
new BufferedReader(new InputStreamReader(ordinalExportsFile.getInputStream()))) {
int ordinalColumnEndIndex = -1;
int nameColumnStartIndex = -1;
int mode = NONE;
String inString;
while ((inString = in.readLine()) != null) {
StringTokenizer tok = new StringTokenizer(inString);
if (!tok.hasMoreElements()) {
continue;
}
String str = tok.nextToken();
if (str.startsWith(";")) {
continue; // comment - skip line
}
if (mode == NONE && inString.trim().startsWith("ordinal")) {
// rely on column header labels to establish ordinal and name column start/end
int ordinalColumnStartIndex = inString.indexOf("ordinal");
if (ordinalColumnStartIndex < 0) {
continue;
}
ordinalColumnEndIndex = ordinalColumnStartIndex + 7;
if (str.equals("ordinal")) {
nameColumnStartIndex = inString.indexOf("name");
if (nameColumnStartIndex < 1) {
Msg.error(this,
"Failed to parse ordinal symbol file: " + ordinalExportsFile);
break;
}
mode = ORDINAL;
continue;
}
if (mode != ORDINAL) {
continue;
}
// must be a definition line
// ordinal Name DemangledName
inString = stripComment(inString);
if (inString.length() < nameColumnStartIndex) {
continue;
}
String ordStr = str;
// parse ordinal, if bad parse, then done
// parse ordinal from first token on line, if bad parse, then done
String ordinalStr = inString.substring(0, ordinalColumnEndIndex).trim();
int ordinal;
try {
ordinal = Integer.parseInt(ordStr);
ordinal = Integer.parseInt(ordinalStr);
}
catch (NumberFormatException exc) {
// done parsing
break;
}
if (!tok.hasMoreElements()) {
break;
String nameStr = inString.substring(nameColumnStartIndex).trim();
if (nameStr.length() == 0) {
break; // unexpected
}
String entryName = tok.nextToken();
// if [NONAME] present strip-off and use next field as name (i.e., mangled name)
if (nameStr.startsWith("[NONAME]")) {
nameStr = nameStr.substring(8).trim();
}
LibraryExportedSymbol sym = ordMap.get(new Integer(ordinal));
int index = nameStr.indexOf(' ');
if (index > 0) {
nameStr = nameStr.substring(0, index);
}
if (nameStr.length() == 0) {
continue; // skip if no name
}
LibraryExportedSymbol sym = ordMap.get(Integer.valueOf(ordinal));
if (sym != null) {
symMap.remove(sym.getName());
sym.setName(entryName);
sym.setName(nameStr);
}
else if (addMissingOrdinals) {
sym = new LibraryExportedSymbol(tableName, size, ordinal, entryName, null, null,
sym = new LibraryExportedSymbol(tableName, size, ordinal, nameStr, null, null,
tempPurge, false, null);
symMap.put(entryName, sym);
symMap.put(nameStr, sym);
ordMap.put(ordinal, sym);
}
}
in.close();
ir.close();
}
catch (FileNotFoundException e) {
return;
@ -290,92 +341,9 @@ class LibrarySymbolTable {
}
}
public void applyDefdFile(ResourceFile defFile) {
try {
InputStreamReader ir = new InputStreamReader(defFile.getInputStream());
BufferedReader in = new BufferedReader(ir);
int mode = NONE;
String inString;
while ((inString = in.readLine()) != null) {
StringTokenizer tok = new StringTokenizer(inString);
if (!tok.hasMoreElements()) {
continue;
}
String cmd = tok.nextToken();
if (cmd.startsWith(";")) {
continue;
}
if (cmd.equals("LIBRARY")) {
mode = LIBRARY;
continue;
}
if (cmd.equals("EXPORTS")) {
mode = EXPORTS;
continue;
}
if (mode != EXPORTS) {
continue;
}
// must be a definition line
// entryname[=internalName] [@Ordinal [NONAME]] [PRIVATE] [DATA]
String entryName = cmd;
// search for '='
// none, then no internalName
int eqPos = entryName.indexOf('=');
if (eqPos > 0) {
entryName = entryName.substring(0, eqPos - 1);
}
// search for '@'
// none, then no ordinalName and no NONAME
// @, might be NONAME
// optional PRIVATE and DATA
String nxtStr = tok.nextToken();
String ordStr = null;
if (nxtStr.startsWith("@")) {
if (nxtStr.length() > 1) {
ordStr = nxtStr.substring(1);
}
else {
if (!tok.hasMoreElements()) {
continue;
}
ordStr = tok.nextToken();
}
if (!tok.hasMoreElements()) {
continue;
}
nxtStr = tok.nextToken();
// if (nxtStr.equals("NONAME")) {
// }
}
int ordinal = Integer.parseInt(ordStr);
LibraryExportedSymbol sym = ordMap.get(new Integer(ordinal));
if (sym != null) {
symMap.remove(sym.getName());
sym.setName(entryName);
symMap.put(entryName, sym);
}
else {
Msg.info(this, "* " + ordinal + " : " + entryName);
}
}
in.close();
ir.close();
}
catch (FileNotFoundException e) {
return;
}
catch (IOException e) {
Msg.error(this, "Unexpected Exception: " + e.getMessage(), e);
}
private String stripComment(String str) {
int index = str.indexOf(';');
return index < 0 ? str : str.substring(0, index);
}
List<String> getForwards() {
@ -390,14 +358,14 @@ class LibrarySymbolTable {
* exist.
*/
LibraryExportedSymbol getSymbol(int ordinal) {
return ordMap.get(new Integer(ordinal));
return ordMap.get(ordinal);
}
/**
* Returns the symbol for the specified name
*
* @param symbol the name of the desired symbol
* @return
* @return symbol map entry or null if not found
*/
LibraryExportedSymbol getSymbol(String symbol) {
return symMap.get(symbol);

View File

@ -0,0 +1,83 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.opinion;
import static org.junit.Assert.*;
import java.io.File;
import org.junit.Test;
import generic.jar.ResourceFile;
import resources.ResourceManager;
public class OrdinalFileSymbolLoadTest {
protected static final String ORD_TEST_FILE = "ghidra/app/util/opinion/test.ord";
@Test
public void testORDFileParse() {
File ordFile = ResourceManager.getResourceFile(ORD_TEST_FILE);
assertNotNull(ordFile);
ResourceFile ordResourceFile = new ResourceFile(ordFile);
LibrarySymbolTable symTable = new LibrarySymbolTable("test", 32);
symTable.applyOrdinalFile(ordResourceFile, true);
LibraryExportedSymbol symbol = symTable.getSymbol(1);
assertNotNull(symbol);
assertEquals("SymbolName1", symbol.getName());
assertEquals(-1, symbol.getPurge());
assertEquals("test", symbol.getLibraryName());
symbol = symTable.getSymbol(2);
assertNotNull(symbol);
assertEquals("SymbolName2", symbol.getName());
assertEquals(-1, symbol.getPurge());
assertEquals("test", symbol.getLibraryName());
symbol = symTable.getSymbol(20);
assertNotNull(symbol);
assertEquals("SymbolName3", symbol.getName());
assertEquals(-1, symbol.getPurge());
assertEquals("test", symbol.getLibraryName());
symbol = symTable.getSymbol(30);
assertNotNull(symbol);
assertEquals("SymbolName4", symbol.getName());
assertEquals(-1, symbol.getPurge());
assertEquals("test", symbol.getLibraryName());
symbol = symTable.getSymbol(40);
assertNotNull(symbol);
assertEquals("SymbolName5", symbol.getName());
assertEquals(-1, symbol.getPurge());
assertEquals("test", symbol.getLibraryName());
symbol = symTable.getSymbol(50);
assertNull(symbol);
symbol = symTable.getSymbol(60);
assertNotNull(symbol);
assertEquals("SymbolName6", symbol.getName());
assertEquals(-1, symbol.getPurge());
assertEquals("test", symbol.getLibraryName());
}
}

View File

@ -0,0 +1,31 @@
Microsoft ordinal to symbol mapping file
produced using the DUMPBIN /EXPORTS command
This test file is intended to be representitive
of the ordinal mapping table format which whose
start is identified by the table header starting
with the word 'ordinal'
ordinal hint RVA name
1 xyz SymbolName1
2 2 abc SymbolName2
; comments
20 123 [NONAME] SymbolName3 x y z
30 3 456 [NONAME] SymbolName4 a b c
25 18 AppPolicyGetClrCompat (forwarded to kernelbase.AppPolicyGetClrCompat)
40 567 SymbolName5 (forwarded to ...
; the following symbol will be discarded since it has no name mapping
50 910 [NONAME]
60 899 [NONAME] SymbolName6 x y z
Summary
other summary information