Nixyri/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2024-10-23 05:31:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/GP-4110_James_bsim_script_cleanup--SQUASHED' into Ghidra_11.0

Browse Source
This commit is contained in:
ghidra1 2023-12-08 13:47:31 -05:00
commit ce5c48eba9
22 changed files with 538 additions and 817 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesScript.java Executable file
View File

@ -0,0 +1,174 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This script will print information about a BSim comparison of two functions. Navigate to the
// first function, run this script, navigate to the second function, and run the script again.
// The two functions can be in different (open) programs. Subsequent runs of the script will
// compare the current function and the previous function. For each comparison, the BSim signature
// weights file is chosen based on the architecture of the current function.
//@category BSim
import java.io.*;
import org.xml.sax.SAXException;
import generic.jar.ResourceFile;
import generic.lsh.vector.*;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileOptions;
import ghidra.app.decompiler.signature.SignatureResult;
import ghidra.app.script.GhidraScript;
import ghidra.app.services.ProgramManager;
import ghidra.features.bsim.query.GenSignatures;
import ghidra.program.model.address.Address;
import ghidra.program.model.lang.LanguageID;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.Program;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.NonThreadedXmlPullParserImpl;
import ghidra.xml.XmlPullParser;
import utilities.util.reflection.ReflectionUtilities;
public class CompareBSimSignaturesScript extends GhidraScript {
protected LSHVectorFactory vectorFactory;
@Override
protected void run() throws Exception {
if (isRunningHeadless()) {
popup("This script must be run from the Ghidra GUI");
return;
}
if (currentProgram == null) {
popup("This script requires an open program");
return;
}
Function func = getFunctionContaining(currentAddress);
if (func == null) {
popup("currentAddress must be in a function for this script.");
return;
}
ProgramManager programManager = state.getTool().getService(ProgramManager.class);
String lastProgramIdString = System.getProperty("ghidra.lastprogram");
if (lastProgramIdString == null) {
setProperties(func);
return;
}
Program lastProgram =
getProgram(programManager.getAllOpenPrograms(), Long.parseLong(lastProgramIdString));
if (lastProgram == null) {
setProperties(func);
return;
}
String addrstring = System.getProperty("ghidra.lastaddress");
if (addrstring == null) {
setProperties(func);
return;
}
Address addr = lastProgram.getAddressFactory().getAddress(addrstring);
Function lastfunction = lastProgram.getFunctionManager().getFunctionAt(addr);
if (lastfunction == null) {
setProperties(func);
return;
}
if (!buildLSHVectorFactory()) {
return;
}
LSHVector vec = generateVector(func, currentProgram);
LSHVector lastvector = generateVector(lastfunction, lastProgram);
VectorCompare veccompare = new VectorCompare();
double sim = lastvector.compare(vec, veccompare);
double signif = vectorFactory.calculateSignificance(veccompare);
StringBuilder buf = new StringBuilder();
buf.append("Comparison results:\n");
buf.append(lastProgram.getName());
buf.append(".");
buf.append(lastfunction.getName());
buf.append(" vs. ");
buf.append(currentProgram.getName());
buf.append(".");
buf.append(func.getName());
buf.append("\n Similarity: ");
buf.append(Double.toString(sim));
buf.append("\n Significance: ");
buf.append(Double.toString(signif));
buf.append("\n");
lastvector.compareDetail(vec, buf);
println(buf.toString());
}
private void setProperties(Function func) {
System.setProperty("ghidra.lastprogram",
Long.toString(currentProgram.getUniqueProgramID()));
String addrstring = func.getEntryPoint().toString();
System.setProperty("ghidra.lastaddress", addrstring);
}
private LSHVector generateVector(Function f, Program program) {
DecompInterface decompiler = new DecompInterface();
try {
decompiler.setOptions(new DecompileOptions());
decompiler.toggleSyntaxTree(false);
decompiler.setSignatureSettings(vectorFactory.getSettings());
if (!decompiler.openProgram(program)) {
println("Unable to initalize the Decompiler interface");
println(decompiler.getLastMessage());
return null;
}
SignatureResult sigres = decompiler.generateSignatures(f, false, 10, null);
LSHVector vec = vectorFactory.buildVector(sigres.features);
return vec;
}
finally {
decompiler.closeProgram();
decompiler.dispose();
}
}
private Program getProgram(Program[] progarray, long id) {
if (progarray == null) {
return null;
}
for (Program prog : progarray) {
if (prog.getUniqueProgramID() == id) {
return prog;
}
}
return null;
}
protected static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile)
throws FileNotFoundException, IOException, SAXException {
InputStream input = weightsFile.getInputStream();
XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser",
SpecXmlUtils.getXmlHandler(), false);
vectorFactory.readWeights(parser);
input.close();
}
protected boolean buildLSHVectorFactory() {
vectorFactory = new WeightedLSHCosineVectorFactory();
try {
LanguageID id = currentProgram.getLanguageID();
ResourceFile defaultWeightsFile = GenSignatures.getWeightsFile(id, id);
readWeights(vectorFactory, defaultWeightsFile);
}
catch (IOException | SAXException e) {
printerr("Unexpected Exception...");
printerr(ReflectionUtilities.stackTraceToString(e));
}
return true;
}
}

63
Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesSpecifyWeightsScript.java Executable file
View File

@ -0,0 +1,63 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This script will print information about a BSim comparison of two functions. Navigate to the
// first function, run this script, navigate to the second function, and run the script again.
// The two functions can be in different (open) programs. Subsequent runs of the script will
// compare the current function and the previous function. For each comparison, the user selects
// which weights file to use.
//@category BSim
import java.io.IOException;
import org.xml.sax.SAXException;
import generic.jar.ResourceFile;
import generic.lsh.vector.WeightedLSHCosineVectorFactory;
import ghidra.features.base.values.GhidraValuesMap;
import ghidra.framework.Application;
import ghidra.util.exception.CancelledException;
import utilities.util.reflection.ReflectionUtilities;
public class CompareBSimSignaturesSpecifyWeightsScript extends CompareBSimSignaturesScript {
private static final String[] WEIGHT_FILES = { "lshweights_nosize.xml", "lshweights_32.xml",
"lshweights_64.xml", "lshweights_cpool.xml" };
@Override
protected boolean buildLSHVectorFactory() {
vectorFactory = new WeightedLSHCosineVectorFactory();
GhidraValuesMap values = new GhidraValuesMap();
values.defineChoice("weights file", WEIGHT_FILES[0], WEIGHT_FILES);
try {
askValues("Select Weights File", "Select Weights File", values);
}
catch (CancelledException e) {
return false;
}
String weightsFile = values.getChoice("weights file");
ResourceFile defaultWeightsFile = Application.findDataFileInAnyModule(weightsFile);
try {
readWeights(vectorFactory, defaultWeightsFile);
}
catch (IOException | SAXException e) {
printerr("Unexpected Exception...");
printerr(ReflectionUtilities.stackTraceToString(e));
return false;
}
return true;
}
}

9
Ghidra/Features/BSim/ghidra_scripts/CompareExecutables.java → Ghidra/Features/BSim/ghidra_scripts/CompareExecutablesScript.java
View File

@ -25,19 +25,18 @@ import ghidra.features.bsim.query.FunctionDatabase;
import ghidra.features.bsim.query.client.*;
import ghidra.features.bsim.query.description.ExecutableRecord;
public class CompareExecutables extends GhidraScript {
public class CompareExecutablesScript extends GhidraScript {
private ExecutableComparison exeCompare;
@Override
protected void run() throws Exception {
URL url = BSimClientFactory.deriveBSimURL("ghidra://localhost/repo");
try (FunctionDatabase database = BSimClientFactory.buildClient(url, true)) {
// FileScoreCaching cache = new FileScoreCaching("/tmp/test_scorecacher.txt");
TableScoreCaching cache = new TableScoreCaching(database);
exeCompare =
new ExecutableComparison(database, 1000000, "11111111111111111111111111111111",
cache,
monitor);
exeCompare = new ExecutableComparison(database, 1000000,
"11111111111111111111111111111111", cache, monitor);
// Specify the list of executables to compare by giving their md5 hash
// exeCompare.addExecutable("22222222222222222222222222222222"); // 32 hex-digit string
// exeCompare.addExecutable("33333333333333333333333333333333");

148
Ghidra/Features/BSim/ghidra_scripts/CompareSignatures.java
View File

@ -1,148 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Use the decompiler to generate a signature for the current function containing the cursor
// If we remember the last signature that was generated, compare this signature with
// the last signature and print the similarity
//@category BSim
import java.io.*;
import org.xml.sax.SAXException;
import generic.jar.ResourceFile;
import generic.lsh.vector.*;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileOptions;
import ghidra.app.decompiler.signature.SignatureResult;
import ghidra.app.script.GhidraScript;
import ghidra.app.services.ProgramManager;
import ghidra.features.bsim.query.GenSignatures;
import ghidra.program.model.address.Address;
import ghidra.program.model.lang.LanguageID;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.Program;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.NonThreadedXmlPullParserImpl;
import ghidra.xml.XmlPullParser;
public class CompareSignatures extends GhidraScript {
private LSHVectorFactory vectorFactory;
private LSHVector generateVector(Function f, Program program) {
DecompInterface decompiler = new DecompInterface();
decompiler.setOptions(new DecompileOptions());
decompiler.toggleSyntaxTree(false);
decompiler.setSignatureSettings(vectorFactory.getSettings());
if (!decompiler.openProgram(program)) {
println("Unable to initalize the Decompiler interface");
println(decompiler.getLastMessage());
return null;
}
SignatureResult sigres = decompiler.generateSignatures(f, false, 10, null);
LSHVector vec = vectorFactory.buildVector(sigres.features);
return vec;
}
private Program getProgram(Program[] progarray, String name) {
if ((name == null) || (progarray == null)) {
return null;
}
for (Program prog : progarray) {
if (name.equals(prog.getName())) {
return prog;
}
}
return null;
}
private static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile)
throws FileNotFoundException, IOException, SAXException {
InputStream input = weightsFile.getInputStream();
XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser",
SpecXmlUtils.getXmlHandler(), false);
vectorFactory.readWeights(parser);
input.close();
}
private void buildLSHVectorFactory() {
vectorFactory = new WeightedLSHCosineVectorFactory();
try {
LanguageID id = currentProgram.getLanguageID();
ResourceFile defaultWeightsFile = GenSignatures.getWeightsFile(id, id);
readWeights(vectorFactory, defaultWeightsFile);
}
catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (SAXException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
protected void run() throws Exception {
Function func = this.getFunctionContaining(this.currentAddress);
if (func == null) {
return;
}
buildLSHVectorFactory();
LSHVector vec = generateVector(func, currentProgram);
ProgramManager programManager = state.getTool().getService(ProgramManager.class);
Program[] progarray = programManager.getAllOpenPrograms();
String lastprogram_string = System.getProperty("ghidra.lastprogram");
Program lastprogram = getProgram(progarray, lastprogram_string);
VectorCompare veccompare = new VectorCompare();
if (lastprogram != null) {
String addrstring = System.getProperty("ghidra.lastaddress");
if (addrstring != null) {
Address addr = lastprogram.getAddressFactory().getAddress(addrstring);
Function lastfunction = lastprogram.getFunctionManager().getFunctionAt(addr);
if (lastfunction != null) {
LSHVector lastvector = generateVector(lastfunction, lastprogram);
double sim = lastvector.compare(vec, veccompare);
double signif = vectorFactory.calculateSignificance(veccompare);
StringBuilder buf = new StringBuilder();
buf.append("Comparison results:\n");
buf.append(lastprogram.getName());
buf.append(".");
buf.append(lastfunction.getName());
buf.append(" vs. ");
buf.append(currentProgram.getName());
buf.append(".");
buf.append(func.getName());
buf.append("\n Similarity: ");
buf.append(Double.toString(sim));
buf.append("\n Significance: ");
buf.append(Double.toString(signif));
buf.append("\n");
lastvector.compareDetail(vec, buf);
println(buf.toString());
}
}
}
System.setProperty("ghidra.lastprogram", currentProgram.getName());
String addrstring = func.getEntryPoint().toString();
System.setProperty("ghidra.lastaddress", addrstring);
}
}

155
Ghidra/Features/BSim/ghidra_scripts/CompareSignaturesSpecifyWeights.java
View File

@ -1,155 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Compare the BSim feature vectors of two functions.
//@category BSim
import java.io.*;
import org.xml.sax.SAXException;
import generic.jar.ResourceFile;
import generic.lsh.vector.*;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileOptions;
import ghidra.app.decompiler.signature.SignatureResult;
import ghidra.app.script.GhidraScript;
import ghidra.app.services.ProgramManager;
import ghidra.framework.Application;
import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.Program;
import ghidra.util.exception.CancelledException;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.NonThreadedXmlPullParserImpl;
import ghidra.xml.XmlPullParser;
public class CompareSignaturesSpecifyWeights extends GhidraScript {
private static final String DEFAULT_LSH_WEIGHTS_FILE = "lshweights_nosize.xml";
private LSHVectorFactory vectorFactory;
private LSHVector generateVector(Function f, Program program) {
DecompInterface decompiler = new DecompInterface();
decompiler.setOptions(new DecompileOptions());
decompiler.setSignatureSettings(vectorFactory.getSettings());
decompiler.toggleSyntaxTree(false);
if (!decompiler.openProgram(program)) {
println("Unable to initalize the Decompiler interface");
println(decompiler.getLastMessage());
return null;
}
SignatureResult sigres = decompiler.generateSignatures(f, false, 10, null);
LSHVector vec = vectorFactory.buildVector(sigres.features);
return vec;
}
private static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile)
throws FileNotFoundException, IOException, SAXException {
InputStream input = weightsFile.getInputStream();
XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser",
SpecXmlUtils.getXmlHandler(), false);
vectorFactory.readWeights(parser);
input.close();
}
private boolean buildLSHVectorFactory() {
vectorFactory = new WeightedLSHCosineVectorFactory();
try {
String weightsFile =
askString("Enter weights file name", "weights file", DEFAULT_LSH_WEIGHTS_FILE);
ResourceFile defaultWeightsFile = Application.findDataFileInAnyModule(weightsFile);
readWeights(vectorFactory, defaultWeightsFile);
}
catch (FileNotFoundException e) {
e.printStackTrace();
return false;
}
catch (IOException e) {
e.printStackTrace();
return false;
}
catch (SAXException e) {
e.printStackTrace();
return false;
}
catch (CancelledException e) {
return false;
}
return true;
}
private Program getProgram(Program[] progarray, String name) {
if ((name == null) || (progarray == null)) {
return null;
}
for (Program prog : progarray) {
if (name.equals(prog.getName())) {
return prog;
}
}
return null;
}
@Override
protected void run() throws Exception {
Function func = this.getFunctionContaining(this.currentAddress);
if (func == null) {
return;
}
if (!buildLSHVectorFactory()) {
return;
}
LSHVector vec = generateVector(func, currentProgram);
ProgramManager programManager = state.getTool().getService(ProgramManager.class);
Program[] progarray = programManager.getAllOpenPrograms();
String lastprogram_string = System.getProperty("ghidra.lastprogram");
Program lastprogram = getProgram(progarray, lastprogram_string);
VectorCompare veccompare = new VectorCompare();
if (lastprogram != null) {
String addrstring = System.getProperty("ghidra.lastaddress");
if (addrstring != null) {
Address addr = lastprogram.getAddressFactory().getAddress(addrstring);
Function lastfunction = lastprogram.getFunctionManager().getFunctionAt(addr);
if (lastfunction != null) {
LSHVector lastvector = generateVector(lastfunction, lastprogram);
double sim = lastvector.compare(vec, veccompare);
double signif = vectorFactory.calculateSignificance(veccompare);
StringBuilder buf = new StringBuilder();
buf.append("Comparison results:\n");
buf.append(lastprogram.getName());
buf.append(".");
buf.append(lastfunction.getName());
buf.append(" vs. ");
buf.append(currentProgram.getName());
buf.append(".");
buf.append(func.getName());
buf.append("\n Similarity: ");
buf.append(Double.toString(sim));
buf.append("\n Significance: ");
buf.append(Double.toString(signif));
buf.append("\n");
lastvector.compareDetail(vec, buf);
println(buf.toString());
}
}
}
System.setProperty("ghidra.lastprogram", currentProgram.getName());
String addrstring = func.getEntryPoint().toString();
System.setProperty("ghidra.lastaddress", addrstring);
}
}

72
Ghidra/Features/BSim/ghidra_scripts/DebugSignatures.java
View File

@ -1,72 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.List;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileOptions;
import ghidra.app.decompiler.signature.DebugSignature;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.lang.Language;
import ghidra.program.model.listing.Function;
public class DebugSignatures extends GhidraScript {
private static final int SIGNATURE_SETTINGS = 0x45;
@Override
protected void run() throws Exception {
Function func = this.getFunctionContaining(this.currentAddress);
if (func == null) {
popup("No function selected!");
return;
}
DecompInterface decompiler = new DecompInterface();
decompiler.setOptions(new DecompileOptions());
decompiler.toggleSyntaxTree(false);
decompiler.setSignatureSettings(SIGNATURE_SETTINGS);
if (!decompiler.openProgram(this.currentProgram)) {
println("Unable to initalize the Decompiler interface");
println(decompiler.getLastMessage());
return;
}
Language language = this.currentProgram.getLanguage();
List<DebugSignature> sigres = decompiler.debugSignatures(func, 10, null);
StringBuffer buf = new StringBuffer();
buf.append("\nFunction: ");
buf.append(func.getName());
buf.append("\nentry: ");
buf.append(func.getEntryPoint().toString());
buf.append("\n\n");
if (sigres == null) {
printf("Null sigres!\n");
}
else {
for (int i = 0; i < sigres.size(); ++i) {
sigres.get(i).printRaw(language, buf);
buf.append("\n");
}
}
printf("%s\n", buf.toString());
decompiler.closeProgram();
decompiler.dispose();
}
}

37
Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.java Executable file
View File

@ -0,0 +1,37 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Generate the BSim signature for the function currently containing the cursor
// and dump the feature hashes and debug information to the console.
import java.util.List;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.signature.DebugSignature;
import ghidra.program.model.lang.Language;
import ghidra.program.model.listing.Function;
public class DumpBSimDebugSignaturesScript extends DumpBSimSignaturesScript {
@Override
protected void getSignatures(StringBuffer buf, DecompInterface decompiler, Function func) {
List<DebugSignature> sigres = decompiler.debugSignatures(func, 10, null);
Language language = currentProgram.getLanguage();
for (int i = 0; i < sigres.size(); ++i) {
sigres.get(i).printRaw(language, buf);
buf.append("\n");
}
}
}

48
Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.py Executable file
View File

@ -0,0 +1,48 @@
## ###
# IP: GHIDRA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
# Generate the BSim signature for the function at the current address,
# then dump the signature hashes and debug information to the console
# @category: BSim.python
import ghidra.app.decompiler.DecompInterface as DecompInterface
import ghidra.app.decompiler.DecompileOptions as DecompileOptions
def processFunction(func):
decompiler = DecompInterface()
try:
options = DecompileOptions()
decompiler.setOptions(options)
decompiler.toggleSyntaxTree(False)
decompiler.setSignatureSettings(0x4d)
if not decompiler.openProgram(currentProgram):
print "Unable to initialize the Decompiler interface!"
print "%s" % decompiler.getLastMessage()
return
language = currentProgram.getLanguage()
sigres = decompiler.debugSignatures(func,10,None)
for i,res in enumerate(sigres):
buf = java.lang.StringBuffer()
sigres.get(i).printRaw(language,buf)
print "%s" % buf.toString()
finally:
decompiler.closeProgram()
decompiler.dispose()
func = currentProgram.getFunctionManager().getFunctionContaining(currentAddress)
if func is None:
print "no function at current address"
else:
processFunction(func)

80
Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.java Executable file
View File

@ -0,0 +1,80 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Generate the BSim signature for the function currently
// containing the cursor and dump the feature hashes to the console.
//@category BSim
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileOptions;
import ghidra.app.decompiler.signature.SignatureResult;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.listing.Function;
public class DumpBSimSignaturesScript extends GhidraScript {
protected static final int SIGNATURE_SETTINGS = 0x4d;
@Override
public void run() throws Exception {
if (isRunningHeadless()) {
popup("This script must be run in the Ghidra GUI");
return;
}
if (currentProgram == null) {
popup("This script requires an open program");
return;
}
Function func = getFunctionContaining(currentAddress);
if (func == null) {
popup("No function selected!");
return;
}
DecompInterface decompiler = new DecompInterface();
try {
decompiler.setOptions(new DecompileOptions());
decompiler.toggleSyntaxTree(false);
decompiler.setSignatureSettings(SIGNATURE_SETTINGS);
if (!decompiler.openProgram(currentProgram)) {
println("Unable to initalize the Decompiler interface");
println(decompiler.getLastMessage());
return;
}
StringBuffer buf = new StringBuffer();
buf.append("\nFunction: ");
buf.append(func.getName());
buf.append("\nentry: ");
buf.append(func.getEntryPoint().toString());
buf.append("\n\n");
getSignatures(buf, decompiler, func);
printf("%s\n", buf.toString());
}
finally {
decompiler.closeProgram();
decompiler.dispose();
}
}
protected void getSignatures(StringBuffer buf, DecompInterface decompiler, Function func) {
SignatureResult sigres = decompiler.generateSignatures(func, false, 10, null);
for (int feature : sigres.features) {
buf.append(Integer.toHexString(feature));
buf.append("\n");
}
}
}

42
Ghidra/Features/BSim/ghidra_scripts/DumpDebugSignatures.py → Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.py
View File

@ -13,36 +13,38 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
# Use the decompiler to generate signatures for the function at the current address, then dump the
# signature hashes and debug information to the console
# Generate the BSim signature for the function at the current address, then dump the
# signature hashes to the console
# @category: BSim.python
import ghidra.app.decompiler.tracking.DecompInterfaceTracking as DecompInterfaceTracking
import ghidra.app.decompiler.DecompInterface as DecompInterface
import ghidra.app.decompiler.DecompileOptions as DecompileOptions
import generic.lsh.vector.WeightedLSHCosineVectorFactory as WeightedLSHCosineVectorFactory
import ghidra.query.GenSignatures as GenSignatures
import ghidra.features.bsim.query.GenSignatures as GenSignatures
import ghidra.xml.NonThreadedXmlPullParserImpl as NonThreadedXmlPullParserImpl
import ghidra.util.xml.SpecXmlUtils as SpecXmlUtils
def processFunction(func):
decompiler = DecompInterfaceTracking()
options = DecompileOptions()
decompiler.setOptions(options)
decompiler.toggleSyntaxTree(False)
decompiler.setSignatureSettings(getSettings())
if not decompiler.openProgram(currentProgram):
print "Unable to initialize the Decompiler interface!"
print "%s" % decompiler.getLastMessage()
return
language = currentProgram.getLanguage()
sigres = decompiler.debugSignatures(func,10,None)
for i,res in enumerate(sigres):
decompiler = ghidra.app.decompiler.DecompInterface()
try:
options = ghidra.app.decompiler.DecompileOptions()
decompiler.setOptions(options)
decompiler.toggleSyntaxTree(False)
decompiler.setSignatureSettings(getSettings())
if not decompiler.openProgram(currentProgram):
print "Unable to initialize the Decompiler interface!"
print "%s" % decompiler.getLastMessage()
return
sigres = decompiler.generateSignatures(func, False, 10, None)
buf = java.lang.StringBuffer()
sigres.get(i).printRaw(language,buf)
print "%s" % buf.toString()
decompiler.closeProgram()
decompiler.dispose()
for i,res in enumerate(sigres.features):
buf.append(java.lang.Integer.toHexString(sigres.features[i]))
buf.append("\n")
print buf.toString()
finally:
decompiler.closeProgram()
decompiler.dispose()
def getSettings():
vectorFactory = WeightedLSHCosineVectorFactory()

115
Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.java
View File

@ -1,115 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Use the decompiler to generate signatures for the function currently containing the cursor
// and dump the signature hashes to the console
//@category BSim
import java.io.*;
import java.util.List;
import org.xml.sax.SAXException;
import generic.jar.ResourceFile;
import generic.lsh.vector.LSHVectorFactory;
import generic.lsh.vector.WeightedLSHCosineVectorFactory;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileOptions;
import ghidra.app.decompiler.signature.DebugSignature;
import ghidra.app.decompiler.signature.SignatureResult;
import ghidra.app.script.GhidraScript;
import ghidra.features.bsim.query.GenSignatures;
import ghidra.program.model.lang.Language;
import ghidra.program.model.lang.LanguageID;
import ghidra.program.model.listing.Function;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.NonThreadedXmlPullParserImpl;
import ghidra.xml.XmlPullParser;
public class DumpSignatures extends GhidraScript {
private LSHVectorFactory vectorFactory;
@Override
public void run() throws Exception {
Function func = this.getFunctionContaining(this.currentAddress);
if (func == null) {
return;
}
buildLSHVectorFactory();
boolean debug = false;
DecompInterface decompiler = new DecompInterface();
decompiler.setOptions(new DecompileOptions());
decompiler.setSignatureSettings(vectorFactory.getSettings());
decompiler.toggleSyntaxTree(false);
if (!decompiler.openProgram(this.currentProgram)) {
println("Unable to initalize the Decompiler interface");
println(decompiler.getLastMessage());
return;
}
if (!debug) {
SignatureResult sigres = decompiler.generateSignatures(func, false, 10, null);
StringBuffer buf = new StringBuffer("\n");
for (int feature : sigres.features) {
buf.append(Integer.toHexString(feature));
buf.append("\n");
}
println(buf.toString());
}
else {
Language language = this.currentProgram.getLanguage();
List<DebugSignature> sigres = decompiler.debugSignatures(func, 10, null);
StringBuffer buf = new StringBuffer("\n");
for (int i = 0; i < sigres.size(); ++i) {
sigres.get(i).printRaw(language, buf);
buf.append("\n");
}
println(buf.toString());
}
decompiler.closeProgram();
decompiler.dispose();
}
private static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile)
throws FileNotFoundException, IOException, SAXException {
InputStream input = weightsFile.getInputStream();
XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser",
SpecXmlUtils.getXmlHandler(), false);
vectorFactory.readWeights(parser);
input.close();
}
private void buildLSHVectorFactory() {
vectorFactory = new WeightedLSHCosineVectorFactory();
try {
LanguageID id = currentProgram.getLanguageID();
ResourceFile defaultWeightsFile = GenSignatures.getWeightsFile(id, id);
readWeights(vectorFactory, defaultWeightsFile);
}
catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (SAXException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

61
Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.py
View File

@ -1,61 +0,0 @@
## ###
# IP: GHIDRA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
# Use the decompiler to generate signatures for the function at the current address, then dump the
# signature hashes to the console
# @category: BSim.python
import ghidra.app.decompiler.tracking.DecompInterfaceTracking as DecompInterfaceTracking
import ghidra.app.decompiler.DecompileOptions as DecompileOptions
import generic.lsh.vector.WeightedLSHCosineVectorFactory as WeightedLSHCosineVectorFactory
import ghidra.query.GenSignatures as GenSignatures
import ghidra.xml.NonThreadedXmlPullParserImpl as NonThreadedXmlPullParserImpl
import ghidra.util.xml.SpecXmlUtils as SpecXmlUtils
def processFunction(func):
decompiler = ghidra.app.decompiler.tracking.DecompInterfaceTracking()
options = ghidra.app.decompiler.DecompileOptions()
decompiler.setOptions(options)
decompiler.toggleSyntaxTree(False)
decompiler.setSignatureSettings(getSettings())
if not decompiler.openProgram(currentProgram):
print "Unable to initialize the Decompiler interface!"
print "%s" % decompiler.getLastMessage()
return
sigres = decompiler.generateSignatures(func, False, 10, None)
buf = java.lang.StringBuffer()
for i,res in enumerate(sigres.features):
buf.append(java.lang.Integer.toHexString(sigres.features[i]))
buf.append("\n")
print buf.toString()
decompiler.closeProgram()
decompiler.dispose()
def getSettings():
vectorFactory = WeightedLSHCosineVectorFactory()
id = currentProgram.getLanguageID()
defaultWeightsFile = GenSignatures.getWeightsFile(id,id)
input = defaultWeightsFile.getInputStream()
parser = NonThreadedXmlPullParserImpl(input,"Vector weights parser", SpecXmlUtils.getXmlHandler(),False)
vectorFactory.readWeights(parser)
input.close()
return vectorFactory.getSettings()
func = currentProgram.getFunctionManager().getFunctionContaining(currentAddress)
if func is None:
print "no function at current address"
else:
processFunction(func)

17
Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.java → Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.java
View File

@ -26,24 +26,22 @@ import ghidra.features.bsim.query.protocol.SimilarityVectorResult;
import ghidra.program.database.symbol.FunctionSymbol;
import ghidra.program.model.listing.*;
public class ExampleOverviewQueryScript extends GhidraScript {
private static final double SIMILARITY_BOUND = 0.7;
private static final double SIGNIFICANCE_BOUND = 0.0;
public class ExampleOverviewQuery extends GhidraScript {
private static final double SIMILARITY_BOUND = 0.7;
private static final double SIGNIFICANCE_BOUND = 0.0;
@Override
protected void run() throws Exception {
Program queryingProgram = currentProgram;
HashSet<FunctionSymbol> funcsToQuery = new HashSet<>();
FunctionIterator fIter = queryingProgram.getFunctionManager().getFunctionsNoStubs(true);
for (Function func : fIter){
for (Function func : fIter) {
funcsToQuery.add((FunctionSymbol) func.getSymbol());
}
}
SFOverviewInfo overviewInfo = new SFOverviewInfo(funcsToQuery);
overviewInfo.setSimilarityThreshold(SIMILARITY_BOUND);
overviewInfo.setSimilarityThreshold(SIMILARITY_BOUND);
overviewInfo.setSignificanceThreshold(SIGNIFICANCE_BOUND);
try (SimilarFunctionQueryService queryService =
new SimilarFunctionQueryService(queryingProgram)) {
String DATABASE_URL = askString("Enter database URL", "URL:");
@ -65,5 +63,4 @@ public class ExampleOverviewQuery extends GhidraScript {
printf("%s\n", buf.toString());
}
}
}

6
Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.py → Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.py
View File

@ -16,8 +16,8 @@
# Example of how to perform an overview query in a script
# @category BSim.python
import ghidra.query.facade.SFOverviewInfo as SFOverviewInfo
import ghidra.query.facade.SimilarFunctionQueryService as SimilarFunctionQueryService
import ghidra.features.bsim.query.facade.SFOverviewInfo as SFOverviewInfo
import ghidra.features.bsim.query.facade.SimilarFunctionQueryService as SimilarFunctionQueryService
import java.util.HashSet
SIMILARITY_BOUND = 0.7
@ -37,7 +37,7 @@ DB_URL = askString("Enter database URL", "URL:")
queryService.initializeDatabase(DB_URL)
vectorFactory = queryService.getLSHVectorFactory()
overviewResults = queryService.overviewSimilarFunctions(overviewInfo, monitor)
overviewResults = queryService.overviewSimilarFunctions(overviewInfo, None, monitor)
for result in overviewResults.result:
print "Name: %s" % result.getBase().getFunctionName()

48
Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClient.java → Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClientScript.java
View File

@ -20,18 +20,51 @@ import java.io.StringWriter;
import java.net.URL;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import ghidra.app.script.GhidraScript;
import ghidra.features.bsim.query.BSimClientFactory;
import ghidra.features.bsim.query.FunctionDatabase;
import ghidra.features.base.values.GhidraValuesMap;
import ghidra.features.bsim.query.*;
import ghidra.features.bsim.query.description.*;
import ghidra.features.bsim.query.protocol.*;
import ghidra.util.MessageType;
import ghidra.util.Msg;
public class ExampleQueryClient extends GhidraScript {
public class ExampleQueryClientScript extends GhidraScript {
private static final String URL = "URL";
private static final String EXECUTABLE_NAME = "Executable Name";
private static final String FUNCTION_NAME = "Function Name";
@Override
protected void run() throws Exception {
URL url = BSimClientFactory.deriveBSimURL("ghidra://localhost/repo");
GhidraValuesMap values = new GhidraValuesMap();
values.defineString(URL);
values.defineString(EXECUTABLE_NAME);
values.defineString(FUNCTION_NAME);
values.setValidator((valueMap, status) -> {
String url = valueMap.getString(URL);
if (StringUtils.isBlank(url)) {
status.setStatusText(URL + " cannot be empty!", MessageType.ERROR);
return false;
}
String exe = valueMap.getString(EXECUTABLE_NAME);
if (StringUtils.isBlank(exe)) {
status.setStatusText(EXECUTABLE_NAME + " cannot be empty!", MessageType.ERROR);
return false;
}
String func = valueMap.getString(FUNCTION_NAME);
if (StringUtils.isBlank(func)) {
status.setStatusText(FUNCTION_NAME + " cannot be empty!", MessageType.ERROR);
return false;
}
return true;
});
askValues("BSim Query Info", "BSim Query Info", values);
URL url = BSimClientFactory.deriveBSimURL(values.getString(URL));
try (FunctionDatabase client = BSimClientFactory.buildClient(url, false)) {
if (!client.initialize()) {
Msg.error(this, "Unable to connect to server");
@ -45,15 +78,16 @@ public class ExampleQueryClient extends GhidraScript {
write.flush();
QueryName exequery = new QueryName();
exequery.spec.exename = "libdocdoxygenplugin.so";
exequery.spec.exename = values.getString(EXECUTABLE_NAME);
ResponseName respname = exequery.execute(client);
if (respname == null) {
Msg.error(this, client.getLastError());
return;
}
ExecutableRecord erec = respname.manage.getExecutableRecordSet().first();
FunctionDescription funcrec =
respname.manage.findFunctionByName("DocDoxygenPlugin::createCatalog", erec);
respname.manage.findFunctionByName(values.getString(FUNCTION_NAME), erec);
QueryChildren childquery = new QueryChildren();
childquery.md5sum = funcrec.getExecutableRecord().getMd5();
@ -76,7 +110,7 @@ public class ExampleQueryClient extends GhidraScript {
}
}
write.flush();
Msg.info(this, write.toString());
printf("%s", write.toString());
}
}

43
Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.java
View File

@ -50,24 +50,31 @@ public class GenerateSignatures extends GhidraScript {
final LSHVectorFactory vectorFactory = FunctionDatabase.generateLSHVectorFactory();
final GenSignatures gensig = new GenSignatures(true);
final String templatename =
askString("GenerateSignatures:", "Database template", "medium_nosize");
final Configuration config = FunctionDatabase.loadConfigurationTemplate(templatename);
vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings);
gensig.setVectorFactory(vectorFactory);
gensig.addExecutableCategories(config.info.execats);
gensig.addFunctionTags(config.info.functionTags);
gensig.addDateColumnName(config.info.dateColumnName);
final String repo = "ghidra://localhost/" + state.getProject().getName();
final String path = GenSignatures.getPathFromDomainFile(currentProgram);
gensig.openProgram(this.currentProgram, null, null, null, repo, path);
final FunctionManager fman = currentProgram.getFunctionManager();
final Iterator<Function> iter = fman.getFunctions(true);
gensig.scanFunctions(iter, fman.getFunctionCount(), monitor);
final FileWriter fwrite = new FileWriter(file);
final DescriptionManager manager = gensig.getDescriptionManager();
manager.saveXml(fwrite);
fwrite.close();
try {
final String templatename =
askString("GenerateSignatures:", "Database template", "medium_nosize");
final Configuration config = FunctionDatabase.loadConfigurationTemplate(templatename);
vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings);
gensig.setVectorFactory(vectorFactory);
gensig.addExecutableCategories(config.info.execats);
gensig.addFunctionTags(config.info.functionTags);
gensig.addDateColumnName(config.info.dateColumnName);
final String repo = "ghidra://localhost/" + state.getProject().getName();
final String path = GenSignatures.getPathFromDomainFile(currentProgram);
gensig.openProgram(this.currentProgram, null, null, null, repo, path);
final FunctionManager fman = currentProgram.getFunctionManager();
final Iterator<Function> iter = fman.getFunctions(true);
gensig.scanFunctions(iter, fman.getFunctionCount(), monitor);
try (FileWriter fwrite = new FileWriter(file)) {
final DescriptionManager manager = gensig.getDescriptionManager();
manager.saveXml(fwrite);
fwrite.close();
}
}
finally {
gensig.dispose();
}
}
}

4
Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.py
View File

@ -18,8 +18,8 @@
import java.lang.System as System
import java.io.File as File
import ghidra.query.FunctionDatabase as FunctionDatabase
import ghidra.query.GenSignatures as GenSignatures
import ghidra.features.bsim.query.FunctionDatabase as FunctionDatabase
import ghidra.features.bsim.query.GenSignatures as GenSignatures
import java.io.FileWriter as FileWriter
def run():

19
Ghidra/Features/BSim/ghidra_scripts/QueryFunction.java
View File

@ -25,27 +25,24 @@ import ghidra.features.bsim.query.description.*;
import ghidra.features.bsim.query.protocol.*;
import ghidra.program.model.listing.Function;
public class QueryFunction extends GhidraScript {
//GenSignatures gensig;
//FunctionDatabase database;
private static final int MATCHES_PER_FUNC = 10;
private static final double SIMILARITY_BOUND = 0.7;
private static final double CONFIDENCE_BOUND = 0.0;
@Override
public void run() throws Exception {
if (currentProgram == null) {
return;
}
Function func = this.getFunctionContaining(this.currentAddress);
if (func == null){
popup("No function selected!");
return;
}
String DATABASE_URL = askString("Enter Database URL", "URL");
Function func = this.getFunctionContaining(this.currentAddress);
if (func == null) {
popup("No function selected!");
return;
}
String DATABASE_URL = askString("Enter Database URL", "URL");
URL url = BSimClientFactory.deriveBSimURL(DATABASE_URL);
try (FunctionDatabase database = BSimClientFactory.buildClient(url, false)) {
if (!database.initialize()) {

6
Ghidra/Features/BSim/ghidra_scripts/QueryFunction.py
View File

@ -16,9 +16,9 @@
# Example of performing a BSim query on a single function
# @category BSim.python
import ghidra.query.BSimClientFactory as BSimClientFactory
import ghidra.query.GenSignatures as GenSignatures
import ghidra.query.protocol.QueryNearest as QueryNearest
import ghidra.features.bsim.query.BSimClientFactory as BSimClientFactory
import ghidra.features.bsim.query.GenSignatures as GenSignatures
import ghidra.features.bsim.query.protocol.QueryNearest as QueryNearest
MATCHES_PER_FUNC = 100
SIMILARITY_BOUND = 0.7

20
Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.java
View File

@ -123,7 +123,7 @@ public class QueryWithFiltersScript extends GhidraScript {
// the items are "AND'd together.
//
// ie: "The compiler cannot equal windows AND the compiler cannot equal foo_compiler".
addBsimFilter(new CompilerBSimFilterType(), "windows, foo_compiler");
addBsimFilter(new NotCompilerBSimFilterType(), "windows, foo_compiler");
//connect to the database
try {
@ -149,8 +149,7 @@ public class QueryWithFiltersScript extends GhidraScript {
// set returned from the previous query.
addBsimFilter(new Md5BSimFilterType(), currentProgram.getExecutableMD5());
addBsimFilter(new CompilerBSimFilterType(), "gcc");
addBsimFilter(new FunctionTagBSimFilterType("KNOWN_LIBRARY", queryService),
"false");
//addBsimFilter(new FunctionTagBSimFilterType("KNOWN_LIBRARY", queryService), "false");
// Apply the filters and print results.
List<BSimMatchResult> filteredRows =
@ -194,7 +193,7 @@ public class QueryWithFiltersScript extends GhidraScript {
* @throws CancelledException if the user cancelled the operation
*/
private List<BSimMatchResult> executeQuery(SFQueryInfo qInfo)
throws QueryDatabaseException, CancelledException {
throws QueryDatabaseException, CancelledException {
SFQueryResult queryResults = queryService.querySimilarFunctions(qInfo, null, monitor);
List<BSimMatchResult> resultRows =
@ -228,8 +227,8 @@ public class QueryWithFiltersScript extends GhidraScript {
@Override
public boolean test(Program t, FunctionDescription u) {
return queryService.getLSHVectorFactory()
.getSelfSignificance(
u.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND;
.getSelfSignificance(
u.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND;
}
});
@ -238,9 +237,10 @@ public class QueryWithFiltersScript extends GhidraScript {
// Filters out any functions with a self significance less than a
// certain value.
//
preFilter.addPredicate((x, y) -> queryService.getLSHVectorFactory()
.getSelfSignificance(
y.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND);
preFilter.addPredicate((x,
y) -> queryService.getLSHVectorFactory()
.getSelfSignificance(
y.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND);
//
// Option 3. Static method
@ -276,7 +276,7 @@ public class QueryWithFiltersScript extends GhidraScript {
program.getAddressFactory().getDefaultAddressSpace().getAddress(funcDesc.getAddress());
Function function = program.getFunctionManager().getFunctionAt(address);
if (function == null || function.getName().equals(funcDesc.getFunctionName())) {
if (function == null || !function.getName().equals(funcDesc.getFunctionName())) {
return false;
}
return function.getSymbol().getSource() != SourceType.ANALYSIS;

173
Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.py
View File

@ -1,173 +0,0 @@
## ###
# IP: GHIDRA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
# Advanced example of BSim querying
# @category BSim.python
import ghidra.query.facade.SimilarFunctionQueryService as SimilarFunctionQueryService
import ghidra.query.facade.SFQueryInfo as SFQueryInfo
import ghidra.query.FunctionDatabase as FunctionDatabase
import ghidra.query.facade.QueryDatabaseException as QueryDatabaseException
import java.util.HashSet as HashSet
import ghidra.app.plugin.core.query.QueryNearestRow as QueryNearestRow
import java.util.function.BiPredicate as BiPredicate
import ghidra.query.protocol.FilterTemplate as FilterTemplate
import ghidra.app.plugin.core.query.ExecutableResult as ExecutableResult
import java.util.Comparator as Comparator
import java.util.Arrays as Arrays
import java.lang.Double as Double
#Query thresholds
MAX_NUM_FUNCTIONS = 100
SIMILARITY_BOUND = 0.7
SIGNIFICANCE_BOUND = 0.0
#limit the number of results displayed
NUM_EXES_TO_DISPLAY = 10
#for prefiltering: this number will be used to filter out small functions
SELF_SIGNIFICANCE_BOUND = 40.0
def run():
#get the set of functions to query
funcsToQuery = getFunctionsToQuery()
#sets up the object required for querying the database
queryService = SimilarFunctionQueryService(currentProgram)
queryInfo = SFQueryInfo(funcsToQuery)
bsimFilter = queryInfo.getBsimFilter()
#sets the query parameters.
#change the defined constants to control how fuzzy of
#a match you're willing to accept, and the maximum number
#of matches to return for each function
queryInfo.setMaximumResults(MAX_NUM_FUNCTIONS)
queryInfo.setSimilarityThreshold(SIMILARITY_BOUND)
queryInfo.setSignificanceThreshold(SIGNIFICANCE_BOUND)
#add the prefilters
setPrefilters(queryService, queryInfo)
#add a filter on the date
addBsimFilter(bsimFilter, FilterTemplate.DateLater(""), "01/01/1776")
#add a filter with multiple values. Since this is an "Equal" filter, the results are OR'd together
#so a given executable will pass the main filter if it passes at least one of the subfilters
addBsimFilter(bsimFilter, FilterTemplate.ArchEquals(),"x86:LE:64:default, x86:LE:32:default, ARM:LE:32:v4")
#now add a "notequal" filter
#to pass, the compiler can't be windows and it can't be foo_compiler
addBsimFilter(bsimFilter,FilterTemplate.CompNotEqual(),"windows, foo_compiler")
#establish a connection to the BSim database
try:
dbUrl = askString("","Enter the URL of the BSim database:", "ghidra://localhost/bsimDB")
queryService.initializeDatabase(dbUrl)
error = queryService.getDatabase().getLastError()
if error is not None and (error.category is ErrorCategory.Nodatabase):
print "Database [%s] cannot be found (does it exist?)" % dbUrl
return
except QueryDatabaseException as e:
print e.getMessage()
return
resultRows = executeQuery(queryService,queryInfo)
printFunctionQueryResults(resultRows, "\nFunction-level results before filtering")
#now add some post-query filters, which filters the result set returned by the previous query
addBsimFilter(bsimFilter, FilterTemplate.Md5NotEqual(), currentProgram.getExecutableMD5())
addBsimFilter(bsimFilter, FilterTemplate.CompilerEquals(), "gcc")
addBsimFilter(bsimFilter, FilterTemplate.FunctionTagTemplate("KNOWN_LIBRARY", queryService), "false")
#apply the filters and print the results
filteredRows = QueryNearestRow.filterMatchRows(bsimFilter, resultRows)
printFunctionQueryResults(filteredRows, "\nFunction-level results after filtering")
printExecutableInformation(filteredRows)
return
#collect the functions to query from currentProgram
def getFunctionsToQuery():
functions = HashSet();
fIter = currentProgram.getFunctionManager().getFunctionsNoStubs(True)
for func in fIter:
functions.add(func.getSymbol())
return functions
#query the database
def executeQuery(queryService,queryInfo):
queryResults = queryService.querySimilarFunctions(queryInfo,monitor)
resultRows = QueryNearestRow.generate(queryResults.getSimilarityResults(),currentProgram)
return resultRows
def printFunctionQueryResults(resultRows, title):
print "%s: %d\n\n" % (title, resultRows.size())
for row in resultRows:
print " queried function: %s" % row.getOriginalFunctionDescription().getFunctionName()
print " matching function: %s" % row.getMatchFunctionDescription().getFunctionName()
print " executable of matching function: %s" % row.getMatchFunctionDescription().getExecutableRecord().getNameExec()
print " similarity: %f" % row.getSimilarity()
print " significance: %f\n" % row.getSignificance()
#Prefilters are used to filter out functions before sending a query to the database
#A typical use case would be to collect all functions in a binary, then use a
#prefilter to remove the functions with low self-significance (which is the
#"BSim way" to remove small functions)
def setPrefilters(queryService, queryInfo):
preFilter = queryInfo.getPreFilter();
selfSigFilter = ExampleFilter(queryService)
preFilter.addPredicate(selfSigFilter)
class ExampleFilter(BiPredicate):
def __init__(self, queryService):
self.queryService = queryService
def test(self,program, fdesc):
return self.queryService.getLSHVectorFactory().getSelfSignificance(fdesc.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND
def addBsimFilter(bsimFilter, filterTemplate, values):
for value in values.split(","):
if len(value.strip()) > 0:
bsimFilter.addAtom(filterTemplate, value.strip(), FilterTemplate.Blank())
#calls the methods to aggregate executable-level information about the matches
def printExecutableInformation(filteredRows):
execrows = ExecutableResult.generateFromMatchRows(filteredRows)
results = execrows.toArray()
sorter = Sorter()
Arrays.sort(results,sorter)
print "Executable-level results:"
numExes = min(len(results),NUM_EXES_TO_DISPLAY)
for i in range (numExes):
print " MD5: %s" % results[i].getExecutableRecord().getMd5()
print " Executable Name: %s" % results[i].getExecutableRecord().getNameExec()
print " Function Count: %d" % results[i].getFunctionCount()
print " Significance Sum: %f\n" % results[i].getSignificanceSum()
return
class Sorter(Comparator):
def __init__(self):
return
def compare(self,o1,o2):
return Double.compare(o2.getSignificanceSum(), o1.getSignificanceSum())
run()

15
Ghidra/Features/BSim/ghidra_scripts/TailoredAnalysis.java
View File

@ -21,13 +21,14 @@ import ghidra.program.model.listing.Program;
// suitable for BSim ingest process. Intended to be invoked as an analyzeHeadless -preScript
//@category BSim
//note: script mentioned in BSim documentation by name
public class TailoredAnalysis extends GhidraScript {
@Override
public void run() throws Exception {
Options pl = currentProgram.getOptions(Program.ANALYSIS_PROPERTIES);
pl.setBoolean("Decompiler Parameter ID", false);
// These analyzers generate lots of cross references, which are not necessary for
// signature analysis, and take time to run. On the other hand, you may want
// them in general to facilitate general analysis
@ -42,9 +43,15 @@ public class TailoredAnalysis extends GhidraScript {
// analyzerOptions.setBoolean("Commit Function Signatures", false);
// You really want these options turned on
pl.setBoolean("Shared Return Calls",true);
pl.setBoolean("Function Start Search", true);
pl.setBoolean("DWARF", false);
pl.setBoolean("Shared Return Calls", true);
pl.setBoolean("Function Start Search", true);
//The DWARF analyzer can take a long time, so for mass ingest it might be worth
//turning it off
//Moreover, the DWARF analyzer can lock prototypes, which can change the BSim signature
//of a function. This can negatively impact matches between executables with DWARF
//and executables without it.
pl.setBoolean("DWARF", false);
// Options analyzerOptions = pl.getOptions("Function Start Search");
// analyzerOptions.setBoolean("Search Data Blocks", true);
// analyzerOptions = pl.getOptions("Function Start Search After Code");