From a2b786919913b44f756ee38d2bf18fa788ef3a6d Mon Sep 17 00:00:00 2001 From: dragonmacher <48328597+dragonmacher@users.noreply.github.com> Date: Mon, 29 Mar 2021 11:16:38 -0400 Subject: [PATCH 1/8] gradle test option tweaks --- gradle/javaTestProject.gradle | 20 +++++++++++++++++--- gradle/root/test.gradle | 7 +++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/gradle/javaTestProject.gradle b/gradle/javaTestProject.gradle index 525ce3e32b..d2fb80849e 100644 --- a/gradle/javaTestProject.gradle +++ b/gradle/javaTestProject.gradle @@ -156,7 +156,7 @@ def initTestJVM(Task task, String rootDirName) { // -javaagent:/path/to/jmockit.jar task.doFirst { def jmockitPath = configurations.jmockitAgent.singleFile - + task.jvmArgs '-DupgradeProgramErrorMessage=' + upgradeProgramErrorMessage, '-DupgradeTimeErrorMessage=' + upgradeTimeErrorMessage, '-Dlog4j.configuration=' + logPropertiesUrl, @@ -177,11 +177,15 @@ def initTestJVM(Task task, String rootDirName) { '-Duser.country=US', '-Duser.language=en', '-Djdk.attach.allowAttachSelf', - '-javaagent:' + jmockitPath, - '-DLock.DEBUG=true', + '-javaagent:' + jmockitPath, + '-noverify', + '-XX:TieredStopAtLevel=1', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=' + debugPort + // Note: this args are used to speed-up the tests, but are not safe for production code + // -noverify and -XX:TieredStopAtLevel=1 + // Note: modern remote debug invocation; // -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 @@ -191,6 +195,16 @@ def initTestJVM(Task task, String rootDirName) { // -Xnoagent // -Djava.compiler=NONE // -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8000 + + // + // TODO Future Updates: + // The test configuration should be updated to support all known modes of operation: + // command-line test execution, CI test execution of a branch upon request, and full + // CI test execution (this is slow and may need to run overnight). We do not currently + // support well running tests via the command-line. See discussion at github 2854. + // For better command-line usage we will need to update tests such that they can + // share a VM, enabling us to elimnate the use of 'forEver 1' in this file. + // } } /********************************************************************************* diff --git a/gradle/root/test.gradle b/gradle/root/test.gradle index 1284ec7229..cd50d17fb6 100644 --- a/gradle/root/test.gradle +++ b/gradle/root/test.gradle @@ -345,10 +345,13 @@ def initTestJVM(Task task, String rootDirName) { '-Duser.country=US', '-Duser.language=en', '-Djdk.attach.allowAttachSelf', - '-javaagent:' + jmockitPath, - '-DLock.DEBUG=true', + '-javaagent:' + jmockitPath, + '-noverify', + '-XX:TieredStopAtLevel=1', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=' + debugPort + // Note: this args are used to speed-up the tests, but are not safe for production code + // -noverify and -XX:TieredStopAtLevel=1 // Note: modern remote debug invocation; // -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 From bc8e56bd60ecdca7e6fca2e04ab4b910aaeec334 Mon Sep 17 00:00:00 2001 From: ghidra1 Date: Thu, 25 Feb 2021 18:33:00 -0500 Subject: [PATCH 2/8] GP-356 refactor of EditFunctionSignatureDialog and the various extensions. --- .../datamgr/editor/DataTypeEditorManager.java | 192 +++---- .../AbstractEditFunctionSignatureDialog.java | 435 ++++++++++++++++ .../function/EditFunctionSignatureDialog.java | 470 +++++------------- .../FunctionDataTypeHTMLRepresentation.java | 2 +- .../datamgr/DataTypeManagerPluginTest.java | 12 +- .../actions/OverridePrototypeAction.java | 194 +++++--- 6 files changed, 778 insertions(+), 527 deletions(-) create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/AbstractEditFunctionSignatureDialog.java diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/datamgr/editor/DataTypeEditorManager.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/datamgr/editor/DataTypeEditorManager.java index 2743e42c38..e52ef5cd58 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/datamgr/editor/DataTypeEditorManager.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/datamgr/editor/DataTypeEditorManager.java @@ -18,25 +18,21 @@ package ghidra.app.plugin.core.datamgr.editor; import java.util.ArrayList; import java.util.List; -import javax.swing.ComboBoxModel; -import javax.swing.JPanel; - import docking.ComponentProvider; import docking.actions.DockingToolActions; import docking.actions.SharedDockingActionPlaceholder; -import docking.widgets.checkbox.GCheckBox; -import docking.widgets.combobox.GhidraComboBox; -import docking.widgets.label.GLabel; import ghidra.app.plugin.core.compositeeditor.*; import ghidra.app.plugin.core.datamgr.DataTypeManagerPlugin; -import ghidra.app.plugin.core.function.EditFunctionSignatureDialog; +import ghidra.app.plugin.core.function.AbstractEditFunctionSignatureDialog; import ghidra.framework.model.DomainObject; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.data.*; import ghidra.program.model.data.Enum; -import ghidra.program.model.listing.*; +import ghidra.program.model.listing.FunctionSignature; +import ghidra.program.model.listing.Program; import ghidra.util.*; -import ghidra.util.exception.*; +import ghidra.util.exception.CancelledException; +import ghidra.util.exception.DuplicateNameException; /** * Manages program and archive data type editors. @@ -238,8 +234,8 @@ public class DataTypeEditorManager list.add(editor); } } - for (int i = 0; i < list.size(); i++) { - dismissEditor(list.get(i)); + for (EditorProvider element : list) { + dismissEditor(element); } } @@ -519,55 +515,10 @@ public class DataTypeEditorManager editFunctionSignature(category, functionDefinition); } - private void editFunctionSignature(final Category category, - final FunctionDefinition functionDefinition) { - - Function function = - new UndefinedFunction(plugin.getProgram(), plugin.getProgram().getMinAddress()) { - @Override - public String getCallingConventionName() { - if (functionDefinition == null) { - return super.getCallingConventionName(); - } - return functionDefinition.getGenericCallingConvention().toString(); - } - - @Override - public void setCallingConvention(String name) throws InvalidInputException { - // no-op; we handle this in the editor dialog - } - - @Override - public void setInline(boolean isInline) { - // can't edit this from the DataTypeManager - } - - @Override - public void setNoReturn(boolean hasNoReturn) { - // can't edit this from the DataTypeManager - } - - @Override - public FunctionSignature getSignature() { - if (functionDefinition != null) { - return functionDefinition; - } - return super.getSignature(); - } - - @Override - public String getName() { - if (functionDefinition != null) { - return functionDefinition.getName(); - } - return "newFunction"; - } - }; - - // DT how do I do the same as the other creates. + private void editFunctionSignature(Category category, FunctionDefinition functionDefinition) { PluginTool tool = plugin.getTool(); DTMEditFunctionSignatureDialog editSigDialog = new DTMEditFunctionSignatureDialog( - plugin.getTool(), "Edit Function Signature", function, category, functionDefinition); + plugin.getTool(), "Edit Function Signature", category, functionDefinition); editSigDialog.setHelpLocation( new HelpLocation("DataTypeManagerPlugin", "Function_Definition")); tool.showDialog(editSigDialog); @@ -577,64 +528,73 @@ public class DataTypeEditorManager // Inner Classes //================================================================================================== - private class DTMEditFunctionSignatureDialog extends EditFunctionSignatureDialog { + /** + * DTMEditFunctionSignatureDialog provides the ability to edit the + * function signature associated with a specific {@link FunctionDefinition}. + * Use of this editor requires the presence of the tool-based datatype manager service. + */ + private class DTMEditFunctionSignatureDialog extends AbstractEditFunctionSignatureDialog { + private final FunctionDefinition functionDefinition; + private final FunctionSignature oldSignature; private final Category category; - private final FunctionDefinition functionDefinitionDataType; - DTMEditFunctionSignatureDialog(PluginTool pluginTool, String title, Function function, - Category category, FunctionDefinition functionDefinition) { - super(pluginTool, title, function); + DTMEditFunctionSignatureDialog(PluginTool pluginTool, String title, Category category, + FunctionDefinition functionDefinition) { + super(pluginTool, title, false, false, false); + this.functionDefinition = functionDefinition; this.category = category; - this.functionDefinitionDataType = functionDefinition; - - if (functionDefinitionDataType != null) { - setCallingConvention( - functionDefinitionDataType.getGenericCallingConvention().toString()); - } + this.oldSignature = buildSignature(); } - @Override - protected void installCallingConventionWidget(JPanel parentPanel) { - callingConventionComboBox = new GhidraComboBox<>(); - GenericCallingConvention[] values = GenericCallingConvention.values(); - String[] choices = new String[values.length]; - for (int i = 0; i < values.length; i++) { - choices[i] = values[i].toString(); - } - - setCallingConventionChoices(choices); - parentPanel.add(new GLabel("Calling Convention:")); - parentPanel.add(callingConventionComboBox); - } - - @Override - protected void installInlineWidget(JPanel parentPanel) { - inlineCheckBox = new GCheckBox("Inline"); - } - - @Override - protected void installNoReturnWidget(JPanel parentPanel) { - noReturnCheckBox = new GCheckBox("No Return"); - } - - @Override - protected void installCallFixupWidget(JPanel parentPanel) { - // don't add this panel - } - - @Override - protected void setCallingConvention(String callingConvention) { - ComboBoxModel model = callingConventionComboBox.getModel(); - int size = model.getSize(); - for (int i = 0; i < size; i++) { - Object item = model.getElementAt(i); - if (item.equals(callingConvention)) { - callingConventionComboBox.setSelectedItem(callingConvention); - return; + private FunctionSignature buildSignature() { + if (functionDefinition != null) { + if (category.getDataTypeManager() != functionDefinition.getDataTypeManager()) { + throw new IllegalArgumentException( + "functionDefinition and category must have same Datatypemanager"); } + return functionDefinition; } + return new FunctionDefinitionDataType("newFunction"); + } - callingConventionComboBox.setSelectedItem(GenericCallingConvention.unknown); + @Override + protected String[] getSupportedCallFixupNames() { + return null; // Call fixup not supported on FunctionDefinition + } + + @Override + protected String getCallFixupName() { + return null; // Call fixup not supported on FunctionDefinition + } + + @Override + protected FunctionSignature getFunctionSignature() { + return oldSignature; + } + + @Override + protected String getPrototypeString() { + return getFunctionSignature().getPrototypeString(); + } + + @Override + protected String getCallingConventionName() { + return getFunctionSignature().getGenericCallingConvention().toString(); + } + + @Override + protected List getCallingConventionNames() { + GenericCallingConvention[] values = GenericCallingConvention.values(); + List choices = new ArrayList<>(); + for (GenericCallingConvention value : values) { + choices.add(value.toString()); + } + return choices; + } + + @Override + protected DataTypeManager getDataTypeManager() { + return category.getDataTypeManager(); } @Override @@ -657,9 +617,9 @@ public class DataTypeEditorManager GenericCallingConvention.getGenericCallingConvention(getCallingConvention()); newDefinition.setGenericCallingConvention(callingConvention); - DataTypeManager manager = category.getDataTypeManager(); + DataTypeManager manager = getDataTypeManager(); SourceArchive sourceArchive = manager.getLocalSourceArchive(); - if (functionDefinitionDataType == null) { + if (functionDefinition == null) { newDefinition.setSourceArchive(sourceArchive); newDefinition.setCategoryPath(category.getCategoryPath()); int id = manager.startTransaction("Create Function Definition"); @@ -669,14 +629,14 @@ public class DataTypeEditorManager else { int id = manager.startTransaction("Edit Function Definition"); try { - if (!functionDefinitionDataType.getName().equals(newDefinition.getName())) { - functionDefinitionDataType.setName(newDefinition.getName()); + if (!functionDefinition.getName().equals(newDefinition.getName())) { + functionDefinition.setName(newDefinition.getName()); } - functionDefinitionDataType.setArguments(newDefinition.getArguments()); - functionDefinitionDataType.setGenericCallingConvention( + functionDefinition.setArguments(newDefinition.getArguments()); + functionDefinition.setGenericCallingConvention( newDefinition.getGenericCallingConvention()); - functionDefinitionDataType.setReturnType(newDefinition.getReturnType()); - functionDefinitionDataType.setVarArgs(newDefinition.hasVarArgs()); + functionDefinition.setReturnType(newDefinition.getReturnType()); + functionDefinition.setVarArgs(newDefinition.hasVarArgs()); } catch (InvalidNameException | DuplicateNameException e) { // not sure why we are squashing this? ...assuming this can't happen diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/AbstractEditFunctionSignatureDialog.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/AbstractEditFunctionSignatureDialog.java new file mode 100644 index 0000000000..43e106e0f1 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/AbstractEditFunctionSignatureDialog.java @@ -0,0 +1,435 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.core.function; + +import java.awt.Component; +import java.awt.event.ItemEvent; +import java.util.List; + +import javax.swing.*; + +import docking.DialogComponentProvider; +import docking.widgets.checkbox.GCheckBox; +import docking.widgets.combobox.GhidraComboBox; +import docking.widgets.label.GDLabel; +import docking.widgets.label.GLabel; +import ghidra.app.services.DataTypeManagerService; +import ghidra.app.util.cparser.C.ParseException; +import ghidra.app.util.parser.FunctionSignatureParser; +import ghidra.framework.plugintool.PluginTool; +import ghidra.program.model.data.DataTypeManager; +import ghidra.program.model.data.FunctionDefinitionDataType; +import ghidra.program.model.listing.FunctionSignature; +import ghidra.util.exception.CancelledException; + +/** + * EditFunctionSignatureDialog provides an abstract implementation + * a function signature editor. Use of this editor requires the presence of the tool-based + * datatype manager service. + */ +public abstract class AbstractEditFunctionSignatureDialog extends DialogComponentProvider { + + private static final String NONE_CHOICE = "-NONE-"; + private static int SIGNATURE_COLUMNS = 60; + + protected JLabel signatureLabel; + protected JTextField signatureField; + protected JComboBox callingConventionComboBox; + protected JComboBox callFixupComboBox; + protected JCheckBox inlineCheckBox; + protected JCheckBox noReturnCheckBox; + + protected boolean allowInLine; + protected boolean allowNoReturn; + protected boolean allowCallFixup; + + protected PluginTool tool; + + // Due to delayed initialization and tests not actually displaying dialog + // we will track function info initialization + boolean initialized = false; + + /** + * Abstract function signature editor + * + * @param tool A reference to the active tool. + * @param title The title of the dialog. + * @param allowInLine true if in-line attribute control should be included + * @param allowNoReturn true if no-return attribute control should be added + * @param allowCallFixup true if call-fixup choice should be added + */ + public AbstractEditFunctionSignatureDialog(PluginTool tool, String title, boolean allowInLine, + boolean allowNoReturn, boolean allowCallFixup) { + + super(title, true, true, true, false); + this.tool = tool; + this.allowInLine = allowInLine; + this.allowNoReturn = allowNoReturn; + this.allowCallFixup = allowCallFixup; + + addWorkPanel(buildMainPanel()); + addOKButton(); + addCancelButton(); + setDefaultButton(okButton); + setRememberSize(true); + } + + @Override + public JComponent getComponent() { + setFunctionInfo(); //delay update for after construction + return super.getComponent(); + } + + /** + * @return DataTypeManager associated with function or function definition + */ + protected abstract DataTypeManager getDataTypeManager(); + + /** + * @return optional initial function signature which can assist parse with + * identifying referenced datatypes within signature + */ + protected abstract FunctionSignature getFunctionSignature(); + + /** + * @return the initial signature string for the dialog + */ + protected abstract String getPrototypeString(); + + /** + * @return initial calling convention name + */ + protected abstract String getCallingConventionName(); + + /** + * @return list of acceptable calling convention names + */ + protected abstract List getCallingConventionNames(); + + /** + * @return initial in-line attribute value + */ + protected boolean isInline() { + return false; + } + + /** + * @return initial no-return attribute value + */ + protected boolean hasNoReturn() { + return false; + } + + /** + * @return initial call-fixup name or null if n/a + */ + protected abstract String getCallFixupName(); + + /** + * @return array of allowed call fixup names or null + */ + protected abstract String[] getSupportedCallFixupNames(); + + /** + * Method must be invoked following construction to fetch function info + * and update components. + */ + private void setFunctionInfo() { + if (initialized) { + return; + } + initialized = true; + + signatureField.setText(getPrototypeString()); + setCallingConventionChoices(); + callingConventionComboBox.setSelectedItem(getCallingConventionName()); + if (allowInLine) { + inlineCheckBox.setSelected(isInline()); + } + if (allowNoReturn) { + noReturnCheckBox.setSelected(hasNoReturn()); + } + if (allowCallFixup) { + setCallFixupChoices(); + + String callFixupName = getCallFixupName(); + if (callFixupName != null) { + callFixupComboBox.setSelectedItem(callFixupName); + } + } + } + + private JPanel buildMainPanel() { + JPanel mainPanel = new JPanel(); + mainPanel.setLayout(new BoxLayout(mainPanel, BoxLayout.Y_AXIS)); + mainPanel.setBorder(BorderFactory.createEmptyBorder(2, 5, 2, 2)); + mainPanel.add(buildSignaturePanel()); + mainPanel.add(buildAttributePanel()); + if (allowCallFixup) { + installCallFixupWidget(mainPanel); + } + return mainPanel; + } + + private void installCallFixupWidget(JPanel parentPanel) { + JPanel callFixupPanel = buildCallFixupPanel(); + parentPanel.add(callFixupPanel != null ? callFixupPanel : buildSpacerPanel()); + } + + private JPanel buildSignaturePanel() { + JPanel signaturePanel = new JPanel(); + signaturePanel.setLayout(new BoxLayout(signaturePanel, BoxLayout.X_AXIS)); + + signatureField = new JTextField(SIGNATURE_COLUMNS); + signatureLabel = new GDLabel("Signature:"); + signaturePanel.add(signatureLabel); + signaturePanel.add(signatureField); + + signaturePanel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); + + return signaturePanel; + } + + private Component buildSpacerPanel() { + JPanel panel = new JPanel(); + + panel.setLayout(new BoxLayout(panel, BoxLayout.X_AXIS)); + panel.add(Box.createVerticalStrut(20)); + + return panel; + } + + private JPanel buildAttributePanel() { + JPanel attributePanel = new JPanel(); + attributePanel.setLayout(new BoxLayout(attributePanel, BoxLayout.X_AXIS)); + attributePanel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); + + installCallingConventionWidget(attributePanel); + if (allowInLine) { + installInlineWidget(attributePanel); + } + if (allowNoReturn) { + installNoReturnWidget(attributePanel); + } + attributePanel.add(Box.createGlue()); + + return attributePanel; + } + + private void installCallingConventionWidget(JPanel parentPanel) { + callingConventionComboBox = new GhidraComboBox<>(); + parentPanel.add(new GLabel("Calling Convention:")); + parentPanel.add(callingConventionComboBox); + } + + private void installInlineWidget(JPanel parentPanel) { + inlineCheckBox = new GCheckBox("Inline"); + inlineCheckBox.addChangeListener(e -> { + if (inlineCheckBox.isSelected() && callFixupComboBox != null) { + callFixupComboBox.setSelectedItem(NONE_CHOICE); + } + }); + parentPanel.add(inlineCheckBox); + } + + private void installNoReturnWidget(JPanel parentPanel) { + noReturnCheckBox = new GCheckBox("No Return"); + parentPanel.add(noReturnCheckBox); + } + + private JPanel buildCallFixupPanel() { + + if (allowCallFixup) { + return null; + } + + JPanel callFixupPanel = new JPanel(); + callFixupPanel.setLayout(new BoxLayout(callFixupPanel, BoxLayout.X_AXIS)); + + callFixupComboBox = new GhidraComboBox<>(); + callFixupComboBox.addItemListener(e -> { + if (e.getStateChange() == ItemEvent.DESELECTED) { + return; + } + if (!NONE_CHOICE.equals(e.getItem())) { + inlineCheckBox.setSelected(false); + } + }); + + callFixupPanel.add(new GLabel("Call-Fixup:")); + callFixupPanel.add(callFixupComboBox); + + callFixupPanel.add(Box.createGlue()); + callFixupPanel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); + + return callFixupPanel; + } + + /** + * @return plugin tool for which dialog was constructed + */ + protected PluginTool getTool() { + return tool; + } + + private String getSignature() { + return signatureField.getText(); + } + + private void setCallingConventionChoices() { + callingConventionComboBox.removeAllItems(); + for (String element : getCallingConventionNames()) { + callingConventionComboBox.addItem(element); + } + } + + /** + * @return current calling convention selection from dialog + */ + protected String getCallingConvention() { + return (String) callingConventionComboBox.getSelectedItem(); + } + + /** + * @return current in-line attribute value from dialog + */ + protected boolean isInlineSelected() { + return inlineCheckBox != null ? inlineCheckBox.isSelected() : false; + } + + /** + * @return current no-return attribute value from dialog + */ + protected boolean hasNoReturnSelected() { + return noReturnCheckBox != null ? noReturnCheckBox.isSelected() : false; + } + + private void setCallFixupChoices() { + String[] callFixupNames = getSupportedCallFixupNames(); + callFixupComboBox.addItem(NONE_CHOICE); + if (callFixupNames != null) { + for (String element : callFixupNames) { + callFixupComboBox.addItem(element); + } + } + } + + /** + * @return current call fixup selection from dialog or null + */ + protected String getCallFixupSelection() { + if (callFixupComboBox != null) { + String callFixup = (String) callFixupComboBox.getSelectedItem(); + if (callFixup != null && !NONE_CHOICE.equals(callFixup)) { + return callFixup; + } + } + return null; + } + + /** + * This method gets called when the user clicks on the OK Button. The base + * class calls this method. This method will invoke {@link #applyChanges()} + * and close dialog if that method returns true. If false is returned, the + * {@link #applyChanges()} method should display a status message to indicate + * the failure. + */ + @Override + protected void okCallback() { + // only close the dialog if the user made valid changes + try { + if (applyChanges()) { + close(); + } + } + catch (CancelledException e) { + // ignore - do not close + } + } + + @Override + protected void cancelCallback() { + setStatusText(""); + close(); + } + + /** + * Called when the user initiates changes that need to be applied to the + * underlying function or function definition + * + * @return true if applied successfully, otherwise false which will keep + * dialog displayed (a status message should bet set) + * @throws CancelledException if operation cancelled by user + */ + protected abstract boolean applyChanges() throws CancelledException; + + /** + * Perform parse of current user-specified function signature (see {@link #getSignature()}) + * and return valid {@link FunctionDefinitionDataType} if parse successful. + * @return function definition data type if parse successful, otherwise null + * @throws CancelledException if function signature entry cancelled + */ + protected final FunctionDefinitionDataType parseSignature() throws CancelledException { + setFunctionInfo(); // needed for testing which never shows dialog + FunctionSignatureParser parser = new FunctionSignatureParser( + getDataTypeManager(), tool.getService(DataTypeManagerService.class)); + try { + // FIXME: Parser returns FunctionDefinition which only supports GenericCallingConventions + return parser.parse(getFunctionSignature(), getSignature()); + } + catch (ParseException e) { + setStatusText("Invalid Signature: " + e.getMessage()); + } + return null; + } + + /** + * Determine if user-specified function signature has been modified from original + * @return true if modified signature has been entered, else false + */ + protected final boolean isSignatureChanged() { + return !getSignature().equals(getPrototypeString()); + } + + /** + * Determine if user has changed the selected calling convention from the original + * @return true if a change in the selected calling convention has been made + */ + protected final boolean isCallingConventionChanged() { + String current = getCallingConventionName(); + if (current == null && this.getCallingConvention() == null) { + return false; + } + if (current == null && this.getCallingConvention().equals("default")) { + return false; + } + if (current == null && this.getCallingConvention().equals("unknown")) { + return false; + } + if (current == null) { + return true; + } + if (current.equals(getCallingConvention())) { + return false; + } + return true; + } + + @Override + protected void dialogShown() { + signatureField.selectAll(); + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/EditFunctionSignatureDialog.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/EditFunctionSignatureDialog.java index dcd4bfdcba..270b9440a7 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/EditFunctionSignatureDialog.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/function/EditFunctionSignatureDialog.java @@ -15,300 +15,122 @@ */ package ghidra.app.plugin.core.function; -import java.awt.Component; -import java.awt.event.ItemEvent; import java.util.List; -import javax.swing.*; - -import docking.DialogComponentProvider; -import docking.widgets.checkbox.GCheckBox; -import docking.widgets.combobox.GhidraComboBox; -import docking.widgets.label.GDLabel; -import docking.widgets.label.GLabel; import ghidra.app.cmd.function.ApplyFunctionSignatureCmd; -import ghidra.app.services.DataTypeManagerService; -import ghidra.app.util.cparser.C.ParseException; -import ghidra.app.util.parser.FunctionSignatureParser; import ghidra.framework.cmd.Command; import ghidra.framework.cmd.CompoundCmd; import ghidra.framework.model.DomainObject; import ghidra.framework.plugintool.PluginTool; +import ghidra.program.model.data.DataTypeManager; import ghidra.program.model.data.FunctionDefinitionDataType; -import ghidra.program.model.lang.PrototypeModel; import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Program; +import ghidra.program.model.listing.FunctionSignature; import ghidra.program.model.symbol.SourceType; import ghidra.util.Msg; import ghidra.util.exception.CancelledException; import ghidra.util.exception.InvalidInputException; /** - * EditFunctionSignatureDialog provides the ability to edit function - * signatures. Use of this editor requires the presence of the tool-based - * datatype manager service. + * EditFunctionSignatureDialog provides the ability to edit the + * function signature associated with a specific {@link Function}. + * Use of this editor requires the presence of the tool-based datatype manager service. */ -public class EditFunctionSignatureDialog extends DialogComponentProvider { +public class EditFunctionSignatureDialog extends AbstractEditFunctionSignatureDialog { - private static final String NONE_CHOICE = "-NONE-"; - - protected JLabel signatureLabel; - protected JTextField signatureField; - protected JComboBox callingConventionComboBox; - protected JComboBox callFixupComboBox; - protected JCheckBox inlineCheckBox; - protected JCheckBox noReturnCheckBox; - - protected PluginTool tool; - protected Function function; - protected String oldFunctionName; - protected String oldFunctionSignature; + protected final Function function; + protected final String oldFunctionSignature; /** - * This class is not meant to be instantiated directly, but rather by - * subclasses. - * - * @param plugin A reference to the FunctionPlugin. + * Edit function signature for a specified Function + * @param tool A reference to the active tool. * @param title The title of the dialog. * @param function the function which is having its signature edited. */ public EditFunctionSignatureDialog(PluginTool tool, String title, final Function function) { - - super(title, true, true, true, false); - this.tool = tool; + super(tool, title, allowInLine(function), true, allowCallFixup(function)); this.function = function; - this.oldFunctionName = function.getName(); this.oldFunctionSignature = function.getSignature().getPrototypeString(); - - addWorkPanel(buildMainPanel()); - addOKButton(); - addCancelButton(); - setDefaultButton(okButton); - setFunctionInfo(); - setRememberSize(true); } - protected void setFunctionInfo() { - setSignature(function.getSignature().getPrototypeString()); - setCallingConvention(function.getCallingConventionName()); - setInlineSelected(function.isInline()); - inlineCheckBox.setEnabled(!getAffectiveFunction(function).isExternal()); - setNoReturnSelected(function.hasNoReturn()); + protected EditFunctionSignatureDialog(PluginTool tool, String title, final Function function, + boolean allowInLine, boolean allowNoReturn, boolean allowCallFixup) { + super(tool, title, allowInLine, allowNoReturn, allowCallFixup); + this.function = function; + this.oldFunctionSignature = function.getSignature().getPrototypeString(); + } + + @Override + protected FunctionSignature getFunctionSignature() { + return function.getSignature(); + } + + @Override + protected String getPrototypeString() { + return oldFunctionSignature; + } + + @Override + protected String getCallingConventionName() { + return function.getCallingConventionName(); + } + + @Override + protected List getCallingConventionNames() { + return function.getProgram().getFunctionManager().getCallingConventionNames(); + } + + @Override + protected boolean isInline() { + return function.isInline(); + } + + @Override + protected boolean hasNoReturn() { + return function.hasNoReturn(); + } + + @Override + protected String getCallFixupName() { + return function.getCallFixup(); + } + + private static String[] getCallFixupNames(Function function) { + String[] callFixupNames = + function.getProgram().getCompilerSpec().getPcodeInjectLibrary().getCallFixupNames(); + if (callFixupNames.length == 0) { + return null; + } + return callFixupNames; + } + + @Override + protected String[] getSupportedCallFixupNames() { + return getCallFixupNames(function); + } + + @Override + protected DataTypeManager getDataTypeManager() { + return function.getProgram().getDataTypeManager(); } /** * Get the effective function to which changes will be made. This * will be the same as function unless it is a thunk in which case * the returned function will be the ultimate non-thunk function. - * @param f + * @param f function * @return non-thunk function */ - protected Function getAffectiveFunction(Function f) { + private static Function getEffectiveFunction(Function f) { return f.isThunk() ? f.getThunkedFunction(true) : f; } - private JPanel buildMainPanel() { - JPanel mainPanel = new JPanel(); - mainPanel.setLayout(new BoxLayout(mainPanel, BoxLayout.Y_AXIS)); - mainPanel.setBorder(BorderFactory.createEmptyBorder(2, 5, 2, 2)); - mainPanel.add(buildSignaturePanel()); - mainPanel.add(buildAttributePanel()); - - installCallFixupWidget(mainPanel); - - return mainPanel; + private static boolean allowInLine(Function function) { + return !getEffectiveFunction(function).isExternal(); } - protected void installCallFixupWidget(JPanel parentPanel) { - JPanel callFixupPanel = buildCallFixupPanel(); - parentPanel.add(callFixupPanel != null ? callFixupPanel : buildSpacerPanel()); - } - - private JPanel buildSignaturePanel() { - JPanel signaturePanel = new JPanel(); - signaturePanel.setLayout(new BoxLayout(signaturePanel, BoxLayout.X_AXIS)); - - String signature = function.getPrototypeString(false, false); - signatureField = new JTextField(signature.length()); // add some extra room to edit - signatureField.setText(signature); - signatureLabel = new GDLabel("Signature:"); - signaturePanel.add(signatureLabel); - signaturePanel.add(signatureField); - - signaturePanel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); - - return signaturePanel; - } - - private Component buildSpacerPanel() { - JPanel panel = new JPanel(); - - panel.setLayout(new BoxLayout(panel, BoxLayout.X_AXIS)); - panel.add(Box.createVerticalStrut(20)); - - return panel; - } - - private JPanel buildAttributePanel() { - JPanel attributePanel = new JPanel(); - attributePanel.setLayout(new BoxLayout(attributePanel, BoxLayout.X_AXIS)); - attributePanel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); - - installCallingConventionWidget(attributePanel); - installInlineWidget(attributePanel); - installNoReturnWidget(attributePanel); - attributePanel.add(Box.createGlue()); - - return attributePanel; - } - - protected void installCallingConventionWidget(JPanel parentPanel) { - callingConventionComboBox = new GhidraComboBox<>(); - List callingConventions = - function.getProgram().getFunctionManager().getCallingConventionNames(); - String[] choices = callingConventions.toArray(new String[callingConventions.size()]); - setCallingConventionChoices(choices); - parentPanel.add(new GLabel("Calling Convention:")); - parentPanel.add(callingConventionComboBox); - } - - protected void installInlineWidget(JPanel parentPanel) { - inlineCheckBox = new GCheckBox("Inline"); - inlineCheckBox.addChangeListener(e -> { - if (inlineCheckBox.isSelected() && callFixupComboBox != null) { - callFixupComboBox.setSelectedItem(NONE_CHOICE); - } - }); - parentPanel.add(inlineCheckBox); - } - - protected void installNoReturnWidget(JPanel parentPanel) { - noReturnCheckBox = new GCheckBox("No Return"); - parentPanel.add(noReturnCheckBox); - } - - private JPanel buildCallFixupPanel() { - - String[] callFixupNames = - function.getProgram().getCompilerSpec().getPcodeInjectLibrary().getCallFixupNames(); - if (callFixupNames.length == 0) { - return null; - } - - JPanel callFixupPanel = new JPanel(); - callFixupPanel.setLayout(new BoxLayout(callFixupPanel, BoxLayout.X_AXIS)); - - callFixupComboBox = new GhidraComboBox<>(); - callFixupComboBox.addItem(NONE_CHOICE); - for (String element : callFixupNames) { - callFixupComboBox.addItem(element); - } - - callFixupComboBox.addItemListener(e -> { - if (e.getStateChange() == ItemEvent.DESELECTED) { - return; - } - if (!NONE_CHOICE.equals(e.getItem())) { - inlineCheckBox.setSelected(false); - } - }); - - String callFixupName = function.getCallFixup(); - if (callFixupName != null) { - callFixupComboBox.setSelectedItem(callFixupName); - } - - callFixupPanel.add(new GLabel("Call-Fixup:")); - callFixupPanel.add(callFixupComboBox); - - callFixupPanel.add(Box.createGlue()); - callFixupPanel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); - - return callFixupPanel; - } - - protected PluginTool getTool() { - return tool; - } - - protected Program getProgram() { - return function.getProgram(); - } - - protected Function getFunction() { - return function; - } - - public String getSignature() { - return signatureField.getText(); - } - - protected void setSignature(String signature) { - signatureField.setText(signature); - } - - protected void setCallingConventionChoices(String[] callingConventions) { - callingConventionComboBox.removeAllItems(); - for (String element : callingConventions) { - callingConventionComboBox.addItem(element); - } - } - - protected String getCallingConvention() { - return (String) callingConventionComboBox.getSelectedItem(); - } - - protected void setCallingConvention(String callingConvention) { - callingConventionComboBox.setSelectedItem(callingConvention); - } - - protected boolean isInlineSelected() { - return inlineCheckBox.isSelected(); - } - - protected void setInlineSelected(boolean selected) { - inlineCheckBox.setSelected(selected); - } - - protected boolean hasNoReturnSelected() { - return noReturnCheckBox.isSelected(); - } - - protected void setNoReturnSelected(boolean selected) { - noReturnCheckBox.setSelected(selected); - } - - protected String getCallFixupSelection() { - if (callFixupComboBox != null) { - String callFixup = (String) callFixupComboBox.getSelectedItem(); - if (callFixup != null && !NONE_CHOICE.equals(callFixup)) { - return callFixup; - } - } - return null; - } - - /** - * This method gets called when the user clicks on the OK Button. The base - * class calls this method. - */ - @Override - protected void okCallback() { - // only close the dialog if the user made valid changes - try { - if (applyChanges()) { - close(); - } - } - catch (CancelledException e) { - // ignore - do not close - } - } - - @Override - protected void cancelCallback() { - setStatusText(""); - close(); + private static boolean allowCallFixup(Function function) { + return getCallFixupNames(function) != null; } /** @@ -318,6 +140,7 @@ public class EditFunctionSignatureDialog extends DialogComponentProvider { * @return true if the command was successfully created. * @throws CancelledException if operation cancelled by user */ + @Override protected boolean applyChanges() throws CancelledException { // create the command Command command = createCommand(); @@ -327,7 +150,7 @@ public class EditFunctionSignatureDialog extends DialogComponentProvider { } // run the command - if (!getTool().execute(command, getProgram())) { + if (!getTool().execute(command, function.getProgram())) { setStatusText(command.getStatusMsg()); return false; } @@ -336,25 +159,16 @@ public class EditFunctionSignatureDialog extends DialogComponentProvider { return true; } - protected FunctionDefinitionDataType parseSignature() throws CancelledException { - FunctionSignatureParser parser = new FunctionSignatureParser( - getProgram().getDataTypeManager(), tool.getService(DataTypeManagerService.class)); - try { - return parser.parse(getFunction().getSignature(), getSignature()); - } - catch (ParseException e) { - setStatusText("Invalid Signature: " + e.getMessage()); - } - return null; - } - private Command createCommand() throws CancelledException { Command cmd = null; - if (!getSignature().equals(this.oldFunctionSignature) || !isSameCallingConvention() || + if (isSignatureChanged() || isCallingConventionChanged() || (function.getSignatureSource() == SourceType.DEFAULT)) { FunctionDefinitionDataType definition = parseSignature(); + if (definition == null) { + return null; + } cmd = new ApplyFunctionSignatureCmd(function.getEntryPoint(), definition, SourceType.USER_DEFINED, true, true); } @@ -394,85 +208,67 @@ public class EditFunctionSignatureDialog extends DialogComponentProvider { return errMsg; } }); - compoundCommand.add(new Command() { - @Override - public boolean applyTo(DomainObject obj) { - function.setInline(isInlineSelected()); - return true; - } + if (allowInLine) { + compoundCommand.add(new Command() { + @Override + public boolean applyTo(DomainObject obj) { + function.setInline(isInlineSelected()); + return true; + } - @Override - public String getName() { - return "Update Function Inline Flag"; - } + @Override + public String getName() { + return "Update Function Inline Flag"; + } - @Override - public String getStatusMsg() { - return null; - } - }); - compoundCommand.add(new Command() { - @Override - public boolean applyTo(DomainObject obj) { - function.setNoReturn(hasNoReturnSelected()); - return true; - } + @Override + public String getStatusMsg() { + return null; + } + }); + } + if (allowNoReturn) { + compoundCommand.add(new Command() { + @Override + public boolean applyTo(DomainObject obj) { + function.setNoReturn(hasNoReturnSelected()); + return true; + } - @Override - public String getName() { - return "Update Function No Return Flag"; - } + @Override + public String getName() { + return "Update Function No Return Flag"; + } - @Override - public String getStatusMsg() { - return null; - } - }); - compoundCommand.add(new Command() { - @Override - public boolean applyTo(DomainObject obj) { - function.setCallFixup(getCallFixupSelection()); - return true; - } + @Override + public String getStatusMsg() { + return null; + } + }); + } + if (allowCallFixup) { + compoundCommand.add(new Command() { + @Override + public boolean applyTo(DomainObject obj) { + function.setCallFixup(getCallFixupSelection()); + return true; + } - @Override - public String getName() { - return "Update Function Call-Fixup"; - } + @Override + public String getName() { + return "Update Function Call-Fixup"; + } - @Override - public String getStatusMsg() { - return null; - } - }); + @Override + public String getStatusMsg() { + return null; + } + }); + } if (cmd != null) { compoundCommand.add(cmd); } return compoundCommand; } - private boolean isSameCallingConvention() { - PrototypeModel conv = function.getCallingConvention(); - if (conv == null && this.getCallingConvention() == null) { - return true; - } - if (conv == null && this.getCallingConvention().equals("default")) { - return true; - } - if (conv == null && this.getCallingConvention().equals("unknown")) { - return true; - } - if (conv == null) { - return false; - } - if (conv.getName().equals(this.getCallingConvention())) { - return true; - } - return false; - } - - @Override - protected void dialogShown() { - signatureField.selectAll(); - } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/html/FunctionDataTypeHTMLRepresentation.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/html/FunctionDataTypeHTMLRepresentation.java index d2c780c871..ef8083b8fa 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/html/FunctionDataTypeHTMLRepresentation.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/html/FunctionDataTypeHTMLRepresentation.java @@ -88,7 +88,7 @@ public class FunctionDataTypeHTMLRepresentation extends HTMLDataTypeRepresentati GenericCallingConvention genericCallingConvention = functionDefinition.getGenericCallingConvention(); String modifier = genericCallingConvention != GenericCallingConvention.unknown - ? (" " + genericCallingConvention.name()) + ? (" " + genericCallingConvention.getDeclarationName()) : ""; return new TextLine( HTMLUtilities.friendlyEncodeHTML(returnDataType.getDisplayName()) + modifier); diff --git a/Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/datamgr/DataTypeManagerPluginTest.java b/Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/datamgr/DataTypeManagerPluginTest.java index e7adb329df..d91ff9750d 100644 --- a/Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/datamgr/DataTypeManagerPluginTest.java +++ b/Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/datamgr/DataTypeManagerPluginTest.java @@ -50,7 +50,7 @@ import ghidra.app.plugin.core.datamgr.actions.CreateTypeDefDialog; import ghidra.app.plugin.core.datamgr.archive.Archive; import ghidra.app.plugin.core.datamgr.archive.DataTypeManagerHandler; import ghidra.app.plugin.core.datamgr.tree.*; -import ghidra.app.plugin.core.function.EditFunctionSignatureDialog; +import ghidra.app.plugin.core.function.AbstractEditFunctionSignatureDialog; import ghidra.app.plugin.core.programtree.ProgramTreePlugin; import ghidra.app.services.ProgramManager; import ghidra.app.util.datatype.DataTypeSelectionEditor; @@ -695,11 +695,9 @@ public class DataTypeManagerPluginTest extends AbstractGhidraHeadedIntegrationTe DataType dt = iter.next(); listTwo.add(dt); } - for (int i = 0; i < listOne.size(); i++) { - DataType dt = listOne.get(i); + for (DataType dt : listOne) { boolean found = false; - for (int j = 0; j < listTwo.size(); j++) { - DataType dt2 = listTwo.get(j); + for (DataType dt2 : listTwo) { if (dt.isEquivalent(dt2)) { found = true; break; @@ -807,8 +805,8 @@ public class DataTypeManagerPluginTest extends AbstractGhidraHeadedIntegrationTe assertTrue(action.isEnabledForContext(treeContext)); performAction(action, treeContext, false); - EditFunctionSignatureDialog dialog = - waitForDialogComponent(EditFunctionSignatureDialog.class); + AbstractEditFunctionSignatureDialog dialog = + waitForDialogComponent(AbstractEditFunctionSignatureDialog.class); JTextField textField = (JTextField) getInstanceField("signatureField", dialog); setText(textField, newSignature); diff --git a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/plugin/core/decompile/actions/OverridePrototypeAction.java b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/plugin/core/decompile/actions/OverridePrototypeAction.java index 39ddda0173..1d133909e2 100644 --- a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/plugin/core/decompile/actions/OverridePrototypeAction.java +++ b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/plugin/core/decompile/actions/OverridePrototypeAction.java @@ -29,59 +29,12 @@ import ghidra.program.model.data.*; import ghidra.program.model.listing.*; import ghidra.program.model.pcode.*; import ghidra.program.model.symbol.Reference; +import ghidra.program.model.symbol.SourceType; import ghidra.util.*; import ghidra.util.exception.CancelledException; public class OverridePrototypeAction extends AbstractDecompilerAction { - public class ProtoOverrideDialog extends EditFunctionSignatureDialog { - private FunctionDefinition functionDefinition; - - public FunctionDefinition getFunctionDefinition() { - return functionDefinition; - } - - public ProtoOverrideDialog(PluginTool tool, Function func, String signature, String conv) { - super(tool, "Override Signature", func); - setHelpLocation(new HelpLocation(HelpTopics.DECOMPILER, "ActionOverrideSignature")); - setSignature(signature); - setCallingConvention(conv); - } - - /** - * This method gets called when the user clicks on the OK Button. The base - * class calls this method. - */ - @Override - protected void okCallback() { - // only close the dialog if the user made valid changes - if (parseFunctionDefinition()) { - close(); - } - } - - private boolean parseFunctionDefinition() { - - functionDefinition = null; - - try { - functionDefinition = parseSignature(); - } - catch (CancelledException e) { - // ignore - } - - if (functionDefinition == null) { - return false; - } - - GenericCallingConvention convention = - GenericCallingConvention.guessFromName(getCallingConvention()); - functionDefinition.setGenericCallingConvention(convention); - return true; - } - } - public OverridePrototypeAction() { super("Override Signature"); setHelpLocation(new HelpLocation(HelpTopics.DECOMPILER, "ActionOverrideSignature")); @@ -183,11 +136,30 @@ public class OverridePrototypeAction extends AbstractDecompilerAction { return null; } - private String generateSignature(PcodeOp op, String name) { + private String generateSignature(PcodeOp op, String name, Function calledfunc) { + + // TODO: If an override has already be placed-down it should probably be used + // for the initial signature. HighFunction does not make it easy to grab + // existing override prototype + + if (calledfunc != null) { + SourceType signatureSource = calledfunc.getSignatureSource(); + if (signatureSource == SourceType.DEFAULT || signatureSource == SourceType.ANALYSIS) { + calledfunc = null; // ignore + } + } + StringBuffer buf = new StringBuffer(); + Varnode vn = op.getOutput(); DataType dt = null; - if (vn != null) { + if (calledfunc != null) { + dt = calledfunc.getReturnType(); + if (Undefined.isUndefined(dt)) { + dt = null; + } + } + if (dt == null && vn != null) { dt = vn.getHigh().getDataType(); } if (dt != null) { @@ -198,26 +170,48 @@ public class OverridePrototypeAction extends AbstractDecompilerAction { } buf.append(' ').append(name).append('('); - for (int i = 1; i < op.getNumInputs(); ++i) { - vn = op.getInput(i); - dt = null; - if (vn != null) { - dt = vn.getHigh().getDataType(); - } - if (dt != null) { - buf.append(dt.getDisplayName()); - } - else { - buf.append("BAD"); - } - if (i != op.getNumInputs() - 1) { - buf.append(','); + + int index = 1; + if (calledfunc != null) { + for (Parameter p : calledfunc.getParameters()) { + String dtName = getInputDataTypeName(op, index, p.getDataType()); + if (index++ != 1) { + buf.append(", "); + } + buf.append(dtName); + if (p.getSource() != SourceType.DEFAULT) { + buf.append(' '); + buf.append(p.getName()); + } } } + + for (int i = index; i < op.getNumInputs(); ++i) { + if (i != 1) { + buf.append(", "); + } + buf.append(getInputDataTypeName(op, i, null)); + } + buf.append(')'); return buf.toString(); } + private String getInputDataTypeName(PcodeOp op, int inIndex, DataType preferredDt) { + if (preferredDt != null && !Undefined.isUndefined(preferredDt)) { + return preferredDt.getDisplayName(); + } + Varnode vn = op.getInput(inIndex); + DataType dt = null; + if (vn != null) { + dt = vn.getHigh().getDataType(); + } + if (dt != null) { + return dt.getDisplayName(); + } + return "BAD"; + } + @Override protected boolean isEnabledForDecompilerContext(DecompilerActionContext context) { Function function = context.getFunction(); @@ -257,9 +251,10 @@ public class OverridePrototypeAction extends AbstractDecompilerAction { conv = calledfunc.getCallingConventionName(); } - String signature = generateSignature(op, name); + String signature = generateSignature(op, name, calledfunc); PluginTool tool = context.getTool(); - ProtoOverrideDialog dialog = new ProtoOverrideDialog(tool, func, signature, conv); + ProtoOverrideDialog dialog = + new ProtoOverrideDialog(tool, calledfunc != null ? calledfunc : func, signature, conv); tool.showDialog(dialog); FunctionDefinition fdef = dialog.getFunctionDefinition(); if (fdef == null) { @@ -279,4 +274,71 @@ public class OverridePrototypeAction extends AbstractDecompilerAction { program.endTransaction(transaction, commit); } } + + /** + * ProtoOverrideDialog provides the ability to edit the + * function signature associated with a specific function definition override + * at a sub-function callsite. + * Use of this editor requires the presence of the tool-based datatype manager service. + */ + private class ProtoOverrideDialog extends EditFunctionSignatureDialog { + private FunctionDefinition functionDefinition; + private final String initialSignature; + private final String initialConvention; + + /** + * Construct signature override for called function + * @param tool active tool + * @param func function from which program access is achieved and supply of preferred + * datatypes when parsing signature + * @param signature initial prototype signature to be used + * @param conv initial calling convention + */ + public ProtoOverrideDialog(PluginTool tool, Function func, String signature, String conv) { + super(tool, "Override Signature", func, false, false, false); + setHelpLocation(new HelpLocation(HelpTopics.DECOMPILER, "ActionOverrideSignature")); + this.initialSignature = signature; + this.initialConvention = conv; + } + + @Override + protected String getPrototypeString() { + return initialSignature; + } + + @Override + protected String getCallingConventionName() { + return initialConvention; + } + + @Override + protected boolean applyChanges() throws CancelledException { + return parseFunctionDefinition(); + } + + private boolean parseFunctionDefinition() { + + functionDefinition = null; + + try { + functionDefinition = parseSignature(); + } + catch (CancelledException e) { + // ignore + } + + if (functionDefinition == null) { + return false; + } + + GenericCallingConvention convention = + GenericCallingConvention.guessFromName(getCallingConvention()); + functionDefinition.setGenericCallingConvention(convention); + return true; + } + + public FunctionDefinition getFunctionDefinition() { + return functionDefinition; + } + } } From 01c24376456fc4d9b0395aa4a4fc58ea83031a51 Mon Sep 17 00:00:00 2001 From: ghidra1 Date: Mon, 5 Apr 2021 17:48:36 -0400 Subject: [PATCH 3/8] GP-809 Improved demangler treatment of existing thunk functions --- .../analysis/AbstractDemanglerAnalyzer.java | 48 ++++++++--- .../app/util/demangler/DemangledFunction.java | 81 +++++++++++++++++-- 2 files changed, 115 insertions(+), 14 deletions(-) diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/AbstractDemanglerAnalyzer.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/AbstractDemanglerAnalyzer.java index 7763217f21..966e4fa931 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/AbstractDemanglerAnalyzer.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/AbstractDemanglerAnalyzer.java @@ -18,8 +18,7 @@ package ghidra.app.plugin.core.analysis; import ghidra.app.services.*; import ghidra.app.util.demangler.*; import ghidra.app.util.importer.MessageLog; -import ghidra.program.model.address.Address; -import ghidra.program.model.address.AddressSetView; +import ghidra.program.model.address.*; import ghidra.program.model.listing.*; import ghidra.program.model.symbol.*; import ghidra.util.exception.CancelledException; @@ -41,6 +40,9 @@ import ghidra.util.task.TaskMonitor; */ public abstract class AbstractDemanglerAnalyzer extends AbstractAnalyzer { + private static final AddressSetView EXTERNAL_SET = new AddressSet( + AddressSpace.EXTERNAL_SPACE.getMinAddress(), AddressSpace.EXTERNAL_SPACE.getMaxAddress()); + public AbstractDemanglerAnalyzer(String name, String description) { super(name, description, AnalyzerType.BYTE_ANALYZER); setPriority(AnalysisPriority.DATA_TYPE_PROPOGATION.before().before().before()); @@ -59,6 +61,8 @@ public abstract class AbstractDemanglerAnalyzer extends AbstractAnalyzer { try { monitor.setIndeterminate(true); + // NOTE: demangling of Externals may lose mangled name if original + // imported name has already been assigned to the External symbol (e.g., ordinal based name) return doAdded(program, set, monitor, log); } finally { @@ -75,18 +79,42 @@ public abstract class AbstractDemanglerAnalyzer extends AbstractAnalyzer { log.appendMsg(getName(), "Invalid demangler options--cannot demangle"); return false; } + + // Demangle external symbols after memory symbols. + // This is done to compensate for cases where the mangled name on externals may be lost + // after demangling when an alternate Ordinal symbol exists. The external mangled + // name is helpful in preserving thunk relationships when a mangled symbols have been + // placed on a thunk. It is assumed that analyzer is presented with entire + // EXTERNAL space in set (all or none). + boolean demangleExternals = set.contains(EXTERNAL_SET.getMinAddress()); + if (demangleExternals) { + set = set.subtract(EXTERNAL_SET); + } - int count = 0; + String baseMonitorMessage = monitor.getMessage(); + int memorySymbolCount = + demangleSymbols(program, set, 0, baseMonitorMessage, options, log, monitor); + if (demangleExternals) { + // process external symbols last + demangleSymbols(program, EXTERNAL_SET, memorySymbolCount, baseMonitorMessage, options, + log, monitor); + } - String defaultMessage = monitor.getMessage(); + return true; + } + private int demangleSymbols(Program program, AddressSetView set, int initialCount, + String baseMonitorMessage, DemanglerOptions options, MessageLog log, + TaskMonitor monitor) throws CancelledException { + + int count = initialCount; SymbolTable symbolTable = program.getSymbolTable(); SymbolIterator it = symbolTable.getPrimarySymbolIterator(set, true); while (it.hasNext()) { monitor.checkCanceled(); if (++count % 100 == 0) { - monitor.setMessage(defaultMessage + " - " + count + " symbols"); + monitor.setMessage(baseMonitorMessage + " - " + count + " symbols"); } Symbol symbol = it.next(); @@ -101,8 +129,7 @@ public abstract class AbstractDemanglerAnalyzer extends AbstractAnalyzer { apply(program, address, demangled, options, log, monitor); } } - - return true; + return count; } /** @@ -152,10 +179,13 @@ public abstract class AbstractDemanglerAnalyzer extends AbstractAnalyzer { return true; } - // Someone has already added arguments or return to the function signature + // Someone has already added arguments or return to the function signature. + // Treatment of thunks must be handled later since thunk relationship may + // need to be broken if (symbol.getSymbolType() == SymbolType.FUNCTION) { Function function = (Function) symbol.getObject(); - if (function.getSignatureSource().isHigherPriorityThan(SourceType.ANALYSIS)) { + if (!function.isThunk() && + function.getSignatureSource().isHigherPriorityThan(SourceType.ANALYSIS)) { return true; } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemangledFunction.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemangledFunction.java index cc0e591505..60a759ac82 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemangledFunction.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemangledFunction.java @@ -349,14 +349,26 @@ public class DemangledFunction extends DemangledObject { Function function = createFunction(program, address, options.doDisassembly(), monitor); if (function == null) { - // no function whose signature we need to update - // NOTE: this does not make much sense - // renameExistingSymbol(program, address, symbolTable); - // maybeCreateUndefined(program, address); + // No function whose signature we need to update return false; } - //if existing function signature is user defined - add demangled label only + if (function.isThunk()) { + // If thunked function has same mangled name we can discard our + // symbol if no other symbols at this address (i.e., rely entirely on + // thunked function). + // NOTE: mangled name on external may be lost once it is demangled. + if (shouldThunkBePreserved(function)) { + // Preserve thunk and remove mangled symbol. Allow to proceed normally by returning true. + function.getSymbol().setName(null, SourceType.DEFAULT); + return true; + } + + // Break thunk relationship and continue applying demangle function below + function.setThunkedFunction(null); + } + + // If existing function signature is user defined - add demangled label only boolean makePrimary = (function.getSignatureSource() != SourceType.USER_DEFINED); Symbol demangledSymbol = @@ -395,6 +407,65 @@ public class DemangledFunction extends DemangledObject { return true; } + /** + * Determine if existing thunk relationship should be preserved and mangled symbol + * discarded. This is the case when the thunk function mangled name matches + * the thunked function since we want to avoid duplicate symbol names. + * @param thunkFunction thunk function with a mangled symbol which is currently + * being demangled. + * @return true if thunk should be preserved and mangled symbol discarded, otherwise + * false if thunk relationship should be eliminated and demangled function information + * should be applied as normal. + */ + private boolean shouldThunkBePreserved(Function thunkFunction) { + Program program = thunkFunction.getProgram(); + SymbolTable symbolTable = program.getSymbolTable(); + if (thunkFunction.getSymbol().isExternalEntryPoint()) { + return false; // entry point should retain its own symbol + } + Symbol[] symbols = symbolTable.getSymbols(thunkFunction.getEntryPoint()); + if (symbols.length > 1) { + return false; // too many symbols present to preserve thunk + } + // NOTE: order of demangling unknown - thunked function may, or may not, have + // already been demangled + Function thunkedFunction = thunkFunction.getThunkedFunction(true); + if (mangled.equals(thunkedFunction.getName())) { + // thunked function has matching mangled name + return true; + } + if (thunkedFunction.isExternal()) { + if (thunkedFunction.getParentNamespace() instanceof Library) { + // Thunked function does not have mangled name, if it did it would have + // matched name check above or now reside in a different namespace + return false; + } + // assume external contained with specific namespace + ExternalLocation externalLocation = + program.getExternalManager().getExternalLocation(thunkedFunction.getSymbol()); + String originalImportedName = externalLocation.getOriginalImportedName(); + if (originalImportedName == null) { + // assume external manually manipulated without use of mangled name + return false; + } + if (mangled.equals(externalLocation.getOriginalImportedName())) { + // matching mangled name also resides at thunked function location + return true; + } + + // TODO: carefully compare signature in absense of matching mangled name + return false; + } + + if (symbolTable.getSymbol(mangled, thunkedFunction.getEntryPoint(), + program.getGlobalNamespace()) != null) { + // matching mangled name also resides at thunked function location + return true; + } + + return false; + } + private boolean hasVarArgs() { if (parameters.isEmpty()) { return false; From c49402afc05f4d0e36aef44ba434c1a75b359a14 Mon Sep 17 00:00:00 2001 From: ghidra1 Date: Mon, 5 Apr 2021 18:57:38 -0400 Subject: [PATCH 4/8] GP-839 correct space name case sensitivity for AddressMapImpl --- .../program/model/address/AddressMapImpl.java | 15 +++++++++------ .../program/model/address/AddressMapImplTest.java | 10 ++++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressMapImpl.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressMapImpl.java index fa68939881..2afe61480e 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressMapImpl.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressMapImpl.java @@ -15,10 +15,10 @@ */ package ghidra.program.model.address; -import ghidra.util.UniversalIdGenerator; - import java.util.*; +import ghidra.util.UniversalIdGenerator; + /** * AddressMapImpl provides a stand-alone AddressMap. * An AddressMapImpl instance should only be used to decode keys which it has generated. @@ -94,6 +94,7 @@ public class AddressMapImpl { * start of a key range. */ private Comparator addressInsertionKeyRangeComparator = new Comparator() { + @Override public int compare(Object keyRangeObj, Object addrObj) { KeyRange range = (KeyRange) keyRangeObj; Address addr = (Address) addrObj; @@ -158,7 +159,7 @@ public class AddressMapImpl { } void checkAddressSpace(AddressSpace addrSpace) { - String name = addrSpace.getName().toUpperCase(); + String name = addrSpace.getName(); AddressSpace existingSpace = spaceMap.get(name); if (existingSpace == null) { spaceMap.put(name, addrSpace); @@ -245,10 +246,12 @@ public class AddressMapImpl { private void addKeyRanges(List keyRangeList, Address start, Address end) { int index = Arrays.binarySearch(sortedBaseStartAddrs, start); - if (index < 0) + if (index < 0) { index = -index - 2; - if (index < 0) + } + if (index < 0) { index++; + } while (index < sortedBaseStartAddrs.length && end.compareTo(sortedBaseStartAddrs[index]) >= 0) { Address addr1 = max(start, sortedBaseStartAddrs[index]); @@ -309,7 +312,7 @@ public class AddressMapImpl { } for (AddressSpace space : remapSpaces.values()) { - spaceMap.put(space.getName().toUpperCase(), space); + spaceMap.put(space.getName(), space); } for (int i = 0; i < baseAddrs.length; i++) { diff --git a/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressMapImplTest.java b/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressMapImplTest.java index c890ec9659..00a17bff09 100644 --- a/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressMapImplTest.java +++ b/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressMapImplTest.java @@ -15,7 +15,7 @@ */ package ghidra.program.model.address; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import org.junit.*; @@ -26,6 +26,7 @@ public class AddressMapImplTest extends AbstractGenericTest { AddressSpace sp16; AddressSpace sp32; AddressSpace sp64; + AddressSpace ov64; AddressSpace regSpace; AddressSpace stackSpace; SegmentedAddressSpace segSpace1; @@ -42,6 +43,8 @@ public class AddressMapImplTest extends AbstractGenericTest { sp32 = new GenericAddressSpace("THREE", 32, AddressSpace.TYPE_RAM, 2); sp64 = new GenericAddressSpace("FOUR", 64, AddressSpace.TYPE_RAM, 2); + ov64 = new OverlayAddressSpace("four", sp64, 100, 0x1000, 0x1fff); + segSpace1 = new SegmentedAddressSpace("SegSpaceOne", 3); segSpace2 = new SegmentedAddressSpace("SegSpaceTwo", 4); @@ -50,7 +53,7 @@ public class AddressMapImplTest extends AbstractGenericTest { map = new AddressMapImpl(); - addrs = new Address[29]; + addrs = new Address[31]; addrs[0] = sp8.getAddress(0); addrs[1] = sp8.getAddress(0x0ff); addrs[2] = sp16.getAddress(0); @@ -84,6 +87,9 @@ public class AddressMapImplTest extends AbstractGenericTest { addrs[27] = stackSpace.getAddress(0); addrs[28] = stackSpace.getAddress(0x80000000); + addrs[29] = ov64.getAddress(0x1100); + addrs[30] = ov64.getAddress(0x2000); + } @Test From b1d55498ecfe70c061cea8db1b601b5cdb9057c8 Mon Sep 17 00:00:00 2001 From: dragonmacher <48328597+dragonmacher@users.noreply.github.com> Date: Tue, 6 Apr 2021 09:27:23 -0400 Subject: [PATCH 5/8] GP-830 - Updated the threaded table model to allow subclasses to change how data adds and removes are performed. This fixes and exception and increases performance for the Symbol Table --- .../SymbolTableAddRemoveStrategy.java | 85 +++++++++++++++++++ .../core/symtable/SymbolTableModel.java | 8 ++ .../threaded/DefaultAddRemoveStrategy.java | 61 +++++++++++++ .../threaded/TableAddRemoveStrategy.java | 40 +++++++++ .../widgets/table/threaded/TableData.java | 6 +- .../table/threaded/TableUpdateJob.java | 29 ++----- .../table/threaded/ThreadedTableModel.java | 30 +++++++ 7 files changed, 233 insertions(+), 26 deletions(-) create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableAddRemoveStrategy.java create mode 100644 Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/DefaultAddRemoveStrategy.java create mode 100644 Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableAddRemoveStrategy.java diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableAddRemoveStrategy.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableAddRemoveStrategy.java new file mode 100644 index 0000000000..f6cbefa1e2 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableAddRemoveStrategy.java @@ -0,0 +1,85 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.core.symtable; + +import java.util.*; + +import docking.widgets.table.AddRemoveListItem; +import docking.widgets.table.threaded.TableAddRemoveStrategy; +import docking.widgets.table.threaded.TableData; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.TaskMonitor; + +/** + * This strategy attempts to optimize removal of db objects that have been deleted. The issue with + * deleted db objects is that they may no longer have their attributes, which means we cannot + * use any of those attributes that may have been used as the basis for sorting. We use the + * table's sort to perform a binary search of existing symbols for removal. If the binary search + * does not work, then removal operations will require slow list traversal. Additionally, + * some clients use proxy objects in add/remove list to signal which object needs to be removed, + * since the original object is no longer available to the client. Using these proxy objects + * in a binary search may lead to exceptions if the proxy has unsupported methods called when + * searching. + * + *

This strategy will has guilty knowledge of client proxy object usage. The proxy objects + * are coded such that the {@code hashCode()} and {@code equals()} methods will match those + * methods of the data's real objects. + * + * @param the row type + */ +public class SymbolTableAddRemoveStrategy implements TableAddRemoveStrategy { + + @Override + public void process(List> addRemoveList, TableData tableData, + TaskMonitor monitor) throws CancelledException { + + // + // Hash map the existing values so that we can use any object inside the add/remove list + // as a key into this map to get the matching existing value. + // + Map hashed = new HashMap<>(); + for (T t : tableData) { + hashed.put(t, t); + } + + int n = addRemoveList.size(); + monitor.setMessage("Adding/Removing " + n + " items..."); + monitor.initialize(n); + for (int i = 0; i < n; i++) { + AddRemoveListItem item = addRemoveList.get(i); + T value = item.getValue(); + if (item.isChange()) { + T toRemove = hashed.get(value); + if (toRemove != null) { + tableData.remove(toRemove); + } + tableData.insert(value); + } + else if (item.isRemove()) { + T toRemove = hashed.get(value); + if (toRemove != null) { + tableData.remove(toRemove); + } + } + else if (item.isAdd()) { + tableData.insert(value); + } + monitor.checkCanceled(); + monitor.setProgress(i); + } + monitor.setMessage("Done adding/removing"); + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableModel.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableModel.java index fb922b22c0..c20783ca38 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableModel.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/symtable/SymbolTableModel.java @@ -18,6 +18,7 @@ package ghidra.app.plugin.core.symtable; import java.util.*; import docking.widgets.table.*; +import docking.widgets.table.threaded.TableAddRemoveStrategy; import ghidra.app.cmd.function.DeleteFunctionCmd; import ghidra.app.cmd.label.DeleteLabelCmd; import ghidra.app.cmd.label.RenameLabelCmd; @@ -60,6 +61,8 @@ class SymbolTableModel extends AddressBasedTableModel { private ReferenceManager refMgr; private Symbol lastSymbol; private SymbolFilter filter; + private TableAddRemoveStrategy deletedDbObjectAddRemoveStrategy = + new SymbolTableAddRemoveStrategy<>(); SymbolTableModel(SymbolProvider provider, PluginTool tool) { super("Symbols", tool, null, null); @@ -88,6 +91,11 @@ class SymbolTableModel extends AddressBasedTableModel { return descriptor; } + @Override + protected TableAddRemoveStrategy getAddRemoveStrategy() { + return deletedDbObjectAddRemoveStrategy; + } + void setFilter(SymbolFilter filter) { this.filter = filter; reload(); diff --git a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/DefaultAddRemoveStrategy.java b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/DefaultAddRemoveStrategy.java new file mode 100644 index 0000000000..17c085b223 --- /dev/null +++ b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/DefaultAddRemoveStrategy.java @@ -0,0 +1,61 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package docking.widgets.table.threaded; + +import java.util.List; + +import docking.widgets.table.AddRemoveListItem; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.TaskMonitor; + +/** + * A strategy that uses the table's sort state to perform a binary search of items to be added + * and removed. + * + * @param the row type + */ +public class DefaultAddRemoveStrategy implements TableAddRemoveStrategy { + + @Override + public void process(List> addRemoveList, TableData updatedData, + TaskMonitor monitor) throws CancelledException { + + int n = addRemoveList.size(); + monitor.setMessage("Adding/Removing " + n + " items..."); + monitor.initialize(n); + + // Note: this class does not directly perform a binary such, but instead relies on that + // work to be done by the call to TableData.remove() + for (int i = 0; i < n; i++) { + AddRemoveListItem item = addRemoveList.get(i); + T value = item.getValue(); + if (item.isChange()) { + updatedData.remove(value); + updatedData.insert(value); + } + else if (item.isRemove()) { + updatedData.remove(value); + } + else if (item.isAdd()) { + updatedData.insert(value); + } + monitor.checkCanceled(); + monitor.setProgress(i); + } + monitor.setMessage("Done adding/removing"); + } + +} diff --git a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableAddRemoveStrategy.java b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableAddRemoveStrategy.java new file mode 100644 index 0000000000..3bb91bb580 --- /dev/null +++ b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableAddRemoveStrategy.java @@ -0,0 +1,40 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package docking.widgets.table.threaded; + +import java.util.List; + +import docking.widgets.table.AddRemoveListItem; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.TaskMonitor; + +/** + * A strategy to perform table add and remove updates + * + * @param the row type + */ +public interface TableAddRemoveStrategy { + + /** + * Adds to and removes from the table data those items in the given add/remove list + * @param addRemoveList the items to add/remove + * @param tableData the table's data + * @param monitor the monitor + * @throws CancelledException if the monitor is cancelled + */ + public void process(List> addRemoveList, TableData tableData, + TaskMonitor monitor) throws CancelledException; +} diff --git a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableData.java b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableData.java index 72b24ff65d..e59de08958 100644 --- a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableData.java +++ b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableData.java @@ -137,7 +137,7 @@ public class TableData implements Iterable { * @param t the item * @return the index */ - int indexOf(ROW_OBJECT t) { + public int indexOf(ROW_OBJECT t) { if (!sortContext.isUnsorted()) { Comparator comparator = sortContext.getComparator(); return Collections.binarySearch(data, t, comparator); @@ -153,7 +153,7 @@ public class TableData implements Iterable { return -1; } - boolean remove(ROW_OBJECT t) { + public boolean remove(ROW_OBJECT t) { if (source != null) { source.remove(t); } @@ -186,7 +186,7 @@ public class TableData implements Iterable { * * @param value the row Object to insert */ - void insert(ROW_OBJECT value) { + public void insert(ROW_OBJECT value) { if (source != null) { // always update the master data diff --git a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableUpdateJob.java b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableUpdateJob.java index 08f09b1ab7..94df233f11 100644 --- a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableUpdateJob.java +++ b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableUpdateJob.java @@ -553,31 +553,14 @@ public class TableUpdateJob { */ private void doProcessAddRemoves() throws CancelledException { - int n = addRemoveList.size(); - monitor.setMessage("Adding/Removing " + n + " items..."); - monitor.initialize(n); - initializeSortCache(); - - for (int i = 0; i < n; i++) { - AddRemoveListItem item = addRemoveList.get(i); - T value = item.getValue(); - if (item.isChange()) { - updatedData.remove(value); - updatedData.insert(value); - } - else if (item.isRemove()) { - updatedData.remove(value); - } - else if (item.isAdd()) { - updatedData.insert(value); - } - monitor.checkCanceled(); - monitor.setProgress(i); + try { + TableAddRemoveStrategy strategy = model.getAddRemoveStrategy(); + strategy.process(addRemoveList, updatedData, monitor); + } + finally { + clearSortCache(); } - monitor.setMessage("Done adding/removing"); - - clearSortCache(); } /** When sorting we cache column value lookups to increase speed. */ diff --git a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/ThreadedTableModel.java b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/ThreadedTableModel.java index 6439ef4edd..67ac2cc585 100644 --- a/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/ThreadedTableModel.java +++ b/Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/ThreadedTableModel.java @@ -90,6 +90,8 @@ public abstract class ThreadedTableModel private volatile Worker worker; // only created as needed (if we are incremental) private int minUpdateDelayMillis; private int maxUpdateDelayMillis; + private TableAddRemoveStrategy binarySearchAddRemoveStrategy = + new DefaultAddRemoveStrategy<>(); protected ThreadedTableModel(String modelName, ServiceProvider serviceProvider) { this(modelName, serviceProvider, null); @@ -510,6 +512,17 @@ public abstract class ThreadedTableModel /** * Removes the specified object from this model and schedules an update. + * + *

Note: for this method to function correctly, the given object must compare as + * {@link #equals(Object)} and have the same {@link #hashCode()} as the object to be removed + * from the table data. This allows clients to create proxy objects to pass into this method, + * as long as they honor those requirements. + * + *

If this model's data is sorted, then a binary search will be used to locate the item + * to be removed. However, for this to work, all field used to sort the data must still be + * available from the original object and must be the same values. If this is not true, then + * the binary search will not work and a brute force search will be used. + * * @param obj the object to remove */ public void removeObject(ROW_OBJECT obj) { @@ -786,6 +799,23 @@ public abstract class ThreadedTableModel updateManager.setTaskMonitor(monitor); } + /** + * Returns the strategy to use for performing adds and removes to this table. Subclasses can + * override this method to customize this process for their particular type of data. See + * the implementations of {@link TableAddRemoveStrategy} for details. + * + *

Note: The default add/remove strategy assumes that objects to be removed will be the + * same instance that is in the list of this model. This allows the {@link #equals(Object)} + * and {@link #hashCode()} to be used when removing the object from the list. If you model + * does not pass the same instance into {@link #removeObject(Object)}, then you will need to + * update your add/remove strategy accordingly. + * + * @return the strategy + */ + protected TableAddRemoveStrategy getAddRemoveStrategy() { + return binarySearchAddRemoveStrategy; + } + public void setIncrementalTaskMonitor(TaskMonitor monitor) { SystemUtilities.assertTrue(loadIncrementally, "Cannot set an incremental task monitor " + "on a table that was not constructed to load incrementally"); From 316ab7c5abdaba94315e870e2b963178430c63ac Mon Sep 17 00:00:00 2001 From: dragonmacher <48328597+dragonmacher@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:02:27 -0400 Subject: [PATCH 6/8] GP-835 - reduced the amount of work done by the Defined Strings table Closes #2889 --- .../ViewStringsPlugin/ViewStringsPlugin.htm | 6 ++ .../core/strings/ViewStringsPlugin.java | 80 ++++++++++--------- .../model/data/StringDataInstance.java | 7 +- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/Ghidra/Features/Base/src/main/help/help/topics/ViewStringsPlugin/ViewStringsPlugin.htm b/Ghidra/Features/Base/src/main/help/help/topics/ViewStringsPlugin/ViewStringsPlugin.htm index d49b242d8b..ddd6a49794 100644 --- a/Ghidra/Features/Base/src/main/help/help/topics/ViewStringsPlugin/ViewStringsPlugin.htm +++ b/Ghidra/Features/Base/src/main/help/help/topics/ViewStringsPlugin/ViewStringsPlugin.htm @@ -71,6 +71,12 @@ option, OR

  • Select the button on the tool bar.
    • + +

    The refresh icon on the toolbar will + appear grayed-out by default. If potential changes to string data are detected, + the icon will become green in color. The toolbar button can be pressed in either state + for a full table reload. +

    Settings... and Default Settings...

    diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/strings/ViewStringsPlugin.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/strings/ViewStringsPlugin.java index 9ee8a34e80..e9f54b7a95 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/strings/ViewStringsPlugin.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/strings/ViewStringsPlugin.java @@ -15,12 +15,11 @@ */ package ghidra.app.plugin.core.strings; -import javax.swing.ImageIcon; +import javax.swing.Icon; import docking.ActionContext; import docking.action.*; import ghidra.app.CorePluginPackage; -import ghidra.app.events.ProgramSelectionPluginEvent; import ghidra.app.plugin.PluginCategoryNames; import ghidra.app.plugin.ProgramPlugin; import ghidra.app.plugin.core.data.DataSettingsDialog; @@ -38,6 +37,7 @@ import ghidra.util.table.SelectionNavigationAction; import ghidra.util.table.actions.MakeProgramSelectionAction; import ghidra.util.task.SwingUpdateManager; import resources.Icons; +import resources.ResourceManager; /** * Plugin that provides the "Defined Strings" table, where all the currently defined @@ -57,7 +57,11 @@ import resources.Icons; //@formatter:on public class ViewStringsPlugin extends ProgramPlugin implements DomainObjectListener { - private DockingAction selectAction; + private static Icon REFRESH_ICON = Icons.REFRESH_ICON; + private static Icon REFRESH_NOT_NEEDED_ICON = + ResourceManager.getDisabledIcon(Icons.REFRESH_ICON, 60); + + private DockingAction refreshAction; private DockingAction showSettingsAction; private DockingAction showDefaultSettingsAction; private SelectionNavigationAction linkNavigationAction; @@ -82,7 +86,7 @@ public class ViewStringsPlugin extends ProgramPlugin implements DomainObjectList } private void createActions() { - DockingAction refreshAction = new DockingAction("Refresh Strings", getName()) { + refreshAction = new DockingAction("Refresh Strings", getName()) { @Override public boolean isEnabledForContext(ActionContext context) { @@ -91,12 +95,14 @@ public class ViewStringsPlugin extends ProgramPlugin implements DomainObjectList @Override public void actionPerformed(ActionContext context) { + getToolBarData().setIcon(REFRESH_NOT_NEEDED_ICON); reload(); } }; - ImageIcon refreshIcon = Icons.REFRESH_ICON; - refreshAction.setDescription("Reloads all string data from the program"); - refreshAction.setToolBarData(new ToolBarData(refreshIcon)); + refreshAction.setToolBarData(new ToolBarData(REFRESH_NOT_NEEDED_ICON)); + refreshAction.setDescription( + "Push at any time to refresh the current table of strings.
    " + + "This button is highlighted when the data may be stale.
    "); refreshAction.setHelpLocation(new HelpLocation("ViewStringsPlugin", "Refresh")); tool.addLocalAction(provider, refreshAction); @@ -152,13 +158,6 @@ public class ViewStringsPlugin extends ProgramPlugin implements DomainObjectList } - private void selectData(ProgramSelection selection) { - ProgramSelectionPluginEvent pspe = - new ProgramSelectionPluginEvent("Selection", selection, currentProgram); - firePluginEvent(pspe); - processEvent(pspe); - } - @Override public void dispose() { reloadUpdateMgr.dispose(); @@ -186,45 +185,50 @@ public class ViewStringsPlugin extends ProgramPlugin implements DomainObjectList } } + private void markDataAsStale() { + provider.getComponent().repaint(); + refreshAction.getToolBarData().setIcon(REFRESH_ICON); + } + @Override public void domainObjectChanged(DomainObjectChangedEvent ev) { + if (ev.containsEvent(DomainObject.DO_OBJECT_RESTORED) || ev.containsEvent(ChangeManager.DOCR_MEMORY_BLOCK_MOVED) || ev.containsEvent(ChangeManager.DOCR_MEMORY_BLOCK_REMOVED) || - ev.containsEvent(ChangeManager.DOCR_CODE_REMOVED) || ev.containsEvent(ChangeManager.DOCR_DATA_TYPE_CHANGED)) { - reload(); - + markDataAsStale(); + return; } - else if (ev.containsEvent(ChangeManager.DOCR_CODE_ADDED)) { - for (int i = 0; i < ev.numRecords(); ++i) { - DomainObjectChangeRecord doRecord = ev.getChangeRecord(i); - Object newValue = doRecord.getNewValue(); - switch (doRecord.getEventType()) { - case ChangeManager.DOCR_CODE_REMOVED: - case ChangeManager.DOCR_COMPOSITE_ADDED: - ProgramChangeRecord pcRec = (ProgramChangeRecord) doRecord; - provider.remove(pcRec.getStart(), pcRec.getEnd()); - break; - case ChangeManager.DOCR_CODE_ADDED: - if (newValue instanceof Data) { - provider.add((Data) newValue); - } - break; - default: - //Msg.info(this, "Unhandled event type: " + doRecord.getEventType()); - break; - } + + for (int i = 0; i < ev.numRecords(); ++i) { + + DomainObjectChangeRecord doRecord = ev.getChangeRecord(i); + Object newValue = doRecord.getNewValue(); + switch (doRecord.getEventType()) { + case ChangeManager.DOCR_CODE_REMOVED: + ProgramChangeRecord pcRec = (ProgramChangeRecord) doRecord; + provider.remove(pcRec.getStart(), pcRec.getEnd()); + break; + case ChangeManager.DOCR_CODE_ADDED: + if (newValue instanceof Data) { + provider.add((Data) newValue); + } + break; + default: + //Msg.info(this, "Unhandled event type: " + doRecord.getEventType()); + break; } } - else if (ev.containsEvent(ChangeManager.DOCR_DATA_TYPE_SETTING_CHANGED)) { + + if (ev.containsEvent(ChangeManager.DOCR_DATA_TYPE_SETTING_CHANGED)) { // Unusual code: because the table model goes directly to the settings values // during each repaint, we don't need to figure out which row was changed. provider.getComponent().repaint(); } } - void reload() { + private void reload() { reloadUpdateMgr.update(); } } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringDataInstance.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringDataInstance.java index c50c779e16..f36e147f63 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringDataInstance.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringDataInstance.java @@ -162,7 +162,7 @@ public class StringDataInstance { return ((AbstractStringDataType) dt).getStringDataInstance(data, data, data.getLength()); } - if (dt instanceof Array && !data.isInitializedMemory()) { + if (dt instanceof Array && data.isInitializedMemory()) { ArrayStringable arrayStringable = ArrayStringable.getArrayStringable(((Array) dt).getDataType()); if (arrayStringable != null && arrayStringable.hasStringValue(data)) { @@ -918,8 +918,9 @@ public class StringDataInstance { if (byteOffset + charSize > stringBytes.length) { return false; } - long origCodePointValue = DataConverter.getInstance(buf.isBigEndian()).getValue(stringBytes, - byteOffset, charSize); + long origCodePointValue = DataConverter.getInstance(buf.isBigEndian()) + .getValue(stringBytes, + byteOffset, charSize); return origCodePointValue == StringUtilities.UNICODE_REPLACEMENT; } From 3507820e03191fd4fce34aae4e921778f1e12ccc Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Wed, 7 Apr 2021 10:51:05 -0400 Subject: [PATCH 7/8] GP-678, GP-811: Refactored fetchDependencies.gradle, now downloads FID datasets from ghidra-data repo, and revisied depencies/flatRepo layout. --- .gitignore | 2 +- DevGuide.md | 72 ++- Ghidra/Features/FunctionID/build.gradle | 9 +- Ghidra/Features/GhidraServer/build.gradle | 9 +- .../GhidraDev/GhidraDevPlugin/build.gradle | 18 +- build.gradle | 9 +- gradle/root/svg.gradle | 1 - gradle/support/fetchDependencies.gradle | 543 ++++++++---------- 8 files changed, 289 insertions(+), 374 deletions(-) diff --git a/.gitignore b/.gitignore index a2896ae8a3..75845fcbfb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ ghidra.repos.config # Misc files produced while executing application repositories/ -flatRepo/ +dependencies/ Ghidra/.ghidraSvrKeys wrapper.log* diff --git a/DevGuide.md b/DevGuide.md index eceb003b58..9876a5ab8f 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -33,7 +33,7 @@ You may not need all of these, depending on which portions you are building or d - https://adoptopenjdk.net/releases.html?variant=openjdk11&jvmVariant=hotspot - Amazon Corretto - https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/downloads-list.html -* Gradle 5.0 or later - We use version 5.0, and tested with up to 5.6.3. +* Gradle 5.0 or later - We use version 5.0, and tested with up to 6.8.3. - https://gradle.org/next-steps/?version=5.0&format=bin * A C/C++ compiler - We use GCC on Linux, Xcode (Clang) on macOS, and Visual Studio (2017 or later) on Windows. - https://gcc.gnu.org/ @@ -61,9 +61,9 @@ You may not need all of these, depending on which portions you are building or d - https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/ * Eclipse PDE - Environment for developing the GhidraDev plugin. - https://www.eclipse.org/pde/ -* Eclipse CDT. We use version 8.6.0 - Build dependency for the GhidraDev plugin. +* Eclipse CDT. We build against version 8.6.0 - Build dependency for the GhidraDev plugin. - https://www.eclipse.org/cdt/ -* PyDev. We use version 6.3.1 - Build dependency for the GhidraDev plugin. +* PyDev. We build against version 6.3.1 - Build dependency for the GhidraDev plugin. - https://sourceforge.net/projects/pydev/files/pydev/ There are many, many others automatically downloaded by Gradle from Maven Central and Bintray JCenter when building and/or setting up the development environment. @@ -110,64 +110,64 @@ or manually by downloading the required dependencies. Choose one of the two fol The flat directory-style repository can be setup automatically by running a simple Gradle script. Navigate to `~/git/ghidra` and run the following: ``` -gradle --init-script gradle/support/fetchDependencies.gradle init +gradle -I gradle/support/fetchDependencies.gradle init ``` The Gradle task to be executed, in this case _init_, is unimportant. The point is to have Gradle execute -the `fetchDependencies.gradle` script. If it ran correctly you will have a new `~/git/ghidra/flatRepo/` -directory populated with the following jar files: - * AXMLPrinter2 - * csframework - * dex-ir-2.0 - * dex-reader-2.0 - * dex-reader-api-2.0 - * dex-tools-2.0 - * dex-translator-2.0 - * dex-writer-2.0 - * hfsx - * hfsx_dmglib - * iharder-base64 - -There will also be a new archive files at: - * ~/git/ghidra/Ghidra/Features/GhidraServer/build/`yajsw-stable-12.12.zip` - * ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/`PyDev 6.3.1.zip` - * ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/`cdt-8.6.0.zip` +the `fetchDependencies.gradle` script. If it ran correctly you will have a new `~/git/ghidra/dependencies/` +directory populated with the following files: + * flatRepo/AXMLPrinter2.jar + * flatRepo/csframework.jar + * flatRepo/dex-ir-2.0.jar + * flatRepo/dex-reader-2.0.jar + * flatRepo/dex-reader-api-2.0.jar + * flatRepo/dex-tools-2.0.jar + * flatRepo/dex-translator-2.0.jar + * flatRepo/dex-writer-2.0.jar + * flatRepo/hfsx.jar + * flatRepo/hfsx_dmglib.jar + * flatRepo/iharder-base64.jar + * cdt-8.6.0.zip + * PyDev 6.3.1.zip + * yajsw-stable-12.12.zip + * fid/*.fidb If you see these, congrats! Skip to [building](#building-ghidra) or [developing](#developing-ghidra). If not, continue with manual download instructions below... ### Manual Download Instructions -Create the `~/git/ghidra/flatRepo/` directory to hold the manually-downloaded dependencies: +Create the `~/git/ghidra/dependencies/` and `~/git/ghidra/dependencies/flatRepo` directories to hold the manually-downloaded dependencies: ```bash -mkdir ~/git/ghidra/flatRepo +mkdir ~/git/ghidra/dependencies +mkdir ~/git/ghidra/dependencies/flatRepo ``` #### Get Dependencies for FileFormats: Download `dex-tools-2.0.zip` from the dex2jar project's releases page on GitHub. -Unpack the `dex-*.jar` files from the `lib` directory to `~/git/ghidra/flatRepo`: +Unpack the `dex-*.jar` files from the `lib` directory to `~/git/ghidra/dependencies/flatRepo`: ```bash cd ~/Downloads # Or wherever curl -OL https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip unzip dex-tools-2.0.zip -cp dex2jar-2.0/lib/dex-*.jar ~/git/ghidra/flatRepo/ +cp dex2jar-2.0/lib/dex-*.jar ~/git/ghidra/dependencies/flatRepo/ ``` Download `AXMLPrinter2.jar` from the "android4me" archive on code.google.com. -Place it in `~/git/ghidra/flatRepo`: +Place it in `~/git/ghidra/dependencies/flatRepo`: ```bash -cd ~/git/ghidra/flatRepo +cd ~/git/ghidra/dependencies/flatRepo curl -OL https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar ``` #### Get Dependencies for DMG: Download `hfsexplorer-0_21-bin.zip` from www.catacombae.org. -Unpack the `lib` directory to `~/git/ghidra/flatRepo`: +Unpack the `lib` directory to `~/git/ghidra/dependencies/flatRepo`: ```bash cd ~/Downloads # Or wherever @@ -176,35 +176,33 @@ mkdir hfsx cd hfsx unzip ../hfsexplorer-0_21-bin.zip cd lib -cp csframework.jar hfsx_dmglib.jar hfsx.jar iharder-base64.jar ~/git/ghidra/flatRepo/ +cp csframework.jar hfsx_dmglib.jar hfsx.jar iharder-base64.jar ~/git/ghidra/dependencies/flatRepo/ ``` #### Get Dependencies for GhidraServer Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12. Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in: -`~/git/ghidra/Ghidra/Features/GhidraServer/build`: +`~/git/ghidra/dependencies/`: ```bash cd ~/Downloads # Or wherever curl -OL https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip -mkdir -p ~/git/ghidra/Ghidra/Features/GhidraServer/build/ -cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra/Ghidra/Features/GhidraServer/build/ +cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra/dependencies/ ``` #### Get Dependencies for GhidraDev Building the GhidraDev plugin for Eclipse requires the CDT and PyDev plugins for Eclipse. Download `cdt-8.6.0.zip` from The Eclipse Foundation, and place it in: -`~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/`: +`~/git/ghidra/dependencies/`: ```bash cd ~/Downloads # Or wherever curl -OL 'https://archive.eclipse.org/tools/cdt/releases/8.6/cdt-8.6.0.zip' curl -o 'cdt-8.6.0.zip.sha512' -L --retry 3 'https://www.eclipse.org/downloads/sums.php?type=sha512&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' shasum -a 512 -c 'cdt-8.6.0.zip.sha512' -mkdir -p ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ -cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ +cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra/dependencies/ ``` Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same directory: @@ -212,7 +210,7 @@ Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same director ```bash cd ~/Downloads # Or wherever curl -L -o 'PyDev 6.3.1.zip' https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip -cp ~/Downloads/'PyDev 6.3.1.zip' ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ +cp ~/Downloads/'PyDev 6.3.1.zip' ~/git/ghidra/dependencies/ ``` ## Building Ghidra diff --git a/Ghidra/Features/FunctionID/build.gradle b/Ghidra/Features/FunctionID/build.gradle index 8345178732..a67ae99435 100644 --- a/Ghidra/Features/FunctionID/build.gradle +++ b/Ghidra/Features/FunctionID/build.gradle @@ -31,10 +31,11 @@ dependencies { } -// All *.fidb files located in the BIN repo under src/main/fidb will be unpacked -def fidbSrcDir = "${getProjectLocationInBinRepo(project)}/src/main/fidb" - -def fidDbFiles = fileTree(fidbSrcDir) { +// All *.fidb files located in the dependencies/fid directory OR the +// BIN repo under src/main/fidb will be unpacked +def depsDir = file("${DEPS_DIR}/fidb") +def binRepoDir = "${getProjectLocationInBinRepo(project)}/src/main/fidb" +def fidDbFiles = fileTree(depsDir.exists() ? depsDir : binRepoDir) { include '**/*.fidb' } diff --git a/Ghidra/Features/GhidraServer/build.gradle b/Ghidra/Features/GhidraServer/build.gradle index ec08f05fcc..484be39516 100644 --- a/Ghidra/Features/GhidraServer/build.gradle +++ b/Ghidra/Features/GhidraServer/build.gradle @@ -40,12 +40,11 @@ addExports([ ]) CopySpec yajswCopySpec = copySpec { - File localFile = file("build/${yajswRelease}.zip") - File binFile = file("${BIN_REPO}/Ghidra/Features/GhidraServer/${yajswRelease}.zip") + File depsFile = file("${DEPS_DIR}/GhidraServer/${yajswRelease}.zip") + File binRepoFile = file("${BIN_REPO}/Ghidra/Features/GhidraServer/${yajswRelease}.zip") - // First check if the file was downloaded and dropped in locally. If not, check in the bin - // repo. - def yajswZipTree = localFile.exists() ? zipTree(localFile) : zipTree(binFile) + // First check if the file is in the dependencies repo. If not, check in the bin repo. + def yajswZipTree = depsFile.exists() ? zipTree(depsFile) : zipTree(binRepoFile) from(yajswZipTree) { include "${yajswRelease}/lib/core/**" diff --git a/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle b/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle index aecbcd515e..bb4516d155 100644 --- a/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle +++ b/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle @@ -80,12 +80,11 @@ task pyDevUnpack(type:Copy) { !pyDevDestDir.exists() } - File localFile = file("build/PyDev 6.3.1.zip") - File binFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/PyDev 6.3.1.zip") + File depsFile = file("${DEPS_DIR}/GhidraDev/PyDev 6.3.1.zip") + File binRepoFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/PyDev 6.3.1.zip") - // First check if the file was downloaded and dropped in locally. If not, check in the bin - // repo. - def pyDevZipTree = localFile.exists() ? zipTree(localFile) : zipTree(binFile) + // First check if the file is in the dependencies repo. If not, check in the bin repo. + def pyDevZipTree = depsFile.exists() ? zipTree(depsFile) : zipTree(binRepoFile) from pyDevZipTree exclude "**/.project", "**/.pydevproject" @@ -104,12 +103,11 @@ task cdtUnpack(type:Copy) { !cdtDestDir.exists() } - File localFile = file("build/cdt-8.6.0.zip") - File binFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/cdt-8.6.0.zip") + File depsFile = file("${DEPS_DIR}/GhidraDev/cdt-8.6.0.zip") + File binRepoFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/cdt-8.6.0.zip") - // First check if the file was downloaded and dropped in locally. If not, check in the bin - // repo. - def cdtZipTree = localFile.exists() ? zipTree(localFile) : zipTree(binFile) + // First check if the file is in the dependencies repo. If not, check in the bin repo. + def cdtZipTree = depsFile.exists() ? zipTree(depsFile) : zipTree(binRepoFile) from cdtZipTree diff --git a/build.gradle b/build.gradle index 066143562d..6ab7f43302 100644 --- a/build.gradle +++ b/build.gradle @@ -51,9 +51,9 @@ if ("32".equals(System.getProperty("sun.arch.data.model"))) { * Define the location of bin repo *********************************************************************************/ project.ext.GHIDRA_GROUP = "Z Ghidra" -project.ext.BIN_REPO = file("${projectDir}/../ghidra.bin").absolutePath project.ext.ROOT_PROJECT_DIR = projectDir.absolutePath -project.ext.BIN_REPO_PATH = BIN_REPO // TODO make path names consistent +project.ext.BIN_REPO = file("${projectDir}/../ghidra.bin").absolutePath +project.ext.DEPS_DIR = file("${projectDir}/dependencies") /********************************************************************************* * Prevent forked Java processes from stealing focus @@ -67,13 +67,14 @@ allprojects { /********************************************************************************* * Use flat directory-style repository if flatRepo directory is present. *********************************************************************************/ -if (file("flatRepo").isDirectory()) { +def flatRepo = file("${DEPS_DIR}/flatRepo") +if (flatRepo.isDirectory()) { allprojects { repositories { mavenLocal() mavenCentral() jcenter() - flatDir name: "flat", dirs:["$rootProject.projectDir/flatRepo"] + flatDir name: "flat", dirs:["$flatRepo"] } } } diff --git a/gradle/root/svg.gradle b/gradle/root/svg.gradle index 153b8c4720..20309a2ea6 100644 --- a/gradle/root/svg.gradle +++ b/gradle/root/svg.gradle @@ -43,7 +43,6 @@ task rasterizeSvg(type: JavaExec) { // added these in the individual projects which use this task (eg: to the 'compile' // configuration) but since this is the only task which requires them, it seemed // appropriate to just add them here. - def BIN_REPO = rootProject.file(BIN_REPO_PATH).toString() classpath = files ( BIN_REPO + "/ExternalLibraries/libsforBuild/batik-all-1.7.jar", BIN_REPO + "/ExternalLibraries/libsforBuild/xml-apis-ext.jar") diff --git a/gradle/support/fetchDependencies.gradle b/gradle/support/fetchDependencies.gradle index 6b4a7912ad..a9a8db3b0a 100644 --- a/gradle/support/fetchDependencies.gradle +++ b/gradle/support/fetchDependencies.gradle @@ -22,29 +22,13 @@ * immediately after cloning the Ghidra repository before any other gradle * * tasks are run. * * * - * Specifically, this task: * + * usage: from the command line in the main ghidra repository directory, run * + * the following: * * * - * 1. Downloads various dependencies required by the ghidra build and * - * puts them in /build/downloads/. From here they are * - * unzipped and/or copied to their final locations. The files to be * - * downloaded: * - * - dex-tools-2.0.zip * - * - AXMLPrinter2.jar * - * - hfsexplorer-0_21-bin.zip * - * - yajsw-stable-12.12.zip * - * - cdt-8.6.0.zip * - * - PyDev 6.3.1.zip * - * * - * 2. Creates a directory at /flatRepo which is used as a * - * flat directory-style respository for the files extracted above. * - * * - * usage: from the command line in the main ghidra repository * - * directory, run the following: * - * * - * gradle --init-script gradle/support/fetchDependencies.gradle init * + * gradle -I gradle/support/fetchDependencies.gradle init * * * * Note: When running the script, files will only be downloaded if * - * necessary (eg: they are not already in the build/downloads/ * + * necessary (eg: they are not already in the dependencies/downloads/ * * directory). * * * *******************************************************************************/ @@ -52,138 +36,226 @@ import java.util.zip.*; import java.nio.file.*; import java.security.MessageDigest; -import org.apache.commons.io.*; -import org.apache.commons.io.filefilter.*; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.WildcardFileFilter; -ext.HOME_DIR = System.getProperty('user.home') -ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getParentFile().getParentFile() -ext.FLAT_REPO_DIR = new File(REPO_DIR, "flatRepo") -ext.DOWNLOADS_DIR = new File(REPO_DIR, "build/downloads") - -// Stores the size of the file being downloaded (for formatting print statements) -ext.FILE_SIZE = 0; - -// The URLs for each of the dependencies -ext.DEX_ZIP = 'https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip' -ext.AXML_ZIP = 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar' -ext.HFS_ZIP = 'https://sourceforge.net/projects/catacombae/files/HFSExplorer/0.21/hfsexplorer-0_21-bin.zip' -ext.YAJSW_ZIP = 'https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip' -ext.PYDEV_ZIP = 'https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip' -ext.CDT_ZIP = 'https://archive.eclipse.org/tools/cdt/releases/8.6/cdt-8.6.0.zip' - -// The SHA-256s for each of the dependencies -ext.DEX_SHA_256 = '7907eb4d6e9280b6e17ddce7ee0507eae2ef161ee29f70a10dbc6944fdca75bc' -ext.AXML_SHA_256 = '00ed038eb6abaf6ddec8d202a3ed7a81b521458f4cd459948115cfd02ff59d6d' -ext.HFS_SHA_256 = '90c9b54798abca5b12f4a678db7d0a4c970f4702cb153c11919536d0014dedbf' -ext.YAJSW_SHA_256 = '1398fcb1e93abb19992c4fa06d7fe5758aabb4c45781d7ef306c6f57ca7a7321' -ext.PYDEV_SHA_256 = '4d81fe9d8afe7665b8ea20844d3f5107f446742927c59973eade4f29809b0699' -ext.CDT_SHA_256 = '81b7d19d57c4a3009f4761699a72e8d642b5e1d9251d2bb98df438b1e28f8ba9' - -// Number of times to try and establish a connection when downloading files before -// failing -ext.NUM_RETRIES = 2 - -// Set up a maven repository configuration so we can get access to Apache FileUtils for -// copying/deleting files. initscript { - repositories { - mavenCentral() - } - dependencies { - classpath 'commons-io:commons-io:2.5' - } + repositories { mavenCentral() } + dependencies { classpath 'commons-io:commons-io:2.8.0' } } -// This is where the real flow of the script starts... -try { - createDirs() - populateFlatRepo() +ext.NUM_RETRIES = 3 // # of times to try to download a file before failing +ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getParentFile().getParentFile() +ext.DEPS_DIR = file("${REPO_DIR}/dependencies") +ext.DOWNLOADS_DIR = file("${DEPS_DIR}/downloads") +ext.FID_DIR = file("${DEPS_DIR}/fidb") +ext.FLAT_REPO_DIR = file("${DEPS_DIR}/flatRepo") + +ext.deps = [ + [ + name: 'dex-tools-2.0.zip', + url: 'https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip', + sha256: '7907eb4d6e9280b6e17ddce7ee0507eae2ef161ee29f70a10dbc6944fdca75bc', + destination: { + unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, 'dex-tools-2.0.zip') + FileUtils.copyDirectory(new File(DOWNLOADS_DIR, 'dex2jar-2.0/lib/'), FLAT_REPO_DIR, new WildcardFileFilter("dex-*")); + } + ], + [ + name: 'hfsexplorer-0_21-bin.zip', + url: 'https://sourceforge.net/projects/catacombae/files/HFSExplorer/0.21/hfsexplorer-0_21-bin.zip', + sha256: '90c9b54798abca5b12f4a678db7d0a4c970f4702cb153c11919536d0014dedbf', + destination: { + def hfsxdir = new File (DOWNLOADS_DIR, "hfsx") + hfsxdir.mkdir() + unzip (DOWNLOADS_DIR, hfsxdir, 'hfsexplorer-0_21-bin.zip') + FileUtils.copyFileToDirectory(new File(DOWNLOADS_DIR, "hfsx/lib/csframework.jar"), FLAT_REPO_DIR); + FileUtils.copyFileToDirectory(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx_dmglib.jar"), FLAT_REPO_DIR); + FileUtils.copyFileToDirectory(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx.jar"), FLAT_REPO_DIR); + FileUtils.copyFileToDirectory(new File(DOWNLOADS_DIR, "hfsx/lib/iharder-base64.jar"), FLAT_REPO_DIR); + } + ], + [ + name: 'AXMLPrinter2.jar', + url: 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar', + sha256: '00ed038eb6abaf6ddec8d202a3ed7a81b521458f4cd459948115cfd02ff59d6d', + destination: FLAT_REPO_DIR + ], + [ + name: 'yajsw-stable-12.12.zip', + url: 'https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip', + sha256: '1398fcb1e93abb19992c4fa06d7fe5758aabb4c45781d7ef306c6f57ca7a7321', + destination: file("${DEPS_DIR}/GhidraServer") + ], + [ + name: 'PyDev 6.3.1.zip', + url: 'https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip', + sha256: '4d81fe9d8afe7665b8ea20844d3f5107f446742927c59973eade4f29809b0699', + destination: file("${DEPS_DIR}/GhidraDev") + ], + [ + name: 'cdt-8.6.0.zip', + url: 'https://archive.eclipse.org/tools/cdt/releases/8.6/cdt-8.6.0.zip', + sha256: '81b7d19d57c4a3009f4761699a72e8d642b5e1d9251d2bb98df438b1e28f8ba9', + destination: file("${DEPS_DIR}/GhidraDev") + ], + [ + name: 'vs2012_x64.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2012_x64.fidb', + sha256: 'f26548a6df6b6963a418d8c83ac216d9e196b180d944a52b8123c457d472b7c9', + destination: FID_DIR + ], + [ + name: 'vs2012_x86.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2012_x86.fidb', + sha256: '0a8962cf3699d5b8d4b3a79400382462519edc26570a46b2085200e38534f900', + destination: FID_DIR + ], + [ + name: 'vs2015_x64.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2015_x64.fidb', + sha256: '187248f87fc1deb695bc3051b2d92f9b7482023a356821154db22478eed13088', + destination: FID_DIR + ], + [ + name: 'vs2015_x86.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2015_x86.fidb', + sha256: '1d05afa070e9c09b83ee15d544c8559ed0d2b53d7eac476f8f5f8849543b3812', + destination: FID_DIR + ], + [ + name: 'vs2017_x64.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2017_x64.fidb', + sha256: '1784ad6b25571177ff8212871867559998c6b8256bb1dbaeee864b580c1b2d6a', + destination: FID_DIR + ], + [ + name: 'vs2017_x86.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2017_x86.fidb', + sha256: 'bc9bf30621190e0eb56c4db5ec30ad0401ca7be0311f5a2ce3d894178eafd19c', + destination: FID_DIR + ], + [ + name: 'vs2019_x64.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2019_x64.fidb', + sha256: 'aab04eefd1142f7b3c3f86c8d766abe361b167b4fe4157c36fad18777b2a6fbd', + destination: FID_DIR + ], + [ + name: 'vs2019_x86.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vs2019_x86.fidb', + sha256: '0a2282ac3479ffc022e6cdb4e32e057bc10f0394cfb0f8016d7145be0167f5f7', + destination: FID_DIR + ], + [ + name: 'vsOlder_x64.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vsOlder_x64.fidb', + sha256: 'fe1856c0acad297d9ba4fb6a2df1d32ba34df766d9f1a2a16da0ca2b375e23dd', + destination: FID_DIR + ], + [ + name: 'vsOlder_x86.fidb', + url: 'https://github.com/NationalSecurityAgency/ghidra-data/raw/master/FunctionID/vsOlder_x86.fidb', + sha256: '46e56bc82ba68ad4e9a3c6a2e4ecd3428e2c390c7de0a379fa0165a58d46e115', + destination: FID_DIR + ] +] + +// Download dependencies (if necessary) and verify their hashes +DOWNLOADS_DIR.mkdirs() +deps.each { + File file = new File(DOWNLOADS_DIR, it.name) + if (!it.sha256.equals(generateHash(file))) { + download(it.url, file.path) + assert(it.sha256.equals(generateHash(file))); + } } -finally { - cleanup() + +// Copies the downloaded dependencies to their required destination. +// Some downloads require pre-processing before their relevant pieces can be copied. +deps.each { + if (it.destination instanceof File) { + println("Copying " + it.name + " to " + it.destination) + it.destination.mkdirs() + FileUtils.copyFile(new File(DOWNLOADS_DIR, it.name), new File(it.destination, it.name)); + } + else if (it.destination instanceof Closure) { + println("Processing " + it.name) + it.destination() + } + else { + throw new GradleException("Unexpected destination type: " + it.destination) + } } +//-------------------------------------Helper methods---------------------------------------------- /** - * Creates the directories where the dependencies will be downloaded and stored - */ -def createDirs() { - if (!DOWNLOADS_DIR.exists()) { - DOWNLOADS_DIR.mkdirs() - } - if (!FLAT_REPO_DIR.exists()) { - FLAT_REPO_DIR.mkdirs() - } -} - -/** - * Downloads a file from a URL. If there is a problem connecting to the given - * URL the attempt will be retried NUM_RETRIES times before failing. + * Downloads a file from a URL. The download attempt will be tried NUM_RETRIES times before failing. * - * Progress is shown on the command line in the form of the number of bytes - * downloaded and a percentage of the total. + * Progress is shown on the command line in the form of the number of bytes downloaded and a + * percentage of the total. * - * Note: We do not validate that the number of bytes downloaded matches the - * expected total here; any discrepencies will be caught when checking - * the SHA-256s later on. + * Note: We do not validate that the number of bytes downloaded matches the expected total here; any + * discrepencies will be caught when checking the SHA-256s later on. * * @param url the file to download * @param filename the local file to create for the download */ def download(url, filename) { - println("File: " + url) - BufferedInputStream istream = establishConnection(url, NUM_RETRIES); - assert istream != null : " ***CONNECTION FAILURE***\n max attempts exceeded; exiting\n" - - FileOutputStream ostream = new FileOutputStream(filename); - def dataBuffer = new byte[1024]; - int bytesRead; - int totalRead; - while ((bytesRead = istream.read(dataBuffer, 0, 1024)) != -1) { - - ostream.write(dataBuffer, 0, bytesRead); - totalRead += bytesRead - - print("\r") - if (FILE_SIZE.equals("unknown")) { - print(" Downloading: " + totalRead + " of " + FILE_SIZE) - } - else { - int pctComplete = (totalRead / FILE_SIZE) * 100 - print(" Downloading: " + totalRead + " of " + FILE_SIZE + " (" + pctComplete + "%)") - } - System.out.flush() - } - println("") - - istream.close(); - ostream.close(); + println("URL: " + url) + def(InputStream istream, size) = establishConnection(url, NUM_RETRIES); + assert istream != null : " ***CONNECTION FAILURE***\n max attempts exceeded; exiting\n" + + FileOutputStream ostream = new FileOutputStream(filename); + def dataBuffer = new byte[1024]; + int bytesRead; + int totalRead; + while ((bytesRead = istream.read(dataBuffer, 0, 1024)) != -1) { + ostream.write(dataBuffer, 0, bytesRead); + totalRead += bytesRead + print("\r") + print(" Downloading: " + totalRead + " of " + size) + if (!size.equals("???")) { + int pctComplete = (totalRead / size) * 100 + print(" (" + pctComplete + "%)") + } + print(" ") // overwrite gradle timer output + System.out.flush() + } + println() + istream.close(); + ostream.close(); } /** - * Attemps to establish a connection to the given URL. + * Attempts to establish a connection to the given URL * - * @param url the site to connect to - * @param retries the number of times to attempt to reconnect if there is a failure - * @return the InputStream for the URL + * @param url the URL to connect to + * @param retries the number of times to attempt to connect if there are failures + * @return the InputStream for the URL, and the size of the download in bytes as a string */ def establishConnection(url, retries) { - for (int i=0; i - (e.name as File).with { f -> - if (f.parentFile != null) { - File destPath = new File(targetDir.path, f.parentFile.path) - destPath.mkdirs() - File targetFile = new File(destPath.path, f.name) - targetFile.withOutputStream { w -> - w << zip.getInputStream(e) - } - } - } - } + def zip = new ZipFile(new File(sourceDir, zipFileName)) + zip.entries().findAll { !it.directory }.each { e -> + (e.name as File).with { f -> + if (f.parentFile != null) { + File destPath = new File(targetDir.path, f.parentFile.path) + destPath.mkdirs() + File targetFile = new File(destPath.path, f.name) + targetFile.withOutputStream { w -> + w << zip.getInputStream(e) + } + } + } + } + zip.close() } /** - * Downloads and stores the necessary dependencies in the local flat repository. + * Generates the SHA-256 hash for the given file * - * If the dependency already exists in the downloads folder (DOWNLOADS_DIR) and has the - * proper checksum, it will NOT be re-downloaded. + * @param file the file to generate the SHA-256 hash for + * @return the generated SHA-256 hash, or null if the file does not exist */ -def populateFlatRepo() { - - // 1. Download all the dependencies and verify their checksums. If the dependency has already - // been download, do NOT download again. - File file = new File(DOWNLOADS_DIR, 'dex-tools-2.0.zip') - if (!DEX_SHA_256.equals(generateChecksum(file))) { - download (DEX_ZIP, file.path) - validateChecksum(generateChecksum(file), DEX_SHA_256); - } - - file = new File(DOWNLOADS_DIR, 'AXMLPrinter2.jar') - if (!AXML_SHA_256.equals(generateChecksum(file))) { - download (AXML_ZIP, file.path) - validateChecksum(generateChecksum(file), AXML_SHA_256); - } - - file = new File(DOWNLOADS_DIR, 'hfsexplorer-0_21-bin.zip') - if (!HFS_SHA_256.equals(generateChecksum(file))) { - download (HFS_ZIP, file.path) - validateChecksum(generateChecksum(file), HFS_SHA_256); - } - - file = new File(DOWNLOADS_DIR, 'yajsw-stable-12.12.zip') - if (!YAJSW_SHA_256.equals(generateChecksum(file))) { - download (YAJSW_ZIP, file.path) - validateChecksum(generateChecksum(file), YAJSW_SHA_256); - } - - file = new File(DOWNLOADS_DIR, 'PyDev 6.3.1.zip') - if (!PYDEV_SHA_256.equals(generateChecksum(file))) { - download (PYDEV_ZIP, file.path) - validateChecksum(generateChecksum(file), PYDEV_SHA_256); - } - - file = new File(DOWNLOADS_DIR, 'cdt-8.6.0.zip') - if (!CDT_SHA_256.equals(generateChecksum(file))) { - download (CDT_ZIP, file.path) - validateChecksum(generateChecksum(file), CDT_SHA_256); - } - - // 2. Unzip the dependencies - unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, "dex-tools-2.0.zip") - unzipHfsx() - - // 3. Copy the necessary jars to the flatRepo directory. Yajsw, CDT, and PyDev go directly into - // the source repository. - copyDexTools() - copyAXML() - copyHfsx() - copyYajsw() - copyPyDev() - copyCdt() -} - -/** - * Generates the SHA-256 for the given file - * - * @param file the file to generate the checksum for - * @return the generated checksum - */ -def generateChecksum(file) { - if (!file.exists()) { - return null - } - MessageDigest md = MessageDigest.getInstance("SHA-256"); - md.update(Files.readAllBytes(Paths.get(file.path))); - byte[] digest = md.digest(); - StringBuilder sb = new StringBuilder(); - for (byte b : digest) { - sb.append(String.format("%02x", b)); - } - - return sb.toString(); -} - -/** - * Compares two checksums and generates an assert failure if they do not match - * - * @param sourceSha256 the checksum to validate - * @param expectedSha256 the expected checksum - */ -def validateChecksum(sourceSha256, expectedSha256) { - assert(sourceSha256.equals(expectedSha256)); -} - -/** - * Unzips the hfsx zip file - */ -def unzipHfsx() { - def hfsxdir = getOrCreateTempHfsxDir() - unzip (DOWNLOADS_DIR, hfsxdir, "hfsexplorer-0_21-bin.zip") -} - -/** - * Copies the dex-tools jars to the flat repository - * - * Note: This will only copy files beginning with "dex-" - */ -def copyDexTools() { - FileUtils.copyDirectory(new File(DOWNLOADS_DIR, 'dex2jar-2.0/lib/'), FLAT_REPO_DIR, new WildcardFileFilter("dex-*")); -} - -/** - * Copies the AXMLPrinter2 jar to the flat repository - */ -def copyAXML() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, 'AXMLPrinter2.jar'), new File(FLAT_REPO_DIR, "AXMLPrinter2.jar")); -} - -/** - * Copies the necessary hfsx jars to the flat repository - */ -def copyHfsx() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/csframework.jar"), new File(FLAT_REPO_DIR, "csframework.jar")); - FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx_dmglib.jar"), new File(FLAT_REPO_DIR, "hfsx_dmglib.jar")); - FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx.jar"), new File(FLAT_REPO_DIR, "hfsx.jar")); - FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/iharder-base64.jar"), new File(FLAT_REPO_DIR, "iharder-base64.jar")); -} - -/** - * Copies the yajswdir zip to its location in the GhidraServer project. - */ -def copyYajsw() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, "yajsw-stable-12.12.zip"), new File(REPO_DIR, "Ghidra/Features/GhidraServer/build/yajsw-stable-12.12.zip")); -} - -/** - * Copies the pydev zip to its bin repository location - */ -def copyPyDev() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, 'PyDev 6.3.1.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/PyDev 6.3.1.zip")); -} - -/** - * Copies the cdt zip to its bin repository location - */ -def copyCdt() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, 'cdt-8.6.0.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/cdt-8.6.0.zip")); -} - -/** - * Creates a temporary folder to house the hfsx zip contents - * - * @return the newly-created hfsx directory object - */ -def getOrCreateTempHfsxDir() { - def hfsxdir = new File (DOWNLOADS_DIR, "hfsx") - if (!hfsxdir.exists()) { - hfsxdir.mkdir() - } - - return hfsxdir; -} - -/** - * Performs any cleanup operations that need to be performed after the flat repo has - * been populated. - */ -def cleanup() { - // Uncomment this if we want to delete the downloads folder. For now, leave this and - // depend on a gradle clean to wipe it out. - // - //if (DOWNLOADS_DIR.exists()) { - // FileUtils.deleteDirectory(DOWNLOADS_DIR) - //} +def generateHash(file) { + if (!file.exists()) { + return null + } + MessageDigest md = MessageDigest.getInstance("SHA-256"); + md.update(Files.readAllBytes(Paths.get(file.path))); + byte[] digest = md.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); } From deffc912a067b37b446f8865fe5532c73580d3ef Mon Sep 17 00:00:00 2001 From: ghidra1 Date: Thu, 8 Apr 2021 10:38:42 -0400 Subject: [PATCH 8/8] GP-0 Corrected swing related issues affecting testing --- .../Base/src/main/java/ghidra/app/script/GhidraState.java | 5 ++++- .../java/ghidra/app/script/GhidraScriptAskMethodsTest.java | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraState.java b/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraState.java index 2ec73d6898..32b7161418 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraState.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraState.java @@ -29,6 +29,7 @@ import ghidra.program.model.address.AddressSet; import ghidra.program.model.listing.Program; import ghidra.program.util.ProgramLocation; import ghidra.program.util.ProgramSelection; +import ghidra.util.Swing; import ghidra.util.SystemUtilities; /** @@ -64,7 +65,9 @@ public class GhidraState { this.currentHighlight = highlight; this.isGlobalState = true; if (!SystemUtilities.isInHeadlessMode()) { - gatherParamPanel = new GatherParamPanel(this); + Swing.runNow(() -> { + gatherParamPanel = new GatherParamPanel(this); + }); } } diff --git a/Ghidra/Features/Base/src/test.slow/java/ghidra/app/script/GhidraScriptAskMethodsTest.java b/Ghidra/Features/Base/src/test.slow/java/ghidra/app/script/GhidraScriptAskMethodsTest.java index 507a99625a..af8f7bc57b 100644 --- a/Ghidra/Features/Base/src/test.slow/java/ghidra/app/script/GhidraScriptAskMethodsTest.java +++ b/Ghidra/Features/Base/src/test.slow/java/ghidra/app/script/GhidraScriptAskMethodsTest.java @@ -84,8 +84,10 @@ public class GhidraScriptAskMethodsTest extends AbstractGhidraHeadedIntegrationT } private void clearScriptCachedValues() { - Map map = (Map) TestUtils.getInstanceField("askMap", script); - map.clear(); + if (script != null) { + Map map = (Map) TestUtils.getInstanceField("askMap", script); + map.clear(); + } } @Test