From 7f515c5e0ef4f0568e88c7172a4e8cb9849ec7bf Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Tue, 5 Nov 2024 08:30:54 -0500 Subject: [PATCH] GP-5093: Converting 'What's New' to Markdown --- .../Public_Release/build.gradle | 5 +- .../Public_Release/certification.manifest | 2 +- .../src/global/docs/WhatsNew.html | 167 ------------------ .../src/global/docs/WhatsNew.md | 118 +++++++++++++ .../src/main/java/help/HelpBuildUtils.java | 7 +- gradle/distributableGhidraModule.gradle | 18 +- gradle/helpProject.gradle | 35 +++- 7 files changed, 174 insertions(+), 178 deletions(-) delete mode 100644 Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.html create mode 100644 Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.md diff --git a/Ghidra/Configurations/Public_Release/build.gradle b/Ghidra/Configurations/Public_Release/build.gradle index defe20000a..632a51d42a 100644 --- a/Ghidra/Configurations/Public_Release/build.gradle +++ b/Ghidra/Configurations/Public_Release/build.gradle @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,5 +17,6 @@ apply from: "$rootProject.projectDir/gradle/distributableGhidraModule.gradle" apply from: "$rootProject.projectDir/gradle/javaProject.gradle" apply from: "$rootProject.projectDir/gradle/jacocoProject.gradle" apply from: "$rootProject.projectDir/gradle/javaTestProject.gradle" +apply from: "$rootProject.projectDir/gradle/helpProject.gradle" apply plugin: 'eclipse' eclipse.project.name = 'Z Public Release' diff --git a/Ghidra/Configurations/Public_Release/certification.manifest b/Ghidra/Configurations/Public_Release/certification.manifest index bf35c85f28..9ad85924a3 100644 --- a/Ghidra/Configurations/Public_Release/certification.manifest +++ b/Ghidra/Configurations/Public_Release/certification.manifest @@ -4,7 +4,7 @@ README.md||GHIDRA||||END| data/PDB_SYMBOL_SERVER_URLS.pdburl||GHIDRA||||END| src/global/docs/ChangeHistory.html||GHIDRA||||END| src/global/docs/UserAgreement.html||GHIDRA||||END| -src/global/docs/WhatsNew.html||GHIDRA||||END| +src/global/docs/WhatsNew.md||GHIDRA||||END| src/main/resources/UserAgreement.html||GHIDRA||||END| src/main/resources/defaultTools/CodeBrowser.tool||GHIDRA||||END| src/main/resources/splash.txt||GHIDRA||||END| diff --git a/Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.html b/Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.html deleted file mode 100644 index 82c9225a03..0000000000 --- a/Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.html +++ /dev/null @@ -1,167 +0,0 @@ - - - - - Ghidra What's New - - - - - -

Ghidra: NSA Reverse Engineering Software

- -

- Ghidra is a software reverse engineering (SRE) framework developed by NSA's Research Directorate. - This framework includes a suite of full-featured, high-end software analysis tools that enable - users to analyze compiled code on a variety of platforms including Windows, MacOS, and Linux. - Capabilities include disassembly, assembly, decompilation, debugging, emulation, graphing, and scripting, along with - hundreds of other features. Ghidra supports a wide variety of processor instruction sets and - executable formats and can be run in both user-interactive and automated modes. Users may also - develop their own Ghidra plug-in components and/or scripts using the exposed API. In addition there are - numerous ways to extend Ghidra such as new processors, loaders/exporters, automated analyzers, - and new visualizations. -

- -

- In support of NSA's Cybersecurity mission, Ghidra was built to solve scaling and teaming problems - on complex SRE efforts and to provide a customizable and extensible SRE research platform. NSA - has applied Ghidra SRE capabilities to a variety of problems that involve analyzing malicious - code and generating deep insights for NSA analysts who seek a better understanding of potential - vulnerabilities in networks and systems. -

-
- -

What's New in Ghidra 11.2

- -

This release includes new features, enhancements, performance improvements, quite a few bug fixes, and many pull-request - contributions. Thanks to all those who have contributed their time, thoughts, and code. The Ghidra user community thanks you too!

- -

The not-so-fine print: Please Read!

- -

Ghidra 11.2 is fully backward compatible with project data from previous releases. - However, programs and data type archives which are created or modified in 11.2 will not be usable by an earlier Ghidra version.

- -

IMPORTANT: Ghidra 11.2 requires at minimum JDK 21 to run.

- -

IMPORTANT: To use the Debugger or do a full source distribution build, you will need Python3 (3.9 to 3.12 supported) installed on your system.

- -

NOTE: There have been reports of certain features causing the XWindows server to crash. A fix for - CVE-2024-31083 in X.org software in April 2024 introduced a regression, which has been fixed in xwayland 23.2.6 and xorg-server 21.1.13. If you experience - any crashing of Ghidra, most likely causing a full logout, check if your xorg-server has been updated to at least the noted version.

- -

NOTE: Each build distribution will include native components (e.g., decompiler) for at least one platform (e.g., Windows x86-64). - If you have another platform that is not included in the build distribution, you can build - native components for your platform directly from the distribution. - See the Ghidra Installation Guide for additional information. - Users running with older shared libraries and operating systems (e.g., CentOS 7.x) may also run into - compatibility errors when launching native executables such as the Decompiler and GNU Demangler which - may necessitate a rebuild of native components.

- -

NOTE: Ghidra Server: The Ghidra 11.x server is compatible with Ghidra 9.2 and later Ghidra clients. Ghidra 11.x - clients are compatible with all 10.x and 9.x servers. Although, due to potential Java version differences, it is recommended - that Ghidra Server installations older than 10.2 be upgraded. Those using 10.2 and newer should not need a server upgrade.

- -

NOTE: Any programs imported with a Ghidra beta version or code built directly from source code outside of a release tag may not be compatible, - and may have flaws that won't be corrected by using this new release. Any programs analyzed from a beta or other local master source build should be considered - experimental and re-imported and analyzed with a release version.

- -

Programs imported with previous release versions should upgrade correctly through various - automatic upgrade mechanisms. However, there may be improvements or bug fixes in the import and analysis process that will provide better results than prior - Ghidra versions. You might consider comparing a fresh import of any program you will continue to reverse engineer to see if the latest Ghidra - provides better results.

- -

Memory Search

- -

The Search Memory feature in Ghidra has been updated substantially to provide two new features:

-
- -
- -

Set operations, accessible from the pull-down menu under Search, allow you to augment - results by performing boolean operations on an existing search. For example, you might search for the hex pattern "DE AD" using Search, - add "BE EF" to the pattern field, and then select "A-B" to retrieve a list of byte sequences that begin with "DE AD" but do not include "DE AD BE EF". - Scanning for changes is most useful in a dynamic environment, such as the Debugger. Given an existing search, you can look for values that have changed, - increased, decreased, or remained the same. Simple examples might include looking for counters while a process is running, checking for areas of decompressed - memory, or identifying active areas of the heap.

- -

PDB

- -

The PDB Symbol Server Search Config dialog has been changed, allowing the user to mark symbol servers as trusted or untrusted. - This is an improvement over the previous mechanism that based trust on the symbol server's connection type.

- -

Debugger

- -

ATTENTION: Please either delete and re-import the default Emulator tool, or - manually remove the TraceRmiPlugin from your EmulatorTool!

- -

There are new launchers/features for the traceRMI version of dbgeng, including extended launch options, kernel debugging, and - remote process server connections.

- -

Decompiler

- -

The Decompiler can now automatically recover strings built on the stack and initial support for optimized heap strings has been added. - Stack strings are typically found in optimized code and obfuscated malware.

- -

A new Search All action has been added which displays a table containing the results found within the current function.

- -

Programming Languages

- -

Golang support for versions 1.15 and 1.16 have been added. This brings the supported Golang versions to 1.15 thru 1.22.

- -

Processors

- -

There have been quite a few improvements to the Sparc processor specification, including additional instructions, 64-bit relocation support, and better - handling of call/return detection through tracking of the o7 link register. In addition, the calling convention for both sparc 32 and 64 bit binaries - have had an overhaul to support hidden structure return and much improved parameter allocation of floating point and general data types.

- -

The Intel M16C/60/80 sleigh processor specifications have been added. In addition, there have been numerous fixes to the - ARM, RX, M68000, PIC16, PPC, and x86 processor specifications.

- -

Other Improvements

- -

Actions have been added to compare functions directly from the Listing, Decompiler, or Functions Table via popup menu items. If there - is already a Function Comparison window showing, there are two actions: one to add the selected function(s) to the existing comparison, and - one to create a new Function Comparison Window. This allows a workflow where users can build up a set of functions to compare as they browse - around instead of having to select them all at once.

- -

For Ghidra script and plugin developers who would prefer to use Visual Studio Code, a new script VSCodeProjectScript will create a new - Visual Studio Code project that is setup to do Ghidra scripting and module development. The capabilities are similar to the Eclipse - GhidraDev plugin.

- -

There have been major speed improvements when creating or modifying large structures within the structure editor. In general large structure manipulation - should perform fluidly no matter the size of the structure. If the structure contains a large number of defined data, there could still be some degradation in - speed. Some fixed performance issues include: resizing a structure smaller or larger, clicking on an item to select a row, and defining a data type either with keyboard actions or dragging - and dropping from the data type manager. In addition, the behavior of automatically growing the size of a structure has been made consistent. Defining data on the last element of a structure - is allowed to automatically grow the structure to fit the data type. Defining data anywhere other than the last element isn't allowed if the data type does not fit because - of defined data that would need to be cleared, or there are not enough undefined bytes.

- -

Additional Bug Fixes and Enhancements

- -

Numerous other new features, improvements, and bug fixes are fully listed in the ChangeHistory file.

- -
- https://www.nsa.gov/ghidra -

-
- - - diff --git a/Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.md b/Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.md new file mode 100644 index 0000000000..3f79a8c0c1 --- /dev/null +++ b/Ghidra/Configurations/Public_Release/src/global/docs/WhatsNew.md @@ -0,0 +1,118 @@ +# What's New in Ghidra 11.2 +This release includes new features, enhancements, performance improvements, quite a few bug fixes, +and many pull-request contributions. Thanks to all those who have contributed their time, thoughts, +and code. The Ghidra user community thanks you too! + +### The not-so-fine print: Please Read! +Ghidra 11.2 is fully backward compatible with project data from previous releases. However, programs +and data type archives which are created or modified in 11.2 will not be usable by an earlier Ghidra +version. + +__IMPORTANT:__ Ghidra 11.2 requires at minimum JDK 21 to run. + +__IMPORTANT:__ To use the Debugger or do a full source distribution build, you will need Python3 +(3.9 to 3.12 supported) installed on your system. + +__NOTE:__ There have been reports of certain features causing the XWindows server to crash. A fix +for `CVE-2024-31083` in X.org software in April 2024 introduced a regression, which has been fixed +in xwayland 23.2.6 and xorg-server 21.1.13. If you experience any crashing of Ghidra, most likely +causing a full logout, check if your xorg-server has been updated to at least the noted version. + +__NOTE:__ Each build distribution will include native components (e.g., decompiler) for at least one +platform (e.g., Windows x86-64). If you have another platform that is not included in the build +distribution, you can build native components for your platform directly from the distribution. +See the `Installation Guide` for additional information. Users running with older shared libraries +and operating systems (e.g., CentOS 7.x) may also run into compatibility errors when launching +native executables such as the Decompiler and GNU Demangler which may necessitate a rebuild of +native components. + +__NOTE:__ Ghidra Server: The Ghidra 11.x server is compatible with Ghidra 9.2 and later Ghidra +clients. Ghidra 11.x clients are compatible with all 10.x and 9.x servers. Although, due to +potential Java version differences, it is recommended that Ghidra Server installations older than +10.2 be upgraded. Those using 10.2 and newer should not need a server upgrade. + +__NOTE:__ Any programs imported with a Ghidra beta version or code built directly from source code +outside of a release tag may not be compatible, and may have flaws that won't be corrected by using +this new release. Any programs analyzed from a beta or other local master source build should be +considered experimental and re-imported and analyzed with a release version. + +Programs imported with previous release versions should upgrade correctly through various automatic +upgrade mechanisms. However, there may be improvements or bug fixes in the import and analysis +process that will provide better results than prior Ghidra versions. You might consider comparing a +fresh import of any program you will continue to reverse engineer to see if the latest Ghidra +provides better results. + +## Memory Search +The __Search Memory__ feature in Ghidra has been updated substantially to provide two new features: +* The ability to perform set operations on successive searches +* The ability to (re)scan memory for changes in value + +Set operations, accessible from the pull-down menu under `Search`, allow you to augment results by +performing boolean operations on an existing search. For example, you might search for the hex +pattern `DE AD` using `Search`, add `BE EF` to the pattern field, and then select `A-B` to retrieve +a list of byte sequences that begin with `DE AD` but do not include `DE AD BE EF`. Scanning for +changes is most useful in a dynamic environment, such as the Debugger. Given an existing search, +you can look for values that have changed, increased, decreased, or remained the same. Simple +examples might include looking for counters while a process is running, checking for areas of +decompressed memory, or identifying active areas of the heap. + +## PDB +The `PDB Symbol Server Search Config` dialog has been changed, allowing the user to mark symbol +servers as trusted or untrusted. This is an improvement over the previous mechanism that based trust +on the symbol server's connection type. + +## Debugger +__ATTENTION:__ Please either delete and re-import the default `Emulator` tool, or manually remove +the `TraceRmiPlugin` from your EmulatorTool! + +There are new launchers/features for the traceRMI version of dbgeng, including extended launch +options, kernel debugging, and remote process server connections. + +## Decompiler +* The Decompiler can now automatically recover strings built on the stack and initial support for + optimized heap strings has been added. Stack strings are typically found in optimized code and + obfuscated malware. + +* A new Search All action has been added which displays a table containing the results found within + the current function. + +## Programming Languages +Golang support for versions `1.15` and `1.16` have been added. This brings the supported Golang +versions to `1.15` through `1.22`. + +## Processors +* There have been quite a few improvements to the `Sparc` processor specification, including + additional instructions, 64-bit relocation support, and better handling of call/return detection + through tracking of the `o7` link register. In addition, the calling convention for both + sparc 32 and 64 bit binaries have had an overhaul to support hidden structure return and much + improved parameter allocation of floating point and general data types. + +* The `Intel M16C/60/80` sleigh processor specifications have been added. In addition, there have + been numerous fixes to the `ARM`, `RX`, `M68000`, `PIC16`, `PPC`, and `x86` processor + specifications. + +## Other Improvements +* Actions have been added to compare functions directly from the Listing, Decompiler, or Functions + Table via popup menu items. If there is already a Function Comparison window showing, there are + two actions: one to add the selected function(s) to the existing comparison, and one to create a + new Function Comparison Window. This allows a workflow where users can build up a set of functions + to compare as they browse around instead of having to select them all at once. + +* For Ghidra script and plugin developers who would prefer to use Visual Studio Code, a new script + `VSCodeProjectScript.java` will create a new Visual Studio Code project that is setup to do Ghidra + scripting and module development. The capabilities are similar to the Eclipse GhidraDev plugin. + +* There have been major speed improvements when creating or modifying large structures within the + structure editor. In general large structure manipulation should perform fluidly no matter the + size of the structure. If the structure contains a large number of defined data, there could + still be some degradation in speed. Some fixed performance issues include: resizing a structure + smaller or larger, clicking on an item to select a row, and defining a data type either with + keyboard actions or dragging and dropping from the data type manager. In addition, the behavior + of automatically growing the size of a structure has been made consistent. Defining data on the + last element of a structure is allowed to automatically grow the structure to fit the data type. + Defining data anywhere other than the last element isn't allowed if the data type does not fit + because of defined data that would need to be cleared, or there are not enough undefined bytes. + +## Additional Bug Fixes and Enhancements +Numerous other new features, improvements, and bug fixes are fully listed in the +[Change History](ChangeHistory.html) file. diff --git a/Ghidra/Framework/Help/src/main/java/help/HelpBuildUtils.java b/Ghidra/Framework/Help/src/main/java/help/HelpBuildUtils.java index 18d8d3af76..b58b1f29f3 100644 --- a/Ghidra/Framework/Help/src/main/java/help/HelpBuildUtils.java +++ b/Ghidra/Framework/Help/src/main/java/help/HelpBuildUtils.java @@ -207,8 +207,13 @@ public class HelpBuildUtils { ResourceFile file = null; if (SystemUtilities.isInDevelopmentMode()) { - // example: "docs/WhatsNew.html", which lives in a source dir in dev mode + // Look for HTML files that live in global docs dir, such as 'docs/README_PDB.html'. file = findModuleFile("src/global/" + updatedPath); + if (file == null) { + // Look for HTML files that get built to the global docs dir (such as + // 'docs/WhatsNew.md' -> 'WhatsNew.html') + file = findModuleFile("build/src/global/" + updatedPath); + } } else { // diff --git a/gradle/distributableGhidraModule.gradle b/gradle/distributableGhidraModule.gradle index 62f2866a0a..96353c9e0e 100644 --- a/gradle/distributableGhidraModule.gradle +++ b/gradle/distributableGhidraModule.gradle @@ -98,8 +98,18 @@ rootProject.assembleDistribution { // fileTree.each { File file -> String filePath = getGlobalFilePathSubDirName(file) - from (file) { - into filePath + + if (file.name.toLowerCase().endsWith(".md")) { + rootProject.assembleMarkdownToHtml { + from (file) { + into filePath + } + } + } + else { + from (file) { + into filePath + } } } } @@ -194,7 +204,7 @@ plugins.withType(JavaPlugin) { into { zipPath + "/ghidra_scripts" } } - // External Libraries + // External Libraries gradle.taskGraph.whenReady { taskGraph -> List externalPaths = getExternalRuntimeDependencies(p) externalPaths.each { path -> @@ -203,8 +213,6 @@ plugins.withType(JavaPlugin) { } } } - - } } diff --git a/gradle/helpProject.gradle b/gradle/helpProject.gradle index 6438e0fc0d..6ec5d8cd77 100644 --- a/gradle/helpProject.gradle +++ b/gradle/helpProject.gradle @@ -235,6 +235,8 @@ tasks.register('indexHelp', JavaExec) { File helpRootDir = file('src/main/help/help') File outputFile = file("build/help/main/help/${project.name}_JavaHelpSearch") + onlyIf ("There is no help root directory") { helpRootDir.exists() } + inputs.dir helpRootDir outputs.dir outputFile @@ -282,11 +284,37 @@ tasks.register('indexHelp', JavaExec) { } } +// Task for building Markdown in src/global/docs to HTML +// - the files generated will be placed in a build directory usable during development mode +tasks.register('buildGlobalMarkdown') { + group "private" + dependsOn ':MarkdownSupport:classes' + + FileTree markdownFiles = this.project.fileTree('src/global/docs') { + include '*.md' + } + + onlyIf ("There are no markdown files") { !markdownFiles.isEmpty() } + + inputs.files markdownFiles + + doFirst { + markdownFiles.each { f -> + def htmlName = f.name[0..-3] + "html" + javaexec { + classpath = project(':MarkdownSupport').sourceSets.main.runtimeClasspath + mainClass = 'ghidra.markdown.MarkdownToHtml' + args f + args file("build/src/global/docs/${htmlName}") + } + } + } +} // Task for building Ghidra help files // - depends on the output from the help indexer // - validates help -// - the files generated will be placed in a diretory usable during development mode and will +// - the files generated will be placed in a directory usable during development mode and will // eventually be placed in: // - the .jar file in production mode, or // - the -help.jar file in development mode @@ -305,7 +333,9 @@ tasks.register('buildHelpFiles', JavaExec) { File helpRootDir = file('src/main/help/help') File outputDir = file('build/help/main/help') - + + onlyIf ("There is no help root directory") { helpRootDir.exists() } + // // Inputs (used for incremental building): // 1) Java files in the Help module used to build help @@ -397,6 +427,7 @@ tasks.register('buildHelp', Jar) { description " Builds the help for this module. [gradle/helpProject.gradle]\n" dependsOn tasks.named('buildHelpFiles') + dependsOn tasks.named('buildGlobalMarkdown') duplicatesStrategy 'exclude' from "build/help/main" // include the generated help and index files