ghidra/gradle/root/distribution.gradle

/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
import org.apache.tools.ant.filters.*

/*********************************************************************************
 * distribution.gradle
 *
 * This contains gradle tasks for packaging Ghidra artifacts for distribution. To
 * run a full distribution, execute the buildGhidra task. This will build for the
 * current platform only. Add -PallPlatforms to build a multiplatform build.
 *
 *********************************************************************************/

apply from: "$rootProject.projectDir/gradle/support/sbom.gradle"

/********************************************************************************
 * Local Vars
 *********************************************************************************/
def currentPlatform = getCurrentPlatformName()
def PROJECT_DIR = file (rootProject.projectDir.absolutePath)
ext.DISTRIBUTION_DIR = file("$buildDir/dist")
ext.ZIP_NAME_PREFIX = "${rootProject.DISTRO_PREFIX}_${rootProject.BUILD_DATE_SHORT}"
ext.ZIP_DIR_PREFIX = "${rootProject.DISTRO_PREFIX}"
ext.ALL_REPOS = [rootProject.file('.').getName()]

// Add any additional repos to the ALL_REPOS array
File extensionsList = file("ghidra.repos.config")
if (extensionsList.isFile()) {
	extensionsList.eachLine { line ->
		line = line.trim()
		if (line == "" || line.startsWith("#")) {
			return // Skip just this one
		}
		ALL_REPOS += "$line"
	}
}

ext.ghidraPath = files()

/********************************************************************************
 * Local Methods
 *********************************************************************************/
 
/** 
 * Returns the git commit version of the given git repository. If the
 * path is invalid (doesn't exist or isn't in a git repository), an empty
 * string will be returned.
 */
def getGitRev(repoPath) { 
        
   println("getting git commit for $repoPath")
   
   // If the path doesn't exist, the exec command will fail before it can
   // even run the 'git' command, so short-circuit the whole thing here.
   if (!new File(repoPath).exists()) {
      return ""
   }
   
   // Check to see if the given repo is a git repository. No need to exec
   // if it isn't.
   if (!new File(repoPath + "/.git").exists()) {
      return ""
   }
          
   // Exec the git command to get the commit hash. Note the try/catch - this is
   // necessary to catch catastrophic errors on the exec command (eg: 
   // if the git command is not available). This is necessary because the 
   // 'ignoreExitValue' attribute only applies to the return value of the 
   // command being executed (eg: git); it doesn't apply to the return value of
   // the exec command itself.
   def stdout = new ByteArrayOutputStream()
   try {
	   exec {
	      ignoreExitValue = true
	      workingDir repoPath
	      commandLine 'git', 'rev-parse', 'HEAD'
	      standardOutput = stdout
	   }
   }
   catch (Exception e) {
      println("ERROR: gradle exec failed to run 'git rev-parse': is git installed on this system?")
   }
      
   // Return the commit hash
   println(stdout)
   return stdout.toString().trim()
}

/*********************************************************************************
 * JAVADOCS - RAW
 *
 * Creates javadocs for all source defined in the above 'javadocFiles' file tree.
 *
 * Note:	Artifacts are placed in a temporary folder that is deleted upon 
 * 		completion of the build.
 *
 *********************************************************************************/
task createJavadocs(type: Javadoc, description: 'Generate javadocs for all projects', group: 'Documentation') {
	
	destinationDir file(rootProject.projectDir.toString() + "/build/tmp/javadoc")
	
	failOnError false

	// the "source" property must be set in individual project's build.gradle files.
	// projects that want to be included in the Jsondocs should add the following to 
	// their build.gradle file:
	//
	//	apply from: "$rootProject.projectDir/gradle/javadoc.gradle"
	//
		
	// Must add classpath for main and test source sets. Javadoc will fail if it cannot
	// find referenced classes.
	classpath =  rootProject.ext.ghidraPath

	// generate documentation using html5
	options.addBooleanOption("html5", true)

	options.addBooleanOption('Xdoclint:none', true)
	
	// Some internal packages are not public and need to be exported.
	options.addMultilineStringsOption("-add-exports").setValue(["java.desktop/sun.awt=ALL-UNNAMED"])
}


 /*********************************************************************************
 * JSONDOCS - RAW
 *
 * Creates JSON docs for all source defined in the above 'javadocFiles' file tree. 
 * These documents are used by Python to show system documentation (whereas Java will 
 * use Javadoc files).
 *
 * Note:	Artifacts are placed in a temporary folder that is deleted upon 
 * 		completion of the build.
 *
 *********************************************************************************/
 
configurations {
	jsondoc
}

dependencies {
	jsondoc project('Doclets')
}

 
task createJsondocs(type: Javadoc, description: 'Generate JSON docs for all projects', group: 'Documentation') {
	
	group 'private'
	
	String ROOT_PROJECT_DIR = rootProject.projectDir.toString()
	
	destinationDir file(ROOT_PROJECT_DIR + "/build/tmp/jsondoc")
	
	failOnError false

	// Must add classpath for main and test source sets. Javadoc will fail if it cannot
	// find referenced classes.
	classpath =  rootProject.ext.ghidraPath

	// the "source" property must be set in individual project's build.gradle files.
	// projects that want to be included in the Jsondocs should add the following to 
	// their build.gradle file:
	//
	//	apply from: "$rootProject.projectDir/gradle/javadoc.gradle"
	//
	
	// Generate at package level because user may try to get help directly on an object they have 
	// rather than its public interface.
	options.addBooleanOption("package", true)

	// Newer versions of gradle set this to true by default.
	// The JsonDoclet doesn't have the -notimestamp option so ensure it isn't set.
	options.setNoTimestamp(false)
	
	// Some internal packages are not public and need to be exported.
	options.addMultilineStringsOption("-add-exports").setValue(["java.desktop/sun.awt=ALL-UNNAMED"])

	options.doclet = "ghidra.doclets.json.JsonDoclet"
	doFirst {
		options.docletpath = new ArrayList(configurations.jsondoc.files)
	}
}

task createPythonTypeStubs(type: Javadoc, description: 'Generate pyi stubs for all projects', group: 'Documentation') {

	group 'private'

	String ROOT_PROJECT_DIR = rootProject.projectDir.toString()

	destinationDir file(ROOT_PROJECT_DIR + "/build/typestubs/src")

	failOnError false

	// Must add classpath for main and test source sets. Javadoc will fail if it cannot
	// find referenced classes.
	classpath =  rootProject.ext.ghidraPath


	// Generate at package level because user may try to get help directly on an object they have
	// rather than its public interface.
	options.addBooleanOption("package", true)

	// Set the ghidra flag to enable the creation of the ghidra_builtins pseudo package
	options.addBooleanOption("ghidra", true)

	// Newer versions of gradle set this to true by default.
	// The JsonDoclet doesn't have the -notimestamp option so ensure it isn't set.
	options.setNoTimestamp(false)

	// Some internal packages are not public and need to be exported.
	options.addMultilineStringsOption("-add-exports").setValue(["java.desktop/sun.awt=ALL-UNNAMED"])

	options.doclet = "ghidra.doclets.typestubs.PythonTypeStubDoclet"
	doFirst {
		options.docletpath = new ArrayList(configurations.jsondoc.files)
	}
}


task createGhidraStubsWheel {
	group 'private'
	description "Creates the ghidra-stubs wheel for the Ghidra api.  [gradle/root/distribution.gradle]"

	dependsOn("createPythonTypeStubs")

	String ROOT_PROJECT_DIR = rootProject.projectDir.toString()

	def cwd = file(ROOT_PROJECT_DIR + "/build/typestubs")
	def destinationDir = file(cwd.toString() + "/dist")

	it.outputs.file(destinationDir.toString() + "/ghidra_stubs-${project.version}-py3-none-any.whl")

	doFirst {
		copy {
			from(file(ROOT_PROJECT_DIR + "/LICENSE"))
			into cwd
		}

		def manifest = file(cwd.toString() + "/MANIFEST.in" )
		manifest.write("graft src\n")

		def pyproject = file(cwd.toString() + "/pyproject.toml" )
		pyproject.write("""\
			[build-system]
			requires = ["setuptools", "wheel"]
			build-backend = "setuptools.build_meta"

			[project]
			name = "ghidra-stubs"
			version = "${project.version}"
			classifiers = [
				"License :: OSI Approved :: Apache Software License",
				"Typing :: Stubs Only",
			]
			""".stripIndent()
		)
	}

	doLast {
		File setuptools = project(":Debugger-rmi-trace").findPyDep(".")
		
		if (rootProject.PYTHON3 == null) {
			throw new GradleException("A supported version of Python ${SUPPORTED_PY_VERSIONS} was not found!")
		}
		
		exec {
			workingDir { cwd.toString() }
			commandLine rootProject.PYTHON3
			args "-m", "pip", "wheel", "-w", destinationDir.toString(), "--no-index", "-f", setuptools, "."
		}
	}
}


/*********************************************************************************
 * JAVADOCS - ZIP
 *
 * Creates a zip file of all javadocs to be put in the release.
 *
 * Note: Artifacts are placed in a temporary folder, deleted at build completion
 *
 *********************************************************************************/
task zipJavadocs(type: Zip) {
	group 'private'
	archiveFileName = 'GhidraAPI_javadoc.zip'	
	destinationDirectory = file(rootProject.projectDir.toString() + "/build/tmp")

	from createJavadocs {
		into "api"
	}
	
	from createJsondocs {
		into "api"
	}
	
	description "Zips javadocs for Ghidra api.  [gradle/root/distribution.gradle]"
}



/**********************************************************************************************
 *
 * Copies platform independent files to the distribution staging area in preparation
 * for the distribution zip
 *
 **********************************************************************************************/
task assembleDistribution (type: Copy) {
	// force this task to always be "out of date"
	// Not sure why this is necessary, but without it, gradle thinks this task is "up to date"
	// every other time it is run even though in both cases the output directory has been removed
	outputs.upToDateWhen {false}

	group 'private'
	description "Copies core files/folders to the distribution location."
	destinationDir file(DISTRIBUTION_DIR.getPath() + "/" +  ZIP_DIR_PREFIX)

	// Make sure that we don't try to copy the same file with the same path.
	duplicatesStrategy 'exclude'
			
	exclude "**/certification.manifest"
	exclude "**/certification.local.manifest"
	exclude "**/.project"
	exclude "**/.classpath"
	exclude "**/.pydevproject"
	exclude "**/delete.me"
	exclude "**/.vs/**"
	exclude "**/*.vcxproj.user"
	exclude "**/.settings/**"
		
	/////////////////////////////
	// COPY all GPL support files
	// (modules with build.gradle handled separately)
	/////////////////////////////
	from (ROOT_PROJECT_DIR + "/GPL") {
		include "*.*"
		include "Icons/**"
		include "licenses/**"
		into "GPL"
	}
	
	//////////////////////////////
	// LICENSE SUPPORT
	//////////////////////////////
	from ("licenses") {
		into ("licenses")
		exclude "**/certification.manifest"
	}

	from (ROOT_PROJECT_DIR) {
		include "LICENSE"
	}
	
	/////////////////
	// DB DIR LOCK FILE
	//
	// This lock file must be created to prevent users from modifying script files.  We 
	// create it here, copy it to the archive, then delete it.
	/////////////////
	File dbLockFile = file('Ghidra/.dbDirLock')

	from ('Ghidra') {
		include '.dbDirLock'
		into 'Ghidra'
		
		doFirst {
			dbLockFile.withWriter { out ->
				out.writeLine("lock file to prevent modification of core ghidra scripts")
			}
		}
		doLast {
			dbLockFile.delete()
		}
	}

	/////////////////
	// APPLICATION PROPERTIES
	/////////////////
	from (ROOT_PROJECT_DIR + "/Ghidra/application.properties") {
		def buildDateFile = file("$buildDir/build_date.properties")
		def gitRevFile = file("$buildDir/git-rev.properties")
		doFirst {
			file("$buildDir").mkdirs()
				
			// Get the build dates and add to the build file
			buildDateFile.text = ""
			if (rootProject.BUILD_DATES_NEEDED) {
				buildDateFile.text += "application.build.date=" + rootProject.BUILD_DATE + "\n"
				buildDateFile.text += "application.build.date.short=" + rootProject.BUILD_DATE_SHORT + "\n"
			}
				
			// Get the git revisions and add to the git file
			gitRevFile.text = ""
			if (rootProject.GIT_REVS_NEEDED) {
				ALL_REPOS.each { 
					def rev = getGitRev(ROOT_PROJECT_DIR + "/../${it}")
					gitRevFile.text += "application.revision.${it}=" + "$rev" + "\n"
				}
			}
		}
		doLast {
			delete buildDateFile
			delete gitRevFile
		}
		into "Ghidra"
			
		// Add the build and git info to the application.properties file
		filter (ConcatFilter, prepend: buildDateFile)
		filter (ConcatFilter, prepend: gitRevFile)
	}
	
	/////////////////
	// JAVADOCS
	/////////////////
	from (zipJavadocs) {
		into 'docs'
	}

	////////////////////////////
	// Ghidra Python type stubs
	////////////////////////////
	from (createGhidraStubsWheel) {
		into 'docs'
	}
	
	////////////////
	// Patch Readme
	////////////////
	from (ROOT_PROJECT_DIR + "/GhidraBuild/patch") {
		into "Ghidra/patch"
	}
	
	/////////////////////////////////
	// Eclipse formatters
	/////////////////////////////////
	from (ROOT_PROJECT_DIR + "/eclipse") {
		exclude "certification.manifest"
		into "support/eclipse"
	}

	//////////////////////////////////////
	// Software Bill of Materials (SBOM)
	//////////////////////////////////////
	doLast {
		def bomFile = file("${destinationDir}/bom.json")
		writeSoftwareBillOfMaterials(destinationDir, bomFile)
	}
}

/*********************************************************************************
 * NATIVES
 *
 * Creates copy tasks for each platform, to move native files to the
 * distribution staging folder.
 *
 * Input:	Native executables created during the build phase. It is assumed that
 *			these have already been built and are located in the proper location.
 *
 *********************************************************************************/
project.PLATFORMS.each { platform ->
	task ("assembleDistribution_${platform.name}", type: Copy ) { 	
		// force this task to always be "out of date"
		// Not sure why this is necessary, but without it, gradle thinks this task is "up to date"
		// every other time it is run even though in both cases the output directory has been removed
		outputs.upToDateWhen {false}

		
		// delete the gradle ziptree temp directory because of gradle bug not cleaning up its temp files.
		delete rootProject.file("build/tmp/expandedArchives")
		
		
		group 'private'
		description "Copies the platform-dependent files/folders to the distribution location."
		destinationDir file(DISTRIBUTION_DIR.getPath() + "/" +  ZIP_DIR_PREFIX)
				
		// Make sure that we don't try to copy the same file with the same path into the
		// zip (this can happen!)
		duplicatesStrategy 'exclude'
	}
}

/*********************************************************************************
 *
 * Copies source zips for projects to the distribution staging folder.
 *
 **********************************************************************************/
task assembleSource (type: Copy) {
	group 'private'
	description "Copies source zips for all core projects to the distribution folder"
	destinationDir DISTRIBUTION_DIR
	
}

/************************************************************************************
 *
 * Copies a markdown file and a generaterated html file into the distribution folder.
 *
 ************************************************************************************/
task assembleMarkdownToHtml (type: Copy) {
	group 'private'
	description "Copies a markdown file and a generaterated html file into the distribution folder"
	
	dependsOn ':MarkdownSupport:classes'
	outputs.upToDateWhen {false}
	destinationDir file(DISTRIBUTION_DIR.getPath() + "/" +  ZIP_DIR_PREFIX)
	
	eachFile { f ->
		def htmlName = f.sourceName[0..-3] + "html"
		def htmlPath = f.relativePath.replaceLastName(htmlName).pathString 
		
		javaexec {		
			classpath = project(':MarkdownSupport').sourceSets.main.runtimeClasspath
			mainClass = 'ghidra.markdown.MarkdownToHtml'
			args f.file
			args file("${destinationDir.path}/${htmlPath}")
		}
	}
}

/*********************************************************************************
 * 
 * Creates a directory of extensions that are external from the installation zip.
 *
 **********************************************************************************/
task createExternalExtensions(type: Copy) {
	
	group 'private'
	description "Creates directory of extensions that are external to the installation zip (does not clean up artifacts) [gradle/root/distribution.gradle]"
	
	dependsOn assembleSource
	
	destinationDir new File(DISTRIBUTION_DIR.getPath(), "external_extensions")
	
	// Make sure that we don't try to copy the same file with the same path.
	duplicatesStrategy 'exclude'
	
}

import groovy.io.FileType
import java.nio.file.Path
import java.nio.file.Files
import java.nio.file.attribute.FileTime
import java.time.OffsetDateTime
import java.util.concurrent.TimeUnit
import java.time.ZoneId

/*********************************************************************************
 * Update sla file timestamps to current time plus timeOffsetMinutes value.
 * 
 * distributionDirectoryPath - Contains files/folders used by gradle zip task.
 * timeOffsetMinutes - Number of minutes to increase sla file timestamp.
 * 
**********************************************************************************/
def updateSlaFilesTimestamp(String distributionDirectoryPath, int timeOffsetMinutes) {
	logger.debug("updateSlaFilesTimestamp: distributionDirectoryPath = '$distributionDirectoryPath' and timeOffsetMinutes = '$timeOffsetMinutes',")

	if (timeOffsetMinutes <= 0) {
		throw new GradleException("updateSlaFilesTimestamp: timeOffsetMinutes value of '$timeOffsetMinutes' is invalid.")
	}

	// path to sla files in distribution directory
	def directory = new File(distributionDirectoryPath)

	if (!directory.exists()) {
		throw new GradleException("updateSlaFilesTimestamp: path to sla files '$directory' does not exist.")
	}

	OffsetDateTime dt = OffsetDateTime.now(ZoneId.of("UTC")).plusMinutes(timeOffsetMinutes);

	int numFilesAdded = 0;

	// For each .sla file, update timestamp attributes.
	directory.eachFileRecurse(FileType.FILES) { file ->
		if(file.name.endsWith('sla')) {
			Files.setAttribute(file.toPath(), "creationTime", FileTime.from(dt.toEpochSecond(), TimeUnit.SECONDS ));
			Files.setAttribute(file.toPath(), "lastModifiedTime", FileTime.from(dt.toEpochSecond(), TimeUnit.SECONDS ));
			Files.setAttribute(file.toPath(), "lastAccessTime", FileTime.from(dt.toEpochSecond(), TimeUnit.SECONDS ));

			logger.debug("updateSlaFilesTimestamp: Updating $file.name with timestamp attributes of " + new Date(file.lastModified()))

			numFilesAdded++
		}
	}

	println "updateSlaFilesTimestamp: Updated timestamps to $numFilesAdded .sla files."
}

/*********************************************************************************
 *
 * Assemble all targets.
 *
 **********************************************************************************/
task assembleAll() {

	group 'private'
	description "Assembles all targets to one installation directory (does not clean up artifacts) [gradle/root/distribution.gradle]"

	dependsOn assembleDistribution
	dependsOn assembleSource
	dependsOn assembleMarkdownToHtml
	dependsOn "assembleDistribution_$currentPlatform"

	if (project.hasProperty("allPlatforms")) {
		project.PLATFORMS.each { platform ->
			dependsOn ":assembleDistribution_${platform.name}"
		}
	}

	doFirst {
		// We always want the extensions directory to exist in the zip, even if there's nothing
		// installed there.
		new File( DISTRIBUTION_DIR.getPath() + "/" + ZIP_DIR_PREFIX + "/Ghidra/Extensions").mkdirs()


		// The dependent tasks copy the sla and slaspec files into "extractTo/<release name>/ghidra/"
		// and then later to "extractTo/<release name>/dist/", which this zip task compresses. The copy 
		// tasks do not preserve the file modification times. If slaspec timestamp > sla timestamp, 
		// a sleigh compile is triggered on Ghidra app startup. Calling this method before files are zipped
		// will ensure the zip archive has sla files newer than slaspec. Give new timestamp of now plus 
		// two minutes.
		updateSlaFilesTimestamp(DISTRIBUTION_DIR.getPath(), 2)
	}
}

/*********************************************************************************
 *
 * Creates the local installation zip.
 *
 **********************************************************************************/
task createInstallationZip(type: Zip) { t ->

	group 'private'
	description "Creates local installation zip (does not clean up artifacts) [gradle/root/distribution.gradle]"

	dependsOn assembleAll

	if (project.hasProperty("allPlatforms")) {
		archiveFileName = "${ZIP_NAME_PREFIX}.zip"
	}
	else {
		archiveFileName = "${ZIP_NAME_PREFIX}_${currentPlatform}.zip"
	}
	destinationDirectory = DISTRIBUTION_DIR

	// Make sure that we don't try to copy the same file with the same path.
	duplicatesStrategy 'exclude'

	from (DISTRIBUTION_DIR.getPath() + "/" +  ZIP_DIR_PREFIX) {
		into ZIP_DIR_PREFIX
	}
	
	doLast {
		delete file(DISTRIBUTION_DIR.getPath() + "/" + ZIP_DIR_PREFIX)
	}
}


/*********************************************************************************
 * 
 * Builds the Ghidra installation zip file for the local platform
 *
 **********************************************************************************/
task buildGhidra() {
	description "Builds Ghidra for the current platform. The resulting zip will be in build/dist"

	if (project.hasProperty("externalExtensions")) {
		dependsOn createExternalExtensions
	}
	dependsOn createInstallationZip
	
}