mirror of
https://github.com/NationalSecurityAgency/ghidra.git
synced 2025-02-16 07:30:16 +00:00
Merge remote-tracking branch 'origin/adamopolous_test_script_fixes' into Ghidra_9.1
This commit is contained in:
Ryan Kurtz
commit
0ed8fb63d7
@ -367,7 +367,7 @@ def createTestTask(Project subproject, String testType, String bucketName, int t
|
||||
group "test"
|
||||
testClassesDirs = files subproject.sourceSets["$testType"].output.classesDirs
|
||||
classpath = subproject.sourceSets["$testType"].runtimeClasspath
|
||||
|
||||
|
||||
maxParallelForks = numMaxParallelForks
|
||||
|
||||
initTestJVM(t, testRootDirName)
|
||||
@ -411,17 +411,13 @@ def createTestTask(Project subproject, String testType, String bucketName, int t
|
||||
*********************************************************************************/
|
||||
configure(subprojects.findAll {parallelMode == true}) { subproject ->
|
||||
afterEvaluate {
|
||||
// "subprojects { afterEvaluate { evaluate()"
|
||||
// forces evaluation of subproject configuration. Needed for inheriting excludes
|
||||
// from non-parallel counterpart.
|
||||
// subproject.evaluate()
|
||||
|
||||
if (!shouldSkipTestTaskCreation(subproject)) {
|
||||
logger.info("parallelCombinedTestReport: Creating 'test' tasks for " + subproject.name + " subproject.")
|
||||
|
||||
Map<String,Map> testMap = getTestsForSubProject(subproject.sourceSets.test.java)
|
||||
Map<String,List> testMap = getTestsForSubProject(subproject.sourceSets.test.java)
|
||||
|
||||
for (Map.Entry<String,Map> classMap : testMap.entrySet()) {
|
||||
for (Map.Entry<String,List> classMap : testMap.entrySet()) {
|
||||
|
||||
String bucketName = classMap.getKey();
|
||||
|
||||
@ -429,12 +425,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject ->
|
||||
int taskNameCounter = 1 // task suffix
|
||||
int numMaxParallelForks = 40 // unit tests are fast; 40 seems to be a reasonable number
|
||||
|
||||
Map<String,Long> tests = classMap.getValue();
|
||||
def sorted = tests.sort { a, b -> b.value <=> a.value };
|
||||
List<String> classesList = new ArrayList(sorted.keySet());
|
||||
List<String> tests = classMap.getValue();
|
||||
|
||||
while (classesListPosition < classesList.size()) {
|
||||
createTestTask(subproject, "test", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks)
|
||||
while (classesListPosition < tests.size()) {
|
||||
createTestTask(subproject, "test", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks)
|
||||
classesListPosition+=numMaxParallelForks
|
||||
taskNameCounter+=1; // "test_1_appConfig", "test_2_appConfig, etc.
|
||||
}
|
||||
@ -444,9 +438,9 @@ configure(subprojects.findAll {parallelMode == true}) { subproject ->
|
||||
if (!shouldSkipIntegrationTestTaskCreation(subproject)) {
|
||||
logger.info("parallelCombinedTestReport: Creating 'integrationTest' tasks for " + subproject.name + " subproject.")
|
||||
|
||||
Map<String,Map> testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java)
|
||||
Map<String,List> testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java)
|
||||
|
||||
for (Map.Entry<String,Map> classMap : testMap.entrySet()) {
|
||||
for (Map.Entry<String,List> classMap : testMap.entrySet()) {
|
||||
|
||||
String bucketName = classMap.getKey();
|
||||
|
||||
@ -458,12 +452,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject ->
|
||||
// 20 seems like a good balance of throughput vs resource usage for ghidratest server.
|
||||
int numMaxParallelForks = 20
|
||||
|
||||
Map<String,Long> tests = classMap.getValue();
|
||||
def sorted = tests.sort { a, b -> b.value <=> a.value };
|
||||
List<String> classesList = new ArrayList(sorted.keySet());
|
||||
|
||||
while (classesListPosition < classesList.size()) {
|
||||
createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks)
|
||||
List<String> tests = classMap.getValue();
|
||||
|
||||
while (classesListPosition < tests.size()) {
|
||||
createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks)
|
||||
classesListPosition+=numMaxParallelForks
|
||||
taskNameCounter+=1; // "integrationTest_1_appConfig", "integrationTest_2_appConfig, etc.
|
||||
}
|
||||
|
||||
@ -82,6 +82,7 @@ AbstractToolSavingTest
|
||||
AbstractVersionControlActionTest
|
||||
AbstractVTCorrelatorTest
|
||||
AbstractVTMarkupItemTest
|
||||
CallTreePluginTest
|
||||
DiffTestAdapter
|
||||
DWARFTestBase
|
||||
AbstractSelfSimilarCorrelatorTest
|
||||
|
||||
@ -4,9 +4,10 @@ import java.lang.reflect.Constructor;
|
||||
import java.lang.*;
|
||||
import java.io.*;
|
||||
|
||||
// This is a map of configuration type names (integration vs. non-integration vs. docking etc..)
|
||||
// to tests (test name, duration)
|
||||
ext.testReport = null;
|
||||
ext.integrationConfigs = new ArrayList<>();
|
||||
ext.dockingConfigs = new ArrayList<>();
|
||||
ext.appConfigs = new ArrayList<>();
|
||||
ext.ghidraConfigs = new ArrayList<>();
|
||||
|
||||
/*
|
||||
* Checks if html test report for an individual test class has a valid name.
|
||||
@ -15,190 +16,6 @@ boolean hasValidTestReportClassName(String name) {
|
||||
return name.endsWith("Test.html") && !name.contains("Suite")
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns duration for a test class report.
|
||||
*/
|
||||
long getDurationFromTestReportClass(String fileContents, String fileName) {
|
||||
/* The duration for the entire test class appears in the test report as (multiline):
|
||||
* <div class="infoBox" id="duration">
|
||||
* <div class="counter">0s</div>
|
||||
* The duration value may appear in the format of: 1m2s, 1m2.3s, 3.4s
|
||||
*/
|
||||
Pattern p = Pattern.compile("(?<=id=\"duration\">[\r\n]<div\\sclass=\"counter\">)(.*?)(?=</div)",
|
||||
Pattern.MULTILINE);
|
||||
Matcher m = p.matcher(fileContents);
|
||||
assert m.find() == true
|
||||
String duration = m.group()
|
||||
assert duration != null && duration.trim().length() > 0
|
||||
|
||||
long durationInMillis
|
||||
|
||||
// Parse out the duration
|
||||
if (duration.contains("m") && duration.contains("s")) { // has minute and seconds
|
||||
int minutes = Integer.parseInt(duration.substring(0, duration.indexOf("m")))
|
||||
double seconds = Double.parseDouble(duration.substring(duration.indexOf("m") + 1
|
||||
, duration.length()-1))
|
||||
durationInMillis = (minutes * 60 * 1000) + (seconds * 1000)
|
||||
} else if (!duration.contains("m") && duration.contains("s")) { // has only seconds
|
||||
double seconds = Double.parseDouble(duration.substring(0, duration.length()-1))
|
||||
durationInMillis = (seconds * 1000)
|
||||
} else { // unknown format
|
||||
assert false : "getDurationFromTestReportClass: Unknown duration format in $fileName. 'duration' value is $duration"
|
||||
}
|
||||
logger.debug("getDurationFromTestReportClass: durationInMillis = '"+ durationInMillis
|
||||
+"' parsed from duration = '" + duration + "' in $fileName")
|
||||
return durationInMillis
|
||||
}
|
||||
|
||||
/*
|
||||
* Creates a map of tests to their durations, organized by the type of
|
||||
* application configuration they require.
|
||||
*
|
||||
* When creating groups of tests to run, we have to ensure that we not only
|
||||
* group them by duration (to make the parallelization more efficient) but also
|
||||
* by the type of application config they require (to avoid a catastrophic test
|
||||
* failure).
|
||||
*
|
||||
* This timing information is gleaned by parsing the html results of a previous
|
||||
* test run.
|
||||
*
|
||||
* The application config information is contained in the resource file
|
||||
* app_config_breakout.txt.
|
||||
*
|
||||
* eg: GhidraAppConfiguration -> DiffTestTypeAdapter, 0.135s
|
||||
*/
|
||||
def Map<String, Map<String, Long>> getTestReport() {
|
||||
|
||||
// If we have already created the test report, do not waste time creating
|
||||
// it again. Just return it.
|
||||
if (project.testReport != null) {
|
||||
return project.testReport;
|
||||
}
|
||||
|
||||
logger.debug("getTestReport: Populating 'testReport' using '$testTimeParserInputDir'")
|
||||
|
||||
testReport = new HashMap<String,Map>();
|
||||
|
||||
List<String> integrationConfigs = new ArrayList<>();
|
||||
List<String> dockingConfigs = new ArrayList<>();
|
||||
List<String> appConfigs = new ArrayList<>();
|
||||
List<String> ghidraConfigs = new ArrayList<>();
|
||||
parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs);
|
||||
|
||||
File classesReportDir = new File(testTimeParserInputDir)
|
||||
if(!classesReportDir.exists()) {
|
||||
logger.info("getTestReport: The path '$testTimeParserInputDir' does not exist on the file system." +
|
||||
" Returning empty testReport map.")
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
// These are the configuration 'buckets' that each test class will be dumped
|
||||
// into. Only tests in the same bucket will be run together.
|
||||
Map dockingBucket = new HashMap<String, Long>();
|
||||
Map integrationBucket = new HashMap<String, Long>();
|
||||
Map appBucket = new HashMap<String, Long>();
|
||||
Map ghidraBucket = new HashMap<String, Long>();
|
||||
Map unknownBucket = new HashMap<String, Long>();
|
||||
|
||||
int excludedHtmlFiles = 0 // counter
|
||||
int totalHtmlFiles = 0
|
||||
String excludedHtmlFileNames = "" // for log.info summary message
|
||||
|
||||
classesReportDir.eachFileRecurse (FileType.FILES) { file ->
|
||||
|
||||
totalHtmlFiles++
|
||||
// Only read html file for a Test and not a test Suite
|
||||
if(hasValidTestReportClassName(file.name)) {
|
||||
String fileContents = file.text
|
||||
/* The fully qualified class name appears in the test report as:
|
||||
* <h1>Class ghidra.app.plugin.assembler.sleigh.BuilderTest</h1>
|
||||
*/
|
||||
String fqNameFromTestReport = fileContents.find("(?<=<h1>Class\\s).*?(?=</h1>)")
|
||||
int nameIndex = fqNameFromTestReport.lastIndexOf('.')
|
||||
String shortName = fqNameFromTestReport.substring(nameIndex+1);
|
||||
long durationInMillis = getDurationFromTestReportClass(fileContents, file.name)
|
||||
|
||||
|
||||
File rootDir = project.rootDir.getParentFile();
|
||||
File foundFile;
|
||||
fileTree(rootDir.getAbsolutePath()).visit { FileVisitDetails details ->
|
||||
if (details.getName().contains(shortName + ".java")) {
|
||||
foundFile = details.getFile();
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundFile.exists()) {
|
||||
// throw error
|
||||
}
|
||||
|
||||
String javaFileContents = foundFile.text;
|
||||
|
||||
if (javaFileContents.contains(shortName)) {
|
||||
|
||||
// Match the word right after "extends", if there is one
|
||||
Pattern p = Pattern.compile("extends\\W+(\\w+)");
|
||||
Matcher m = p.matcher(javaFileContents);
|
||||
String extendsClass = "";
|
||||
while (m.find()) {
|
||||
extendsClass = m.group(1);
|
||||
break;
|
||||
}
|
||||
|
||||
if (extendsClass.isEmpty()) {
|
||||
unknownBucket.put(fqNameFromTestReport, durationInMillis);
|
||||
}
|
||||
else {
|
||||
if (integrationConfigs.contains(extendsClass)) {
|
||||
integrationBucket.put(fqNameFromTestReport, durationInMillis);
|
||||
}
|
||||
else if (dockingConfigs.contains(extendsClass)) {
|
||||
dockingBucket.put(fqNameFromTestReport, durationInMillis);
|
||||
}
|
||||
else if (appConfigs.contains(extendsClass)) {
|
||||
appBucket.put(fqNameFromTestReport, durationInMillis);
|
||||
}
|
||||
else if (ghidraConfigs.contains(extendsClass)) {
|
||||
ghidraBucket.put(fqNameFromTestReport, durationInMillis);
|
||||
}
|
||||
else {
|
||||
unknownBucket.put(fqNameFromTestReport, durationInMillis);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
testReport.put("integration", integrationBucket);
|
||||
testReport.put("docking", dockingBucket);
|
||||
testReport.put("app", appBucket);
|
||||
testReport.put("ghidra", ghidraBucket);
|
||||
testReport.put("unknown", unknownBucket);
|
||||
|
||||
logger.debug("getTestReport: Added to testReport: class name = '"
|
||||
+ fqNameFromTestReport + "' and durationInMillis = '"+ durationInMillis
|
||||
+"' from " + file.name)
|
||||
}
|
||||
else {
|
||||
logger.debug("getTestReport: Excluding " + file.name + " from test report parsing.")
|
||||
excludedHtmlFileNames += file.name + ", "
|
||||
excludedHtmlFiles++
|
||||
}
|
||||
}
|
||||
|
||||
int processedFiles = 0;
|
||||
for (Map.Entry<String,Map> entry : testReport.entrySet()) {
|
||||
Map<String,Long> testMap = entry.getValue();
|
||||
processedFiles += testMap.size();
|
||||
}
|
||||
assert totalHtmlFiles != 0 : "getTestReport: Did not parse any valid html files in $testTimeParserInputDir. Directory might be empty"
|
||||
assert totalHtmlFiles == (processedFiles + excludedHtmlFiles) : "Not all html files processed."
|
||||
logger.info("getTestReport:\n" +
|
||||
"\tIncluded " + testReport.size() + " and excluded " + excludedHtmlFiles
|
||||
+ " html files out of " + totalHtmlFiles + " in Junit test report.\n"
|
||||
+ "\tExcluded html file names are: " + excludedHtmlFileNames + "\n"
|
||||
+ "\tParsed test report located at " + testTimeParserInputDir)
|
||||
|
||||
return project.testReport
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the file containing the mapping of test classes to application configs and assigns those
|
||||
* classes to the appropriate lists.
|
||||
@ -299,12 +116,11 @@ String constructFullyQualifiedClassName(String fileContents, String fileName) {
|
||||
return packageName + "." + fileName.replace(".java","")
|
||||
}
|
||||
|
||||
/* Creates a list of test classes, sorted by duration, for a subproject.
|
||||
* First parses JUnit test report located at 'testTimeParserInputDir' for <fully qualified class name, duration in milliseconds> .
|
||||
* Then traverses a test sourceSet for a subproject for a test to include and assigns a duration value.
|
||||
* Returns a sorted list of test classes for the sourceSet parameter.
|
||||
/*
|
||||
* Creates a map of config types to the test classes for that type. This should be
|
||||
* used to creates sets of tests that can be run in parallel.
|
||||
*/
|
||||
def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet) {
|
||||
def Map<String, List> getTestsForSubProject(SourceDirectorySet sourceDirectorySet) {
|
||||
|
||||
def testsForSubProject = new HashMap<String,LinkedHashMap>();
|
||||
|
||||
@ -317,9 +133,18 @@ def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet
|
||||
logger.debug("getTestsForSubProject: Found " + sourceDirectorySet.files.size()
|
||||
+ " file(s) in source set to process.")
|
||||
|
||||
Map<String,Map> testReports = getTestReport();
|
||||
assert (testReports != null) : "getTestsForSubProject: testReport should not be null"
|
||||
|
||||
// Read in the config file that indicates which base test classes are associated with
|
||||
// which application configs. This is not a comprehensive list of all test classes
|
||||
// in Ghidra - it's the list of all classes that are extended in Ghidra.
|
||||
parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs);
|
||||
|
||||
// "Buckets" that delineate which test classes should be run together.
|
||||
List dockingBucket = new ArrayList<String>();
|
||||
List integrationBucket = new ArrayList<String>();
|
||||
List appBucket = new ArrayList<String>();
|
||||
List ghidraBucket = new ArrayList<String>();
|
||||
List unknownBucket = new ArrayList<String>();
|
||||
|
||||
for (File file : sourceDirectorySet.getFiles()) {
|
||||
logger.debug("getTestsForSubProject: Found file in sourceSet = " + file.name)
|
||||
|
||||
@ -331,7 +156,7 @@ def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet
|
||||
}
|
||||
|
||||
String fileContents = file.text
|
||||
|
||||
|
||||
// Must not have a Category annotation
|
||||
if (hasCategoryExcludes(fileContents)) {
|
||||
logger.debug("getTestsForSubProject: Found category exclude for '"
|
||||
@ -339,75 +164,68 @@ def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet
|
||||
excludedClassFilesCategory++
|
||||
continue
|
||||
}
|
||||
|
||||
String fqName = constructFullyQualifiedClassName( fileContents, file.name)
|
||||
|
||||
boolean foundTest = false;
|
||||
for (Map.Entry<String,Map> entry : testReports.entrySet()) {
|
||||
String configName = entry.getKey();
|
||||
Map<String,Long> tests = entry.getValue();
|
||||
// Get any extending class so we can see what bucket it belongs to. Do this
|
||||
// by grabbing the next word after "extends".
|
||||
Pattern p = Pattern.compile("extends\\W+(\\w+)");
|
||||
Matcher m = p.matcher(fileContents);
|
||||
String extendsClass = "";
|
||||
while (m.find()) {
|
||||
extendsClass = m.group(1);
|
||||
break;
|
||||
}
|
||||
|
||||
// Get full package name of the class.
|
||||
Pattern p2 = Pattern.compile("package\\s+([a-zA_Z_][\\.\\w]*);");
|
||||
Matcher m2 = p2.matcher(fileContents);
|
||||
String packageName = "";
|
||||
while (m2.find()) {
|
||||
packageName = m2.group(1);
|
||||
break;
|
||||
}
|
||||
|
||||
// Construct a var of the form "<package name>.<class name>". This will
|
||||
// be stored in the appropriate bucket and be used when creating test
|
||||
// tasks later on.
|
||||
String className = packageName + "." + file.name
|
||||
className = className.replace(".java", "")
|
||||
|
||||
if (tests.containsKey(fqName)) {
|
||||
foundTest = true;
|
||||
if (!testsForSubProject.containsKey(configName)) {
|
||||
Map<String,Map> configToTestMap = new LinkedHashMap<>();
|
||||
testsForSubProject.put(configName, configToTestMap);
|
||||
}
|
||||
|
||||
Map<String,Long> subTests = testsForSubProject.get(configName);
|
||||
|
||||
long duration = tests.get(fqName);
|
||||
|
||||
if (duration > 0) {
|
||||
subTests.put(fqName,duration);
|
||||
logger.debug("getTestsForSubProject: Adding '" + fqName + "'")
|
||||
includedClassFilesInTestReport++
|
||||
}
|
||||
else {
|
||||
logger.debug("getTestsForSubProject: Excluding '" + fqName
|
||||
+ "' because duration from test report is " + duration
|
||||
+ "ms. Probably because all test methods are @Ignore'd." )
|
||||
excludedClassAllTestsIgnored++
|
||||
}
|
||||
if (extendsClass.isEmpty()) {
|
||||
unknownBucket.add(className);
|
||||
}
|
||||
else {
|
||||
if (integrationConfigs.contains(extendsClass)) {
|
||||
integrationBucket.add(className);
|
||||
}
|
||||
else if (dockingConfigs.contains(extendsClass)) {
|
||||
dockingBucket.add(className);
|
||||
}
|
||||
else if (appConfigs.contains(extendsClass)) {
|
||||
appBucket.add(className);
|
||||
}
|
||||
else if (ghidraConfigs.contains(extendsClass)) {
|
||||
ghidraBucket.add(className);
|
||||
}
|
||||
else {
|
||||
unknownBucket.add(className);
|
||||
}
|
||||
}
|
||||
if (!foundTest) {
|
||||
// Don't know what this test is so put it in the "unknown" bucket
|
||||
if (!testsForSubProject.containsKey("unknown")) {
|
||||
Map<String,Map> configToTestMap = new LinkedHashMap<>();
|
||||
testsForSubProject.put("unknown", configToTestMap);
|
||||
}
|
||||
|
||||
Map<String,Long> subTests = testsForSubProject.get("unknown");
|
||||
|
||||
logger.debug("getTestsForSubProject: Found test class not in test report."
|
||||
+ " Bumping to front of tasks '" + fqName + "'")
|
||||
subTests.put(fqName, 3600000) // cheap way to bump to front of (eventually) sorted list
|
||||
includedClassFilesNotInTestReport++
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by duration
|
||||
for (Map.Entry<String,Map> entry : testsForSubProject.entrySet()) {
|
||||
Map<String,Long> testMap = entry.getValue();
|
||||
testMap.sort { a, b -> b.value <=> a.value }
|
||||
}
|
||||
|
||||
/* logger.info ("getTestsForSubProject:\n"
|
||||
+ "\tIncluding " + includedClassFilesInTestReport + " test classes for this sourceSet because they are in the test report.\n"
|
||||
+ "\tIncluding/bumping " + includedClassFilesNotInTestReport + " not in test report.\n"
|
||||
+ "\tExcluding "+ excludedClassFilesBadName +" based on name not ending in 'Test' or contains 'Abstract' or 'Suite', " + excludedClassFilesCategory
|
||||
+ " based on '@Category, " + excludedClassAllTestsIgnored + " because duration = 0ms.\n"
|
||||
+ "\tReturning sorted list of size "+ sorted.size() + " out of " + sourceDirectorySet.files.size()
|
||||
+ " total files found in sourceSet.")
|
||||
*/
|
||||
|
||||
int filesProcessed = includedClassFilesNotInTestReport + includedClassFilesInTestReport +
|
||||
excludedClassFilesBadName + excludedClassFilesCategory + excludedClassAllTestsIgnored
|
||||
|
||||
assert sourceDirectorySet.files.size() == filesProcessed : "getTestsForSubProject did not process every file in sourceSet"
|
||||
|
||||
Map<String,List> testBuckets = new HashMap<String,List>();
|
||||
testBuckets.put("docking", dockingBucket)
|
||||
testBuckets.put("integration", integrationBucket)
|
||||
testBuckets.put("app", appBucket)
|
||||
testBuckets.put("ghidra", ghidraBucket)
|
||||
testBuckets.put("unknown", unknownBucket)
|
||||
|
||||
logger.debug("integration bucket: " + integrationBucket)
|
||||
logger.debug("docking bucket: " + dockingBucket)
|
||||
logger.debug("app bucket: " + appBucket)
|
||||
logger.debug("ghidra bucket: " + ghidraBucket)
|
||||
logger.debug("unknown bucket: " + unknownBucket)
|
||||
|
||||
return testsForSubProject;
|
||||
return testBuckets;
|
||||
}
|
||||
|
||||
/*********************************************************************************
|
||||
|
||||
Loading…
Reference in New Issue
Block a user