Nixyri/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2024-11-22 12:11:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch

Browse Source 
'origin/GP-2908_improvie_table_sorting_performance--SQUASHED'
(Closes #4782)
This commit is contained in:
Ryan Kurtz 2022-12-02 01:01:52 -05:00
commit e96f223df0
5 changed files with 113 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/datamgr/util/DataTypeUtils.java
View File

@ -28,7 +28,6 @@ import ghidra.program.database.data.ProjectDataTypeManager;
import ghidra.program.model.data.*;
import ghidra.program.model.data.Enum;
import ghidra.util.Msg;
import ghidra.util.datastruct.Algorithms;
import ghidra.util.exception.AssertException;
import resources.MultiIcon;
import resources.ResourceManager;
@ -322,10 +321,10 @@ public class DataTypeUtils {
searchTextStart = prepareSearchText(searchTextStart);
searchTextEnd = prepareSearchText(searchTextEnd);
int startIndex = Algorithms.binarySearchWithDuplicates(dataTypeList, searchTextStart,
int startIndex = binarySearchWithDuplicates(dataTypeList, searchTextStart,
DATA_TYPE_LOOKUP_COMPARATOR);
int endIndex = Algorithms.binarySearchWithDuplicates(dataTypeList, searchTextEnd,
int endIndex = binarySearchWithDuplicates(dataTypeList, searchTextEnd,
DATA_TYPE_LOOKUP_COMPARATOR);
return dataTypeList.subList(startIndex, endIndex);
@ -454,6 +453,41 @@ public class DataTypeUtils {
}
Msg.showInfo(DataTypeUtils.class, parent, title, msg);
}
public static int binarySearchWithDuplicates(List<DataType> data,
String searchItem, Comparator<Object> comparator) {
int index = Collections.binarySearch(data, searchItem, comparator);
// the binary search returns a negative, incremented position if there is no match in the
// list for the given search
if (index < 0) {
index = -index - 1;
}
else {
index = findTrueStartIndex(searchItem, data, index, comparator);
}
return index;
}
// finds the index of the first element in the given list--this is used in conjunction with
// the binary search, which doesn't produce the desired results when searching lists with
// duplicates
private static int findTrueStartIndex(String searchItem, List<DataType> dataList,
int startIndex, Comparator<Object> comparator) {
if (startIndex < 0) {
return startIndex;
}
for (int i = startIndex; i >= 0; i--) {
if (comparator.compare(dataList.get(i), searchItem) != 0) {
return ++i; // previous index
}
}
return 0; // this means that the search text matches the first element in the lists
}
}
//==================================================================================================

67
Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/TableUpdateJob.java
View File

@ -21,7 +21,6 @@ import java.util.*;
import docking.widgets.table.*;
import ghidra.util.*;
import ghidra.util.datastruct.Algorithms;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -59,6 +58,7 @@ public class TableUpdateJob<T> {
ADD_REMOVING,
SORTING,
APPLYING,
CANCELLED,
DONE
}
//@formatter:on
@ -279,6 +279,9 @@ public class TableUpdateJob<T> {
pendingRequestedState = null;
monitor.clearCanceled();
}
else if (currentState != CANCELLED) {
setState(CANCELLED);
}
else {
setState(DONE);
}
@ -314,6 +317,7 @@ public class TableUpdateJob<T> {
case SORTING:
return APPLYING;
case APPLYING:
case CANCELLED:
default:
return DONE;
}
@ -341,6 +345,9 @@ public class TableUpdateJob<T> {
case APPLYING:
applyData();
break;
case CANCELLED:
notifyCancelled();
break;
default:
}
}
@ -512,10 +519,15 @@ public class TableUpdateJob<T> {
int size = data.size();
monitor.setMessage("Sorting " + model.getName() + " (" + size + " rows)" + "...");
monitor.initialize(size);
Comparator<T> comparator = newSortContext.getComparator();
Algorithms.mergeSort(data, comparator, monitor);
Comparator<T> monitoredComparator = new MonitoredComparator<>(comparator, monitor, size);
try {
Collections.sort(data, monitoredComparator);
}
catch (SortCancelledException e) {
// do nothing, the old data will remain
}
monitor.setMessage("Done sorting");
}
@ -664,6 +676,13 @@ public class TableUpdateJob<T> {
}
}
private void notifyCancelled() {
Swing.runNow(() -> {
model.backgroundWorkCancelled();
});
}
public synchronized void cancel() {
isFired = true; // let the job die, ignoring any issues that may arise
pendingRequestedState = DONE;
@ -682,4 +701,46 @@ public class TableUpdateJob<T> {
}
return buffy.toString();
}
/**
* Wraps a comparator<T> to add progress monitoring and cancel checking
*
* @param <T> The type of data being sorted
*/
private static class MonitoredComparator<T> implements Comparator<T> {
private Comparator<T> delegate;
private TaskMonitor monitor;
private long comparisonCount;
private long expectedComparisons;
MonitoredComparator(Comparator<T> delegate, TaskMonitor monitor, int size) {
this.delegate = delegate;
this.monitor = monitor;
// After testing the number of comparisons needed to sort random data for the
// sort used by Collections, the max seems to be less then O(N (log(n)-1).
// This seems to be a reasonable approximation for random data. For sorted data
// the number drops to exactly N-1 comparisons, but that just means the progress
// bar only be part way complete when the sort completes.
// log base 2 of N = natural log N / natural log 2
long logN = (long) (Math.log(size) / Math.log(2));
expectedComparisons = size * (logN - 1);
expectedComparisons = Math.max(1, expectedComparisons); // make sure it is never 0
monitor.initialize(100);
}
@Override
public int compare(T o1, T o2) {
if (monitor.isCancelled()) {
throw new SortCancelledException();
}
long percentCompleted = ++comparisonCount * 100 / expectedComparisons;
monitor.setProgress(percentCompleted);
return delegate.compare(o1, o2);
}
}
private static class SortCancelledException extends RuntimeException {
// special version of RuntimeException for MontitoredComparator
}
}

12
Ghidra/Framework/Docking/src/main/java/docking/widgets/table/threaded/ThreadedTableModel.java
View File

@ -534,6 +534,17 @@ public abstract class ThreadedTableModel<ROW_OBJECT, DATA_SOURCE>
updateManager.updateNow();
}
/**
* Called when a {@link TableUpdateJob} is cancelled by the user via the Gui. (Disposing of the
* table takes a different path.) This is not called when using an incrementally loading
* table model.
*/
protected void backgroundWorkCancelled() {
pendingSortContext = null;
sortCompleted(null);
notifyModelSorted(false);
}
protected void setModelState(TableData<ROW_OBJECT> allData,
TableData<ROW_OBJECT> filteredData) {
@ -994,4 +1005,5 @@ public abstract class ThreadedTableModel<ROW_OBJECT, DATA_SOURCE>
delegate.loadingFinished(wasCancelled);
}
}
}

219
Ghidra/Framework/Generic/src/main/java/ghidra/util/datastruct/Algorithms.java
View File

@ -1,219 +0,0 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.datastruct;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import ghidra.util.task.TaskMonitorAdapter;
import java.util.*;
/**
* <CODE>Algorithms</CODE> is a class containing static methods that implement
* general algorithms based on objects returned from a data model.
*/
public class Algorithms {
@SuppressWarnings({ "unchecked", "rawtypes" })
public static int binarySearchWithDuplicates(List data, Object searchItem, Comparator comparator) {
int index = Collections.binarySearch(data, searchItem, comparator);
// the binary search returns a negative, incremented position if there is no match in the
// list for the given search
if (index < 0) {
index = -index - 1;
}
else {
index = findTrueStartIndex(searchItem, data, index, comparator);
}
return index;
}
// finds the index of the first element in the given list--this is used in conjunction with
// the binary search, which doesn't produce the desired results when searching lists with
// duplicates
private static <T> int findTrueStartIndex(T searchItem, List<T> dataList, int startIndex,
Comparator<T> comparator) {
if (startIndex < 0) {
return startIndex;
}
for (int i = startIndex; i >= 0; i--) {
if (comparator.compare(dataList.get(i), searchItem) != 0) {
return ++i; // previous index
}
}
return 0; // this means that the search text matches the first element in the lists
}
public static <T> void bubbleSort(List<T> data, int low, int high, Comparator<T> comparator) {
try {
doBubbleSort(data, low, high, comparator, TaskMonitorAdapter.DUMMY_MONITOR);
}
catch (CancelledException e) {
// do nothing--cancelled
}
}
private static <T> void doBubbleSort(List<T> data, int low, int high, Comparator<T> comparator,
TaskMonitor monitor) throws CancelledException {
for (int i = high; i > low; --i) {
monitor.checkCanceled();
boolean swapped = false;
for (int j = low; j < i; j++) {
if (comparator.compare(data.get(j), data.get(j + 1)) > 0) {
Collections.swap(data, j, j + 1);
swapped = true;
}
}
if (!swapped) {
return;
}
}
}
public static <T> void mergeSort(List<T> data, Comparator<T> c, TaskMonitor monitor) {
List<T> aux = new ArrayList<T>(data);
mergeSort(aux, data, 0, data.size(), c, monitor);
}
private static <T> void mergeSort(List<T> src, List<T> dest, int low, int high,
Comparator<T> c, TaskMonitor monitor) {
try {
doMergeSort(src, dest, low, high, c, monitor);
}
catch (CancelledException e) {
// do nothing--cancelled
}
}
private static <T> void doMergeSort(List<T> src, List<T> dest, int low, int high,
Comparator<T> c, TaskMonitor monitor) throws CancelledException {
monitor.checkCanceled();
monitor.setProgress(low);
int length = high - low;
if (length < 7) {
doBubbleSort(dest, low, high - 1, c, monitor);
return;
}
// Recursively sort halves of dest into src
int mid = (low + high) >> 1;
doMergeSort(dest, src, low, mid, c, monitor);
doMergeSort(dest, src, mid, high, c, monitor);
// If list is already sorted, just copy from src to dest. This is an
// optimization that results in faster sorts for nearly ordered lists.
if (c.compare(src.get(mid - 1), src.get(mid)) <= 0) {
for (int i = low; i < high; i++) {
monitor.checkCanceled();
dest.set(i, src.get(i));
}
return;
}
// Merge sorted halves (now in src) into dest
for (int i = low, p = low, q = mid; i < high; i++) {
monitor.checkCanceled();
if (q >= high || p < mid && c.compare(src.get(p), src.get(q)) <= 0) {
dest.set(i, src.get(p++));
}
else {
dest.set(i, src.get(q++));
}
}
}
// /**
// * Performs a quick sort on an array of long values.
// * The entire array is sorted using the provided comparator.
// * @param model the index based model containing the data to be searched.
// * @param monitor provides feedback about the sort progress and allows user to cancel sort.
// * @return true if the qsort completed the sort without being cancelled.
// */
// public static <T> void qsort(List<T> data, Comparator<T> comparator, TaskMonitor monitor) {
// qsort(data, 0, data.size()-1, comparator, monitor);
// }
// /**
// * Performs a quick sort on a portion of an array of long values.
// * The array is sorted between the low index and high index inclusive
// * using the provided comparator.
// * @param model the index based model containing the data to be searched.
// * @param low the index for the low side of the range of indexes to sort.
// * @param high the index for the high side of the range of indexes to sort.
// * @param monitor provides feedback about the sort progress and allows user to cancel sort.
// * @return true if the qsort completed the sort without being cancelled.
// */
// public static <T> void qsort(List<T> data, int low, int high, Comparator<T> comparator, TaskMonitor monitor) {
// if (monitor.isCancelled()) {
// return;
// }
// if (low+6 > high) {
// bubbleSort(data, low, high, comparator);
// return;
// }
// if (high <= low) {
// return;
// }
// monitor.setProgress(low);
// swapMiddleValueToEnd(data, low, high, comparator);
// Collections.swap(data, (low+high)/2, high);
// T pivotObj = data.get(high-1);
//
// int i=low;
// int j=high;
// while(i<j) {
// while(comparator.compare(data.get(++i), pivotObj) < 0){
// if (monitor.isCancelled()) {
// return;
// }
// }
// while(comparator.compare(pivotObj, data.get(--j)) < 0) {
// if (monitor.isCancelled()) {
// return;
// }
// }
// if (i < j) {
// Collections.swap(data, i, j);
// }
// }
// Collections.swap(data, i, high);
// qsort(data, low, i-1, comparator, monitor);
// qsort(data, i+1, high, comparator, monitor);
// }
//
// private static <T> void swapMiddleValueToEnd(List<T> data, int low, int high, Comparator<T> comparator) {
// int middle = (low+high)/2;
// if (comparator.compare(data.get(middle), data.get(low)) < 0) {
// Collections.swap(data, middle, low);
// }
// if (comparator.compare(data.get(high), data.get(low)) < 0) {
// Collections.swap(data, high, low);
// }
// if (comparator.compare(data.get(high), data.get(middle)) < 0) {
// Collections.swap(data, high, middle);
// }
// Collections.swap(data, middle, high-1);
// }
}

136
Ghidra/Framework/Generic/src/test/java/ghidra/util/datastruct/AlgorithmsTest.java
View File

@ -1,136 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.datastruct;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.*;
import org.junit.Test;
import generic.test.AbstractGenericTest;
import ghidra.util.task.TaskMonitorAdapter;
public class AlgorithmsTest extends AbstractGenericTest {
Comparator<Long> comparator;
public AlgorithmsTest() {
super();
comparator = new Comparator<Long>() {
@Override
public int compare(Long a, Long b) {
if (a < b) {
return -1;
}
else if (a > b) {
return 1;
}
return 0;
}
};
}
private List<Long> getList(long[] data) {
List<Long> list = new ArrayList<Long>(data.length);
for (int i = 0; i < data.length; i++) {
list.add(data[i]);
}
return list;
}
@Test
public void testBubbleSort() {
List<Long> data = getList(new long[] { 5, 8, 10, 2, 10, 3, 3, 7, 10, 23, 0, 15, 22 });
int low = 3;
int high = 8;
Algorithms.bubbleSort(data, low, high, comparator);
long[] expected = new long[] { 5, 8, 10, 2, 3, 3, 7, 10, 10, 23, 0, 15, 22 };
for (int i = 0; i < expected.length; i++) {
assertEquals(new Long(expected[i]), data.get(i));
}
}
@Test
public void testmergeSort() {
List<Long> data = getList(new long[] { 5, 8, 10, 2, 10, 3, 3, 7, 10, 23, 0, 15, 22 });
Algorithms.mergeSort(data, comparator, TaskMonitorAdapter.DUMMY_MONITOR);
long[] expected = new long[] { 0, 2, 3, 3, 5, 7, 8, 10, 10, 10, 15, 22, 23 };
for (int i = 0; i < expected.length; i++) {
assertEquals(new Long(expected[i]), data.get(i));
}
}
@Test
public void testmergeSort2() {
List<Long> data = getList(new long[] { 0, 1, 2, 3, 4, 0, 0, 0 });
Algorithms.mergeSort(data, comparator, TaskMonitorAdapter.DUMMY_MONITOR);
long[] expected = new long[] { 0, 0, 0, 0, 1, 2, 3, 4 };
for (int i = 0; i < expected.length; i++) {
assertEquals(new Long(expected[i]), data.get(i));
}
}
@Test
public void testmergeSort3() {
List<Long> data = getList(new long[] { 0, 1, 2, 3, 4, 4, 4, 4 });
Algorithms.mergeSort(data, comparator, TaskMonitorAdapter.DUMMY_MONITOR);
long[] expected = new long[] { 0, 1, 2, 3, 4, 4, 4, 4 };
for (int i = 0; i < expected.length; i++) {
assertEquals(new Long(expected[i]), data.get(i));
}
}
@Test
public void testmergeSort4() {
List<Long> data = getList(new long[] { 1, 1, 1, 1, 1, 1, 1, 1 });
Algorithms.mergeSort(data, comparator, TaskMonitorAdapter.DUMMY_MONITOR);
long[] expected = new long[] { 1, 1, 1, 1, 1, 1, 1, 1 };
for (int i = 0; i < expected.length; i++) {
assertEquals(new Long(expected[i]), data.get(i));
}
}
@Test
public void testmergeSort5() {
long[] l = new long[100000];
Random r = new Random();
for (int i = 0; i < l.length; i++) {
l[i] = r.nextLong();
}
List<Long> data = getList(l);
Algorithms.mergeSort(data, comparator, TaskMonitorAdapter.DUMMY_MONITOR);
for (int i = 0; i < l.length - 1; i++) {
assertTrue("i = " + i, data.get(i) <= data.get(i + 1));
}
}
@Test
public void testBinarySearch() {
List<Long> data = getList(new long[] { 0, 2, 3, 3, 5, 7, 8, 10, 10, 10, 15, 22, 23 });
assertEquals(0, Collections.binarySearch(data, new Long(0)));
assertEquals(4, Collections.binarySearch(data, new Long(5)));
assertEquals(12, Collections.binarySearch(data, new Long(23)));
assertEquals(-8, Collections.binarySearch(data, new Long(9)));
assertEquals(-1, Collections.binarySearch(data, new Long(-12)));
assertEquals(-14, Collections.binarySearch(data, new Long(50)));
}
}