Merge remote-tracking branch 'origin/GP-4483_Dan_staleEmuADDs--SQUASHED' into patch

Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/mapping/AbstractDebuggerPlatformMapper.java

@@ -17,6 +17,7 @@ package ghidra.app.plugin.core.debug.mapping;
 
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.Map.Entry;
 import java.util.stream.Collectors;
 
 import ghidra.app.plugin.core.debug.disassemble.DisassemblyInject;
@@ -27,7 +28,11 @@ import ghidra.framework.plugintool.PluginTool;
 import ghidra.program.model.address.*;
 import ghidra.program.model.lang.Endian;
 import ghidra.trace.model.Trace;
+import ghidra.trace.model.TraceAddressSnapRange;
 import ghidra.trace.model.guest.TracePlatform;
+import ghidra.trace.model.listing.TraceInstruction;
+import ghidra.trace.model.memory.TraceMemoryOperations;
+import ghidra.trace.model.memory.TraceMemoryState;
 import ghidra.trace.model.target.TraceObject;
 import ghidra.trace.model.thread.TraceThread;
 import ghidra.util.classfinder.ClassSearcher;
@@ -61,7 +66,13 @@ public abstract class AbstractDebuggerPlatformMapper implements DebuggerPlatform
 	}
 
 	protected boolean isCancelSilently(Address start, long snap) {
-		return trace.getCodeManager().instructions().getAt(snap, start) != null;
+		TraceInstruction exists = trace.getCodeManager().instructions().getAt(snap, start);
+		if (exists == null) {
+			return false;
+		}
+		var states = trace.getMemoryManager().getStates(snap, exists.getRange());
+		return TraceMemoryOperations.isStateEntirely(exists.getRange(), states,
+			TraceMemoryState.KNOWN);
 	}
 
 	protected Collection<DisassemblyInject> getDisassemblyInjections(TracePlatform platform) {

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceInstructionsView.java

@@ -16,6 +16,7 @@
 package ghidra.trace.database.listing;
 
 import java.util.*;
+import java.util.Map.Entry;
 
 import org.apache.commons.lang3.tuple.Pair;
 
@@ -29,9 +30,13 @@ import ghidra.program.model.util.CodeUnitInsertionException;
 import ghidra.trace.database.context.DBTraceRegisterContextManager;
 import ghidra.trace.database.context.DBTraceRegisterContextSpace;
 import ghidra.trace.database.guest.InternalTracePlatform;
+import ghidra.trace.database.memory.DBTraceMemoryManager;
+import ghidra.trace.database.memory.DBTraceMemorySpace;
 import ghidra.trace.model.*;
 import ghidra.trace.model.guest.TracePlatform;
 import ghidra.trace.model.listing.*;
+import ghidra.trace.model.memory.TraceMemoryOperations;
+import ghidra.trace.model.memory.TraceMemoryState;
 import ghidra.trace.util.*;
 import ghidra.util.LockHold;
 import ghidra.util.exception.CancelledException;
@@ -92,7 +97,7 @@ public class DBTraceInstructionsView extends AbstractBaseDBTraceDefinedUnitsView
 
 		protected void truncateOrDelete(TraceInstruction exists) {
 			if (exists.getStartSnap() < lifespan.lmin()) {
-				exists.setEndSnap(lifespan.lmin());
+				exists.setEndSnap(lifespan.lmin() - 1);
 			}
 			else {
 				exists.delete();
@@ -471,6 +476,15 @@ public class DBTraceInstructionsView extends AbstractBaseDBTraceDefinedUnitsView
 			conflict, conflictCodeUnit, overwrite);
 	}
 
+	protected boolean isKnown(DBTraceMemorySpace ms, long snap, CodeUnit cu) {
+		if (ms == null) {
+			return false;
+		}
+		AddressRange range = new AddressRangeImpl(cu.getMinAddress(), cu.getMaxAddress());
+		var states = ms.getStates(snap, range);
+		return TraceMemoryOperations.isStateEntirely(range, states, TraceMemoryState.KNOWN);
+	}
+
 	/**
 	 * Checks the intended locations for conflicts with existing units.
 	 * 
@@ -486,6 +500,7 @@ public class DBTraceInstructionsView extends AbstractBaseDBTraceDefinedUnitsView
 			Set<Address> skipDelaySlots) {
 		// NOTE: Partly derived from CodeManager#checkInstructionSet()
 		// Attempted to factor more fluently
+		DBTraceMemoryManager mm = space.trace.getMemoryManager();
 		for (InstructionBlock block : instructionSet) {
 			// If block contains a known error, record its address, and do not proceed beyond it
 			Address errorAddress = null;
@@ -519,6 +534,12 @@ public class DBTraceInstructionsView extends AbstractBaseDBTraceDefinedUnitsView
 					lastProtoInstr = protoInstr;
 				}
 				CodeUnit existsCu = overlap.getRight();
+				DBTraceMemorySpace ms =
+					mm.getMemorySpace(existsCu.getAddress().getAddressSpace(), false);
+				if (!isKnown(ms, startSnap, existsCu) && existsCu instanceof TraceCodeUnit tcu) {
+					tcu.delete();
+					continue;
+				}
 				int cmp = existsCu.getMinAddress().compareTo(protoInstr.getMinAddress());
 				boolean existsIsInstruction = (existsCu instanceof TraceInstruction);
 				if (cmp == 0 && existsIsInstruction) {
@@ -552,7 +573,7 @@ public class DBTraceInstructionsView extends AbstractBaseDBTraceDefinedUnitsView
 				}
 				// NOTE: existsIsInstruction implies cmp != 0, so record as off-cut conflict
 				block.setCodeUnitConflict(existsCu.getAddress(), protoInstr.getAddress(),
-					flowFromAddress, existsIsInstruction, existsIsInstruction);
+					flowFromAddress, existsIsInstruction, true);
 			}
 		}
 	}

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/DBTraceAddressSnapRangePropertyMapAddressSetView.java

@@ -39,13 +39,17 @@ public class DBTraceAddressSnapRangePropertyMapAddressSetView<T> extends Abstrac
 	private final Predicate<? super T> predicate;
 
 	/**
-	 * TODO Document me
+	 * Construct an {@link AddressSetView} based on the given map of entries and predicate.
 	 * 
-	 * The caller must reduce the map if only a certain range is desired.
+	 * <p>
+	 * The spatial map is a 2-dimensional collection of entries, but only the address dimension is
+	 * considered. This set behaves as the union of address ranges for all entries whose values pass
+	 * the predicate. Typically, the caller reduces the map first.
 	 * 
-	 * @param lock
+	 * @param space the address space of the given map
-	 * @param map
+	 * @param lock a lock to ensure access to the underlying database is synchronized
-	 * @param predicate
+	 * @param map the map whose entries to test
+	 * @param predicate the predicate for testing entry values
 	 */
 	public DBTraceAddressSnapRangePropertyMapAddressSetView(AddressSpace space, ReadWriteLock lock,
 			SpatialMap<TraceAddressSnapRange, T, TraceAddressSnapRangeQuery> map,

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/DBTraceAddressSnapRangePropertyMapTree.java

@@ -515,6 +515,12 @@ public class DBTraceAddressSnapRangePropertyMapTree<T, DR extends AbstractDBTrac
 				Rectangle2DDirection.TOPMOST, TraceAddressSnapRangeQuery::new);
 		}
 
+		public static TraceAddressSnapRangeQuery mostRecent(AddressRange range, Lifespan span) {
+			return intersecting(
+				new ImmutableTraceAddressSnapRange(range, span),
+				Rectangle2DDirection.TOPMOST, TraceAddressSnapRangeQuery::new);
+		}
+
 		public static TraceAddressSnapRangeQuery equalTo(TraceAddressSnapRange shape) {
 			return equalTo(shape, null, TraceAddressSnapRangeQuery::new);
 		}

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemoryManager.java

@@ -279,6 +279,13 @@ public class DBTraceMemoryManager extends AbstractDBTraceSpaceBasedManager<DBTra
 			m -> m.getMostRecentStateEntry(snap, address));
 	}
 
+	@Override
+	public Entry<TraceAddressSnapRange, TraceMemoryState> getViewMostRecentStateEntry(long snap,
+			AddressRange range, Predicate<TraceMemoryState> predicate) {
+		return delegateRead(range.getAddressSpace(),
+			m -> m.getViewMostRecentStateEntry(snap, range, predicate));
+	}
+
 	@Override
 	public Entry<TraceAddressSnapRange, TraceMemoryState> getViewMostRecentStateEntry(long snap,
 			Address address) {

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemorySpace.java

@@ -406,11 +406,17 @@ public class DBTraceMemorySpace
 	@Override
 	public Entry<TraceAddressSnapRange, TraceMemoryState> getViewMostRecentStateEntry(long snap,
 			Address address) {
+		return getViewMostRecentStateEntry(snap, new AddressRangeImpl(address, address), s -> true);
+	}
+
+	@Override
+	public Entry<TraceAddressSnapRange, TraceMemoryState> getViewMostRecentStateEntry(long snap,
+			AddressRange range, Predicate<TraceMemoryState> predicate) {
+		assertInSpace(range);
 		for (Lifespan span : viewport.getOrderedSpans(snap)) {
-			Entry<TraceAddressSnapRange, TraceMemoryState> entry =
+			var entry = stateMapSpace.reduce(TraceAddressSnapRangeQuery.mostRecent(range, span))
-				stateMapSpace.reduce(TraceAddressSnapRangeQuery.mostRecent(address, span))
+					.firstEntry();
-						.firstEntry();
+			if (entry != null && predicate.test(entry.getValue())) {
-			if (entry != null) {
 				return entry;
 			}
 		}

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/AbstractDBTraceProgramViewListing.java

@@ -22,6 +22,7 @@ import org.apache.commons.collections4.IteratorUtils;
 
 import generic.NestedIterator;
 import ghidra.program.database.ProgramDB;
+import ghidra.program.database.code.InstructionDB;
 import ghidra.program.database.function.OverlappingFunctionException;
 import ghidra.program.model.address.*;
 import ghidra.program.model.data.DataType;
@@ -41,6 +42,7 @@ import ghidra.trace.database.thread.DBTraceThread;
 import ghidra.trace.model.*;
 import ghidra.trace.model.listing.*;
 import ghidra.trace.model.memory.TraceMemoryRegion;
+import ghidra.trace.model.memory.TraceMemoryState;
 import ghidra.trace.model.program.TraceProgramView;
 import ghidra.trace.model.program.TraceProgramViewListing;
 import ghidra.trace.model.property.TracePropertyMapOperations;
@@ -717,9 +719,21 @@ public abstract class AbstractDBTraceProgramViewListing implements TraceProgramV
 	public Instruction createInstruction(Address addr, InstructionPrototype prototype,
 			MemBuffer memBuf, ProcessorContextView context, int forcedLengthOverride)
 			throws CodeUnitInsertionException {
-		// TODO: Why memBuf? Can it vary from program memory?
+		int checkLengthOverride =
+			InstructionDB.checkLengthOverride(forcedLengthOverride, prototype);
+		int length = checkLengthOverride != 0 ? checkLengthOverride : prototype.getLength();
+		AddressRange range;
+		try {
+			range = new AddressRangeImpl(addr, length);
+		}
+		catch (AddressOverflowException e) {
+			throw new CodeUnitInsertionException("Code unit would extend beyond address space");
+		}
+		var mostRecent = program.memory.memoryManager.getViewMostRecentStateEntry(program.snap,
+			range, s -> s == TraceMemoryState.KNOWN);
+		long snap = mostRecent == null ? program.snap : mostRecent.getKey().getY2();
 		return codeOperations.instructions()
-				.create(Lifespan.nowOn(program.snap), addr, platform, prototype, context,
+				.create(Lifespan.nowOn(snap), addr, platform, prototype, context,
 					forcedLengthOverride);
 	}
 

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/DBTraceProgramView.java

@@ -1635,6 +1635,17 @@ public class DBTraceProgramView implements TraceProgramView {
 	}
 
 	protected boolean isCodeVisible(TraceCodeUnit cu, Lifespan lifespan) {
+		try {
+			byte[] cubytes = cu.getBytes();
+			byte[] mmbytes = new byte[cubytes.length];
+			memory.getBytes(cu.getAddress(), mmbytes);
+			if (!Arrays.equals(cubytes, mmbytes)) {
+				return false;
+			}
+		}
+		catch (MemoryAccessException e) {
+			throw new AssertionError(e);
+		}
 		return viewport.isCompletelyVisible(cu.getRange(), lifespan, cu,
 			getCodeOcclusion(cu.getTraceSpace()));
 	}

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/listing/TraceInstructionsView.java

@@ -48,7 +48,13 @@ public interface TraceInstructionsView extends TraceBaseDefinedUnitsView<TraceIn
 	/**
 	 * Create an instruction for the host platform
 	 * 
-	 * @see #create(Lifespan, Address, TracePlatform, InstructionPrototype, ProcessorContextView)
+	 * @param lifespan the lifespan for the instruction unit
+	 * @param address the starting address of the instruction
+	 * @param prototype the instruction prototype
+	 * @param context the input disassembly context for the instruction
+	 * @param forcedLengthOverride reduced instruction byte-length (1..7) or 0 to use default length
+	 * @return the new instruction
+	 * @throws CodeUnitInsertionException if the instruction cannot be created
 	 */
 	default TraceInstruction create(Lifespan lifespan, Address address,
 			InstructionPrototype prototype, ProcessorContextView context, int forcedLengthOverride)
@@ -76,7 +82,14 @@ public interface TraceInstructionsView extends TraceBaseDefinedUnitsView<TraceIn
 	/**
 	 * Create several instructions for the host platform
 	 * 
-	 * @see #addInstructionSet(Lifespan, TracePlatform, InstructionSet, boolean)
+	 * <p>
+	 * <b>NOTE:</b> This does not throw {@link CodeUnitInsertionException}. Conflicts are instead
+	 * recorded in the {@code instructionSet}.
+	 * 
+	 * @param lifespan the lifespan for all instruction units
+	 * @param instructionSet the set of instructions to add
+	 * @param overwrite true to replace conflicting instructions
+	 * @return the (host) address set of instructions actually added
 	 */
 	default AddressSetView addInstructionSet(Lifespan lifespan, InstructionSet instructionSet,
 			boolean overwrite) {

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryOperations.java

@@ -62,6 +62,38 @@ import ghidra.util.task.TaskMonitor;
  * accidentally rely on implied temporal relationships in scratch space.
  */
 public interface TraceMemoryOperations {
+	/**
+	 * Check if the return value of {@link #getStates(long, AddressRange)} or similar represents a
+	 * single entry of the given state.
+	 * 
+	 * <p>
+	 * This method returns false if there is not exactly one entry of the given state whose range
+	 * covers the given range. As a special case, an empty collection will cause this method to
+	 * return true iff state is {@link TraceMemoryState#UNKNOWN}.
+	 * 
+	 * @param range the range to check, usually that passed to
+	 *            {@link #getStates(long, AddressRange)}.
+	 * @param stateEntries the collection returned by {@link #getStates(long, AddressRange)}.
+	 * @param state the expected state
+	 * @return true if the state matches
+	 */
+	static boolean isStateEntirely(AddressRange range,
+			Collection<Entry<TraceAddressSnapRange, TraceMemoryState>> stateEntries,
+			TraceMemoryState state) {
+		if (stateEntries.isEmpty()) {
+			return state == TraceMemoryState.UNKNOWN;
+		}
+		if (stateEntries.size() != 1) {
+			return false;
+		}
+		Entry<TraceAddressSnapRange, TraceMemoryState> ent = stateEntries.iterator().next();
+		if (ent.getValue() != state) {
+			return false;
+		}
+		AddressRange entRange = ent.getKey().getRange();
+		return entRange.contains(range.getMinAddress()) && entRange.contains(range.getMaxAddress());
+	}
+
 	/**
 	 * Get the trace to which the memory manager belongs
 	 * 
@@ -261,13 +293,25 @@ public interface TraceMemoryOperations {
 	 * Get the entry recording the most recent state at the given snap and address, following
 	 * schedule forks
 	 * 
-	 * @param snap the time
+	 * @param snap the latest time to consider
-	 * @param address the location
+	 * @param address the address
-	 * @return the state
+	 * @return the most-recent entry
 	 */
 	Entry<TraceAddressSnapRange, TraceMemoryState> getViewMostRecentStateEntry(long snap,
 			Address address);
 
+	/**
+	 * Get the entry recording the most recent state since the given snap within the given range
+	 * that satisfies a given predicate, following schedule forks
+	 * 
+	 * @param snap the latest time to consider
+	 * @param range the range of addresses
+	 * @param predicate a predicate on the state
+	 * @return the most-recent entry
+	 */
+	Entry<TraceAddressSnapRange, TraceMemoryState> getViewMostRecentStateEntry(long snap,
+			AddressRange range, Predicate<TraceMemoryState> predicate);
+
 	/**
 	 * Get at least the subset of addresses having state satisfying the given predicate
 	 * 

Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/program/DBTraceProgramViewListingTest.java

@@ -19,13 +19,16 @@ import static ghidra.lifecycle.Unfinished.TODO;
 import static org.junit.Assert.*;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.*;
 
 import org.junit.*;
 
 import db.Transaction;
+import ghidra.app.plugin.assembler.*;
 import ghidra.program.database.ProgramBuilder;
-import ghidra.program.model.address.AddressSet;
+import ghidra.program.disassemble.Disassembler;
+import ghidra.program.model.address.*;
 import ghidra.program.model.data.*;
 import ghidra.program.model.lang.*;
 import ghidra.program.model.listing.*;
@@ -35,6 +38,13 @@ import ghidra.test.AbstractGhidraHeadlessIntegrationTest;
 import ghidra.trace.database.ToyDBTraceBuilder;
 import ghidra.trace.database.listing.DBTraceCodeManager;
 import ghidra.trace.database.memory.DBTraceMemoryManager;
+import ghidra.trace.model.Lifespan;
+import ghidra.trace.model.memory.TraceMemoryFlag;
+import ghidra.trace.model.thread.TraceThread;
+import ghidra.trace.model.time.TraceSnapshot;
+import ghidra.trace.model.time.TraceTimeManager;
+import ghidra.trace.model.time.schedule.TraceSchedule;
+import ghidra.util.Msg;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
 
@@ -48,9 +58,11 @@ public class DBTraceProgramViewListingTest extends AbstractGhidraHeadlessIntegra
 	DBTraceCodeManager code;
 
 	protected static void assertUndefined(CodeUnit cu) {
-		Data data = (Data) cu;
+		if (cu instanceof Data data && DataType.DEFAULT.equals(data.getDataType()) &&
-		assertEquals(DataType.DEFAULT, data.getDataType());
+			!data.isDefined()) {
-		assertFalse(data.isDefined());
+			return;
+		}
+		fail("Expected undefined unit, but was '%s'".formatted(cu));
 	}
 
 	protected <T> List<T> takeN(int n, Iterator<T> it) {
@@ -896,4 +908,81 @@ public class DBTraceProgramViewListingTest extends AbstractGhidraHeadlessIntegra
 		assertArrayEquals(b.arr(1), cu0.getBytes());
 		assertArrayEquals(b.arr(8), cu1.getBytes());
 	}
+
+	@Test
+	public void testGetCodeUnitsInScratchView() throws Throwable {
+		TraceTimeManager tm = b.trace.getTimeManager();
+		Address entry = b.addr(0x00400000);
+		AddressSetView set = b.set(b.range(0x00400000, 0x00400003));
+		Assembler asm = Assemblers.getAssembler(b.language);
+
+		AssemblyBuffer buf = new AssemblyBuffer(asm, entry);
+		buf.assemble("imm r1, #234");
+		buf.assemble("add r1, r1");
+
+		final long snap;
+		try (Transaction tx = b.startTransaction()) {
+			TraceThread thread = b.getOrAddThread("Threads[1]", 0);
+			tm.getSnapshot(0, true);
+			memory.addRegion("Memory[test]", Lifespan.nowOn(0), b.range(0x00400000, 0x00400fff),
+				TraceMemoryFlag.READ, TraceMemoryFlag.EXECUTE);
+
+			TraceSnapshot scratch = tm.getSnapshot(Long.MIN_VALUE, true);
+			snap = scratch.getKey();
+			scratch.setSchedule(TraceSchedule.ZERO.steppedForward(thread, 1));
+
+			view.setSnap(snap);
+			Disassembler dis =
+				Disassembler.getDisassembler(view, TaskMonitor.DUMMY, msg -> Msg.error(this, msg));
+			AddressSetView result = dis.disassemble(entry, set);
+			assertEquals(set, result);
+
+			assertEquals(4, memory.putBytes(0, entry, ByteBuffer.wrap(buf.getBytes())));
+			// No disassembly at snap 0
+		}
+
+		byte[] arr = new byte[4];
+		view.getMemory().getBytes(entry, arr);
+		assertArrayEquals(buf.getBytes(), arr);
+		assertUndefined(listing.getCodeUnitAt(entry));
+	}
+
+	@Test
+	public void testCreateCodeUnitsInScratchViewAfterBytesChanged() throws Throwable {
+		TraceTimeManager tm = b.trace.getTimeManager();
+		Address entry = b.addr(0x00400000);
+		AddressSetView set = b.set(b.range(0x00400000, 0x00400003));
+		Assembler asm = Assemblers.getAssembler(b.language);
+
+		AssemblyBuffer buf = new AssemblyBuffer(asm, entry);
+		buf.assemble("imm r1, #234");
+		buf.assemble("add r1, r1");
+
+		final long snap;
+		try (Transaction tx = b.startTransaction()) {
+			TraceThread thread = b.getOrAddThread("Threads[1]", 0);
+			tm.getSnapshot(0, true);
+			memory.addRegion("Memory[test]", Lifespan.nowOn(0), b.range(0x00400000, 0x00400fff),
+				TraceMemoryFlag.READ, TraceMemoryFlag.EXECUTE);
+
+			TraceSnapshot scratch = tm.getSnapshot(Long.MIN_VALUE, true);
+			snap = scratch.getKey();
+			scratch.setSchedule(TraceSchedule.ZERO.steppedForward(thread, 1));
+
+			view.setSnap(snap);
+			Disassembler dis =
+				Disassembler.getDisassembler(view, TaskMonitor.DUMMY, msg -> Msg.error(this, msg));
+			AddressSetView result = dis.disassemble(entry, set);
+			assertEquals(set, result);
+
+			assertEquals(4, memory.putBytes(0, entry, ByteBuffer.wrap(buf.getBytes())));
+			// No disassembly at snap 0
+
+			// Attempt re-disassembly at scratch snap
+			result = dis.disassemble(entry, set);
+			assertEquals(set, result);
+		}
+
+		assertEquals("imm r1,#0xea", listing.getCodeUnitAt(entry).toString());
+	}
 }