GP-3506 Corrected various issues with data organization retention and

revision/upgrade
handling for datatypes

Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/datamgr/tree/ArchiveNode.java

@@ -21,9 +21,12 @@ import javax.swing.Icon;
 
 import docking.widgets.tree.GTree;
 import docking.widgets.tree.GTreeNode;
+import generic.theme.GColor;
 import generic.theme.GThemeDefaults.Colors.Messages;
 import ghidra.app.plugin.core.datamgr.archive.Archive;
 import ghidra.program.model.data.*;
+import ghidra.program.model.data.StandAloneDataTypeManager.ArchiveWarning;
+import ghidra.program.model.data.StandAloneDataTypeManager.ArchiveWarningLevel;
 import ghidra.util.HTMLUtilities;
 import ghidra.util.task.SwingUpdateManager;
 
@@ -63,26 +66,29 @@ public class ArchiveNode extends CategoryNode {
 			buf.append(HTMLUtilities.HTML_SPACE);
 			buf.append(HTMLUtilities.HTML_SPACE);
 			buf.append(HTMLUtilities.escapeHTML(programArchSummary));
-			addArchiveWarnings(dtm, buf);
 		}
 		else {
 			buf.append(DEFAULT_DATA_ORG_DESCRIPTION);
 		}
+		addArchiveWarnings(dtm, buf);
 		return buf.toString();
 	}
 
 	private void addArchiveWarnings(DataTypeManager dtm, StringBuilder buf) {
 		if (dtm instanceof StandAloneDataTypeManager archiveDtm) {
-			if (archiveDtm.isProgramArchitectureMissing()) {
+			ArchiveWarning warning = archiveDtm.getWarning();
+			if (warning != ArchiveWarning.NONE) {
+				GColor c = Messages.NORMAL;
+				ArchiveWarningLevel level = warning.level();
+				if (level == ArchiveWarningLevel.ERROR) {
+					c = Messages.ERROR;
+				}
+				else if (level == ArchiveWarningLevel.WARN) {
+					c = Messages.WARNING;
+				}
+				String msg = archiveDtm.getWarningMessage(false);
 				buf.append(HTMLUtilities.BR);
-				buf.append(
-					"<font color=\"" + Messages.ERROR +
-						"\">** Missing Language/Compiler Specification **</font>");
-			}
-			else if (archiveDtm.isProgramArchitectureUpgradeRequired()) {
-				buf.append(HTMLUtilities.BR);
-				buf.append("<font color=\"" + Messages.WARNING +
-					"\">** Language Upgrade Required **</font>");
+				buf.append("<font color=\"" + c + "\">** " + msg + " **</font>");
 			}
 		}
 	}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProgramDB.java

@@ -1775,9 +1775,11 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 
 		monitor.checkCancelled();
 
-		// Upgrade Function Manager 
-		if (openMode == UPGRADE && oldFunctionMgr != null) {
-			oldFunctionMgr.upgrade(this, monitor);
+		if (openMode == UPGRADE) {
+			if (oldFunctionMgr != null) {
+				// Upgrade Function Manager 
+				oldFunctionMgr.upgrade(this, monitor);
+			}
 		}
 
 		for (int i = 0; i < NUM_MANAGERS; i++) {

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProjectDataTypeManager.java

@@ -69,7 +69,7 @@ public class ProjectDataTypeManager extends StandAloneDataTypeManager
 			TaskMonitor monitor) throws CancelledException, VersionException, IOException {
 		super(handle, openMode, errHandler, lock, monitor);
 		this.dataTypeArchive = dataTypeArchive;
-		reportWarning();
+		logWarning();
 	}
 
 	@Override

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/data/CompositeDB.java

@@ -683,13 +683,11 @@ abstract class CompositeDB extends DataTypeDB implements CompositeInternal {
 	}
 	
 	/**
-	 * Perform any neccessary component adjustments based on
-	 * sizes and alignment of components differing from their 
-	 * specification which may be influenced by the data organization.
-	 * If this composite changes parents will not be
-	 * notified - handling this is the caller's responsibility.
-	 * It is assumed that this method is invoked on composites
-	 * in dependency order.
+	 * Perform any neccessary component adjustments based on sizes of components differing from 
+	 * their specification which may be influenced by the data organization.  This method
+	 * does not consider alignment changes and should be used on non-packed structures only.
+	 * If this composite changes parents will not be notified - handling this is the caller's 
+	 * responsibility. It is assumed that this method is invoked on composites in dependency order.
 	 * @throws IOException if database IO error occurs
 	 */
 	protected abstract void fixupComponents() throws IOException;

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/data/DataTypeManagerDB.java

@@ -238,16 +238,18 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 	/**
 	 * Constructor for a data-type manager backed by a packed database file. When
 	 * opening for UPDATE an automatic upgrade will be performed if required.
-	 * NOTE: default DataOrganization will be used.
+	 * NOTE: Default DataOrganization will be used for new archive.
 	 * 
 	 * @param packedDBfile packed datatype archive file (i.e., *.gdt resource).
 	 * @param openMode     open mode CREATE, READ_ONLY or UPDATE (see
 	 *                     {@link DBConstants}).
+	 * @param monitor task monitor
 	 * @throws IOException a low-level IO error. This exception may also be thrown
 	 *                     when a version error occurs (cause is VersionException).
+	 * @throws CancelledException if task cancelled
 	 */
-	protected DataTypeManagerDB(ResourceFile packedDBfile, int openMode)
-			throws IOException {
+	protected DataTypeManagerDB(ResourceFile packedDBfile, int openMode, TaskMonitor monitor)
+			throws IOException, CancelledException {
 
 		this.errHandler = new DbErrorHandler();
 		this.lock = new Lock("DataTypeManagerDB");
@@ -268,19 +270,17 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 					DataTypeArchiveContentHandler.DATA_TYPE_ARCHIVE_CONTENT_TYPE);
 			}
 			else {
-				pdb = PackedDatabase.getPackedDatabase(packedDBfile, false, TaskMonitor.DUMMY);
-				if (openMode == DBConstants.UPDATE) {
-					dbHandle = pdb.openForUpdate(TaskMonitor.DUMMY);
+				pdb = PackedDatabase.getPackedDatabase(packedDBfile, false, monitor);
+
+				if (openMode == DBConstants.READ_ONLY) {
+					dbHandle = pdb.open(monitor);
 				}
-				else {
-					dbHandle = pdb.open(TaskMonitor.DUMMY);
+				else { // UPDATE mode (allows upgrade use)
+					dbHandle = pdb.openForUpdate(monitor);
 				}
 			}
 			openSuccess = true;
 		}
-		catch (CancelledException e1) {
-			throw new AssertException(e1); // can't happen--dummy monitor
-		}
 		finally {
 			if (!openSuccess && pdb != null) {
 				pdb.dispose(); // dispose on error
@@ -290,21 +290,15 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 		// Initialize datatype manager and save new archive on CREATE
 		boolean initSuccess = false;
 		try {
-
-			initPackedDatabase(packedDBfile, openMode); // performs upgrade if needed
-
+			initPackedDatabase(packedDBfile, openMode, monitor); // performs upgrade if needed
 			if (openMode == DBConstants.CREATE) {
 				// preserve UniversalID if it has been established
 				Long uid = universalID != null ? universalID.getValue() : null;
 				((PackedDBHandle) dbHandle).saveAs("Archive", file.getParentFile(),
-					packedDBfile.getName(), uid, TaskMonitor.DUMMY);
+					packedDBfile.getName(), uid, monitor);
 			}
-
 			initSuccess = true;
 		}
-		catch (CancelledException e) {
-			throw new AssertException(e); // can't happen--dummy monitor
-		}
 		finally {
 			if (!initSuccess) {
 				dbHandle.close(); // close on error (packed database will also be disposed)
@@ -312,41 +306,43 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 		}
 	}
 
-	private void initPackedDatabase(ResourceFile packedDBfile, int openMode)
+	private void initPackedDatabase(ResourceFile packedDBfile, int openMode, TaskMonitor monitor)
 			throws CancelledException, IOException {
-		int id = startTransaction("");
-		try {
-			init(openMode, TaskMonitor.DUMMY);
+		try (Transaction tx = openTransaction("")) {
+			init(openMode, monitor);
+
+			if (openMode != DBConstants.CREATE && hasDataOrganizationChange()) {
+				// check for data organization change with possible upgrade
+				handleDataOrganizationChange(openMode, monitor);
+			}
+
+			if (openMode == DBConstants.UPGRADE) {
+				migrateOldFlexArrayComponentsIfRequired(monitor);
+
+				Msg.showInfo(this, null, "Archive Upgraded",
+					"Data type archive has been upgraded: " + packedDBfile.getName());
+			}
 		}
 		catch (VersionException e) {
 			if (openMode == DBConstants.UPDATE && e.isUpgradable()) {
-				try {
-					init(DBConstants.UPGRADE, TaskMonitor.DUMMY);
-					migrateOldFlexArrayComponentsIfRequired(TaskMonitor.DUMMY);
-
-					Msg.showInfo(this, null, "Archive Upgraded",
-						"Data type archive schema has been upgraded: " + packedDBfile.getName());
-				}
-				catch (VersionException ve) {
-					throw new IOException(e); // unexpected
-				}
+				initPackedDatabase(packedDBfile, DBConstants.UPGRADE, monitor);
 			}
 			else {
-				// TODO: Unable to handle required upgrade for read-only without API change
+				// Unable to handle required upgrade
 				throw new IOException(e);
 			}
 		}
-		finally {
-			endTransaction(id, true);
-		}
 	}
 
 	/**
 	 * Constructor for a database-backed <code>DataTypeManagerDB</code> extension.
+	 * NOTE: This does not check for and handle data organization changes which must be
+	 * handled later (use {@link #hasDataOrganizationChange()} and 
+	 * {@link #compilerSpecChanged(TaskMonitor)} to check for and initiate response to changes).
 	 * 
 	 * @param handle     database handle
 	 * @param addrMap    address map (may be null)
-	 * @param openMode   open mode CREATE, READ_ONLY or UPDATE (see {@link DBConstants}).
+	 * @param openMode   open mode CREATE, READ_ONLY, UPDATE, UPGRADE (see {@link DBConstants}).
 	 * @param tablePrefix DB table prefix to be applied to all associated table names.  This 
 	 *                    need only be specified when using multiple instances with the same
 	 *                    DB handle (null or empty string for no-prefix).
@@ -528,6 +524,17 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 		// do nothing
 	}
 
+	protected void handleDataOrganizationChange(int openMode, TaskMonitor monitor)
+			throws IOException, LanguageVersionException, CancelledException {
+		if (openMode == DBConstants.UPDATE) {
+			throw new LanguageVersionException("Data organization change detected", true);
+		}
+		if (openMode == DBConstants.UPGRADE) {
+			compilerSpecChanged(monitor);
+		}
+		// NOTE: No change for READ_ONLY mode
+	}
+
 	/**
 	 * Build Parent/Child table for tracking dataType usage by other dataTypes
 	 * (e.g., arrays, pointers, etc.). Only used to populate the ParentChildAdapter
@@ -777,6 +784,10 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 	 * true, to reflect any changes in the data organization.
 	 * The caller is resposible for ensuring that this setting is done consistent 
 	 * with the {@link #addrMap} setting used during construction if applicable.
+	 * <br>
+	 * If not storing caller may need to check for data organization change to communicate
+	 * change or to facilitate an upgrade situation.
+	 * 
 	 * @param programArchitecture program architecture details (may be null) in which case
 	 * default data organization will be used.
 	 * @param variableStorageMgr variable storage manager (within same database) or null
@@ -805,8 +816,13 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 			saveDataOrganization();
 		}
 		else if (store) {
-			compilerSpecChanged(monitor);
-			updateLastChangeTime();
+			try {
+				compilerSpecChanged(monitor);
+				updateLastChangeTime();
+			}
+			finally {
+				invalidateCache();
+			}
 		}
 	}
 
@@ -822,46 +838,52 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 	 * @throws CancelledException if processing cancelled - data types may not properly reflect
 	 * updated compiler specification
 	 */
-	private void compilerSpecChanged(TaskMonitor monitor) throws IOException, CancelledException {
+	protected void compilerSpecChanged(TaskMonitor monitor) throws IOException, CancelledException {
 		
 		if (mode == DBConstants.READ_ONLY) {
 			throw new ReadOnlyException();
 		}
 
-		DataOrganization oldDataOrganization = readDataOrganization();
+		boolean hasDataOrgChange = hasDataOrganizationChange();
 
-		try {
-			saveDataOrganization();
-
-			if (oldDataOrganization != null &&
-				!oldDataOrganization.equals(dataOrganization)) {
-				Msg.info(this,
-					"Fixing datatypes to reflect data organization change: " + getPath());
-				doCompositeFixup(monitor);
-			}
-
-			// FUTURE: may need to handle calling convention and data organization change impact
-			// on function definitions
+		saveDataOrganization();
 
+		if (hasDataOrgChange) {
+			doCompositeFixup(monitor);
 		}
-		finally {
-			invalidateCache();
-		}
+
+		// FUTURE: may need to handle calling convention and data organization change impact
+		// on function definitions
 	}
 
-	private void saveDataOrganization() throws IOException {
-		if (dataOrganization == null) {
-			return;
-		}
-		DataOrganizationImpl.save(dataOrganization, getDataMap(true), "dataOrg.");
+	protected final boolean hasDataOrganizationChange() throws IOException {
+		// compare DB-stored data organization with the one in affect
+		return !Objects.equals(readDataOrganization(), getDataOrganization());
 	}
 
-	private DataOrganization readDataOrganization() throws IOException {
+	protected void saveDataOrganization() throws IOException {
+		DataOrganizationImpl.save(getDataOrganization(), getDataMap(true), "dataOrg.");
+	}
+
+	/**
+	 * Read the DB-serialized data organization.  If one has not been stored a suitable
+	 * default will be returned.
+	 * @return stored data organization or suitable default.
+	 * @throws IOException if DB error orccurs
+	 */
+	protected DataOrganization readDataOrganization() throws IOException {
 		DBStringMapAdapter dataMap = getDataMap(false);
 		if (dataMap == null) {
 			return null;
 		}
-		return DataOrganizationImpl.restore(dataMap, "dataOrg.");
+
+		DataOrganization dataOrg = DataOrganizationImpl.restore(dataMap, "dataOrg.");
+		if (dataOrg == null) {
+			ProgramArchitecture arch = getProgramArchitecture();
+			return DataOrganizationImpl
+					.getDefaultOrganization(arch != null ? arch.getLanguage() : null);
+		}
+		return dataOrg;
 	}
 
 	@Override
@@ -4235,7 +4257,12 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
 		int count = 0;
 		for (CompositeDB c : orderedComposites) {
 			monitor.checkCancelled();
-			c.fixupComponents();
+			if (c.isPackingEnabled()) {
+				c.repack(true, false);
+			}
+			else {
+				c.fixupComponents();
+			}
 			monitor.setProgress(++count);
 		}
 	}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/data/ProgramDataTypeManager.java

@@ -81,6 +81,9 @@ public class ProgramDataTypeManager extends ProgramBasedDataTypeManagerDB
 		try {
 			setProgramArchitecture(p, p.getSymbolTable().getVariableStorageManager(), false,
 				TaskMonitor.DUMMY);
+			// NOTE: Due to late manner in which program architecture is established, any
+			// response to a data organization change must be handled during a language
+			// upgrade and setLanguage
 		}
 		catch (CancelledException e) {
 			throw new AssertException(e); // unexpected - no IO performed

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/BitFieldPackingImpl.java

@@ -19,6 +19,7 @@ import static ghidra.program.model.pcode.AttributeId.*;
 import static ghidra.program.model.pcode.ElementId.*;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import ghidra.program.database.DBStringMapAdapter;
 import ghidra.program.model.pcode.Encoder;
@@ -189,4 +190,23 @@ public class BitFieldPackingImpl implements BitFieldPacking {
 		}
 		parser.end();
 	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(typeAlignmentEnabled, useMSConvention, zeroLengthBoundary);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		BitFieldPackingImpl other = (BitFieldPackingImpl) obj;
+		return typeAlignmentEnabled == other.typeAlignmentEnabled &&
+			useMSConvention == other.useMSConvention &&
+			zeroLengthBoundary == other.zeroLengthBoundary;
+	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/DataOrganizationImpl.java

@@ -701,15 +701,26 @@ public class DataOrganizationImpl implements DataOrganization {
 	 * Restore a data organization from the specified DB data map.
 	 * @param dataMap DB data map
 	 * @param keyPrefix key prefix for all map entries
-	 * @return data organization
+	 * @return stored data organization or null if not stored
 	 * @throws IOException if an IO error occurs
 	 */
 	public static DataOrganizationImpl restore(DBStringMapAdapter dataMap, String keyPrefix)
 			throws IOException {
 
+		boolean containsDataOrgEntries = false;
+		for (String key : dataMap.keySet()) {
+			if (key.startsWith(keyPrefix)) {
+				containsDataOrgEntries = true;
+				break;
+			}
+		}
+		if (!containsDataOrgEntries) {
+			return null;
+		}
+
 		DataOrganizationImpl dataOrg = new DataOrganizationImpl();
 		
-		dataOrg.bigEndian = dataMap.getBoolean(BIG_ENDIAN_NAME, false);
+		dataOrg.bigEndian = dataMap.getBoolean(keyPrefix + BIG_ENDIAN_NAME, false);
 
 		dataOrg.absoluteMaxAlignment =
 			dataMap.getInt(keyPrefix + ELEM_ABSOLUTE_MAX_ALIGNMENT.name(),
@@ -984,4 +995,35 @@ public class DataOrganizationImpl implements DataOrganization {
 		parser.end();
 
 	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(absoluteMaxAlignment, bigEndian, bitFieldPacking, charSize,
+			defaultAlignment, defaultPointerAlignment, doubleSize, floatSize, integerSize,
+			isSignedChar, longDoubleSize, longLongSize, longSize, machineAlignment, pointerShift,
+			pointerSize, shortSize, sizeAlignmentMap, wideCharSize);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		DataOrganizationImpl other = (DataOrganizationImpl) obj;
+		return absoluteMaxAlignment == other.absoluteMaxAlignment && bigEndian == other.bigEndian &&
+			Objects.equals(bitFieldPacking, other.bitFieldPacking) && charSize == other.charSize &&
+			defaultAlignment == other.defaultAlignment &&
+			defaultPointerAlignment == other.defaultPointerAlignment &&
+			doubleSize == other.doubleSize && floatSize == other.floatSize &&
+			integerSize == other.integerSize && isSignedChar == other.isSignedChar &&
+			longDoubleSize == other.longDoubleSize && longLongSize == other.longLongSize &&
+			longSize == other.longSize && machineAlignment == other.machineAlignment &&
+			pointerShift == other.pointerShift && pointerSize == other.pointerSize &&
+			shortSize == other.shortSize &&
+			Objects.equals(sizeAlignmentMap, other.sizeAlignmentMap) &&
+			wideCharSize == other.wideCharSize;
+	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/DataTypeArchiveIdDumper.java

@@ -39,7 +39,7 @@ public class DataTypeArchiveIdDumper implements GhidraLaunchable {
 
 		FileWriter writer = new FileWriter(outputFile);
 		FileDataTypeManager archive = FileDataTypeManager.openFileArchive(archiveFile, false);
-		archive.reportWarning();
+		archive.logWarning();
 		UniversalID universalID2 = archive.getUniversalID();
 		writer.write("FILE_ID: " + Long.toHexString(universalID2.getValue()));
 		writer.write("\n");

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/FileDataTypeManager.java

@@ -61,15 +61,18 @@ public class FileDataTypeManager extends StandAloneDataTypeManager
 	 * with a warning condition, architecture-specific data may not be available or up-to-date.
 	 * 
 	 * @param packedDbfile file to load or create based upon openMode
-	 * @param openMode one of the DBConstants: CREATE, UPDATE, READ_ONLY, UPGRADE 
+	 * @param openMode one of the DBConstants: CREATE, READ_ONLY or UPDATE
+	 * @param monitor the progress monitor
 	 * @throws IOException if an IO error occurs
+	 * @throws CancelledException if task cancelled
 	 */
-	private FileDataTypeManager(ResourceFile packedDbfile, int openMode) throws IOException {
-		super(validateFilename(packedDbfile), openMode);
+	private FileDataTypeManager(ResourceFile packedDbfile, int openMode, TaskMonitor monitor)
+			throws IOException, CancelledException {
+		super(validateFilename(packedDbfile), openMode, monitor);
 		file = packedDbfile;
 		name = getRootName(file.getName());
 		packedDB = ((PackedDBHandle) dbHandle).getPackedDatabase();
-		reportWarning();
+		logWarning();
 	}
 
 	private static ResourceFile validateFilename(ResourceFile packedDbfile) {
@@ -86,7 +89,13 @@ public class FileDataTypeManager extends StandAloneDataTypeManager
 	 * @throws IOException if an IO error occurs
 	 */
 	public static FileDataTypeManager createFileArchive(File packedDbfile) throws IOException {
-		return new FileDataTypeManager(new ResourceFile(packedDbfile), DBConstants.CREATE);
+		try {
+			return new FileDataTypeManager(new ResourceFile(packedDbfile), DBConstants.CREATE,
+				TaskMonitor.DUMMY);
+		}
+		catch (CancelledException e) {
+			throw new AssertException(e); // unexpected without task monitor use
+		}
 	}
 
 	/**
@@ -129,7 +138,12 @@ public class FileDataTypeManager extends StandAloneDataTypeManager
 	public static FileDataTypeManager openFileArchive(ResourceFile packedDbfile,
 			boolean openForUpdate) throws IOException {
 		int mode = openForUpdate ? DBConstants.UPDATE : DBConstants.READ_ONLY;
-		return new FileDataTypeManager(packedDbfile, mode);
+		try {
+			return new FileDataTypeManager(packedDbfile, mode, TaskMonitor.DUMMY);
+		}
+		catch (CancelledException e) {
+			throw new AssertException(e); // unexpected without task monitor use
+		}
 	}
 
 	/**

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StandAloneDataTypeManager.java

@@ -59,11 +59,16 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 
 	protected String name;
 	
+	public static enum ArchiveWarningLevel {
+		INFO, WARN, ERROR;
+	}
+
 	public static enum ArchiveWarning {
+
 		/**
 		 * {@link #NONE} indicates a normal archive condition
 		 */
-		NONE,
+		NONE(ArchiveWarningLevel.INFO),
 
 		/**
 		 * {@link #UPGRADED_LANGUAGE_VERSION} indicates an archive which has been open for update
@@ -72,7 +77,7 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 		 * which involves significant {@link Register} changes.  Sharing an upgraded archive 
 		 * may impact others who do not have access to the updated {@link Language} module.
 		 */
-		UPGRADED_LANGUAGE_VERSION,
+		UPGRADED_LANGUAGE_VERSION(ArchiveWarningLevel.INFO),
 
 		// programArchitectureSummary must be set for the warnings below
 
@@ -82,7 +87,7 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 		 * a problem being loaded.  The {@link FileDataTypeManager#getWarningDetail()} may provide
 		 * additional insight to the underlying cause. 
 		 */
-		LANGUAGE_NOT_FOUND,
+		LANGUAGE_NOT_FOUND(ArchiveWarningLevel.ERROR),
 
 		/**
 		 * {@link #COMPILER_SPEC_NOT_FOUND} indicates the {@link CompilerSpec}, 
@@ -91,7 +96,7 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 		 * additional insight to the underlying cause.  This condition can only occur if the
 		 * required {@link Language} was found. 
 		 */
-		COMPILER_SPEC_NOT_FOUND,
+		COMPILER_SPEC_NOT_FOUND(ArchiveWarningLevel.ERROR),
 
 		/**
 		 * {@link #LANGUAGE_UPGRADE_REQURED} indicates an archive which has been open read-only
@@ -103,7 +108,27 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 		 * who do not have access to the updated {@link Language} module and should be 
 		 * coordinated with others who may be affected.
 		 */
-		LANGUAGE_UPGRADE_REQURED,
+		LANGUAGE_UPGRADE_REQURED(ArchiveWarningLevel.WARN),
+
+		/**
+		 * {@link #DATA_ORG_CHANGED} indicates an archive which has been open read-only
+		 * requires an upgraded to adjust for changes in the associated data organization.
+		 */
+		DATA_ORG_CHANGED(ArchiveWarningLevel.WARN);
+
+		final ArchiveWarningLevel level;
+
+		ArchiveWarning(ArchiveWarningLevel level) {
+			this.level = level;
+		}
+
+		/**
+		 * Get the warning level
+		 * @return warning level
+		 */
+		public ArchiveWarningLevel level() {
+			return level;
+		}
 	}
 
 	private ArchiveWarning warning;
@@ -134,35 +159,37 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 	 * Constructor for a data-type manager backed by a packed database file.
 	 * When opening for UPDATE an automatic upgrade will be performed if required.
 	 * <p>
-	 * <B>NOTE:</B> {@link #reportWarning()} should be invoked immediately after 
+	 * <B>NOTE:</B> {@link #logWarning()} should be invoked immediately after 
 	 * instantiating a {@link StandAloneDataTypeManager} for an existing database after 
 	 * {@link #getName()} and {@link #getPath()} can be invoked safely.  In addition, it 
 	 * may be appropriate to use {@link #getWarning() check for warnings} prior to use.
 	 * 
 	 * @param packedDbfile packed datatype archive file (i.e., *.gdt resource).
 	 * @param openMode open mode CREATE, READ_ONLY or UPDATE (see {@link DBConstants})
+	 * @param monitor the progress monitor
 	 * @throws IOException a low-level IO error.  This exception may also be thrown
 	 * when a version error occurs (cause is VersionException).
+	 * @throws CancelledException if task cancelled
 	 */
-	protected StandAloneDataTypeManager(ResourceFile packedDbfile, int openMode)
-			throws IOException {
-		super(packedDbfile, openMode);
+	protected StandAloneDataTypeManager(ResourceFile packedDbfile, int openMode,
+			TaskMonitor monitor) throws IOException, CancelledException {
+		super(packedDbfile, openMode, monitor);
 	}
 
 	/**
 	 * Constructor for a data-type manager using a specified DBHandle.
 	 * <p>
-	 * <B>NOTE:</B> {@link #reportWarning()} should be invoked immediately after 
+	 * <B>NOTE:</B> {@link #logWarning()} should be invoked immediately after 
 	 * instantiating a {@link StandAloneDataTypeManager} for an existing database after 
 	 * {@link #getName()} and {@link #getPath()} can be invoked safely.  In addition, it 
 	 * may be appropriate to use {@link #getWarning() check for warnings} prior to use.
 	 * 
 	 * @param handle open database  handle
-	 * @param openMode the program open mode
+	 * @param openMode open mode CREATE, READ_ONLY or UPDATE (see {@link DBConstants})
 	 * @param errHandler the database I/O error handler
 	 * @param lock the program synchronization lock
 	 * @param monitor the progress monitor
-	 * @throws CancelledException if the user cancels an upgrade
+	 * @throws CancelledException if task cancelled
 	 * @throws VersionException if the database does not match the expected version.
 	 * @throws IOException if a database I/O error occurs.
 	 */
@@ -170,6 +197,9 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 			Lock lock, TaskMonitor monitor)
 			throws CancelledException, VersionException, IOException {
 		super(handle, null, openMode, null, errHandler, lock, monitor);
+		if (openMode != DBConstants.CREATE && hasDataOrganizationChange()) {
+			handleDataOrganizationChange(openMode, monitor);
+		}
 	}
 
 	/**
@@ -193,41 +223,77 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 	}
 
 	/**
-	 * Due to the supression of error and warning conditions during instantiation this method should
-	 * be invoked at the end of instatiation when {@link #getName()} and {@link #getPath()} are
-	 * ready to be invoked safely.  Logging will be performed via {@link Msg}.
+	 * Get a suitable warning message.  See {@link #getWarning()} for type and its severity level
+	 * {@link ArchiveWarning#level()}.
+	 * @param includeDetails if false simple message returned, otherwise more details are included.
+	 * @return warning message or null if {@link #getWarning()} is {@link ArchiveWarning#NONE}.
 	 */
-	protected void reportWarning() {
-		String msg;
+	public String getWarningMessage(boolean includeDetails) {
+		String msg = null;
 		switch (warning) {
-			case NONE:
-				break;
 			case LANGUAGE_NOT_FOUND:
-				msg = "Language not found for Archive '" + getName() + "': " +
-					warningDetail.getMessage();
-				Msg.error(this, msg);
+				msg = "Language not found for Archive";
+				if (includeDetails) {
+					msg += " '" + getName() + "': " + warningDetail.getMessage();
+				}
 				break;
 			case COMPILER_SPEC_NOT_FOUND:
-				msg = "Compiler specification not found for Archive '" + getName() + "': " +
-					warningDetail.getMessage();
-				Msg.error(this, msg);
+				msg = "Compiler specification not found for Archive";
+				if (includeDetails) {
+					msg += " '" + getName() + "': " + warningDetail.getMessage();
+				}
 				break;
 			case LANGUAGE_UPGRADE_REQURED:
-				msg = "Language upgrade required for Archive '" + getName() + "': " +
-					programArchitectureSummary;
-				Msg.warn(this, msg);
+				msg = "Language upgrade required for Archive";
+				if (includeDetails) {
+					msg += " '" + getName() + "': " + programArchitectureSummary;
+				}
 				break;
 			case UPGRADED_LANGUAGE_VERSION:
-				ProgramArchitecture arch = getProgramArchitecture();
-				LanguageDescription languageDescription =
-					arch.getLanguage().getLanguageDescription();
-				msg =
-					"Upgraded program-architecture for Archive: '" + getName() +
+				msg = "Upgraded program-architecture for Archive";
+				if (includeDetails) {
+					ProgramArchitecture arch = getProgramArchitecture();
+					LanguageDescription languageDescription =
+						arch.getLanguage().getLanguageDescription();
+					msg += " '" + getName() +
 						"'\n   Language: " +
 						languageDescription.getLanguageID() + " Version " +
 						languageDescription.getVersion() + ".x" +
 						", CompilerSpec: " + arch.getCompilerSpec().getCompilerSpecID();
+				}
+				break;
+			case DATA_ORG_CHANGED:
+				msg = "Data organization upgrade required for Archive";
+				if (includeDetails) {
+					msg += " '" + getName() + "': " + programArchitectureSummary;
+				}
+				break;
+			default:
+				break;
+		}
+		return msg;
+	}
+
+	/**
+	 * Due to the supression of error and warning conditions during instantiation this method should
+	 * be invoked at the end of instatiation when {@link #getName()} and {@link #getPath()} are
+	 * ready to be invoked safely.  Logging will be performed via {@link Msg}.
+	 */
+	protected void logWarning() {
+		String msg = getWarningMessage(true);
+		if (msg == null) {
+			return;
+		}
+		switch (warning.level) {
+			case ERROR:
+				Msg.error(this, msg);
+				break;
+			case WARN:
+				Msg.warn(this, msg);
+				break;
+			default:
 				Msg.info(this, msg);
+				break;
 		}
 
 	}
@@ -238,6 +304,7 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 
 		warning = ArchiveWarning.NONE;
 		if (openMode == DBConstants.CREATE) {
+			saveDataOrganization(); // save default dataOrg
 			return; // optional program architecture is set after initialization is complete
 		}
 
@@ -369,9 +436,18 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
 		if (variableStorageMgr != null) {
 			variableStorageMgr.setProgramArchitecture(getProgramArchitecture());
 		}
-
 	}
 
+	@Override
+	protected void handleDataOrganizationChange(int openMode, TaskMonitor monitor)
+			throws LanguageVersionException, CancelledException, IOException {
+		if (openMode == DBConstants.READ_ONLY) {
+			warning = ArchiveWarning.DATA_ORG_CHANGED;
+		}
+		super.handleDataOrganizationChange(openMode, monitor);
+	}
+
+
 	/**
 	 * Get the program architecture information which has been associated with this 
 	 * datatype manager.  If {@link #getProgramArchitecture()} returns null this method

GhidraDocs/languages/versioning.html

@@ -48,7 +48,15 @@ re-disassembly of all instructions.</p>
 a user to make certain transitions between similar language
 implementations.&nbsp; Such transitions are generally facilitated via a
 default translator, although certain limitations are imposed based upon
-address space sizes and register mappings.<br>
+address space sizes and register mappings.</p>
+<p>Any changes made to a Data Organization could impact the packing of 
+components within a structure or union.  While such changes should be avoided
+due to the possible fallout, any such change to a 
+<span style="font-style: italic;">*.cspec</span> should be made
+in conjuction with a version change to all affected languages within
+the relavent <span style="font-style: italic;">*.ldefs</span> files.  The
+resulting program upgrade will allow affected data types to be updated.
+<br>
 </p>
 <h2><a name="versioning"></a>Language Versioning</h2>
 <p>A language's version is specified as a<span