GP-1068 revised null state for primitive and fixed-length fields for

sparse records
This commit is contained in:
ghidra1 2021-06-25 14:07:21 -04:00
parent 0cba2319fb
commit 7a43d3bdf1
15 changed files with 477 additions and 179 deletions

View File

@ -234,11 +234,15 @@ public class BinaryField extends Field {
@Override
public String toString() {
String classname = getClass().getSimpleName();
String nullState = "";
if (isNull()) {
nullState = "(NULL)";
}
byte[] d = getBinaryData();
if (d == null) {
return classname + ": null";
return classname + nullState + ": null";
}
return classname = "[" + d.length + "] = 0x" + getValueAsString(d);
return classname + nullState + ": [" + d.length + "] = 0x" + getValueAsString(d);
}
@Override

View File

@ -23,7 +23,7 @@ import db.buffers.DataBuffer;
* <code>BooleanField</code> provides a wrapper for boolean data which is read or
* written to a Record.
*/
public final class BooleanField extends Field {
public final class BooleanField extends PrimitiveField {
/**
* Minimum boolean field value (FALSE)
@ -56,14 +56,9 @@ public final class BooleanField extends Field {
this(b, false);
}
@Override
boolean isNull() {
return value == 0;
}
@Override
void setNull() {
checkImmutable();
super.setNull();
value = 0;
}
@ -84,7 +79,7 @@ public final class BooleanField extends Field {
@Override
public void setBooleanValue(boolean b) {
checkImmutable();
updatingPrimitiveValue();
this.value = b ? (byte) 1 : (byte) 0;
}
@ -100,7 +95,7 @@ public final class BooleanField extends Field {
@Override
int read(Buffer buf, int offset) throws IOException {
checkImmutable();
updatingPrimitiveValue();
value = buf.getByte(offset);
return offset + 1;
}
@ -115,11 +110,6 @@ public final class BooleanField extends Field {
return BOOLEAN_TYPE;
}
@Override
public String toString() {
return "BooleanField: " + Boolean.toString(getBooleanValue());
}
@Override
public String getValueAsString() {
return Boolean.toString(getBooleanValue());
@ -127,11 +117,10 @@ public final class BooleanField extends Field {
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof BooleanField)) {
if (!(obj instanceof BooleanField)) {
return false;
}
BooleanField otherField = (BooleanField) obj;
return otherField.value == value;
return ((BooleanField) obj).value == value;
}
@Override
@ -180,10 +169,10 @@ public final class BooleanField extends Field {
@Override
public void setBinaryData(byte[] bytes) {
checkImmutable();
if (bytes.length != 1) {
throw new IllegalFieldAccessException();
}
updatingPrimitiveValue();
value = bytes[0];
}

View File

@ -23,7 +23,7 @@ import db.buffers.DataBuffer;
* <code>ByteField</code> provides a wrapper for single signed byte data
* which is read or written to a Record.
*/
public final class ByteField extends Field {
public final class ByteField extends PrimitiveField {
/**
* Minimum byte field value
@ -71,14 +71,9 @@ public final class ByteField extends Field {
value = b;
}
@Override
boolean isNull() {
return value == 0;
}
@Override
void setNull() {
checkImmutable();
super.setNull();
value = 0;
}
@ -89,7 +84,7 @@ public final class ByteField extends Field {
@Override
public void setByteValue(byte value) {
checkImmutable();
updatingPrimitiveValue();
this.value = value;
}
@ -105,7 +100,7 @@ public final class ByteField extends Field {
@Override
int read(Buffer buf, int offset) throws IOException {
checkImmutable();
updatingPrimitiveValue();
value = buf.getByte(offset);
return offset + 1;
}
@ -120,11 +115,6 @@ public final class ByteField extends Field {
return BYTE_TYPE;
}
@Override
public String toString() {
return "Byte: " + Byte.toString(value);
}
@Override
public String getValueAsString() {
return "0x" + Integer.toHexString(value & 0xff);
@ -132,7 +122,7 @@ public final class ByteField extends Field {
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof ByteField)) {
if (!(obj instanceof ByteField)) {
return false;
}
return ((ByteField) obj).value == value;
@ -189,10 +179,10 @@ public final class ByteField extends Field {
@Override
public void setBinaryData(byte[] bytes) {
checkImmutable();
if (bytes.length != 1) {
throw new IllegalFieldAccessException();
}
updatingPrimitiveValue();
value = bytes[0];
}

View File

@ -52,6 +52,10 @@ public class DBRecord implements Comparable<DBRecord> {
for (int colIndex = 0; colIndex < schemaFields.length; colIndex++) {
try {
fieldValues[colIndex] = schemaFields[colIndex].newField();
if (schema.isSparseColumn(colIndex)) {
// sparse column default to null state/value
fieldValues[colIndex].setNull();
}
}
catch (Exception e) {
throw new AssertException(e);
@ -104,7 +108,7 @@ public class DBRecord implements Comparable<DBRecord> {
/**
* Determine if this record's schema is the same as another record's
* schema. This check factors column count and column field types only.
* @param otherRec
* @param otherRec another record
* @return true if records schemas are the same
*/
public boolean hasSameSchema(DBRecord otherRec) {
@ -123,17 +127,21 @@ public class DBRecord implements Comparable<DBRecord> {
/**
* Determine if this record's schema is compatible with the specified schema.
* This check factors column count and column field types only.
* @param schema other schema
* Index and sparse column checks are not performed.
* @param otherSchema other schema
* @return true if records schemas are the same
*/
public boolean hasSameSchema(Schema schema) {
if (fieldValues.length != schema.getFieldCount()) {
public boolean hasSameSchema(Schema otherSchema) {
if (otherSchema == this.schema) {
return true;
}
if (fieldValues.length != otherSchema.getFieldCount()) {
return false;
}
if (!key.isSameType(schema.getKeyFieldType())) {
if (!key.isSameType(otherSchema.getKeyFieldType())) {
return false;
}
Field[] otherFields = schema.getFields();
Field[] otherFields = otherSchema.getFields();
for (int i = 0; i < fieldValues.length; i++) {
if (!fieldValues[i].isSameType(otherFields[i])) {
return false;
@ -152,8 +160,8 @@ public class DBRecord implements Comparable<DBRecord> {
/**
* Get a copy of the specified field value.
* @param columnIndex
* @return Field
* @param columnIndex field index
* @return Field field value
*/
public Field getFieldValue(int columnIndex) {
Field f = fieldValues[columnIndex];
@ -163,7 +171,7 @@ public class DBRecord implements Comparable<DBRecord> {
/**
* Set the field value for the specified field.
* @param colIndex field index
* @param value field value
* @param value field value (null permitted for sparse column only)
*/
public void setField(int colIndex, Field value) {
if (fieldValues[colIndex].getFieldType() != value.getFieldType()) {
@ -176,7 +184,7 @@ public class DBRecord implements Comparable<DBRecord> {
/**
* Get the specified field. The object returned must not be
* modified.
* @param columnIndex
* @param columnIndex field index
* @return Field
*/
Field getField(int columnIndex) {
@ -195,8 +203,8 @@ public class DBRecord implements Comparable<DBRecord> {
/**
* Determine if the specified field equals the field associated with the
* specified columnIndex.
* @param columnIndex
* @param field
* @param columnIndex field index
* @param field field value to compare with
* @return true if the fields are equal, else false.
*/
public boolean fieldEquals(int columnIndex, Field field) {

View File

@ -20,9 +20,16 @@ import java.io.IOException;
import db.buffers.DataBuffer;
/**
* <code>Field</code> is an abstract data wrapper for use with Records.
* <p><code>Field</code> is an abstract data wrapper for use with Records.
* Note that when comparing two Field instances both must be of the same
* class.
* class.</p>
*
* <p>Fields may take on a null state. In the case of {@link FixedField}
* and {@link PrimitiveField} this state is distinct from value and only
* applies when used for a sparse column within a {@link SparseRecord}.
* In this sparse column situation the {@link SparseRecord#setField(int, Field)}
* method may be passed a null Field argument. Sparse columns with a
* null value/state will not be indexed within a {@link Table}.
*
* <p>Stored Schema Field Type Encoding:</p>
*
@ -376,6 +383,13 @@ public abstract class Field implements Comparable<Field> {
*/
abstract int length();
/**
* Determine if the specified Object is another Field which has the same
* type and value as this Field. When comparing a {@link PrimitiveField},
* with a null state, a value of zero (0) is used.
* @param obj another object
* @return true if this field equals obj
*/
@Override
public abstract boolean equals(Object obj);
@ -407,14 +421,16 @@ public abstract class Field implements Comparable<Field> {
abstract Field getMaxValue();
/**
* Determine if the field value is null (or zero for
* fixed-length fields)
* @return true if null/zero else false
* Determine if the field has been set to a null-state or value.
* @return true if field has been set to a null state or value, else false
*/
abstract boolean isNull();
/**
* Set this field to its null/zero value
* Set this field to its null-state. For variable-length field this will
* generally correspond to a null value, while primitive and fixed-length
* fields will be set to a zero (0) value. This method may only be invoked
* on a sparse column field.
* @throws IllegalFieldAccessException thrown if this field is immutable or is an index field
*/
abstract void setNull();
@ -422,6 +438,8 @@ public abstract class Field implements Comparable<Field> {
/**
* Performs a fast in-place comparison of this field value with another
* field value stored within the specified buffer at the the specified offset.
* NOTE: This method will treat all null primitives as 0 although is not intended
* to support such use.
* @param buffer data buffer
* @param offset field value offset within buffer
* @return comparison value, zero if equal, -1 if this field has a value
@ -430,6 +448,23 @@ public abstract class Field implements Comparable<Field> {
*/
abstract int compareTo(DataBuffer buffer, int offset);
/**
* Compares this Field with another Field for order. Returns a
* negative integer, zero, or a positive integer as this object is less
* than, equal to, or greater than the specified Field.
* <br>
* NOTE: Field objects do not fully comply with the Comparable interface.
* Only the same Field implementations may be compared. In addition, the
* null state is not considered when comparing {@link PrimitiveField}s which have a
* zero (0) value.
* @param otherField another Field which is the same type as this Field
* @return field comparison result (see {@link Comparable#compareTo(Object)}).
* @throws ClassCastException if an attempt to compare dissimilar Fields (e.g.,
* an IntField may not be compared with a ShortField).
*/
@Override
public abstract int compareTo(Field otherField);
/**
* Get the field associated with the specified type value.
* @param fieldType field type index

View File

@ -16,17 +16,25 @@
package db;
/**
* <code>FixedField</code> provides an abstract implementation of a fixed-length
* binary field.
* <code>FixedField</code> provides an abstract implementation of an unsigned fixed-length
* field whose value is specified with a byte-array. This field behaves similar to a
* {@link PrimitiveField} in that a null "state" (see {@link #isNull()}) is supported for
* sparse record column use with a zero (0) value. Unlike a variable-length
* {@link BinaryField} a null "value" (i.e., data byte array) is not permitted.
* <br>
* Implementations may use the internal data byte-array as a lazy storage cache for
* the actual fixed-length value (i.e., invoking {@link #getBinaryData()} may update
* the internal data byte-array if needed).
*/
public abstract class FixedField extends BinaryField {
abstract class FixedField extends BinaryField {
@SuppressWarnings("hiding")
public static final FixedField10 INSTANCE = null;
private boolean isNull = false;
/**
* Construct a fixed-length field
* @param data initial value
* Construct a fixed-length field. A null "state" may only be established
* by invoking the {@link #setNull()} method after construction provided
* the instance is mutable.
* @param data initial storage value (may be null)
* @param immutable true if field value is immutable
*/
FixedField(byte[] data, boolean immutable) {
@ -39,7 +47,25 @@ public abstract class FixedField extends BinaryField {
}
@Override
abstract boolean isNull();
final boolean isNull() {
return isNull;
}
@Override
void setNull() {
checkImmutable();
this.isNull = true;
}
/**
* Invoked prior to setting the field's primitive value this
* method will perform an immutable check and set to a non-null
* state.
*/
final void updatingValue() {
checkImmutable();
this.isNull = false;
}
@Override
void truncate(int length) {

View File

@ -27,21 +27,21 @@ import ghidra.util.BigEndianDataConverter;
*/
public class FixedField10 extends FixedField {
/**
* Zero fixed10 field value
*/
public static final FixedField10 ZERO_VALUE = new FixedField10(0L, (short) 0, true);
/**
* Minimum long field value
*/
public static FixedField10 MIN_VALUE = new FixedField10(0L, (short) 0, true);
public static FixedField10 MIN_VALUE = ZERO_VALUE;
/**
* Maximum long field value
*/
public static FixedField10 MAX_VALUE = new FixedField10(-1L, (short) -1, true);
/**
* Zero fixed10 field value
*/
public static final FixedField10 ZERO_VALUE = new FixedField10(null, true);
/**
* Instance intended for defining a {@link Table} {@link Schema}
*/
@ -65,7 +65,8 @@ public class FixedField10 extends FixedField {
/**
* Construct a 10-byte fixed-length field with an initial value of data.
* @param data initial 10-byte binary value
* @param data initial 10-byte binary value. A null corresponds to zero value
* and does not affect the null-state (see {@link #setNull()} and {@link #isNull()}).
* @throws IllegalArgumentException thrown if data is not 10-bytes in length
*/
public FixedField10(byte[] data) {
@ -74,13 +75,19 @@ public class FixedField10 extends FixedField {
/**
* Construct a 10-byte fixed-length binary field with an initial value of data.
* @param data initial 10-byte binary value
* @param data initial 10-byte binary value. A null corresponds to zero value
* and does not affect the null-state (see {@link #setNull()} and {@link #isNull()}).
* @param immutable true if field value is immutable
* @throws IllegalArgumentException thrown if data is not 10-bytes in length
*/
public FixedField10(byte[] data, boolean immutable) {
super(null, immutable);
setBinaryData(data);
super(data, immutable);
if (data != null) {
if (data.length != 10) {
throw new IllegalArgumentException("Invalid FixedField10 data length");
}
updatePrimitiveValue(data);
}
}
FixedField10(long hi8, short lo2, boolean immutable) {
@ -89,11 +96,6 @@ public class FixedField10 extends FixedField {
this.lo2 = lo2;
}
@Override
boolean isNull() {
return hi8 == 0 && lo2 == 0;
}
@Override
public int compareTo(Field o) {
if (!(o instanceof FixedField10)) {
@ -154,18 +156,27 @@ public class FixedField10 extends FixedField {
}
@Override
public void setBinaryData(byte[] data) {
this.data = data;
if (data == null) {
hi8 = 0;
lo2 = 0;
return;
public void setBinaryData(byte[] d) {
if (d == null || d.length != 10) {
// null value not permitted although null state is (see setNull())
throw new IllegalArgumentException("Invalid FixedField10 data length");
}
if (data.length != 10) {
throw new IllegalArgumentException("Invalid FixedField10 length: " + data.length);
}
hi8 = BigEndianDataConverter.INSTANCE.getLong(data, 0);
lo2 = BigEndianDataConverter.INSTANCE.getShort(data, 8);
updatingValue();
this.data = d;
updatePrimitiveValue(d);
}
void updatePrimitiveValue(byte[] d) {
hi8 = BigEndianDataConverter.INSTANCE.getLong(d, 0);
lo2 = BigEndianDataConverter.INSTANCE.getShort(d, 8);
}
@Override
void setNull() {
super.setNull();
data = null;
hi8 = 0;
lo2 = 0;
}
@Override
@ -184,7 +195,7 @@ public class FixedField10 extends FixedField {
@Override
int read(Buffer buf, int offset) throws IOException {
checkImmutable();
updatingValue();
data = null; // be lazy
hi8 = buf.getLong(offset);
lo2 = buf.getShort(offset + 8);
@ -214,7 +225,7 @@ public class FixedField10 extends FixedField {
if (this == obj) {
return true;
}
if (getClass() != obj.getClass()) {
if (!(obj instanceof FixedField10)) {
return false;
}
FixedField10 other = (FixedField10) obj;

View File

@ -23,7 +23,7 @@ import db.buffers.DataBuffer;
* <code>IntField</code> provides a wrapper for 4-byte signed integer data
* which is read or written to a Record.
*/
public final class IntField extends Field {
public final class IntField extends PrimitiveField {
/**
* Minimum integer field value
@ -71,14 +71,9 @@ public final class IntField extends Field {
value = i;
}
@Override
boolean isNull() {
return value == 0;
}
@Override
void setNull() {
checkImmutable();
super.setNull();
value = 0;
}
@ -89,7 +84,7 @@ public final class IntField extends Field {
@Override
public void setIntValue(int value) {
checkImmutable();
updatingPrimitiveValue();
this.value = value;
}
@ -105,7 +100,7 @@ public final class IntField extends Field {
@Override
int read(Buffer buf, int offset) throws IOException {
checkImmutable();
updatingPrimitiveValue();
value = buf.getInt(offset);
return offset + 4;
}
@ -120,11 +115,6 @@ public final class IntField extends Field {
return INT_TYPE;
}
@Override
public String toString() {
return "IntField: " + Integer.toString(value);
}
@Override
public String getValueAsString() {
return "0x" + Integer.toHexString(value);
@ -132,7 +122,7 @@ public final class IntField extends Field {
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof IntField)) {
if (!(obj instanceof IntField)) {
return false;
}
return ((IntField) obj).value == value;
@ -190,10 +180,10 @@ public final class IntField extends Field {
@Override
public void setBinaryData(byte[] bytes) {
checkImmutable();
if (bytes.length != 4) {
throw new IllegalFieldAccessException();
}
updatingPrimitiveValue();
value = ((bytes[0] & 0xff) << 24) | ((bytes[1] & 0xff) << 16) | ((bytes[2] & 0xff) << 8) |
(bytes[3] & 0xff);
}

View File

@ -23,7 +23,7 @@ import db.buffers.DataBuffer;
* <code>LongField</code> provides a wrapper for 8-byte signed long data
* which is read or written to a Record.
*/
public final class LongField extends Field {
public final class LongField extends PrimitiveField {
/**
* Minimum long field value
@ -71,14 +71,9 @@ public final class LongField extends Field {
value = l;
}
@Override
boolean isNull() {
return value == 0;
}
@Override
void setNull() {
checkImmutable();
super.setNull();
value = 0;
}
@ -89,7 +84,7 @@ public final class LongField extends Field {
@Override
public void setLongValue(long value) {
checkImmutable();
updatingPrimitiveValue();
this.value = value;
}
@ -105,7 +100,7 @@ public final class LongField extends Field {
@Override
int read(Buffer buf, int offset) throws IOException {
checkImmutable();
updatingPrimitiveValue();
value = buf.getLong(offset);
return offset + 8;
}
@ -120,11 +115,6 @@ public final class LongField extends Field {
return LONG_TYPE;
}
@Override
public String toString() {
return "LongField: " + Long.toString(value);
}
@Override
public String getValueAsString() {
return "0x" + Long.toHexString(value);
@ -132,7 +122,7 @@ public final class LongField extends Field {
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof LongField)) {
if (!(obj instanceof LongField)) {
return false;
}
return ((LongField) obj).value == value;
@ -184,10 +174,10 @@ public final class LongField extends Field {
@Override
public void setBinaryData(byte[] bytes) {
checkImmutable();
if (bytes.length != 8) {
throw new IllegalFieldAccessException();
}
updatingPrimitiveValue();
value = (((long) bytes[0] & 0xff) << 56) | (((long) bytes[1] & 0xff) << 48) |
(((long) bytes[2] & 0xff) << 40) | (((long) bytes[3] & 0xff) << 32) |
(((long) bytes[4] & 0xff) << 24) | (((long) bytes[5] & 0xff) << 16) |

View File

@ -0,0 +1,74 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package db;
/**
* <code>PrimitiveField</code> provides a base implementation for
* all primitive value {@link Field}s.
* <br>
* When a {@link PrimitiveField} associated with a {@link SparseRecord}
* has a null state it will have a zero (0) value.
*/
abstract class PrimitiveField extends Field {
private boolean isNull = false;
/**
* Abstract PrimitiveField Constructor for a mutable instance
*/
PrimitiveField() {
super();
}
/**
* Abstract PrimitiveField Constructor
* @param immutable true if field value is immutable
*/
PrimitiveField(boolean immutable) {
super(immutable);
}
@Override
final boolean isNull() {
return isNull;
}
@Override
void setNull() {
checkImmutable();
this.isNull = true;
}
/**
* Invoked prior to setting the field's primitive value this
* method will perform an immutable check and set to a non-null
* state.
*/
final void updatingPrimitiveValue() {
checkImmutable();
this.isNull = false;
}
@Override
public String toString() {
String nullState = "";
if (isNull()) {
nullState = "(NULL)";
}
return getClass().getSimpleName() + nullState + ": " + getValueAsString();
}
}

View File

@ -23,7 +23,7 @@ import db.buffers.DataBuffer;
* <code>ShortField</code> provides a wrapper for 2-byte signed short data
* which is read or written to a Record.
*/
public final class ShortField extends Field {
public final class ShortField extends PrimitiveField {
/**
* Minimum short field value
@ -71,14 +71,9 @@ public final class ShortField extends Field {
value = s;
}
@Override
boolean isNull() {
return value == 0;
}
@Override
void setNull() {
checkImmutable();
super.setNull();
value = 0;
}
@ -89,7 +84,7 @@ public final class ShortField extends Field {
@Override
public void setShortValue(short value) {
checkImmutable();
updatingPrimitiveValue();
this.value = value;
}
@ -105,7 +100,7 @@ public final class ShortField extends Field {
@Override
int read(Buffer buf, int offset) throws IOException {
checkImmutable();
updatingPrimitiveValue();
value = buf.getShort(offset);
return offset + 2;
}
@ -120,11 +115,6 @@ public final class ShortField extends Field {
return SHORT_TYPE;
}
@Override
public String toString() {
return "ShortField: " + Short.toString(value);
}
@Override
public String getValueAsString() {
return "0x" + Integer.toHexString(value & 0xffff);
@ -132,7 +122,7 @@ public final class ShortField extends Field {
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof ShortField)) {
if (!(obj instanceof ShortField)) {
return false;
}
return ((ShortField) obj).value == value;
@ -189,10 +179,10 @@ public final class ShortField extends Field {
@Override
public void setBinaryData(byte[] bytes) {
checkImmutable();
if (bytes.length != 2) {
throw new IllegalFieldAccessException();
}
updatingPrimitiveValue();
value = (short) (((bytes[0] & 0xff) << 8) | (bytes[1] & 0xff));
}

View File

@ -102,6 +102,18 @@ public class SparseRecord extends DBRecord {
return oldSparse != newSparse;
}
@Override
public void setField(int colIndex, Field value) {
if (value == null) {
if (!schema.isSparseColumn(colIndex)) {
throw new IllegalArgumentException("null value supported for sparse column only");
}
value = getField(colIndex).newField();
value.setNull();
}
super.setField(colIndex, value);
}
@Override
public void setLongValue(int colIndex, long value) {
if (changeInSparseStorage(colIndex, value)) {

View File

@ -161,7 +161,7 @@ public final class StringField extends Field {
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof StringField)) {
if (!(obj instanceof StringField)) {
return false;
}
StringField f = (StringField) obj;

View File

@ -32,10 +32,17 @@ public class DBFixedKeySparseIndexedTableTest extends AbstractGenericTest {
private static final int BUFFER_SIZE = 2048;// keep small for chained buffer testing
private static final int CACHE_SIZE = 4 * 1024 * 1024;
private static final int ITER_REC_CNT = 1000;
private static final String table1Name = "TABLE1";
private static final int BOOLEAN_COL = 0; // not indexed
private static final int BYTE_COL = 1; // not indexed
private static final int INT_COL = 2;
private static final int SHORT_COL = 3;
private static final int LONG_COL = 4;
private static final int STR_COL = 5;
private static final int BIN_COL = 6;
private static final int FIXED10_COL = 7;
private File testDir;
private static final String dbName = "test";
@ -125,9 +132,7 @@ public class DBFixedKeySparseIndexedTableTest extends AbstractGenericTest {
assertTrue(!iter.hasNext());
}
@Test
public void testFixedKeyIterator() throws IOException {
private void populateFixedKeySparseRecords() throws IOException {
long txId = dbh.startTransaction();
Table table =
DBTestUtils.createFixedKeyTable(dbh, table1Name, DBTestUtils.ALL_TYPES, true, true);
@ -136,65 +141,239 @@ public class DBFixedKeySparseIndexedTableTest extends AbstractGenericTest {
assertTrue(schema.isSparseColumn(i));
}
// DBRecord r1 = schema.createRecord(FixedField10.ZERO_VALUE);
// System.out.println("Sparse record test columns:");
// for (Field f : r1.getFields()) {
// System.out.println(" " + f.toString());
// }
int cnt = schema.getFieldCount();
for (int i = 0; i < cnt; i++) {
// System.out.println("Write sparse records:");
for (int i = 0; i < cnt + 1; i++) {
Field key = new FixedField10(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 1, (byte) i });
DBRecord r = schema.createRecord(key);
Field f = schema.getField(i);
if (f.isVariableLength()) {
f.setBinaryData(new byte[] { 'X' });
for (Field f : r.getFields()) {
// all fields correspond to a sparse columns and
// should have a null state initially
assertTrue(f.isNull());
}
else {
f = f.getMaxValue();
}
r.setField(i, f);
int nextCol = i + 1;
if (nextCol < cnt) {
f = schema.getField(nextCol);
if (i < cnt) {
Field f = schema.getField(i);
if (f.isVariableLength()) {
f.setBinaryData(new byte[] { 'X' });
}
else {
f = f.getMaxValue();
}
r.setField(i, f);
}
// set min value all fields before i
for (int m = 0; m < i; m++) {
Field f = schema.getField(m);
if (f.isVariableLength()) {
f.setBinaryData(new byte[] { 'x' });
}
else {
f = f.getMinValue();
}
r.setField(nextCol, f);
r.setField(m, f);
}
// // NOTE: sparse columns default to a null state if not explicitly set
// System.out.println("-> " + r.getField(2) + ", " + r.getField(6).toString() + ", " +
// r.getField(7).toString());
table.putRecord(r);
}
assertEquals(cnt + 1, table.getRecordCount());
dbh.endTransaction(txId, true);
saveAsAndReopen(dbName);
}
table = dbh.getTable(table1Name);
assertEquals(cnt, table.getRecordCount());
@Test
public void testFixedKeyIterator() throws IOException {
populateFixedKeySparseRecords();
Table table = dbh.getTable(table1Name);
int cnt = table.getSchema().getFieldCount();
assertEquals(8, cnt); // testing 8 field types as sparse columns in 9 data records
assertEquals(cnt + 1, table.getRecordCount());
// see DBTestUtils for schema column types
// Index does not track null/zero values
assertEquals(0, table.findRecords(IntField.ZERO_VALUE, 2).length);
assertEquals(0, table.findRecords(ShortField.ZERO_VALUE, 3).length);
assertEquals(0, table.findRecords(LongField.ZERO_VALUE, 4).length);
assertEquals(0, table.findRecords(StringField.NULL_VALUE, 5).length);
assertEquals(0, table.findRecords(new BinaryField(), 6).length);
assertEquals(0, table.findRecords(FixedField10.ZERO_VALUE, 7).length);
// System.out.println("Read sparse records:");
int recordIndex = 0;
RecordIterator iterator = table.iterator();
while (iterator.hasNext()) {
DBRecord r = iterator.next();
assertEquals(1, table.findRecords(IntField.MAX_VALUE, 2).length);
assertEquals(1, table.findRecords(ShortField.MAX_VALUE, 3).length);
assertEquals(1, table.findRecords(LongField.MAX_VALUE, 4).length);
assertEquals(1, table.findRecords(new StringField("X"), 5).length);
assertEquals(1, table.findRecords(new BinaryField(new byte[] { 'X' }), 6).length);
assertEquals(1, table.findRecords(FixedField10.MAX_VALUE, 7).length);
Field key =
new FixedField10(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 1, (byte) recordIndex });
assertEquals(key, r.getKeyField());
assertEquals(1, table.findRecords(IntField.MIN_VALUE, 2).length);
assertEquals(1, table.findRecords(ShortField.MIN_VALUE, 3).length);
assertEquals(1, table.findRecords(LongField.MIN_VALUE, 4).length);
assertEquals(1, table.findRecords(new StringField("x"), 5).length);
assertEquals(1, table.findRecords(new BinaryField(new byte[] { 'x' }), 6).length);
assertEquals(0, table.findRecords(FixedField10.MIN_VALUE, 7).length); // same as zero/null
// System.out.println("<- " + r.getField(2) + ", " + r.getField(6).toString() + ", " +
// r.getField(7).toString());
// recordIndex used as walking columnIndex
int columnIndex = recordIndex;
if (columnIndex < cnt) {
Field f = r.getField(columnIndex);
if (f.isVariableLength()) {
Field f2 = f.newField();
f2.setBinaryData(new byte[] { 'X' });
assertEquals(f2, f);
}
else {
assertEquals(f.getMaxValue(), f);
}
}
// set min value all fields before i
for (int m = 0; m < columnIndex; m++) {
Field f = r.getField(m);
if (f.isVariableLength()) {
Field f2 = f.newField();
f2.setBinaryData(new byte[] { 'x' });
assertEquals(f2, f);
}
else {
assertEquals(f.getMinValue(), f);
}
}
for (int n = columnIndex + 1; n < cnt; n++) {
Field f = r.getField(n);
assertTrue(f.isNull());
}
++recordIndex;
}
}
@Test
public void testFixedKeySparseIndex() throws IOException {
populateFixedKeySparseRecords();
Table table = dbh.getTable(table1Name);
int cnt = table.getSchema().getFieldCount();
assertEquals(8, cnt); // testing 8 field types as sparse columns in 9 data records
assertEquals(cnt + 1, table.getRecordCount());
// see DBTestUtils for schema column types
// null state/value not indexed (corresponds to a 0 primitive value)
assertEquals(0, table.findRecords(IntField.ZERO_VALUE, INT_COL).length);
assertEquals(0, table.findRecords(ShortField.ZERO_VALUE, SHORT_COL).length);
assertEquals(0, table.findRecords(LongField.ZERO_VALUE, LONG_COL).length);
assertEquals(0, table.findRecords(StringField.NULL_VALUE, STR_COL).length);
assertEquals(0, table.findRecords(new BinaryField(), BIN_COL).length);
assertEquals(1, table.findRecords(FixedField10.ZERO_VALUE, FIXED10_COL).length); // last record has a FixedField10.ZERO_VALUE
assertEquals(1, table.findRecords(IntField.MAX_VALUE, INT_COL).length);
assertEquals(1, table.findRecords(ShortField.MAX_VALUE, SHORT_COL).length);
assertEquals(1, table.findRecords(LongField.MAX_VALUE, LONG_COL).length);
assertEquals(1, table.findRecords(new StringField("X"), STR_COL).length);
assertEquals(1, table.findRecords(new BinaryField(new byte[] { 'X' }), BIN_COL).length);
assertEquals(1, table.findRecords(FixedField10.MAX_VALUE, FIXED10_COL).length);
assertEquals(6, table.findRecords(IntField.MIN_VALUE, INT_COL).length);
assertEquals(5, table.findRecords(ShortField.MIN_VALUE, SHORT_COL).length);
assertEquals(4, table.findRecords(LongField.MIN_VALUE, LONG_COL).length);
assertEquals(3, table.findRecords(new StringField("x"), STR_COL).length);
assertEquals(2, table.findRecords(new BinaryField(new byte[] { 'x' }), BIN_COL).length);
assertEquals(1, table.findRecords(FixedField10.MIN_VALUE, FIXED10_COL).length); // same as ZERO_VALUE
assertEquals(6, table.getMatchingRecordCount(IntField.MIN_VALUE, INT_COL));
assertEquals(5, table.getMatchingRecordCount(ShortField.MIN_VALUE, SHORT_COL));
assertEquals(4, table.getMatchingRecordCount(LongField.MIN_VALUE, LONG_COL));
assertEquals(3, table.getMatchingRecordCount(new StringField("x"), STR_COL));
assertEquals(2, table.getMatchingRecordCount(new BinaryField(new byte[] { 'x' }), BIN_COL));
assertEquals(1, table.getMatchingRecordCount(FixedField10.MIN_VALUE, FIXED10_COL)); // same as ZERO_VALUE
}
private int count(DBFieldIterator iter) throws IOException {
int count = 0;
while (iter.hasNext()) {
iter.next();
++count;
}
return count;
}
private int count(RecordIterator iter) throws IOException {
int count = 0;
while (iter.hasNext()) {
iter.next();
++count;
}
return count;
}
@Test
public void testFixedKeySparseIndexIterator() throws IOException {
populateFixedKeySparseRecords();
Table table = dbh.getTable(table1Name);
int cnt = table.getSchema().getFieldCount();
assertEquals(8, cnt); // testing 8 field types as sparse columns in 9 data records
assertEquals(cnt + 1, table.getRecordCount());
// see DBTestUtils for schema column types
// null state/value not indexed
assertEquals(7, count(table.indexIterator(INT_COL)));
assertEquals(6, count(table.indexIterator(SHORT_COL)));
assertEquals(5, count(table.indexIterator(LONG_COL)));
assertEquals(4, count(table.indexIterator(STR_COL)));
assertEquals(3, count(table.indexIterator(BIN_COL)));
assertEquals(2, count(table.indexIterator(FIXED10_COL)));
}
@Test
public void testFixedKeySparseIndexFieldIterator() throws IOException {
populateFixedKeySparseRecords();
Table table = dbh.getTable(table1Name);
int cnt = table.getSchema().getFieldCount();
assertEquals(8, cnt); // testing 8 field types as sparse columns in 9 data records
assertEquals(cnt + 1, table.getRecordCount());
// see DBTestUtils for schema column types
// null state/value not indexed - only 2 unique values were used
assertEquals(2, count(table.indexFieldIterator(INT_COL)));
assertEquals(2, count(table.indexFieldIterator(SHORT_COL)));
assertEquals(2, count(table.indexFieldIterator(LONG_COL)));
try {
assertEquals(2, count(table.indexFieldIterator(STR_COL)));
}
catch (UnsupportedOperationException e) {
// expected
}
try {
assertEquals(2, count(table.indexFieldIterator(BIN_COL)));
}
catch (UnsupportedOperationException e) {
// expected
}
assertEquals(2, count(table.indexFieldIterator(FIXED10_COL)));
}
}

View File

@ -134,7 +134,7 @@ public class DBTestUtils {
int indexCnt = 0;
int[] indexedColumns = null;
Schema[] schemas = longKeySchemas;
Schema[] schemas = longKeySchemas.clone();
if (useSparseColumns) {
for (int i = 0; i < schemas.length; i++) {
schemas[i] = createSparseSchema(schemas[i]);
@ -199,7 +199,7 @@ public class DBTestUtils {
int indexCnt = 0;
int[] indexedColumns = null;
Schema[] schemas = fixedKeySchemas;
Schema[] schemas = fixedKeySchemas.clone();
if (useSparseColumns) {
for (int i = 0; i < schemas.length; i++) {
schemas[i] = createSparseSchema(schemas[i]);