Skip to content

Commit 6f4ace9

Browse files
feeblefakiejnmt
feeblefakie
and
jnmt
authored
Backport to branch(3.10) : Fix to handle FLOAT and BLOB data types in PutToMutable function (#311)
Co-authored-by: Jun Nemoto <[email protected]>
1 parent 3c656ae commit 6f4ace9

File tree

4 files changed

+178
-47
lines changed

4 files changed

+178
-47
lines changed

‎generic-contracts/scripts/objects-table-schema.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,14 @@
1111
"object_id": "TEXT",
1212
"version": "TEXT",
1313
"status": "INT",
14-
"registered_at": "BIGINT"
14+
"registered_at": "BIGINT",
15+
"column_boolean": "BOOLEAN",
16+
"column_bigint": "BIGINT",
17+
"column_float": "FLOAT",
18+
"column_double": "DOUBLE",
19+
"column_text": "TEXT",
20+
"column_blob": "BLOB"
1521
},
1622
"compaction-strategy": "LCS"
1723
}
1824
}
19-

‎generic-contracts/src/integration-test/java/com/scalar/dl/genericcontracts/GenericContractEndToEndTest.java‎

Lines changed: 151 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,16 @@
6666
import com.scalar.db.common.error.CoreError;
6767
import com.scalar.db.config.DatabaseConfig;
6868
import com.scalar.db.exception.storage.ExecutionException;
69+
import com.scalar.db.io.BigIntColumn;
70+
import com.scalar.db.io.BlobColumn;
71+
import com.scalar.db.io.BooleanColumn;
72+
import com.scalar.db.io.Column;
73+
import com.scalar.db.io.DataType;
74+
import com.scalar.db.io.DoubleColumn;
75+
import com.scalar.db.io.FloatColumn;
76+
import com.scalar.db.io.IntColumn;
6977
import com.scalar.db.io.Key;
78+
import com.scalar.db.io.TextColumn;
7079
import com.scalar.db.schemaloader.SchemaLoader;
7180
import com.scalar.db.schemaloader.SchemaLoaderException;
7281
import com.scalar.db.service.StorageFactory;
@@ -115,9 +124,6 @@ public class GenericContractEndToEndTest {
115124
private static final String ASSET_ID = "id";
116125
private static final String ASSET_AGE = "age";
117126
private static final String ASSET_OUTPUT = "output";
118-
private static final String DATA_TYPE_INT = "INT";
119-
private static final String DATA_TYPE_BIGINT = "BIGINT";
120-
private static final String DATA_TYPE_TEXT = "TEXT";
121127

122128
private static final String JDBC_TRANSACTION_MANAGER = "jdbc";
123129
private static final String PROP_STORAGE = "scalardb.storage";
@@ -241,6 +247,17 @@ public class GenericContractEndToEndTest {
241247
private static final String SOME_COLUMN_NAME_2 = "version";
242248
private static final String SOME_COLUMN_NAME_3 = "status";
243249
private static final String SOME_COLUMN_NAME_4 = "registered_at";
250+
private static final String SOME_COLUMN_NAME_BOOLEAN = "column_boolean";
251+
private static final String SOME_COLUMN_NAME_BIGINT = "column_bigint";
252+
private static final String SOME_COLUMN_NAME_FLOAT = "column_float";
253+
private static final String SOME_COLUMN_NAME_DOUBLE = "column_double";
254+
private static final String SOME_COLUMN_NAME_TEXT = "column_text";
255+
private static final String SOME_COLUMN_NAME_BLOB = "column_blob";
256+
private static final boolean SOME_BOOLEAN_VALUE = false;
257+
private static final long SOME_BIGINT_VALUE = BigIntColumn.MAX_VALUE;
258+
private static final float SOME_FLOAT_VALUE = Float.MAX_VALUE;
259+
private static final double SOME_DOUBLE_VALUE = Double.MAX_VALUE;
260+
private static final byte[] SOME_BLOB_VALUE = {1, 2, 3, 4, 5};
244261
private static final String SOME_COLLECTION_ID = "set";
245262
private static final ArrayNode SOME_DEFAULT_OBJECT_IDS =
246263
mapper.createArrayNode().add("object1").add("object2").add("object3").add("object4");
@@ -432,41 +449,80 @@ private void prepareCollection() {
432449
prepareCollection(clientService);
433450
}
434451

435-
private JsonNode createColumn(String name, int value) {
452+
private JsonNode createColumn(Column<?> column) {
453+
ObjectNode jsonColumn =
454+
mapper
455+
.createObjectNode()
456+
.put(COLUMN_NAME, column.getName())
457+
.put(DATA_TYPE, column.getDataType().name());
458+
459+
switch (column.getDataType()) {
460+
case BOOLEAN:
461+
jsonColumn.put(VALUE, column.getBooleanValue());
462+
break;
463+
case INT:
464+
jsonColumn.put(VALUE, column.getIntValue());
465+
break;
466+
case BIGINT:
467+
jsonColumn.put(VALUE, column.getBigIntValue());
468+
break;
469+
case FLOAT:
470+
jsonColumn.put(VALUE, column.getFloatValue());
471+
break;
472+
case DOUBLE:
473+
jsonColumn.put(VALUE, column.getDoubleValue());
474+
break;
475+
case TEXT:
476+
jsonColumn.put(VALUE, column.getTextValue());
477+
break;
478+
case BLOB:
479+
jsonColumn.put(VALUE, column.getBlobValueAsBytes());
480+
break;
481+
default:
482+
throw new IllegalArgumentException("Invalid data type: " + column.getDataType());
483+
}
484+
485+
return jsonColumn;
486+
}
487+
488+
private JsonNode createNullColumn(String columnName, DataType dataType) {
436489
return mapper
437490
.createObjectNode()
438-
.put(COLUMN_NAME, name)
439-
.put(VALUE, value)
440-
.put(DATA_TYPE, DATA_TYPE_INT);
491+
.put(COLUMN_NAME, columnName)
492+
.put(DATA_TYPE, dataType.name())
493+
.set(VALUE, null);
441494
}
442495

443-
private JsonNode createColumn(String name, long value) {
496+
private ArrayNode createColumns(int status) {
444497
return mapper
445-
.createObjectNode()
446-
.put(COLUMN_NAME, name)
447-
.put(VALUE, value)
448-
.put(DATA_TYPE, DATA_TYPE_BIGINT);
498+
.createArrayNode()
499+
.add(createColumn(IntColumn.of(SOME_COLUMN_NAME_3, status)))
500+
.add(createColumn(BigIntColumn.of(SOME_COLUMN_NAME_4, SOME_BIGINT_VALUE)))
501+
.add(createColumn(BooleanColumn.of(SOME_COLUMN_NAME_BOOLEAN, SOME_BOOLEAN_VALUE)))
502+
.add(createColumn(BigIntColumn.of(SOME_COLUMN_NAME_BIGINT, SOME_BIGINT_VALUE)))
503+
.add(createColumn(FloatColumn.of(SOME_COLUMN_NAME_FLOAT, SOME_FLOAT_VALUE)))
504+
.add(createColumn(DoubleColumn.of(SOME_COLUMN_NAME_DOUBLE, SOME_DOUBLE_VALUE)))
505+
.add(createColumn(BlobColumn.of(SOME_COLUMN_NAME_BLOB, SOME_BLOB_VALUE)));
449506
}
450507

451-
private JsonNode createColumn(String name, String value) {
508+
private ArrayNode createNullColumns() {
452509
return mapper
453-
.createObjectNode()
454-
.put(COLUMN_NAME, name)
455-
.put(VALUE, value)
456-
.put(DATA_TYPE, DATA_TYPE_TEXT);
510+
.createArrayNode()
511+
.add(createNullColumn(SOME_COLUMN_NAME_3, DataType.INT))
512+
.add(createNullColumn(SOME_COLUMN_NAME_4, DataType.BIGINT))
513+
.add(createNullColumn(SOME_COLUMN_NAME_BOOLEAN, DataType.BOOLEAN))
514+
.add(createNullColumn(SOME_COLUMN_NAME_BIGINT, DataType.BIGINT))
515+
.add(createNullColumn(SOME_COLUMN_NAME_FLOAT, DataType.FLOAT))
516+
.add(createNullColumn(SOME_COLUMN_NAME_DOUBLE, DataType.DOUBLE))
517+
.add(createNullColumn(SOME_COLUMN_NAME_TEXT, DataType.TEXT))
518+
.add(createNullColumn(SOME_COLUMN_NAME_BLOB, DataType.BLOB));
457519
}
458520

459-
private JsonNode createFunctionArguments(
460-
String objectId, String version, int status, long registeredAt) {
521+
private ObjectNode createFunctionArguments(String objectId, String version, ArrayNode columns) {
461522
ArrayNode partitionKey =
462-
mapper.createArrayNode().add(createColumn(SOME_COLUMN_NAME_1, objectId));
523+
mapper.createArrayNode().add(createColumn(TextColumn.of(SOME_COLUMN_NAME_1, objectId)));
463524
ArrayNode clusteringKey =
464-
mapper.createArrayNode().add(createColumn(SOME_COLUMN_NAME_2, version));
465-
ArrayNode columns =
466-
mapper
467-
.createArrayNode()
468-
.add(createColumn(SOME_COLUMN_NAME_3, status))
469-
.add(createColumn(SOME_COLUMN_NAME_4, registeredAt));
525+
mapper.createArrayNode().add(createColumn(TextColumn.of(SOME_COLUMN_NAME_2, version)));
470526

471527
ObjectNode arguments = mapper.createObjectNode();
472528
arguments.put(NAMESPACE, SOME_FUNCTION_NAMESPACE);
@@ -478,6 +534,14 @@ private JsonNode createFunctionArguments(
478534
return arguments;
479535
}
480536

537+
private ObjectNode createFunctionArguments(String objectId, String version, int status) {
538+
return createFunctionArguments(objectId, version, createColumns(status));
539+
}
540+
541+
private ObjectNode createFunctionArgumentsWithNullColumns(String objectId, String version) {
542+
return createFunctionArguments(objectId, version, createNullColumns());
543+
}
544+
481545
private void addObjectsToCollection(
482546
GenericContractClientService clientService, String collectionId, ArrayNode objectIds) {
483547
JsonNode arguments =
@@ -749,9 +813,8 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
749813
.put(OBJECT_ID, SOME_OBJECT_ID)
750814
.put(HASH_VALUE, SOME_HASH_VALUE_1)
751815
.set(METADATA, SOME_METADATA_1);
752-
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0, 1L);
753-
JsonNode functionArguments1 =
754-
createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_1, 1, 1234567890123L);
816+
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0);
817+
JsonNode functionArguments1 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_1, 1);
755818
Scan scan =
756819
Scan.newBuilder()
757820
.namespace(SOME_FUNCTION_NAMESPACE)
@@ -773,11 +836,63 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
773836
assertThat(results.get(0).getText(SOME_COLUMN_NAME_1)).isEqualTo(SOME_OBJECT_ID);
774837
assertThat(results.get(0).getText(SOME_COLUMN_NAME_2)).isEqualTo(SOME_VERSION_ID_0);
775838
assertThat(results.get(0).getInt(SOME_COLUMN_NAME_3)).isEqualTo(0);
776-
assertThat(results.get(0).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(1L);
839+
assertThat(results.get(0).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(SOME_BIGINT_VALUE);
840+
assertThat(results.get(0).getBoolean(SOME_COLUMN_NAME_BOOLEAN)).isEqualTo(SOME_BOOLEAN_VALUE);
841+
assertThat(results.get(0).getBigInt(SOME_COLUMN_NAME_BIGINT)).isEqualTo(SOME_BIGINT_VALUE);
842+
assertThat(results.get(0).getFloat(SOME_COLUMN_NAME_FLOAT)).isEqualTo(SOME_FLOAT_VALUE);
843+
assertThat(results.get(0).getDouble(SOME_COLUMN_NAME_DOUBLE)).isEqualTo(SOME_DOUBLE_VALUE);
844+
assertThat(results.get(0).getBlobAsBytes(SOME_COLUMN_NAME_BLOB)).isEqualTo(SOME_BLOB_VALUE);
777845
assertThat(results.get(1).getText(SOME_COLUMN_NAME_1)).isEqualTo(SOME_OBJECT_ID);
778846
assertThat(results.get(1).getText(SOME_COLUMN_NAME_2)).isEqualTo(SOME_VERSION_ID_1);
779847
assertThat(results.get(1).getInt(SOME_COLUMN_NAME_3)).isEqualTo(1);
780-
assertThat(results.get(1).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(1234567890123L);
848+
assertThat(results.get(1).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(SOME_BIGINT_VALUE);
849+
assertThat(results.get(1).getBoolean(SOME_COLUMN_NAME_BOOLEAN)).isEqualTo(SOME_BOOLEAN_VALUE);
850+
assertThat(results.get(1).getBigInt(SOME_COLUMN_NAME_BIGINT)).isEqualTo(SOME_BIGINT_VALUE);
851+
assertThat(results.get(1).getFloat(SOME_COLUMN_NAME_FLOAT)).isEqualTo(SOME_FLOAT_VALUE);
852+
assertThat(results.get(1).getDouble(SOME_COLUMN_NAME_DOUBLE)).isEqualTo(SOME_DOUBLE_VALUE);
853+
assertThat(results.get(1).getBlobAsBytes(SOME_COLUMN_NAME_BLOB)).isEqualTo(SOME_BLOB_VALUE);
854+
} catch (IOException e) {
855+
throw new RuntimeException(e);
856+
}
857+
}
858+
859+
@Test
860+
public void putObject_FunctionArgumentsWithNullColumnsGiven_ShouldPutRecordToFunctionTable()
861+
throws ExecutionException {
862+
// Arrange
863+
JsonNode contractArguments =
864+
mapper
865+
.createObjectNode()
866+
.put(OBJECT_ID, SOME_OBJECT_ID)
867+
.put(HASH_VALUE, SOME_HASH_VALUE_0)
868+
.set(METADATA, SOME_METADATA_0);
869+
JsonNode functionArguments =
870+
createFunctionArgumentsWithNullColumns(SOME_OBJECT_ID, SOME_VERSION_ID_0);
871+
Scan scan =
872+
Scan.newBuilder()
873+
.namespace(SOME_FUNCTION_NAMESPACE)
874+
.table(SOME_FUNCTION_TABLE)
875+
.partitionKey(Key.ofText(SOME_COLUMN_NAME_1, SOME_OBJECT_ID))
876+
.build();
877+
878+
// Act
879+
clientService.executeContract(
880+
ID_OBJECT_PUT, contractArguments, ID_OBJECT_PUT_MUTABLE, functionArguments);
881+
882+
// Assert
883+
try (Scanner scanner = storage.scan(scan)) {
884+
List<Result> results = scanner.all();
885+
assertThat(results).hasSize(1);
886+
assertThat(results.get(0).getText(SOME_COLUMN_NAME_1)).isEqualTo(SOME_OBJECT_ID);
887+
assertThat(results.get(0).getText(SOME_COLUMN_NAME_2)).isEqualTo(SOME_VERSION_ID_0);
888+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_3)).isTrue();
889+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_4)).isTrue();
890+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_BOOLEAN)).isTrue();
891+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_BIGINT)).isTrue();
892+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_FLOAT)).isTrue();
893+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_DOUBLE)).isTrue();
894+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_TEXT)).isTrue();
895+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_BLOB)).isTrue();
781896
} catch (IOException e) {
782897
throw new RuntimeException(e);
783898
}
@@ -800,7 +915,9 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
800915
.put(TABLE, "foo")
801916
.set(
802917
PARTITION_KEY,
803-
mapper.createArrayNode().add(createColumn(SOME_COLUMN_NAME_1, SOME_OBJECT_ID)));
918+
mapper
919+
.createArrayNode()
920+
.add(createColumn(TextColumn.of(SOME_COLUMN_NAME_1, SOME_OBJECT_ID))));
804921

805922
// Act Assert
806923
assertThatThrownBy(
@@ -832,8 +949,8 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
832949
.put(OBJECT_ID, SOME_OBJECT_ID)
833950
.put(HASH_VALUE, SOME_HASH_VALUE_1)
834951
.set(METADATA, SOME_METADATA_1);
835-
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0, 1L);
836-
JsonNode functionArguments1 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 1, 1L);
952+
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0);
953+
JsonNode functionArguments1 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 1);
837954
Put put =
838955
Put.newBuilder()
839956
.namespace(SOME_FUNCTION_NAMESPACE)

‎generic-contracts/src/main/java/com/scalar/dl/genericcontracts/object/PutToMutableDatabase.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,10 @@ private Column<?> getColumn(JsonNode jsonColumn) {
162162
}
163163

164164
if (dataType.equals(DataType.FLOAT)) {
165-
if (!value.isFloat()) {
165+
// The JSON deserializer does not distinguish between float and double values; all JSON
166+
// numbers with a decimal point are deserialized as double. Therefore, we check for isDouble()
167+
// here even for FLOAT columns.
168+
if (!value.isDouble()) {
166169
throw new ContractContextException(Constants.INVALID_PUT_MUTABLE_FUNCTION_ARGUMENT_FORMAT);
167170
}
168171
return FloatColumn.of(columnName, value.floatValue());
@@ -183,7 +186,9 @@ private Column<?> getColumn(JsonNode jsonColumn) {
183186
}
184187

185188
if (dataType.equals(DataType.BLOB)) {
186-
if (!value.isBinary()) {
189+
// BLOB data is expected as a Base64-encoded string due to JSON limitations. JSON cannot
190+
// represent binary data directly, so BLOBs must be provided as Base64-encoded strings.
191+
if (!value.isTextual()) {
187192
throw new ContractContextException(Constants.INVALID_PUT_MUTABLE_FUNCTION_ARGUMENT_FORMAT);
188193
}
189194
try {

‎generic-contracts/src/test/java/com/scalar/dl/genericcontracts/object/PutToMutableDatabaseTest.java‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ public class PutToMutableDatabaseTest {
4646
private static final String SOME_DOUBLE_COLUMN_NAME = "double_column";
4747
private static final double SOME_DOUBLE_COLUMN_VALUE = 1.23;
4848
private static final String SOME_BLOB_COLUMN_NAME = "blob_column";
49-
private static final byte[] SOME_BLOB_COLUMN_VALUE = {10, 20, 30};
49+
private static final byte[] SOME_BLOB_COLUMN_VALUE = {1, 2, 3, 4, 5};
50+
private static final String SOME_BLOB_COLUMN_TEXT = "AQIDBAU="; // Base64 of {1, 2, 3, 4, 5}
5051
private static final JsonNode SOME_TEXT_COLUMN1 =
5152
mapper
5253
.createObjectNode()
@@ -81,7 +82,10 @@ public class PutToMutableDatabaseTest {
8182
mapper
8283
.createObjectNode()
8384
.put(Constants.COLUMN_NAME, SOME_FLOAT_COLUMN_NAME)
84-
.put(Constants.VALUE, SOME_FLOAT_COLUMN_VALUE)
85+
.put(
86+
Constants.VALUE,
87+
Double.valueOf(
88+
SOME_FLOAT_COLUMN_VALUE)) // float is always converted to double in function args
8589
.put(Constants.DATA_TYPE, "FLOAT");
8690
private static final JsonNode SOME_DOUBLE_COLUMN =
8791
mapper
@@ -93,7 +97,7 @@ public class PutToMutableDatabaseTest {
9397
mapper
9498
.createObjectNode()
9599
.put(Constants.COLUMN_NAME, SOME_BLOB_COLUMN_NAME)
96-
.put(Constants.VALUE, SOME_BLOB_COLUMN_VALUE)
100+
.put(Constants.VALUE, SOME_BLOB_COLUMN_TEXT)
97101
.put(Constants.DATA_TYPE, "BLOB");
98102
private static final JsonNode SOME_NULL_COLUMN =
99103
mapper
@@ -481,9 +485,9 @@ public void invoke_ColumnsWithUnmatchedTypeGiven_ShouldThrowContractContextExcep
481485
mapper
482486
.createObjectNode()
483487
.put(Constants.COLUMN_NAME, SOME_FLOAT_COLUMN_NAME)
484-
.put(Constants.VALUE, SOME_DOUBLE_COLUMN_VALUE)
488+
.put(Constants.VALUE, SOME_INT_COLUMN_VALUE)
485489
.put(Constants.DATA_TYPE, "FLOAT"),
486-
"DOUBLE value with FLOAT data type")
490+
"INT value with FLOAT data type")
487491
.put(
488492
mapper
489493
.createObjectNode()
@@ -495,16 +499,16 @@ public void invoke_ColumnsWithUnmatchedTypeGiven_ShouldThrowContractContextExcep
495499
mapper
496500
.createObjectNode()
497501
.put(Constants.COLUMN_NAME, SOME_TEXT_COLUMN1_NAME)
498-
.put(Constants.VALUE, SOME_BLOB_COLUMN_VALUE)
502+
.put(Constants.VALUE, SOME_INT_COLUMN_VALUE)
499503
.put(Constants.DATA_TYPE, "TEXT"),
500-
"BLOB value with TEXT data type")
504+
"INT value with TEXT data type")
501505
.put(
502506
mapper
503507
.createObjectNode()
504508
.put(Constants.COLUMN_NAME, SOME_BLOB_COLUMN_NAME)
505-
.put(Constants.VALUE, SOME_TEXT_COLUMN1_VALUE)
509+
.put(Constants.VALUE, SOME_BLOB_COLUMN_VALUE)
506510
.put(Constants.DATA_TYPE, "BLOB"),
507-
"TEXT value with BLOB data type");
511+
"BLOB value with BLOB data type");
508512

509513
// Act Assert
510514
builder

0 commit comments

Comments
 (0)