Skip to content

Commit 1e3c868

Browse files
suddendustsuddendust
authored
Support for SELECT on nested JSON fields in Flat Collections (#240)
1 parent e0ada8e commit 1e3c868

25 files changed

+1597
-23
lines changed

‎document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;
9090
import org.hypertrace.core.documentstore.expression.impl.FunctionExpression;
9191
import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression;
92+
import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression;
9293
import org.hypertrace.core.documentstore.expression.impl.KeyExpression;
9394
import org.hypertrace.core.documentstore.expression.impl.LogicalExpression;
9495
import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
@@ -3848,6 +3849,138 @@ void testFlatPostgresCollectionBooleanArrayFilter(String dataStoreName) throws I
38483849
assertDocsAndSizeEqualWithoutOrder(
38493850
dataStoreName, resultIterator, "query/flat_boolean_array_filter_response.json", 2);
38503851
}
3852+
3853+
/**
3854+
* Tests selection of JSONB nested fields using JsonIdentifierExpression on flat collection.
3855+
* Validates selecting simple nested fields, deeply nested fields, and JSONB arrays without any
3856+
* filters.
3857+
*/
3858+
@ParameterizedTest
3859+
@ArgumentsSource(PostgresProvider.class)
3860+
void testFlatCollectionNestedJsonSelections(String dataStoreName) throws IOException {
3861+
Datastore datastore = datastoreMap.get(dataStoreName);
3862+
Collection flatCollection =
3863+
datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT);
3864+
3865+
// Test 1: Select nested STRING field from JSONB column (props.brand)
3866+
Query brandSelectionQuery =
3867+
Query.builder().addSelection(JsonIdentifierExpression.of("props", "brand")).build();
3868+
3869+
Iterator<Document> brandIterator = flatCollection.find(brandSelectionQuery);
3870+
assertDocsAndSizeEqualWithoutOrder(
3871+
dataStoreName, brandIterator, "query/flat_jsonb_brand_selection_response.json", 10);
3872+
3873+
// Test 2: Select deeply nested STRING field from JSONB column (props.seller.address.city)
3874+
Query citySelectionQuery =
3875+
Query.builder()
3876+
.addSelection(JsonIdentifierExpression.of("props", "seller", "address", "city"))
3877+
.build();
3878+
3879+
Iterator<Document> cityIterator = flatCollection.find(citySelectionQuery);
3880+
assertDocsAndSizeEqualWithoutOrder(
3881+
dataStoreName, cityIterator, "query/flat_jsonb_city_selection_response.json", 10);
3882+
3883+
// Test 3: Select STRING_ARRAY field from JSONB column (props.colors)
3884+
Query colorsSelectionQuery =
3885+
Query.builder().addSelection(JsonIdentifierExpression.of("props", "colors")).build();
3886+
3887+
Iterator<Document> colorsIterator = flatCollection.find(colorsSelectionQuery);
3888+
assertDocsAndSizeEqualWithoutOrder(
3889+
dataStoreName, colorsIterator, "query/flat_jsonb_colors_selection_response.json", 10);
3890+
}
3891+
3892+
@ParameterizedTest
3893+
@ArgumentsSource(PostgresProvider.class)
3894+
void testFlatVsNestedCollectionNestedFieldSelections(String dataStoreName) throws IOException {
3895+
Datastore datastore = datastoreMap.get(dataStoreName);
3896+
3897+
Collection nestedCollection = datastore.getCollection(COLLECTION_NAME);
3898+
Collection flatCollection =
3899+
datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT);
3900+
3901+
// Test 1: Select nested field - props.brand
3902+
// Nested collection uses dot notation
3903+
Query nestedBrandQuery =
3904+
Query.builder()
3905+
.addSelection(IdentifierExpression.of("item"))
3906+
.addSelection(IdentifierExpression.of("props.brand"), "brand")
3907+
.addSort(IdentifierExpression.of("item"), ASC)
3908+
.build();
3909+
3910+
// Flat collection uses JsonIdentifierExpression
3911+
// Filter to exclude docs 9-10 to match nested collection dataset
3912+
Query flatBrandQuery =
3913+
Query.builder()
3914+
.addSelection(IdentifierExpression.of("item"))
3915+
.addSelection(JsonIdentifierExpression.of("props", "brand"), "brand")
3916+
.addSort(IdentifierExpression.of("item"), ASC)
3917+
.setFilter(
3918+
RelationalExpression.of(
3919+
IdentifierExpression.of("_id"), LTE, ConstantExpression.of(8)))
3920+
.build();
3921+
3922+
// Assert both match the expected response
3923+
Iterator<Document> nestedBrandIterator = nestedCollection.find(nestedBrandQuery);
3924+
assertDocsAndSizeEqual(dataStoreName, nestedBrandIterator, "query/brand_response.json", 8);
3925+
3926+
Iterator<Document> flatBrandIterator = flatCollection.find(flatBrandQuery);
3927+
assertDocsAndSizeEqual(dataStoreName, flatBrandIterator, "query/brand_response.json", 8);
3928+
3929+
// Test 2: Select nested JSON array - props.colors
3930+
Query nestedColorsQuery =
3931+
Query.builder()
3932+
.addSelection(IdentifierExpression.of("item"))
3933+
.addSelection(IdentifierExpression.of("props.colors"), "colors")
3934+
.addSort(IdentifierExpression.of("item"), ASC)
3935+
.build();
3936+
3937+
// Filter to exclude docs 9-10 to match nested collection dataset
3938+
Query flatColorsQuery =
3939+
Query.builder()
3940+
.addSelection(IdentifierExpression.of("item"))
3941+
.addSelection(JsonIdentifierExpression.of("props", "colors"), "colors")
3942+
.addSort(IdentifierExpression.of("item"), ASC)
3943+
.setFilter(
3944+
RelationalExpression.of(
3945+
IdentifierExpression.of("_id"), LTE, ConstantExpression.of(8)))
3946+
.build();
3947+
3948+
// Assert both match the expected response
3949+
Iterator<Document> nestedColorsIterator = nestedCollection.find(nestedColorsQuery);
3950+
assertDocsAndSizeEqual(dataStoreName, nestedColorsIterator, "query/colors_response.json", 8);
3951+
3952+
Iterator<Document> flatColorsIterator = flatCollection.find(flatColorsQuery);
3953+
assertDocsAndSizeEqual(dataStoreName, flatColorsIterator, "query/colors_response.json", 8);
3954+
3955+
// Test 3: Select nested field WITHOUT alias - should preserve full nested structure
3956+
Query nestedBrandNoAliasQuery =
3957+
Query.builder()
3958+
.addSelection(IdentifierExpression.of("item"))
3959+
.addSelection(IdentifierExpression.of("props.brand"))
3960+
.addSort(IdentifierExpression.of("item"), ASC)
3961+
.build();
3962+
3963+
// Filter to exclude docs 9-10 to match nested collection dataset
3964+
Query flatBrandNoAliasQuery =
3965+
Query.builder()
3966+
.addSelection(IdentifierExpression.of("item"))
3967+
.addSelection(JsonIdentifierExpression.of("props", "brand"))
3968+
.addSort(IdentifierExpression.of("item"), ASC)
3969+
.setFilter(
3970+
RelationalExpression.of(
3971+
IdentifierExpression.of("_id"), LTE, ConstantExpression.of(8)))
3972+
.build();
3973+
3974+
// Assert both match the expected response with nested structure
3975+
Iterator<Document> nestedBrandNoAliasIterator =
3976+
nestedCollection.find(nestedBrandNoAliasQuery);
3977+
assertDocsAndSizeEqual(
3978+
dataStoreName, nestedBrandNoAliasIterator, "query/no_alias_response.json", 8);
3979+
3980+
Iterator<Document> flatBrandNoAliasIterator = flatCollection.find(flatBrandNoAliasQuery);
3981+
assertDocsAndSizeEqual(
3982+
dataStoreName, flatBrandNoAliasIterator, "query/no_alias_response.json", 8);
3983+
}
38513984
}
38523985

38533986
@Nested

‎document-store/src/integrationTest/resources/query/brand_response.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[
2+
{
3+
"item": "Comb"
4+
},
5+
{
6+
"item": "Comb"
7+
},
8+
{
9+
"item": "Mirror"
10+
},
11+
{
12+
"item": "Shampoo",
13+
"brand": "Sunsilk"
14+
},
15+
{
16+
"item": "Shampoo"
17+
},
18+
{
19+
"item": "Soap"
20+
},
21+
{
22+
"item": "Soap",
23+
"brand": "Lifebuoy"
24+
},
25+
{
26+
"item": "Soap",
27+
"brand": "Dettol"
28+
}
29+
]

‎document-store/src/integrationTest/resources/query/colors_response.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
[
2+
{
3+
"item": "Comb"
4+
},
5+
{
6+
"colors": [
7+
],
8+
"item": "Comb"
9+
},
10+
{
11+
"item": "Mirror"
12+
},
13+
{
14+
"colors": [
15+
"Black"
16+
],
17+
"item": "Shampoo"
18+
},
19+
{
20+
"item": "Shampoo"
21+
},
22+
{
23+
"item": "Soap"
24+
},
25+
{
26+
"colors": [
27+
"Orange",
28+
"Blue"
29+
],
30+
"item": "Soap"
31+
},
32+
{
33+
"colors": [
34+
"Blue",
35+
"Green"
36+
],
37+
"item": "Soap"
38+
}
39+
]

‎document-store/src/integrationTest/resources/query/flat_brand_response.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
[
2+
{
3+
"item": "Bottle"
4+
},
5+
{
6+
"item": "Comb"
7+
},
8+
{
9+
"item": "Comb"
10+
},
11+
{
12+
"item": "Cup"
13+
},
14+
{
15+
"item": "Mirror"
16+
},
17+
{
18+
"item": "Shampoo"
19+
},
20+
{
21+
"item": "Shampoo",
22+
"brand": "Sunsilk"
23+
},
24+
{
25+
"item": "Soap",
26+
"brand": "Lifebuoy"
27+
},
28+
{
29+
"item": "Soap"
30+
},
31+
{
32+
"item": "Soap",
33+
"brand": "Dettol"
34+
}
35+
]

‎document-store/src/integrationTest/resources/query/flat_colors_response.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
[
2+
{
3+
"item": "Bottle"
4+
},
5+
{
6+
"item": "Comb"
7+
},
8+
{
9+
"colors": [],
10+
"item": "Comb"
11+
},
12+
{
13+
"item": "Cup"
14+
},
15+
{
16+
"item": "Mirror"
17+
},
18+
{
19+
"item": "Shampoo"
20+
},
21+
{
22+
"colors": [
23+
"Black"
24+
],
25+
"item": "Shampoo"
26+
},
27+
{
28+
"colors": [
29+
"Orange",
30+
"Blue"
31+
],
32+
"item": "Soap"
33+
},
34+
{
35+
"item": "Soap"
36+
},
37+
{
38+
"colors": [
39+
"Blue",
40+
"Green"
41+
],
42+
"item": "Soap"
43+
}
44+
]

‎document-store/src/integrationTest/resources/query/flat_jsonb_add_response.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[
2+
{"item": "Soap", "rating_plus_discount": 14.75},
3+
{"item": "Mirror"},
4+
{"item": "Shampoo", "rating_plus_discount": 9.3},
5+
{"item": "Shampoo"},
6+
{"item": "Soap", "rating_plus_discount": -4.55},
7+
{"item": "Comb"},
8+
{"item": "Comb", "rating_plus_discount": 3.5},
9+
{"item": "Soap"}
10+
]

‎document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[
2+
{
3+
"props": {
4+
"brand": "Dettol"
5+
}
6+
},
7+
{},
8+
{
9+
"props": {
10+
"brand": "Sunsilk"
11+
}
12+
},
13+
{},
14+
{
15+
"props": {
16+
"brand": "Lifebuoy"
17+
}
18+
},
19+
{},
20+
{},
21+
{},
22+
{},
23+
{}
24+
]

‎document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
[
2+
{
3+
"props": {
4+
"seller": {
5+
"address": {
6+
"city": "Mumbai"
7+
}
8+
}
9+
}
10+
},
11+
{
12+
},
13+
{
14+
"props": {
15+
"seller": {
16+
"address": {
17+
"city": "Mumbai"
18+
}
19+
}
20+
}
21+
},
22+
{
23+
},
24+
{
25+
"props": {
26+
"seller": {
27+
"address": {
28+
"city": "Kolkata"
29+
}
30+
}
31+
}
32+
},
33+
{
34+
},
35+
{
36+
"props": {
37+
"seller": {
38+
"address": {
39+
"city": "Kolkata"
40+
}
41+
}
42+
}
43+
},
44+
{},
45+
{},
46+
{}
47+
]

0 commit comments

Comments
 (0)