Skip to content

Commit 6939f82

Browse files
github-actions[bot]github-actions[bot]
committed
[Enhancement] Unify all PPL functions in BuiltinFunctionName (#1062)
* [Enhancement] Unify all PPL functions in BuiltinFunctionName Signed-off-by: Heng Qian <[email protected]> * Fix UT Signed-off-by: Heng Qian <[email protected]> * Tiny change in doc Signed-off-by: Heng Qian <[email protected]> --------- Signed-off-by: Heng Qian <[email protected]> (cherry picked from commit 7c87378) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent 51181fc commit 6939f82

File tree

10 files changed

+198
-129
lines changed

10 files changed

+198
-129
lines changed

‎integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLGeoipITSuite.scala

+4-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package org.opensearch.flint.spark.ppl
77

88
import java.util
99

10+
import org.opensearch.sql.expression.function.BuiltinFunctionName.IP_TO_INT
11+
import org.opensearch.sql.expression.function.BuiltinFunctionName.IS_IPV4
1012
import org.opensearch.sql.expression.function.SerializableUdf.visit
1113
import org.opensearch.sql.ppl.utils.DataTypeTransformer.seq
1214

@@ -57,8 +59,8 @@ class FlintSparkPPLGeoipITSuite
5759
ipAddress: UnresolvedAttribute,
5860
left: LogicalPlan,
5961
right: LogicalPlan): LogicalPlan = {
60-
val is_ipv4 = visit("is_ipv4", util.List.of[Expression](ipAddress))
61-
val ip_to_int = visit("ip_to_int", util.List.of[Expression](ipAddress))
62+
val is_ipv4 = visit(IS_IPV4, util.List.of[Expression](ipAddress))
63+
val ip_to_int = visit(IP_TO_INT, util.List.of[Expression](ipAddress))
6264

6365
val t1 = SubqueryAlias("t1", left)
6466
val t2 = SubqueryAlias("t2", right)

‎integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLJsonFunctionITSuite.scala

+27-23
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ package org.opensearch.flint.spark.ppl
77

88
import java.util
99

10+
import org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_APPEND
11+
import org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_DELETE
12+
import org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTEND
13+
import org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_SET
1014
import org.opensearch.sql.expression.function.SerializableUdf.visit
1115

1216
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
@@ -408,7 +412,7 @@ class FlintSparkPPLJsonFunctionITSuite
408412
val jsonObjExp =
409413
Literal("{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}")
410414
val jsonFunc =
411-
Alias(visit("json_delete", util.List.of(jsonObjExp, keysExpression)), "result")()
415+
Alias(visit(JSON_DELETE, util.List.of(jsonObjExp, keysExpression)), "result")()
412416
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
413417
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
414418
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -429,7 +433,7 @@ class FlintSparkPPLJsonFunctionITSuite
429433
val jsonObjExp =
430434
Literal("{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}")
431435
val jsonFunc =
432-
Alias(visit("json_delete", util.List.of(jsonObjExp, keysExpression)), "result")()
436+
Alias(visit(JSON_DELETE, util.List.of(jsonObjExp, keysExpression)), "result")()
433437
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
434438
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
435439
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -450,7 +454,7 @@ class FlintSparkPPLJsonFunctionITSuite
450454
val jsonObjExp =
451455
Literal("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}")
452456
val jsonFunc =
453-
Alias(visit("json_delete", util.List.of(jsonObjExp, keysExpression)), "result")()
457+
Alias(visit(JSON_DELETE, util.List.of(jsonObjExp, keysExpression)), "result")()
454458
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
455459
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
456460
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -475,7 +479,7 @@ class FlintSparkPPLJsonFunctionITSuite
475479
Literal(
476480
"{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
477481
val jsonFunc =
478-
Alias(visit("json_delete", util.List.of(jsonObjExp, keysExpression)), "result")()
482+
Alias(visit(JSON_DELETE, util.List.of(jsonObjExp, keysExpression)), "result")()
479483
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
480484
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
481485
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -500,7 +504,7 @@ class FlintSparkPPLJsonFunctionITSuite
500504
Literal(
501505
"{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
502506
val jsonFunc =
503-
Alias(visit("json_delete", util.List.of(jsonObjExp, keysExpression)), "result")()
507+
Alias(visit(JSON_DELETE, util.List.of(jsonObjExp, keysExpression)), "result")()
504508
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
505509
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
506510
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -525,7 +529,7 @@ class FlintSparkPPLJsonFunctionITSuite
525529
val jsonObjExp =
526530
Literal("{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}")
527531
val jsonFunc =
528-
Alias(visit("json_set", util.List.of(jsonObjExp, keysExpression)), "result")()
532+
Alias(visit(JSON_SET, util.List.of(jsonObjExp, keysExpression)), "result")()
529533
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
530534
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
531535
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -552,7 +556,7 @@ class FlintSparkPPLJsonFunctionITSuite
552556
val jsonObjExp =
553557
Literal("{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}")
554558
val jsonFunc =
555-
Alias(visit("json_set", util.List.of(jsonObjExp, keysExpression)), "result")()
559+
Alias(visit(JSON_SET, util.List.of(jsonObjExp, keysExpression)), "result")()
556560
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
557561
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
558562
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -573,7 +577,7 @@ class FlintSparkPPLJsonFunctionITSuite
573577
val jsonObjExp =
574578
Literal("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}")
575579
val jsonFunc =
576-
Alias(visit("json_set", util.List.of(jsonObjExp, keysExpression)), "result")()
580+
Alias(visit(JSON_SET, util.List.of(jsonObjExp, keysExpression)), "result")()
577581
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
578582
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
579583
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -598,7 +602,7 @@ class FlintSparkPPLJsonFunctionITSuite
598602
Literal(
599603
"{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
600604
val jsonFunc =
601-
Alias(visit("json_set", util.List.of(jsonObjExp, keysExpression)), "result")()
605+
Alias(visit(JSON_SET, util.List.of(jsonObjExp, keysExpression)), "result")()
602606
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
603607
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
604608
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -625,7 +629,7 @@ class FlintSparkPPLJsonFunctionITSuite
625629
Literal(
626630
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
627631
val jsonFunc =
628-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
632+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
629633
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
630634
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
631635
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -653,7 +657,7 @@ class FlintSparkPPLJsonFunctionITSuite
653657
Literal(
654658
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
655659
val jsonFunc =
656-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
660+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
657661
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
658662
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
659663
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -681,7 +685,7 @@ class FlintSparkPPLJsonFunctionITSuite
681685
Literal(
682686
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
683687
val jsonFunc =
684-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
688+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
685689
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
686690
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
687691
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -709,7 +713,7 @@ class FlintSparkPPLJsonFunctionITSuite
709713
Literal(
710714
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
711715
val jsonFunc =
712-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
716+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
713717
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
714718
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
715719
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -737,7 +741,7 @@ class FlintSparkPPLJsonFunctionITSuite
737741
Literal(
738742
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
739743
val jsonFunc =
740-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
744+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
741745
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
742746
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
743747
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -765,7 +769,7 @@ class FlintSparkPPLJsonFunctionITSuite
765769
Literal(
766770
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
767771
val jsonFunc =
768-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
772+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
769773
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
770774
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
771775
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -797,7 +801,7 @@ class FlintSparkPPLJsonFunctionITSuite
797801
Literal(
798802
"{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")
799803
val jsonFunc =
800-
Alias(visit("json_append", util.List.of(jsonObjExp, keysExpression)), "result")()
804+
Alias(visit(JSON_APPEND, util.List.of(jsonObjExp, keysExpression)), "result")()
801805
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
802806
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
803807
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -824,7 +828,7 @@ class FlintSparkPPLJsonFunctionITSuite
824828
Literal(
825829
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
826830
val jsonFunc =
827-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
831+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
828832
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
829833
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
830834
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -849,7 +853,7 @@ class FlintSparkPPLJsonFunctionITSuite
849853
Literal(
850854
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
851855
val jsonFunc =
852-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
856+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
853857
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
854858
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
855859
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -877,7 +881,7 @@ class FlintSparkPPLJsonFunctionITSuite
877881
Literal(
878882
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
879883
val jsonFunc =
880-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
884+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
881885
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
882886
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
883887
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -905,7 +909,7 @@ class FlintSparkPPLJsonFunctionITSuite
905909
Literal(
906910
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
907911
val jsonFunc =
908-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
912+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
909913
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
910914
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
911915
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -933,7 +937,7 @@ class FlintSparkPPLJsonFunctionITSuite
933937
Literal(
934938
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
935939
val jsonFunc =
936-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
940+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
937941
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
938942
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
939943
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -961,7 +965,7 @@ class FlintSparkPPLJsonFunctionITSuite
961965
Literal(
962966
"{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}")
963967
val jsonFunc =
964-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
968+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
965969
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
966970
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
967971
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)
@@ -993,7 +997,7 @@ class FlintSparkPPLJsonFunctionITSuite
993997
Literal(
994998
"{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")
995999
val jsonFunc =
996-
Alias(visit("json_extend", util.List.of(jsonObjExp, keysExpression)), "result")()
1000+
Alias(visit(JSON_EXTEND, util.List.of(jsonObjExp, keysExpression)), "result")()
9971001
val eval = Project(Seq(UnresolvedStar(None), jsonFunc), table)
9981002
val limit = GlobalLimit(Literal(1), LocalLimit(Literal(1), eval))
9991003
val expectedPlan = Project(Seq(UnresolvedAttribute("result")), limit)

‎ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,10 @@ public enum BuiltinFunctionName {
223223
JSON_EXTRACT(FunctionName.of("json_extract")),
224224
JSON_KEYS(FunctionName.of("json_keys")),
225225
JSON_VALID(FunctionName.of("json_valid")),
226+
JSON_DELETE(FunctionName.of("json_delete")),
227+
JSON_SET(FunctionName.of("json_set")),
228+
JSON_APPEND(FunctionName.of("json_append")),
229+
JSON_EXTEND(FunctionName.of("json_extend")),
226230
// JSON_ARRAY_ALL_MATCH(FunctionName.of("json_array_all_match")),
227231
// JSON_ARRAY_ANY_MATCH(FunctionName.of("json_array_any_match")),
228232
// JSON_ARRAY_FILTER(FunctionName.of("json_array_filter")),
@@ -293,7 +297,13 @@ public enum BuiltinFunctionName {
293297
MULTIMATCHQUERY(FunctionName.of("multimatchquery")),
294298
WILDCARDQUERY(FunctionName.of("wildcardquery")),
295299
WILDCARD_QUERY(FunctionName.of("wildcard_query")),
296-
COALESCE(FunctionName.of("coalesce"));
300+
COALESCE(FunctionName.of("coalesce")),
301+
302+
/** IP Relevance Function. */
303+
CIDR(FunctionName.of("cidr")),
304+
IS_IPV4(FunctionName.of("is_ipv4")),
305+
IP_TO_INT(FunctionName.of("ip_to_int")),
306+
;
297307

298308
private FunctionName name;
299309

0 commit comments

Comments
 (0)