Skip to content

Commit 85b8f16

Browse files
alpass163alpass163
authored
implement the except (distinct | all ) for table model #16760 
1 parent 23be220 commit 85b8f16

File tree

16 files changed

+700
-44
lines changed

16 files changed

+700
-44
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBExceptTableIT.java‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.relational.it.query.recent;
21+
22+
import org.apache.iotdb.it.env.EnvFactory;
23+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
24+
import org.apache.iotdb.itbase.category.TableClusterIT;
25+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
26+
27+
import org.junit.AfterClass;
28+
import org.junit.BeforeClass;
29+
import org.junit.Test;
30+
import org.junit.experimental.categories.Category;
31+
import org.junit.runner.RunWith;
32+
33+
import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
34+
import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
35+
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
36+
37+
@RunWith(IoTDBTestRunner.class)
38+
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
39+
public class IoTDBExceptTableIT {
40+
protected static final String DATABASE_NAME = "test";
41+
protected static final String[] createSqls =
42+
new String[] {
43+
"CREATE DATABASE " + DATABASE_NAME,
44+
"USE " + DATABASE_NAME,
45+
// table1: ('d1', 1, 1) * 3, ('d1', 2, 2) * 1
46+
"create table table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD)",
47+
"insert into table1 values (1, 'd1', 1, 1)",
48+
"insert into table1 values (2, 'd1', 1, 1)",
49+
"insert into table1 values (3, 'd1', 1, 1)",
50+
"insert into table1 values (4, 'd1', 2, 2)",
51+
// table2: ('d1', 1, 1.0) * 1, ('d1', 3, 3.0) * 1
52+
"create table table2(device STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD)",
53+
"insert into table2 values (1, 'd1', 1, 1.0)",
54+
"insert into table2 values (2, 'd1', 3, 3.0)",
55+
// table3: use for testing alias, mirrors table2
56+
"create table table3(device STRING TAG, s1_testName INT64 FIELD, s2_testName DOUBLE FIELD)",
57+
"insert into table3 values (1, 'd1', 1, 1.0)",
58+
"insert into table3 values (2, 'd1', 3, 3.0)",
59+
// table4: test type compatible
60+
"create table table4(device STRING TAG, s1 TEXT FIELD, s2 DOUBLE FIELD)"
61+
};
62+
63+
@BeforeClass
64+
public static void setUp() throws Exception {
65+
EnvFactory.getEnv().initClusterEnvironment();
66+
prepareTableData(createSqls);
67+
}
68+
69+
@AfterClass
70+
public static void tearDown() throws Exception {
71+
EnvFactory.getEnv().cleanClusterEnvironment();
72+
}
73+
74+
@Test
75+
public void normalTest() {
76+
String[] expectedHeader = new String[] {"device", "s1", "s2"};
77+
78+
// --- EXCEPT (DISTINCT) ---
79+
// table1 (distinct): {('d1', 1, 1.0), ('d1', 2, 2.0)}
80+
// table2 (distinct): {('d1', 1, 1.0), ('d1', 3, 3.0)}
81+
// expected one tuple : ('d1', 2, 2.0)
82+
String[] retArray =
83+
new String[] {
84+
"d1,2,2.0,",
85+
};
86+
tableResultSetEqualTest(
87+
"select device, s1, s2 from table1 except select device, s1, s2 from table2",
88+
expectedHeader,
89+
retArray,
90+
DATABASE_NAME);
91+
tableResultSetEqualTest(
92+
"select device, s1, s2 from table1 except distinct select device, s1, s2 from table2",
93+
expectedHeader,
94+
retArray,
95+
DATABASE_NAME);
96+
97+
// --- EXCEPT ALL ---
98+
// Row ('d1', 1, 1.0): table1 has 3, table2 has 1. max(0, 3 - 1) = 2 tuples.
99+
// Row ('d1', 2, 2.0): table1 has 1, table2 has 0. max(0, 1 - 0) = 1 tuple.
100+
// Row ('d1', 3, 3.0): table1 has 0, table2 has 1. max(0, 0 - 1) = 0 tuples.
101+
// expected: 2 * ('d1', 1, 1.0) and 1 * ('d1', 2, 2.0)
102+
retArray = new String[] {"d1,1,1.0,", "d1,1,1.0,", "d1,2,2.0,"};
103+
tableResultSetEqualTest(
104+
"select device, s1, s2 from table1 except all select device, s1, s2 from table2",
105+
expectedHeader,
106+
retArray,
107+
DATABASE_NAME);
108+
// test table3, the column name is different
109+
tableResultSetEqualTest(
110+
"select device, s1, s2 from table1 except all select device, s1_testName, s2_testName from table3",
111+
expectedHeader,
112+
retArray,
113+
DATABASE_NAME);
114+
}
115+
116+
@Test
117+
public void mappingTest() {
118+
// table1 (aliased): (s1 as col_a) -> (1), (1), (1), (2) -> { (1.0) * 3, (2.0) * 1 }
119+
// common value: (1.0)
120+
121+
String[] expectedHeader = new String[] {"col_a"};
122+
123+
// --- EXCEPT (DISTINCT) with alias ---
124+
// t1_distinct = {1.0, 2.0}
125+
// t2_distinct = {1.0, 3.0}
126+
// Result: {2.0}
127+
String[] retArray = new String[] {"2.0,"};
128+
tableResultSetEqualTest(
129+
"select col_a from ((select s1 as col_a, device as col_b from table1) except (select s2, device from table2)) order by col_a",
130+
expectedHeader,
131+
retArray,
132+
DATABASE_NAME);
133+
134+
// --- EXCEPT ALL with alias ---
135+
// Row (1.0): t1 has 3, t2 has 1. max(0, 3 - 1) = 2.
136+
// Row (2.0): t1 has 1, t2 has 0. max(0, 1 - 0) = 1.
137+
// Result: {1.0, 1.0, 2.0} (query has order by)
138+
retArray = new String[] {"1.0,", "1.0,", "2.0,"};
139+
tableResultSetEqualTest(
140+
"select col_a from ((select s1 as col_a, device as col_b from table1) except all (select s2, device from table2)) order by col_a",
141+
expectedHeader,
142+
retArray,
143+
DATABASE_NAME);
144+
}
145+
146+
@Test
147+
public void exceptionTest() {
148+
// type is incompatible (INT32 vs TEXT)
149+
tableAssertTestFail(
150+
"(select * from table1) except all (select * from table4)",
151+
"has incompatible types: INT32, TEXT",
152+
DATABASE_NAME);
153+
154+
tableAssertTestFail(
155+
"(select * from table1) except all (select time from table4)",
156+
"EXCEPT query has different number of fields: 4, 1",
157+
DATABASE_NAME);
158+
}
159+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AssignUniqueId;
7272
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode;
7373
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode;
74+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExceptNode;
7475
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode;
7576
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode;
7677
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode;
@@ -1116,6 +1117,15 @@ public List<String> visitIntersect(IntersectNode node, GraphContext context) {
11161117
return render(node, boxValue, context);
11171118
}
11181119

1120+
@Override
1121+
public List<String> visitExcept(ExceptNode node, GraphContext context) {
1122+
List<String> boxValue = new ArrayList<>();
1123+
boxValue.add(String.format("Except-%s", node.getPlanNodeId().getId()));
1124+
boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols()));
1125+
boxValue.add(String.format("isDistinct: %s", node.isDistinct()));
1126+
return render(node, boxValue, context);
1127+
}
1128+
11191129
private List<String> render(PlanNode node, List<String> nodeBoxString, GraphContext context) {
11201130
Box box = new Box(nodeBoxString);
11211131
List<List<String>> children = new ArrayList<>();

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@
119119
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTreeDeviceViewScanNode;
120120
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AssignUniqueId;
121121
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode;
122+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExceptNode;
122123
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode;
123124
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GroupNode;
124125
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode;
@@ -314,6 +315,7 @@ public enum PlanNodeType {
314315
TABLE_INTO_NODE((short) 1033),
315316
TABLE_UNION_NODE((short) 1034),
316317
TABLE_INTERSECT_NODE((short) 1035),
318+
TABLE_EXCEPT_NODE((short) 1036),
317319

318320
RELATIONAL_INSERT_TABLET((short) 2000),
319321
RELATIONAL_INSERT_ROW((short) 2001),
@@ -705,6 +707,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) {
705707
return UnionNode.deserialize(buffer);
706708
case 1035:
707709
return IntersectNode.deserialize(buffer);
710+
case 1036:
711+
return ExceptNode.deserialize(buffer);
708712
case 2000:
709713
return RelationalInsertTabletNode.deserialize(buffer);
710714
case 2001:

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@
123123
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference;
124124
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTreeDeviceViewScanNode;
125125
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode;
126+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExceptNode;
126127
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode;
127128
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GroupNode;
128129
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode;
@@ -845,4 +846,8 @@ public R visitUnion(UnionNode node, C context) {
845846
public R visitIntersect(IntersectNode node, C context) {
846847
return visitPlan(node, context);
847848
}
849+
850+
public R visitExcept(ExceptNode node, C context) {
851+
return visitPlan(node, context);
852+
}
848853
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.iotdb.db.queryengine.plan.relational.metadata.TreeDeviceViewSchema;
5656
import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils;
5757
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode;
58+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExceptNode;
5859
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
5960
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode;
6061
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.IntersectNode;
@@ -1164,6 +1165,24 @@ protected RelationPlan visitIntersect(Intersect node, Void context) {
11641165
intersectNode, analysis.getScope(node), intersectNode.getOutputSymbols(), outerContext);
11651166
}
11661167

1168+
@Override
1169+
protected RelationPlan visitExcept(Except node, Void context) {
1170+
Preconditions.checkArgument(
1171+
!node.getRelations().isEmpty(), "No relations specified for except");
1172+
SetOperationPlan setOperationPlan = process(node);
1173+
1174+
PlanNode exceptNode =
1175+
new ExceptNode(
1176+
idAllocator.genPlanNodeId(),
1177+
setOperationPlan.getChildren(),
1178+
setOperationPlan.getSymbolMapping(),
1179+
ImmutableList.copyOf(setOperationPlan.getSymbolMapping().keySet()),
1180+
node.isDistinct());
1181+
1182+
return new RelationPlan(
1183+
exceptNode, analysis.getScope(node), exceptNode.getOutputSymbols(), outerContext);
1184+
}
1185+
11671186
private SetOperationPlan process(SetOperation node) {
11681187
RelationType outputFields = analysis.getOutputDescriptor(node);
11691188
List<Symbol> outputs =
@@ -1210,11 +1229,6 @@ protected RelationPlan visitValues(Values node, Void context) {
12101229
throw new IllegalStateException("Values is not supported in current version.");
12111230
}
12121231

1213-
@Override
1214-
protected RelationPlan visitExcept(Except node, Void context) {
1215-
throw new IllegalStateException("Except is not supported in current version.");
1216-
}
1217-
12181232
@Override
12191233
protected RelationPlan visitInsertTablet(InsertTablet node, Void context) {
12201234
final InsertTabletStatement insertTabletStatement = node.getInnerTreeStatement();

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/ImplementExceptAll.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule;
21+
22+
import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction;
23+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
24+
import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments;
25+
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule;
26+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExceptNode;
27+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
28+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns;
29+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
30+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression;
31+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression;
32+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
33+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall;
34+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral;
35+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName;
36+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures;
37+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern;
38+
39+
import com.google.common.collect.ImmutableList;
40+
import org.apache.tsfile.read.common.type.LongType;
41+
42+
import static com.google.common.base.Preconditions.checkState;
43+
import static java.util.Objects.requireNonNull;
44+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.Except.distinct;
45+
import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression.Operator.SUBTRACT;
46+
47+
public class ImplementExceptAll implements Rule<ExceptNode> {
48+
49+
private static final Pattern<ExceptNode> PATTERN =
50+
Patterns.except().with(distinct().equalTo(false));
51+
52+
private final Metadata metadata;
53+
54+
public ImplementExceptAll(Metadata metadata) {
55+
this.metadata = requireNonNull(metadata, "metadata is null");
56+
}
57+
58+
@Override
59+
public Pattern<ExceptNode> getPattern() {
60+
return PATTERN;
61+
}
62+
63+
@Override
64+
public Result apply(ExceptNode node, Captures captures, Context context) {
65+
66+
SetOperationNodeTranslator translator =
67+
new SetOperationNodeTranslator(
68+
metadata, context.getSymbolAllocator(), context.getIdAllocator());
69+
70+
// 1. translate the except(all) node to other planNodes
71+
SetOperationNodeTranslator.TranslationResult translationResult =
72+
translator.makeSetContainmentPlanForAll(node);
73+
74+
checkState(
75+
!translationResult.getCountSymbols().isEmpty(),
76+
"ExceptNode translation result has no count symbols");
77+
78+
// 2. add the filter node above the result node from translation process
79+
// filter condition : row_number <= greatest(...greatest((greatest(count1 - count2, 0) - count3,
80+
// 0))....)
81+
Expression minusCount = translationResult.getCountSymbols().get(0).toSymbolReference();
82+
QualifiedName greatest =
83+
QualifiedName.of(TableBuiltinScalarFunction.GREATEST.getFunctionName());
84+
for (int i = 1; i < translationResult.getCountSymbols().size(); i++) {
85+
minusCount =
86+
new FunctionCall(
87+
greatest,
88+
ImmutableList.of(
89+
new ArithmeticBinaryExpression(
90+
SUBTRACT,
91+
minusCount,
92+
translationResult.getCountSymbols().get(i).toSymbolReference()),
93+
new GenericLiteral(LongType.INT64.getDisplayName(), "0")));
94+
}
95+
96+
FilterNode filterNode =
97+
new FilterNode(
98+
context.getIdAllocator().genPlanNodeId(),
99+
translationResult.getPlanNode(),
100+
new ComparisonExpression(
101+
ComparisonExpression.Operator.LESS_THAN_OR_EQUAL,
102+
translationResult.getRowNumberSymbol().toSymbolReference(),
103+
minusCount));
104+
105+
// 3. add the project node to remove the redundant columns
106+
return Result.ofPlanNode(
107+
new ProjectNode(
108+
context.getIdAllocator().genPlanNodeId(),
109+
filterNode,
110+
Assignments.identity(node.getOutputSymbols())));
111+
}
112+
}

0 commit comments

Comments
 (0)