Skip to content

Commit 6899941

Browse files
QiangCaiQiangCai
committed
reorder expression
1 parent 314db91 commit 6899941

File tree

11 files changed

+495
-2
lines changed

11 files changed

+495
-2
lines changed

‎core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java

+5
Original file line numberDiff line numberDiff line change
@@ -2602,6 +2602,11 @@ private CarbonCommonConstants() {
26022602

26032603
public static final String FILE_HEADER = "fileHeader";
26042604

2605+
@CarbonProperty(dynamicConfigurable = true)
2606+
public static final String CARBON_OPTIMIZE_FILTER = "carbon.optimize.filter";
2607+
2608+
public static final String CARBON_OPTIMIZE_FILTER_DEFAULT = "true";
2609+
26052610
@CarbonProperty(dynamicConfigurable = true)
26062611
public static final String CARBON_REORDER_FILTER = "carbon.reorder.filter";
26072612

‎core/src/main/java/org/apache/carbondata/core/scan/expression/logical/OrExpression.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,6 @@ public String getString() {
5959

6060
@Override
6161
public String getStatement() {
62-
return "(" + left.getString() + " or " + right.getString() + ")";
62+
return "(" + left.getStatement() + " or " + right.getStatement() + ")";
6363
}
6464
}

‎core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/AndMultiExpression.java

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.carbondata.core.scan.expression.optimize;
19+
20+
import org.apache.carbondata.core.scan.expression.Expression;
21+
import org.apache.carbondata.core.scan.expression.logical.AndExpression;
22+
23+
public class AndMultiExpression extends MultiExpression {
24+
25+
@Override
26+
public boolean canMerge(Expression child) {
27+
return child instanceof AndExpression;
28+
}
29+
30+
@Override
31+
public Expression toBinaryExpression() {
32+
return children.stream()
33+
.map(StorageOrdinal::toExpression)
34+
.reduce(AndExpression::new)
35+
.orElse(null);
36+
}
37+
}

‎core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/ExpressionOptimizer.java

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.carbondata.core.scan.expression.optimize;
19+
20+
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
21+
import org.apache.carbondata.core.scan.expression.Expression;
22+
import org.apache.carbondata.core.util.CarbonProperties;
23+
24+
public class ExpressionOptimizer {
25+
26+
public static Expression optimize(CarbonTable table, Expression expression) {
27+
if (!CarbonProperties.isFilterOptimizeEnabled()) {
28+
return expression;
29+
}
30+
MultiExpression multiExpression = MultiExpression.build(expression);
31+
// unsupported expression
32+
if (multiExpression == null) {
33+
return expression;
34+
}
35+
// remove redundancy filter
36+
multiExpression.removeRedundant();
37+
// combine multiple filters to single filter
38+
multiExpression.combine();
39+
// reorder Expression by storage ordinal of columns
40+
if (CarbonProperties.isFilterReorderingEnabled()) {
41+
multiExpression.reorder(table);
42+
}
43+
return multiExpression.toBinaryExpression();
44+
}
45+
}

‎core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/ExpressionWithOrdinal.java

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package org.apache.carbondata.core.scan.expression.optimize;
2+
3+
import java.util.List;
4+
import java.util.Map;
5+
6+
import org.apache.carbondata.core.scan.expression.ColumnExpression;
7+
import org.apache.carbondata.core.scan.expression.Expression;
8+
import org.apache.carbondata.core.scan.expression.UnknownExpression;
9+
import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
10+
11+
public class ExpressionWithOrdinal extends StorageOrdinal {
12+
protected Expression expression;
13+
14+
public ExpressionWithOrdinal(Expression expression) {
15+
this.minOrdinal = Integer.MAX_VALUE;
16+
this.expression = expression;
17+
}
18+
19+
@Override
20+
public void updateMinOrdinal(Map<String, Integer> columnMapOrdinal) {
21+
updateMinOrdinal(expression, columnMapOrdinal);
22+
}
23+
24+
private void updateMinOrdinal(Expression expression, Map<String, Integer> nameMapOrdinal) {
25+
if (expression != null && expression.getChildren() != null) {
26+
if (expression.getChildren().size() == 0) {
27+
if (expression instanceof ConditionalExpression) {
28+
List<ColumnExpression> columnList =
29+
((ConditionalExpression) expression).getColumnList();
30+
for (ColumnExpression columnExpression : columnList) {
31+
updateMinOrdinal(columnExpression.getColumnName(), nameMapOrdinal);
32+
}
33+
}
34+
} else {
35+
for (Expression subExpression : expression.getChildren()) {
36+
if (subExpression instanceof ColumnExpression) {
37+
updateMinOrdinal(((ColumnExpression) subExpression).getColumnName(), nameMapOrdinal);
38+
} else if (expression instanceof UnknownExpression) {
39+
UnknownExpression exp = ((UnknownExpression) expression);
40+
List<ColumnExpression> listOfColExpression = exp.getAllColumnList();
41+
for (ColumnExpression columnExpression : listOfColExpression) {
42+
updateMinOrdinal(columnExpression.getColumnName(), nameMapOrdinal);
43+
}
44+
} else {
45+
updateMinOrdinal(subExpression, nameMapOrdinal);
46+
}
47+
}
48+
}
49+
}
50+
}
51+
52+
private void updateMinOrdinal(String columnName, Map<String, Integer> nameMapOrdinal) {
53+
Integer ordinal = nameMapOrdinal.get(columnName.toLowerCase());
54+
if (ordinal != null && ordinal < minOrdinal) {
55+
minOrdinal = ordinal;
56+
}
57+
}
58+
}

‎core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/MultiExpression.java

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.carbondata.core.scan.expression.optimize;
19+
20+
import java.util.ArrayList;
21+
import java.util.HashMap;
22+
import java.util.List;
23+
import java.util.Map;
24+
25+
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
26+
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
27+
import org.apache.carbondata.core.scan.expression.Expression;
28+
import org.apache.carbondata.core.scan.expression.logical.AndExpression;
29+
import org.apache.carbondata.core.scan.expression.logical.OrExpression;
30+
31+
public abstract class MultiExpression extends StorageOrdinal {
32+
33+
public MultiExpression() {
34+
this.minOrdinal = Integer.MAX_VALUE;
35+
}
36+
37+
protected List<StorageOrdinal> children = new ArrayList<>();
38+
39+
public static MultiExpression build(Expression expression) {
40+
MultiExpression multiExpression = null;
41+
if (expression instanceof AndExpression) {
42+
multiExpression = new AndMultiExpression();
43+
}
44+
if (expression instanceof OrExpression) {
45+
multiExpression = new OrMultiExpression();
46+
}
47+
if (multiExpression == null) {
48+
return null;
49+
}
50+
for (Expression child : expression.getChildren()) {
51+
buildChild(child, multiExpression);
52+
}
53+
return multiExpression;
54+
}
55+
56+
private static void buildChild(Expression expression, MultiExpression parent) {
57+
if (parent.canMerge(expression)) {
58+
// multiple and(or) can be merge into same MultiExpression
59+
for (Expression child : expression.getChildren()) {
60+
buildChild(child, parent);
61+
}
62+
} else {
63+
MultiExpression multiExpression = build(expression);
64+
if (multiExpression == null) {
65+
// it is not and/or expression
66+
parent.addChild(expression);
67+
} else {
68+
// it is and, or expression
69+
parent.addChild(multiExpression);
70+
}
71+
}
72+
}
73+
74+
public abstract boolean canMerge(Expression child);
75+
76+
private void addChild(Expression child) {
77+
addChild(new ExpressionWithOrdinal(child));
78+
}
79+
80+
private void addChild(StorageOrdinal storageOrdinal) {
81+
children.add(storageOrdinal);
82+
}
83+
84+
private Map<String, Integer> columnMapOrdinal(CarbonTable table) {
85+
List<CarbonColumn> createOrderColumns = table.getCreateOrderColumn();
86+
Map<String, Integer> nameMapOrdinal = new HashMap<>(createOrderColumns.size());
87+
int dimensionCount = table.getAllDimensions().size();
88+
for (CarbonColumn column : createOrderColumns) {
89+
if (column.isDimension()) {
90+
nameMapOrdinal.put(column.getColName(), column.getOrdinal());
91+
} else {
92+
nameMapOrdinal.put(column.getColName(), dimensionCount + column.getOrdinal());
93+
}
94+
}
95+
return nameMapOrdinal;
96+
}
97+
98+
public void removeRedundant() {
99+
// TODO remove redundancy filter if exists
100+
}
101+
102+
public void combine() {
103+
// TODO combine multiple filters to single filter if needed
104+
}
105+
106+
public void reorder(CarbonTable table) {
107+
updateMinOrdinal(columnMapOrdinal(table));
108+
sortChildrenByOrdinal();
109+
}
110+
111+
public void sortChildrenByOrdinal() {
112+
children.sort(null);
113+
for (StorageOrdinal child : children) {
114+
if (child instanceof MultiExpression) {
115+
((MultiExpression) child).sortChildrenByOrdinal();
116+
}
117+
}
118+
}
119+
120+
@Override
121+
public void updateMinOrdinal(Map<String, Integer> columnMapOrdinal) {
122+
for (StorageOrdinal child : children) {
123+
child.updateMinOrdinal(columnMapOrdinal);
124+
if (child.minOrdinal < this.minOrdinal) {
125+
this.minOrdinal = child.minOrdinal;
126+
}
127+
}
128+
}
129+
130+
public abstract Expression toBinaryExpression();
131+
132+
}

‎core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/OrMultiExpression.java

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.carbondata.core.scan.expression.optimize;
19+
20+
import org.apache.carbondata.core.scan.expression.Expression;
21+
import org.apache.carbondata.core.scan.expression.logical.OrExpression;
22+
23+
public class OrMultiExpression extends MultiExpression {
24+
@Override
25+
public boolean canMerge(Expression child) {
26+
return child instanceof OrExpression;
27+
}
28+
29+
@Override
30+
public Expression toBinaryExpression() {
31+
return children.stream()
32+
.map(StorageOrdinal::toExpression)
33+
.reduce(OrExpression::new)
34+
.orElse(null);
35+
}
36+
}

‎core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/StorageOrdinal.java

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.carbondata.core.scan.expression.optimize;
19+
20+
import java.util.Map;
21+
22+
import org.apache.carbondata.core.scan.expression.Expression;
23+
24+
public abstract class StorageOrdinal implements Comparable {
25+
26+
protected int minOrdinal;
27+
28+
public abstract void updateMinOrdinal(Map<String, Integer> columnMapOrdinal);
29+
30+
@Override
31+
public int compareTo(Object o) {
32+
return Integer.compare(minOrdinal, ((StorageOrdinal) o).minOrdinal);
33+
}
34+
35+
public Expression toExpression() {
36+
if (this instanceof MultiExpression) {
37+
return ((MultiExpression) this).toBinaryExpression();
38+
} else {
39+
return ((ExpressionWithOrdinal) this).expression;
40+
}
41+
}
42+
}

‎core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java

+7
Original file line numberDiff line numberDiff line change
@@ -2180,6 +2180,13 @@ public boolean isCleanFilesForceAllowed() {
21802180
return Boolean.parseBoolean(configuredValue);
21812181
}
21822182

2183+
public static boolean isFilterOptimizeEnabled() {
2184+
return Boolean.parseBoolean(
2185+
getInstance().getProperty(CarbonCommonConstants.CARBON_OPTIMIZE_FILTER,
2186+
CarbonCommonConstants.CARBON_OPTIMIZE_FILTER_DEFAULT)
2187+
);
2188+
}
2189+
21832190
public static boolean isFilterReorderingEnabled() {
21842191
return Boolean.parseBoolean(
21852192
getInstance().getProperty(CarbonCommonConstants.CARBON_REORDER_FILTER,

0 commit comments

Comments
 (0)