Skip to content

Commit 2baee94

Browse files
committed
Initial checkin for SYSTEM indexes
1 parent 095617f commit 2baee94

File tree

15 files changed

+1069
-36
lines changed

15 files changed

+1069
-36
lines changed

phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java

+34
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.hadoop.hbase.Cell;
2121
import org.apache.hadoop.hbase.CellComparator;
22+
import org.apache.hadoop.hbase.CompareOperator;
2223
import org.apache.hadoop.hbase.client.Scan;
2324
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
2425
import org.apache.hadoop.hbase.util.Pair;
@@ -31,7 +32,9 @@
3132
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
3233
import org.apache.phoenix.jdbc.PhoenixStatement;
3334
import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
35+
import org.apache.phoenix.parse.ComparisonParseNode;
3436
import org.apache.phoenix.parse.CreateIndexStatement;
37+
import org.apache.phoenix.parse.LiteralParseNode;
3538
import org.apache.phoenix.parse.ParseNode;
3639
import org.apache.phoenix.parse.StatelessTraverseAllParseNodeVisitor;
3740
import org.apache.phoenix.parse.SubqueryParseNode;
@@ -87,12 +90,35 @@ public CreateIndexCompiler(PhoenixStatement statement, Operation operation) {
8790
*/
8891
private static class IndexWhereParseNodeVisitor extends StatelessTraverseAllParseNodeVisitor {
8992
private boolean hasSubquery = false;
93+
private boolean hasExcludedSystemSchema = false;
9094

9195
@Override
9296
public Void visit(SubqueryParseNode node) throws SQLException {
9397
hasSubquery = true;
9498
return null;
9599
}
100+
101+
@Override
102+
public boolean visitEnter(ComparisonParseNode node) throws SQLException {
103+
if (node.getFilterOp() == CompareOperator.NOT_EQUAL) {
104+
boolean hasTableSchemColumn = node.getLHS().toString().equals(PhoenixDatabaseMetaData.TABLE_SCHEM) ||
105+
node.getRHS().toString().equals(PhoenixDatabaseMetaData.TABLE_SCHEM);
106+
boolean hasSystemLiteralNode = false;
107+
if (node.getLHS().getClass().isAssignableFrom(LiteralParseNode.class)) {
108+
hasSystemLiteralNode =
109+
SchemaUtil.normalizeLiteral((LiteralParseNode) node.getLHS())
110+
.equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME);
111+
}
112+
if (node.getRHS().getClass().isAssignableFrom(LiteralParseNode.class)) {
113+
hasSystemLiteralNode =
114+
SchemaUtil.normalizeLiteral((LiteralParseNode) node.getRHS())
115+
.equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME);
116+
}
117+
hasExcludedSystemSchema = hasTableSchemColumn && hasSystemLiteralNode;
118+
}
119+
return true;
120+
}
121+
96122
}
97123

98124
private String getValue(PDataType type) {
@@ -138,6 +164,14 @@ private void verifyIndexWhere(ParseNode indexWhere, StatementContext context,
138164
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_INDEX_WHERE_WITH_SUBQUERY).
139165
build().buildException();
140166
}
167+
// Verify that index WHERE clause on SYSTEM.CATALOG table excludes the SYSTEM schema
168+
// TABLE_SCHEM <> 'SYSTEM'
169+
if (SchemaUtil.isMetaTable(dataTableName.getSchemaName(), dataTableName.getTableName())
170+
&& !indexWhereParseNodeVisitor.hasExcludedSystemSchema) {
171+
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_INDEX_WHERE_WITH_SYSTEM_CATALOG_ROWS_NOT_EXCLUDED).
172+
build().buildException();
173+
}
174+
141175

142176
// Verify that index WHERE clause can be evaluated on a single data table row
143177

phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java

+3
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ public SQLException newException(SQLExceptionInfo info) {
113113
" Index where clause cannot include a subquery."),
114114
CANNOT_EVALUATE_INDEX_WHERE(304, "23102",
115115
"Invalid index where clause. It cannot be evaluated on a data table row."),
116+
INVALID_INDEX_WHERE_WITH_SYSTEM_CATALOG_ROWS_NOT_EXCLUDED(305, "23103",
117+
"Invalid index where clause. Index on SYSTEM.CATALOG should always exclude system catalog rows (TABLE_SCHEM <> 'SYSTEM')."),
118+
116119
/**
117120
* Invalid Cursor State (errorcode 04, sqlstate 24)
118121
*/

phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,8 @@ public enum ExpressionType {
202202
BsonValueFunction(BsonValueFunction.class),
203203
PartitionIdFunction(PartitionIdFunction.class),
204204
DecodeBinaryFunction(DecodeBinaryFunction.class),
205-
EncodeBinaryFunction(EncodeBinaryFunction.class);
205+
EncodeBinaryFunction(EncodeBinaryFunction.class),
206+
DecodeViewIdFunction(DecodeViewIndexIdFunction.class);
206207

207208
ExpressionType(Class<? extends Expression> clazz) {
208209
this.clazz = clazz;
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.phoenix.expression.function;
19+
20+
import org.apache.hadoop.hbase.Cell;
21+
import org.apache.hadoop.hbase.CellUtil;
22+
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
23+
import org.apache.phoenix.expression.Determinism;
24+
import org.apache.phoenix.expression.Expression;
25+
import org.apache.phoenix.expression.KeyValueColumnExpression;
26+
import org.apache.phoenix.parse.FunctionParseNode;
27+
import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
28+
import org.apache.phoenix.parse.DecodeViewIndexIdParseNode;
29+
import org.apache.phoenix.parse.PhoenixRowTimestampParseNode;
30+
import org.apache.phoenix.schema.SortOrder;
31+
import org.apache.phoenix.schema.tuple.Tuple;
32+
import org.apache.phoenix.schema.types.PDataType;
33+
import org.apache.phoenix.schema.types.PInteger;
34+
import org.apache.phoenix.schema.types.PLong;
35+
import org.apache.phoenix.schema.types.PSmallint;
36+
37+
import java.sql.Types;
38+
import java.util.List;
39+
40+
import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.NULL_DATA_TYPE_VALUE;
41+
import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN;
42+
43+
/**
44+
* Function to return the timestamp of the empty column which functions as the row timestamp. The
45+
* result returned can be used for debugging(eg. using HBase shell), logging etc.
46+
* Can also be used in sql predicates.
47+
*/
48+
@BuiltInFunction(name = DecodeViewIndexIdFunction.NAME,
49+
nodeClass= DecodeViewIndexIdParseNode.class,
50+
args = {@FunctionParseNode.Argument(allowedTypes = { PLong.class}),
51+
@FunctionParseNode.Argument(allowedTypes = { PInteger.class})
52+
})
53+
public class DecodeViewIndexIdFunction extends ScalarFunction {
54+
55+
public static final String NAME = "DECODE_VIEW_INDEX_ID";
56+
57+
public DecodeViewIndexIdFunction() {
58+
}
59+
60+
/**
61+
* @param children An EMPTY_COLUMN key value expression injected thru
62+
* {@link PhoenixRowTimestampParseNode#create create}
63+
* will cause the empty column key value to be evaluated during scan filter processing.
64+
*/
65+
public DecodeViewIndexIdFunction(List<Expression> children) {
66+
super(children);
67+
68+
// It takes 2 parameters - VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE.
69+
if ((children.size() != 2) || !children.get(0).getClass().isAssignableFrom(
70+
KeyValueColumnExpression.class) || !children.get(1).getClass().isAssignableFrom(
71+
KeyValueColumnExpression.class)) {
72+
throw new IllegalArgumentException(
73+
"DecodeViewIndexIdFunction should only have a "
74+
+ "VIEW_INDEX_ID and a VIEW_INDEX_ID_DATA_TYPE key value expression."
75+
);
76+
}
77+
if (!(children.get(0).getDataType().equals(PLong.INSTANCE))) {
78+
throw new IllegalArgumentException(
79+
"DecodeViewIndexIdFunction should have an "
80+
+ "VIEW_INDEX_ID key value expression of type PLong"
81+
);
82+
}
83+
84+
if (!(children.get(1).getDataType().equals(PInteger.INSTANCE))) {
85+
throw new IllegalArgumentException(
86+
"DecodeViewIndexIdFunction should have an "
87+
+ "VIEW_INDEX_ID_DATA_TYPE key value expression of type PLong"
88+
);
89+
}
90+
}
91+
92+
@Override
93+
public String getName() {
94+
return NAME;
95+
}
96+
97+
@Override
98+
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
99+
if (tuple == null) {
100+
return false;
101+
}
102+
103+
byte[] viewIndexIdCF = ((KeyValueColumnExpression) children.get(0)).getColumnFamily();
104+
byte[] viewIndexIdCQ = ((KeyValueColumnExpression) children.get(0)).getColumnQualifier();
105+
byte[] viewIndexIdTypeCF = ((KeyValueColumnExpression) children.get(1)).getColumnFamily();
106+
byte[] viewIndexIdTypeCQ = ((KeyValueColumnExpression) children.get(1)).getColumnQualifier();
107+
108+
Cell viewIndexIdCell = tuple.getValue(viewIndexIdCF, viewIndexIdCQ);
109+
Cell viewIndexIdDataTypeCell = tuple.getValue(viewIndexIdTypeCF, viewIndexIdTypeCQ);
110+
111+
112+
/*
113+
This is combination of diff client created view index looks like:
114+
client VIEW_INDEX_ID(Cell number of bytes) VIEW_INDEX_ID_DATA_TYPE
115+
pre-4.15 2 bytes NULL
116+
post-4.15[config smallint] 2 bytes 5(smallint)
117+
post-4.15[config bigint] 8 bytes -5(bigint)
118+
119+
VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell representation of the data)
120+
NULL, SMALLINT -> RETRIEVE AND CONVERT TO BIGINT
121+
SMALLINT, SMALLINT -> RETRIEVE AND CONVERT TO BIGINT
122+
BIGINT, BIGINT -> DO NOT CONVERT
123+
124+
*/
125+
126+
if (viewIndexIdCell != null) {
127+
int type = NULL_DATA_TYPE_VALUE;
128+
if (viewIndexIdDataTypeCell != null) {
129+
type = (Integer) PInteger.INSTANCE.toObject(
130+
viewIndexIdDataTypeCell.getValueArray(),
131+
viewIndexIdDataTypeCell.getValueOffset(),
132+
viewIndexIdDataTypeCell.getValueLength(),
133+
PInteger.INSTANCE,
134+
SortOrder.ASC);
135+
}
136+
137+
System.out.println("DecodeViewIndexIdFunction: Type: " + type);
138+
ImmutableBytesWritable columnValue =
139+
new ImmutableBytesWritable(CellUtil.cloneValue(viewIndexIdCell));
140+
if ((type == NULL_DATA_TYPE_VALUE || type == Types.SMALLINT) && (viewIndexIdCell.getValueLength() <
141+
VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN)) {
142+
byte[] newBytes = PLong.INSTANCE.toBytes(PSmallint.INSTANCE.toObject(columnValue.get()));
143+
ptr.set(newBytes, 0, newBytes.length);
144+
} else {
145+
ptr.set(columnValue.get(), columnValue.getOffset(), columnValue.getLength());
146+
}
147+
}
148+
return true;
149+
}
150+
151+
@Override
152+
public PDataType getDataType() {
153+
return PLong.INSTANCE;
154+
}
155+
156+
@Override
157+
public boolean isStateless() {
158+
return false;
159+
}
160+
161+
@Override
162+
public Determinism getDeterminism() {
163+
return Determinism.PER_ROW;
164+
}
165+
166+
}

phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java

+3
Original file line numberDiff line numberDiff line change
@@ -1074,6 +1074,9 @@ public boolean shouldPrepareIndexMutations(Put dataRowState) {
10741074
return true;
10751075
}
10761076
List<Cell> cols = IndexUtil.readColumnsFromRow(dataRowState, getIndexWhereColumns());
1077+
if (cols.isEmpty()) {
1078+
return false;
1079+
}
10771080
// Cells should be sorted as they are searched using a binary search during expression
10781081
// evaluation
10791082
Collections.sort(cols, CellComparator.getInstance());

phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
import org.apache.phoenix.util.PhoenixRuntime;
4040
import org.apache.phoenix.util.ReadOnlyProps;
4141
import org.apache.phoenix.util.ScanUtil;
42+
import org.slf4j.Logger;
43+
import org.slf4j.LoggerFactory;
4244

4345
public class IndexMetaDataCacheClient {
4446

phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,12 @@ protected CreateTableStatement(TableName tableName, ListMultimap<String,Pair<Str
100100
Map<String, Integer> familyCounters, boolean noVerify) {
101101
this.tableName = tableName;
102102
this.props = props == null ? ImmutableListMultimap.<String,Pair<String,Object>>of() : props;
103-
this.tableType = PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(tableName.getSchemaName()) ? PTableType.SYSTEM : tableType;
103+
this.tableType =
104+
(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(
105+
tableName.getSchemaName()) &&
106+
(tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) ?
107+
PTableType.SYSTEM :
108+
tableType);
104109
this.columns = columns == null ? ImmutableList.<ColumnDef>of() : ImmutableList.<ColumnDef>copyOf(columns);
105110
this.pkConstraint = pkConstraint == null ? PrimaryKeyConstraint.EMPTY : pkConstraint;
106111
this.splitNodes = splitNodes == null ? Collections.<ParseNode>emptyList() : ImmutableList.copyOf(splitNodes);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.phoenix.parse;
20+
21+
import org.apache.phoenix.compile.StatementContext;
22+
import org.apache.phoenix.expression.Expression;
23+
import org.apache.phoenix.expression.function.DecodeViewIndexIdFunction;
24+
import org.apache.phoenix.expression.function.FunctionExpression;
25+
import org.apache.phoenix.query.QueryConstants;
26+
import org.apache.phoenix.util.IndexUtil;
27+
28+
import java.sql.SQLException;
29+
import java.util.List;
30+
31+
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
32+
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE;
33+
34+
public class DecodeViewIndexIdParseNode extends FunctionParseNode {
35+
36+
DecodeViewIndexIdParseNode(String name, List<ParseNode> children,
37+
BuiltInFunctionInfo info) {
38+
super(name, children, info);
39+
// It takes 2 parameters - VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE.
40+
if (children.size() != 2) {
41+
throw new IllegalArgumentException(
42+
"DecodeViewIndexIdParseNode should only have "
43+
+ "VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE parse nodes."
44+
);
45+
}
46+
if (children.get(0).getClass().isAssignableFrom(ColumnParseNode.class)
47+
&& children.get(1).getClass().isAssignableFrom(ColumnParseNode.class)
48+
&& (!(((ColumnParseNode) children.get(0)).getName().equals(VIEW_INDEX_ID))
49+
|| !(((ColumnParseNode) children.get(1)).getName().equals(VIEW_INDEX_ID_DATA_TYPE)))
50+
) {
51+
throw new IllegalArgumentException(
52+
"DecodeViewIndexIdParseNode should only have "
53+
+ "VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE parse nodes."
54+
);
55+
}
56+
57+
// CastPastNode is generated during IndexStatement rewriting
58+
if (children.get(0).getClass().isAssignableFrom(CastParseNode.class)
59+
&& children.get(1).getClass().isAssignableFrom(CastParseNode.class)
60+
&& (!((ColumnParseNode) (((CastParseNode) children.get(0)).getChildren().get(0))).getName().equals(
61+
IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, VIEW_INDEX_ID))
62+
|| !((ColumnParseNode) (((CastParseNode) children.get(1)).getChildren().get(0))).getName().equals(
63+
IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, VIEW_INDEX_ID_DATA_TYPE)))
64+
) {
65+
throw new IllegalArgumentException(
66+
"DecodeViewIndexIdParseNode should only have "
67+
+ "VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE parse nodes."
68+
);
69+
}
70+
71+
}
72+
73+
@Override
74+
public FunctionExpression create(List<Expression> children, StatementContext context)
75+
throws SQLException {
76+
return new DecodeViewIndexIdFunction(children);
77+
}
78+
79+
}

0 commit comments

Comments
 (0)