From 8e95c1a52e1fda67bc56fb35db43713bfa0a5d38 Mon Sep 17 00:00:00 2001 From: Divneet Kaur Date: Tue, 24 Sep 2024 12:15:07 -0700 Subject: [PATCH 1/2] PHOENIX-7376 : ViewUtil#findAllDescendantViews should provide two versions to differentiate CQSI initiated by clients and servers --- .../phoenix/compile/CreateTableCompiler.java | 2 +- .../apache/phoenix/schema/MetaDataClient.java | 2 +- .../org/apache/phoenix/util/ViewUtil.java | 22 ++++++++++--------- .../coprocessor/MetaDataEndpointImpl.java | 10 ++++----- .../PhoenixTransformWithViewsInputFormat.java | 2 +- .../end2end/BaseRowKeyMatcherTestIT.java | 2 +- .../phoenix/end2end/ViewMetadataIT.java | 8 +++---- .../apache/phoenix/end2end/ViewUtilIT.java | 8 +++---- 8 files changed, 29 insertions(+), 27 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java index ccda4659c43..a1cc50c37dd 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java @@ -341,7 +341,7 @@ private ViewType setViewTypeToBe(final PhoenixConnection connection, final PTabl List legitimateSiblingViewList = ViewUtil.findAllDescendantViews(childLinkTable, config, parentTenantIdInBytes, parentSchemaNameInBytes, parentToBe.getTableName().getBytes(), - HConstants.LATEST_TIMESTAMP, true).getFirst(); + HConstants.LATEST_TIMESTAMP, true,false).getFirst(); if (!legitimateSiblingViewList.isEmpty()) { PTable siblingView = legitimateSiblingViewList.get(0); Expression siblingViewWhere = getWhereFromView(connection, siblingView); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 243b13ff78b..6a89052007d 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -2156,7 +2156,7 @@ private void verifyIfDescendentViewsExtendPk(PTable tableOrView, Configuration c schemaNameBytes, viewOrTableName, HConstants.LATEST_TIMESTAMP, - false); + false,false); List legitimateChildViews = descViews.getFirst(); int dataTableOrViewPkCols = tableOrView.getPKColumns().size(); if (legitimateChildViews != null && legitimateChildViews.size() > 0) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java index 071bc207697..8d737d219e7 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java @@ -134,26 +134,26 @@ public class ViewUtil { public static Pair, List> findAllDescendantViews( Table sysCatOrsysChildLink, Configuration serverSideConfig, byte[] tenantId, byte[] schemaName, byte[] tableOrViewName, long clientTimeStamp, - boolean findJustOneLegitimateChildView) + boolean findJustOneLegitimateChildView, boolean isServerConnection) throws IOException, SQLException { List legitimateChildViews = new ArrayList<>(); List orphanChildViews = new ArrayList<>(); return findAllDescendantViews(sysCatOrsysChildLink, serverSideConfig, tenantId, schemaName, tableOrViewName, clientTimeStamp, legitimateChildViews, orphanChildViews, - findJustOneLegitimateChildView); + findJustOneLegitimateChildView, isServerConnection); } public static Pair, List> findAllDescendantViews(Table sysCatOrsysChildLink, Configuration serverSideConfig, byte[] parentTenantId, byte[] parentSchemaName, byte[] parentTableOrViewName, long clientTimeStamp, List legitimateChildViews, - List orphanChildViews, boolean findJustOneLegitimateChildView) + List orphanChildViews, boolean findJustOneLegitimateChildView, boolean isServerConnection) throws IOException, SQLException{ return findAllDescendantViews(sysCatOrsysChildLink, null, serverSideConfig, parentTenantId, parentSchemaName, parentTableOrViewName, clientTimeStamp, legitimateChildViews, orphanChildViews, findJustOneLegitimateChildView, - new Pair<>(false, false)); + new Pair<>(false, false), isServerConnection); } @@ -208,7 +208,7 @@ public static Pair, List> findAllDescendantViews( byte[] parentTenantId, byte[] parentSchemaName, byte[] parentTableOrViewName, long clientTimeStamp, List legitimateChildViews, List orphanChildViews, boolean findJustOneLegitimateChildView, - Pair scanSysCatForTTLDefinedOnAnyChildPair) + Pair scanSysCatForTTLDefinedOnAnyChildPair, boolean isServerConnection) throws IOException, SQLException { TableViewFinderResult currentResult = findImmediateRelatedViews(sysCatOrsysChildLink, sysCat, parentTenantId, @@ -226,9 +226,11 @@ public static Pair, List> findAllDescendantViews( if (clientTimeStamp != HConstants.LATEST_TIMESTAMP) { props.setProperty(CURRENT_SCN_ATTRIB, Long.toString(clientTimeStamp)); } - try (PhoenixConnection connection = - QueryUtil.getConnectionOnServer(props, serverSideConfig) - .unwrap(PhoenixConnection.class)) { + try (PhoenixConnection connection = ((isServerConnection) ? + QueryUtil.getConnectionOnServer(props, serverSideConfig) + .unwrap(PhoenixConnection.class) : + QueryUtil.getConnection(props, serverSideConfig) + .unwrap(PhoenixConnection.class))){ try { view = connection.getTableNoCache( SchemaUtil.getTableName(viewSchemaName, viewName)); @@ -257,7 +259,7 @@ public static Pair, List> findAllDescendantViews( viewInfo.getTenantId(), viewInfo.getSchemaName(), viewInfo.getTableName(), clientTimeStamp, legitimateChildViews, orphanChildViews, findJustOneLegitimateChildView, - scanSysCatForTTLDefinedOnAnyChildPair); + scanSysCatForTTLDefinedOnAnyChildPair, isServerConnection); } else { logger.error("Found an orphan parent->child link keyed by this parent." + " Parent Tenant Id: '" + Bytes.toString(parentTenantId) @@ -285,7 +287,7 @@ private static boolean isLegitimateChildView(PTable view, byte[] parentSchemaNam * Returns relatives in a breadth-first fashion. Note that this is not resilient to orphan * linking rows and we also do not try to resolve any of the views to ensure they are valid. * Use {@link ViewUtil#findAllDescendantViews(Table, Configuration, byte[], byte[], byte[], - * long, boolean)} if you are only interested in {@link LinkType#CHILD_TABLE} and need to be + * long, boolean, boolean)} if you are only interested in {@link LinkType#CHILD_TABLE} and need to be * resilient to orphan linking rows. * * @param sysCatOrsysChildLink Table corresponding to either SYSTEM.CATALOG or SYSTEM.CHILD_LINK diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 49a55d78af1..d93b2de2bbc 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -3016,7 +3016,7 @@ public void dropTable(RpcController controller, DropTableRequest request, Pair, List> descendantViews = findAllDescendantViews(hTable, env.getConfiguration(), tenantIdBytes, schemaName, tableOrViewName, clientTimeStamp, - true); + true,true); List legitimateChildViews = descendantViews.getFirst(); List orphanChildViews = descendantViews.getSecond(); if (!legitimateChildViews.isEmpty()) { @@ -3406,16 +3406,16 @@ private MetaDataMutationResult doDropTable(byte[] key, byte[] tenantId, byte[] s * @param childViews child views of table or parent view. Usually this is an empty list * passed to this method, and this method will add child views retrieved using * {@link ViewUtil#findAllDescendantViews(Table, Configuration, byte[], byte[], byte[], - * long, boolean)} + * long, boolean, boolean)} * @param clientVersion client version, used to determine if mutation is allowed. * @return Optional.empty() if mutation is allowed on parent table/view. If not allowed, * returned Optional object will contain metaDataMutationResult with MutationCode. * @throws IOException if something goes wrong while retrieving child views using * {@link ViewUtil#findAllDescendantViews(Table, Configuration, byte[], byte[], byte[], - * long, boolean)} + * long, boolean, boolean)} * @throws SQLException if something goes wrong while retrieving child views using * {@link ViewUtil#findAllDescendantViews(Table, Configuration, byte[], byte[], byte[], - * long, boolean)} + * long, boolean, boolean)} */ private Optional validateIfMutationAllowedOnParent( final PTable parentTable, @@ -3438,7 +3438,7 @@ private Optional validateIfMutationAllowedOnParent( childViews.addAll(findAllDescendantViews(hTable, sysCat, env.getConfiguration(), tenantId, schemaName, tableOrViewName, clientTimeStamp, new ArrayList<>(), new ArrayList<>(), false, - scanSysCatForTTLDefinedOnAnyChildPair) + scanSysCatForTTLDefinedOnAnyChildPair,true) .getFirst()); } diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java index 6410b8cf72e..a5b997e6af8 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java @@ -72,7 +72,7 @@ public List getSplits(JobContext context) throws IOException, Interr String tableName = SchemaUtil.getTableNameFromFullName(oldDataTableFullName); byte[] schemaNameBytes = Strings.isNullOrEmpty(schemaName) ? null : schemaName.getBytes(); Pair, List> allDescendantViews = ViewUtil.findAllDescendantViews(hTable, configuration, null, schemaNameBytes, - tableName.getBytes(), EnvironmentEdgeManager.currentTimeMillis(), false); + tableName.getBytes(), EnvironmentEdgeManager.currentTimeMillis(), false,true); List legitimateDecendants = allDescendantViews.getFirst(); List inputSplits = new ArrayList<>(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java index 0ce628b034d..c4072127aa9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java @@ -592,7 +592,7 @@ private Map assertRowKeyMatchersForTable(String url, String pare ViewUtil.findAllDescendantViews(childLinkTable, cqs.getConfiguration(), EMPTY_BYTE_ARRAY, parentSchemaName.getBytes(), parentTableName.getBytes(), - HConstants.LATEST_TIMESTAMP, false); + HConstants.LATEST_TIMESTAMP, false,true); for (PTable view : allDescendants.getFirst()) { PName tenantId = view.getTenantId(); String viewName = view.getName().getString(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java index c431a021dd8..29c222e3d91 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java @@ -597,7 +597,7 @@ public void testViewHierarchyWithOrphanLinks() throws Exception { cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), parent1TableName.getBytes(), - HConstants.LATEST_TIMESTAMP, false); + HConstants.LATEST_TIMESTAMP, false,true); List legitChildViews = allDescendants.getFirst(); List orphanViews = allDescendants.getSecond(); // All of the orphan links are legit views of the other parent @@ -610,7 +610,7 @@ public void testViewHierarchyWithOrphanLinks() throws Exception { cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), parent2TableName.getBytes(), - HConstants.LATEST_TIMESTAMP, false); + HConstants.LATEST_TIMESTAMP, false,true); legitChildViews = allDescendants.getFirst(); orphanViews = allDescendants.getSecond(); // All of the orphan links are legit views of the other parent @@ -642,7 +642,7 @@ public void testViewHierarchyWithOrphanLinks() throws Exception { cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), parent1TableName.getBytes(), - HConstants.LATEST_TIMESTAMP, false); + HConstants.LATEST_TIMESTAMP, false,true); legitChildViews = allDescendants.getFirst(); orphanViews = allDescendants.getSecond(); assertLegitChildViews(expectedLegitChildViewsListForParent1, @@ -653,7 +653,7 @@ public void testViewHierarchyWithOrphanLinks() throws Exception { cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), parent2TableName.getBytes(), - HConstants.LATEST_TIMESTAMP, false); + HConstants.LATEST_TIMESTAMP, false,true); legitChildViews = allDescendants.getFirst(); orphanViews = allDescendants.getSecond(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewUtilIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewUtilIT.java index 919a71d3065..60150ed2021 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewUtilIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewUtilIT.java @@ -371,7 +371,7 @@ public void testFindLegitChildViews() throws Exception { Pair, List> allDescendants = ViewUtil.findAllDescendantViews(childLinkTable, cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), - parentTable.getBytes(), HConstants.LATEST_TIMESTAMP, true); + parentTable.getBytes(), HConstants.LATEST_TIMESTAMP, true,true); assertTrue("No orphan views expected", allDescendants.getSecond().isEmpty()); List childViews = allDescendants.getFirst(); assertEquals("Just 1 legit child view expected", 1, childViews.size()); @@ -386,7 +386,7 @@ public void testFindLegitChildViews() throws Exception { allDescendants = ViewUtil.findAllDescendantViews(childLinkTable, cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), - parentTable.getBytes(), HConstants.LATEST_TIMESTAMP, false); + parentTable.getBytes(), HConstants.LATEST_TIMESTAMP, false,true); assertTrue("No orphan views expected", allDescendants.getSecond().isEmpty()); childViews = allDescendants.getFirst(); assertEquals("All child views expected", childViewNames.size(), childViews.size()); @@ -418,7 +418,7 @@ public void testOrphanViewDetection() throws Exception { Pair, List> allDescendants = ViewUtil.findAllDescendantViews(childLinkTable, cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), - parent2TableName.getBytes(), HConstants.LATEST_TIMESTAMP, false); + parent2TableName.getBytes(), HConstants.LATEST_TIMESTAMP, false,true); assertTrue("No orphan views expected", allDescendants.getSecond().isEmpty()); assertTrue("No legitimate views expected", allDescendants.getFirst().isEmpty()); @@ -431,7 +431,7 @@ public void testOrphanViewDetection() throws Exception { // orphan parent2->view link should show up as an orphan view of parent2 allDescendants = ViewUtil.findAllDescendantViews(childLinkTable, cqs.getConfiguration(), EMPTY_BYTE_ARRAY, BASE_TABLE_SCHEMA.getBytes(), - parent2TableName.getBytes(), HConstants.LATEST_TIMESTAMP, false); + parent2TableName.getBytes(), HConstants.LATEST_TIMESTAMP, false,true); assertTrue("No legitimate views expected", allDescendants.getFirst().isEmpty()); List orphanViews = allDescendants.getSecond(); assertEquals("1 orphan view expected", 1, orphanViews.size()); From 4843e7bf7a3703cf2473403da8e0fbef88a91625 Mon Sep 17 00:00:00 2001 From: Divneet Kaur Date: Wed, 25 Sep 2024 15:10:46 -0700 Subject: [PATCH 2/2] fixed checkstyle: --- .../apache/phoenix/coprocessor/MetaDataEndpointImpl.java | 4 ++-- .../transform/PhoenixTransformWithViewsInputFormat.java | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index d93b2de2bbc..72680a19278 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -3016,7 +3016,7 @@ public void dropTable(RpcController controller, DropTableRequest request, Pair, List> descendantViews = findAllDescendantViews(hTable, env.getConfiguration(), tenantIdBytes, schemaName, tableOrViewName, clientTimeStamp, - true,true); + true, true); List legitimateChildViews = descendantViews.getFirst(); List orphanChildViews = descendantViews.getSecond(); if (!legitimateChildViews.isEmpty()) { @@ -3438,7 +3438,7 @@ private Optional validateIfMutationAllowedOnParent( childViews.addAll(findAllDescendantViews(hTable, sysCat, env.getConfiguration(), tenantId, schemaName, tableOrViewName, clientTimeStamp, new ArrayList<>(), new ArrayList<>(), false, - scanSysCatForTTLDefinedOnAnyChildPair,true) + scanSysCatForTTLDefinedOnAnyChildPair, true) .getFirst()); } diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java index a5b997e6af8..13ae4c62c94 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/transform/PhoenixTransformWithViewsInputFormat.java @@ -71,8 +71,10 @@ public List getSplits(JobContext context) throws IOException, Interr String schemaName = SchemaUtil.getSchemaNameFromFullName(oldDataTableFullName); String tableName = SchemaUtil.getTableNameFromFullName(oldDataTableFullName); byte[] schemaNameBytes = Strings.isNullOrEmpty(schemaName) ? null : schemaName.getBytes(); - Pair, List> allDescendantViews = ViewUtil.findAllDescendantViews(hTable, configuration, null, schemaNameBytes, - tableName.getBytes(), EnvironmentEdgeManager.currentTimeMillis(), false,true); + Pair, List> allDescendantViews = + ViewUtil.findAllDescendantViews(hTable, configuration, null, schemaNameBytes, + tableName.getBytes(), EnvironmentEdgeManager.currentTimeMillis(), + false, true); List legitimateDecendants = allDescendantViews.getFirst(); List inputSplits = new ArrayList<>();