PHOENIX-7474 Migrate IndexTool tables and make sure they are created

phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java

@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.index;
+
+
+
+import java.io.IOException;
+import java.sql.Connection;
+
+import java.sql.SQLException;
+import java.util.UUID;
+
+
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
+import org.apache.phoenix.query.QueryConstants;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.TableName;
+
+import org.apache.phoenix.jdbc.PhoenixConnection;
+
+import org.apache.phoenix.query.ConnectionQueryServices;
+
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.ClientUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
+
+/**
+ * Utility class to create index tables and/or migrate them.
+ *
+ */
+public class IndexToolTableUtil extends Configured {
+    private static final Logger LOGGER = LoggerFactory.getLogger(IndexToolTableUtil.class);
+
+    public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
+    public static String SYSTEM_OUTPUT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME,
+            OUTPUT_TABLE_NAME);
+
+    public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
+    public static String SYSTEM_RESULT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME,
+            RESULT_TABLE_NAME);
+
+    public static void setIndexToolTableName(Connection connection) throws Exception {
+        ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
+        if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, queryServices.getConfiguration())) {
+            SYSTEM_OUTPUT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME).replace(
+                QueryConstants.NAME_SEPARATOR,
+                QueryConstants.NAMESPACE_SEPARATOR);
+            SYSTEM_RESULT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME).replace(
+                QueryConstants.NAME_SEPARATOR,
+                QueryConstants.NAMESPACE_SEPARATOR);
+        } else {
+            SYSTEM_OUTPUT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME);
+            SYSTEM_RESULT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME);
+        }
+    }
+
+    public static Table createResultTable(Connection connection) throws IOException, SQLException {
+        ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
+        try (Admin admin = queryServices.getAdmin()) {
+            TableName resultTableName = TableName.valueOf(SYSTEM_RESULT_TABLE_NAME);
+            if (SYSTEM_RESULT_TABLE_NAME.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
+                createSystemNamespaceTable(connection);
+            }
+            return createTable(admin, resultTableName);
+        }
+    }
+
+    public static Table createOutputTable(Connection connection) throws IOException, SQLException {
+        ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
+        try (Admin admin = queryServices.getAdmin()) {
+            TableName outputTableName = TableName.valueOf(SYSTEM_OUTPUT_TABLE_NAME);
+            if (SYSTEM_OUTPUT_TABLE_NAME.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
+                createSystemNamespaceTable(connection);
+            }
+            return  createTable(admin, outputTableName);
+        }
+    }
+
+    public static void createSystemNamespaceTable(Connection connection) throws IOException, SQLException {
+        ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
+        if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, queryServices.getConfiguration())) {
+            try (Admin admin = queryServices.getAdmin()) {
+                if (!ClientUtil.isHBaseNamespaceAvailable(admin, SYSTEM_SCHEMA_NAME)) {
+                    NamespaceDescriptor namespaceDescriptor =
+                            NamespaceDescriptor.create(SYSTEM_SCHEMA_NAME).build();
+                    admin.createNamespace(namespaceDescriptor);
+                }
+            }
+        }
+    }
+
+    @VisibleForTesting
+    private static Table createTable(Admin admin, TableName tableName) throws IOException {
+        if (!admin.tableExists(tableName)) {
+            ColumnFamilyDescriptor columnDescriptor =
+                    ColumnFamilyDescriptorBuilder
+                            .newBuilder(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)
+                            .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL)
+                            .build();
+            TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
+                    .setColumnFamily(columnDescriptor).build();
+            try {
+                admin.createTable(tableDescriptor);
+            } catch (TableExistsException e) {
+                LOGGER.warn("Table exists, ignoring", e);
+            }
+        }
+        return admin.getConnection().getTable(tableName);
+    }
+
+
+    public static void createNewIndexToolTables(Connection connection) throws Exception {
+        setIndexToolTableName(connection);
+
+        migrateTable(connection, OUTPUT_TABLE_NAME);
+        migrateTable(connection, RESULT_TABLE_NAME);
+    }
+
+    private static void migrateTable(Connection connection, String tableName) throws Exception {
+        if (!tableName.equals(OUTPUT_TABLE_NAME) && !tableName.equals(RESULT_TABLE_NAME)) {
+            LOGGER.info("Only migrating PHOENIX_INDEX_TOOL tables!");
+        } else {
+            ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
+            try (Admin admin = queryServices.getAdmin()) {
+                TableName oldTableName = TableName.valueOf(tableName);
+                String newTableNameString = tableName.equals(OUTPUT_TABLE_NAME) ?
+                        SYSTEM_OUTPUT_TABLE_NAME : SYSTEM_RESULT_TABLE_NAME;
+
+                TableName newTableName = TableName.valueOf(newTableNameString);
+
+                if (admin.tableExists(oldTableName)) {
+                    String snapshotName = tableName + "_" + UUID.randomUUID();
+                    admin.disableTable(oldTableName);
+                    admin.snapshot(snapshotName, oldTableName);
+                    admin.cloneSnapshot(snapshotName, newTableName);
+                    admin.deleteSnapshot(snapshotName);
+                    admin.deleteTable(oldTableName);
+                } else {
+                    createTable(admin, newTableName);
+                }
+            }
+        }
+    }
+}

phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java

@@ -323,6 +323,8 @@
 import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
 import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
 
+import org.apache.phoenix.mapreduce.index.IndexToolTableUtil;
+
 public class ConnectionQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices {
     private static final Logger LOGGER =
             LoggerFactory.getLogger(ConnectionQueryServicesImpl.class);
@@ -4025,6 +4027,12 @@ private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQ
         try {
             metaConnection.createStatement().executeUpdate(getCDCStreamDDL());
         } catch (TableAlreadyExistsException ignore) {}
+        try {
+            // check if we have old PHOENIX_INDEX_TOOL tables
+            // move data to the new tables under System, or simply create the new tables
+            IndexToolTableUtil.createNewIndexToolTables(metaConnection);
+
+        } catch (Exception ignore) {}
     }
 
     /**
@@ -4589,6 +4597,13 @@ public void upgradeSystemTables(final String url, final Properties props) throws
             // with SYSTEM Namespace
             createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
 
+            try {
+                // check if we have old PHOENIX_INDEX_TOOL tables
+                // move data to the new tables under System, or simply create the new tables
+                IndexToolTableUtil.createNewIndexToolTables(metaConnection);
+
+            } catch (Exception ignore) {}
+
             clearUpgradeRequired();
             success = true;
         } catch (UpgradeInProgressException | UpgradeNotRequiredException e) {

phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java

@@ -38,6 +38,8 @@
 import java.util.Map;
 import java.util.UUID;
 
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.phoenix.thirdparty.com.google.common.base.Strings;
 import org.apache.phoenix.hbase.index.AbstractValueGetter;
@@ -985,10 +987,22 @@ private void setupIndexAndDataTable(Connection connection) throws SQLException,
                             indexTable, qDataTable));
         }
         qSchemaName = SchemaUtil.normalizeIdentifier(schemaName);
+
         pIndexTable = connection.unwrap(PhoenixConnection.class).getTable(
             SchemaUtil.getQualifiedTableName(schemaName, indexTable));
+        if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, getConf())) {
+            pIndexTable = connection.unwrap(PhoenixConnection.class).getTable(
+                    SchemaUtil.getQualifiedTableName(schemaName, indexTable).replace(
+                        QueryConstants.NAME_SEPARATOR,
+                        QueryConstants.NAMESPACE_SEPARATOR));
+        }
+
         indexType = pIndexTable.getIndexType();
         qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable);
+        if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, getConf())) {
+            qIndexTable = qIndexTable.replace(QueryConstants.NAME_SEPARATOR, QueryConstants.NAMESPACE_SEPARATOR);
+        }
+
         if (IndexType.LOCAL.equals(indexType)) {
             isLocalIndexBuild = true;
             if (useSnapshot) {

phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java

@@ -19,20 +19,12 @@
 
 import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
 
-import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -60,8 +52,8 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
         IndexTool.IndexDisableLoggingType.NONE;
     private boolean shouldLogBeyondMaxLookback = true;
 
-    public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
-    public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_NAME);
+    public final static String OUTPUT_TABLE_NAME = IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME;
+    public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME);
     public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
 
     public final static String DATA_TABLE_NAME = "DTName";
@@ -177,26 +169,7 @@ private static byte[] generatePartialOutputTableRowKey(long ts, byte[] indexTabl
     }
 
     public void createOutputTable(Connection connection) throws IOException, SQLException {
-        ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
-        try (Admin admin = queryServices.getAdmin()) {
-            TableName outputTableName = TableName.valueOf(OUTPUT_TABLE_NAME);
-            if (!admin.tableExists(outputTableName)) {
-                ColumnFamilyDescriptor columnDescriptor =
-                    ColumnFamilyDescriptorBuilder
-                        .newBuilder(OUTPUT_TABLE_COLUMN_FAMILY)
-                        .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL)
-                        .build();
-                TableDescriptor tableDescriptor = TableDescriptorBuilder
-                    .newBuilder(TableName.valueOf(OUTPUT_TABLE_NAME))
-                    .setColumnFamily(columnDescriptor).build();
-                try {
-                    admin.createTable(tableDescriptor);
-                } catch (TableExistsException e) {
-                    LOGGER.warn("Table exists, ignoring", e);
-                }
-                outputTable = admin.getConnection().getTable(outputTableName);
-            }
-        }
+        outputTable = IndexToolTableUtil.createOutputTable(connection);
     }
 
     @VisibleForTesting

phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java

@@ -19,27 +19,18 @@
 
 import org.apache.hadoop.hbase.Cell;
 
-import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.coprocessor.IndexToolVerificationResult;
-import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.query.ConnectionQueryServices;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.util.ByteUtil;
 import org.slf4j.Logger;
@@ -58,8 +49,8 @@ public class IndexVerificationResultRepository implements AutoCloseable {
     private Table indexTable;
     public static final String ROW_KEY_SEPARATOR = "|";
     public static final byte[] ROW_KEY_SEPARATOR_BYTE = Bytes.toBytes(ROW_KEY_SEPARATOR);
-    public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
-    public final static byte[] RESULT_TABLE_NAME_BYTES = Bytes.toBytes(RESULT_TABLE_NAME);
+    public static String RESULT_TABLE_NAME = IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME;
+    public static byte[] RESULT_TABLE_NAME_BYTES = Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME);
     public final static byte[] RESULT_TABLE_COLUMN_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
     public final static String SCANNED_DATA_ROW_COUNT = "ScannedDataRowCount";
     public final static byte[] SCANNED_DATA_ROW_COUNT_BYTES = Bytes.toBytes(SCANNED_DATA_ROW_COUNT);
@@ -152,37 +143,18 @@ public IndexVerificationResultRepository(){
     }
 
     public IndexVerificationResultRepository(Connection conn, byte[] indexNameBytes) throws SQLException {
-        resultTable = getTable(conn, RESULT_TABLE_NAME_BYTES);
+        resultTable = getTable(conn, Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME));
         indexTable = getTable(conn, indexNameBytes);
     }
 
     public IndexVerificationResultRepository(byte[] indexName,
                                              HTableFactory hTableFactory) throws IOException {
-        resultTable = hTableFactory.getTable(new ImmutableBytesPtr(RESULT_TABLE_NAME_BYTES));
+        resultTable = hTableFactory.getTable(new ImmutableBytesPtr(Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME)));
         indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName));
     }
 
     public void createResultTable(Connection connection) throws IOException, SQLException {
-        ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
-        try (Admin admin = queryServices.getAdmin()) {
-            TableName resultTableName = TableName.valueOf(RESULT_TABLE_NAME);
-            if (!admin.tableExists(resultTableName)) {
-                ColumnFamilyDescriptor columnDescriptor =
-                    ColumnFamilyDescriptorBuilder
-                        .newBuilder(RESULT_TABLE_COLUMN_FAMILY)
-                        .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL)
-                        .build();
-                TableDescriptor tableDescriptor =
-                    TableDescriptorBuilder.newBuilder(resultTableName)
-                        .setColumnFamily(columnDescriptor).build();
-                try {
-                    admin.createTable(tableDescriptor);
-                } catch (TableExistsException e) {
-                    LOGGER.warn("Table exists, ignoring", e);
-                }
-                resultTable = admin.getConnection().getTable(resultTableName);
-            }
-        }
+        resultTable = IndexToolTableUtil.createResultTable(connection);
     }
 
     private static byte[] generatePartialResultTableRowKey(long ts, byte[] indexTableName) {
@@ -356,7 +328,7 @@ private IndexToolVerificationResult aggregateVerificationResult(
 
     public IndexToolVerificationResult getVerificationResult(Connection conn,
             long ts, byte[] indexTableName) throws IOException, SQLException {
-        try (Table hTable = getTable(conn, RESULT_TABLE_NAME_BYTES)) {
+        try (Table hTable = getTable(conn, Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME))) {
             byte[] startRowKey = generatePartialResultTableRowKey(ts,
                 indexTableName);
             byte[] stopRowKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix(

phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java

@@ -35,8 +35,10 @@
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.mapreduce.util.ConnectionUtil;
 import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.task.Task;
 import org.apache.phoenix.schema.transform.Transform;
 import org.apache.phoenix.util.SchemaUtil;

phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java

@@ -217,8 +217,8 @@ public void initUsersAndTables() {
         view2TableName = tableName + "_V2";
     }
 
-    private static void enablePhoenixHBaseAuthorization(Configuration config,
-                                                        boolean useCustomAccessController) {
+    static void enablePhoenixHBaseAuthorization(Configuration config,
+                                                boolean useCustomAccessController) {
         config.set("hbase.superuser", SUPER_USER + "," + "superUser2");
         config.set("hbase.security.authorization", Boolean.TRUE.toString());
         config.set("hbase.security.exec.permission.checks", Boolean.TRUE.toString());
@@ -242,11 +242,11 @@ private static void enablePhoenixHBaseAuthorization(Configuration config,
         config.set("hbase.regionserver.wal.codec", "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec");
     }
 
-    private static void configureNamespacesOnServer(Configuration conf, boolean isNamespaceMapped) {
+    static void configureNamespacesOnServer(Configuration conf, boolean isNamespaceMapped) {
         conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(isNamespaceMapped));
     }
 
-    private static void configureStatsConfigurations(Configuration conf) {
+    static void configureStatsConfigurations(Configuration conf) {
         conf.set(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
         conf.set(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
         conf.set(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));

phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java

@@ -847,10 +847,20 @@ private static List<String> getArgList (boolean useSnapshot, String schemaName,
     }
 
     private static List<String> getArgList (boolean useSnapshot, String schemaName,
-                            String dataTable, String indxTable, String tenantId,
-                            IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime,
+                                            String dataTable, String indxTable, String tenantId,
+                                            IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime,
                                             IndexTool.IndexDisableLoggingType disableLoggingType,
                                             Long incrementalVerify, boolean useIndexTableAsSource) {
+        return getArgList(useSnapshot, schemaName, dataTable, indxTable, tenantId,verifyType, startTime,
+                endTime, disableLoggingType, incrementalVerify, useIndexTableAsSource, "/tmp/" + UUID.randomUUID().toString());
+    }
+
+    private static List<String> getArgList (boolean useSnapshot, String schemaName,
+                                            String dataTable, String indxTable, String tenantId,
+                                            IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime,
+                                            IndexTool.IndexDisableLoggingType disableLoggingType,
+                                            Long incrementalVerify, boolean useIndexTableAsSource,
+                                            String outputPath) {
         List<String> args = Lists.newArrayList();
         if (schemaName != null) {
             args.add("--schema=" + schemaName);
@@ -895,7 +905,7 @@ private static List<String> getArgList (boolean useSnapshot, String schemaName,
         }
 
         args.add("-op");
-        args.add("/tmp/" + UUID.randomUUID().toString());
+        args.add(outputPath);
         return args;
     }
 
@@ -914,6 +924,14 @@ public static String[] getArgValues(boolean useSnapshot, String schemaName,
         return args.toArray(new String[0]);
     }
 
+    public static String[] getArgValues(boolean useSnapshot, String schemaName,
+                                        String dataTable, String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType,
+                                        IndexTool.IndexDisableLoggingType disableLoggingType, String outputPath) {
+        List<String> args = getArgList(useSnapshot, schemaName, dataTable, indexTable,
+                tenantId, verifyType, null, null, disableLoggingType, null, false, outputPath);
+        return args.toArray(new String[0]);
+    }
+
     public static String [] getArgValues(boolean useSnapshot, String schemaName,
             String dataTable, String indexTable, String tenantId,
             IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime) {
@@ -1010,8 +1028,31 @@ public static IndexTool runIndexTool(Configuration conf, boolean useSnapshot, St
         IndexTool indexingTool = new IndexTool();
         conf.set(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
         indexingTool.setConf(conf);
-        final String[] cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName,
-            indexTableName, tenantId, verifyType, disableLoggingType);
+        boolean additionalArgsContainPath = false;
+        String path = "";
+        List<String> newadditionalArgs = Lists.newArrayList();
+        for (String arg : additionalArgs){
+            if (additionalArgsContainPath == true) {
+                path = arg;
+            }
+            else if ( arg.equals("-op") || arg.equals("-output-path")) {
+                additionalArgsContainPath = true;
+            }
+            else {
+                newadditionalArgs.add(arg);
+            }
+        }
+        additionalArgs = newadditionalArgs.toArray(new String[0]);
+
+        String[] cmdArgs;
+        if (additionalArgsContainPath) {
+            cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName,
+                    indexTableName, tenantId, verifyType, disableLoggingType, path);
+        }
+        else {
+            cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName,
+                    indexTableName, tenantId, verifyType, disableLoggingType);
+        }
         List<String> cmdArgList = new ArrayList<>(Arrays.asList(cmdArgs));
         cmdArgList.addAll(Arrays.asList(additionalArgs));
         LOGGER.info("Running IndexTool with {}", Arrays.toString(cmdArgList.toArray()), new Exception("Stack Trace"));

phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java

@@ -74,22 +74,13 @@ public abstract class LoadSystemTableSnapshotBase extends BaseTest {
     private static final Logger LOGGER = LoggerFactory.getLogger(
             LoadSystemTableSnapshotBase.class);
 
-    public static final String SNAPSHOT_DIR = "snapshots4_7/";
+    public static String SNAPSHOT_DIR;
     public static String rootDir;
 
-    private static final HashMap<String, String> SNAPSHOTS_TO_LOAD;
+    private static HashMap<String, String> SNAPSHOTS_TO_LOAD;
 
     public static final byte[] MUTEX_LOCKED = "MUTEX_LOCKED".getBytes(StandardCharsets.UTF_8);
 
-    static {
-        SNAPSHOTS_TO_LOAD = new HashMap<>();
-        //Add any HBase tables, including Phoenix System tables
-
-        SNAPSHOTS_TO_LOAD.put("SYSTEM.CATALOG_SNAPSHOT", "SYSTEM.CATALOG");
-        SNAPSHOTS_TO_LOAD.put("SYSTEM.FUNCTION_SNAPSHOT", "SYSTEM.FUNCTION");
-        SNAPSHOTS_TO_LOAD.put("SYSTEM.SEQUENCE_SNAPSHOT", "SYSTEM.SEQUENCE");
-        SNAPSHOTS_TO_LOAD.put("SYSTEM.STATS_SNAPSHOT", "SYSTEM.STATS");
-    }
 
     private static void decompress(String in, File out) throws IOException {
         try (TarArchiveInputStream fin = new TarArchiveInputStream(new FileInputStream(in))){
@@ -109,11 +100,24 @@ private static void decompress(String in, File out) throws IOException {
     }
 
     public static synchronized void setupCluster(boolean createBlockUpgradeMutex) throws Exception {
+        //Add any HBase tables, including Phoenix System tables
+        HashMap<String, String> snapshotsToLoad = new HashMap<>();
+        snapshotsToLoad.put("SYSTEM.CATALOG_SNAPSHOT", "SYSTEM.CATALOG");
+        snapshotsToLoad.put("SYSTEM.FUNCTION_SNAPSHOT", "SYSTEM.FUNCTION");
+        snapshotsToLoad.put("SYSTEM.SEQUENCE_SNAPSHOT", "SYSTEM.SEQUENCE");
+        snapshotsToLoad.put("SYSTEM.STATS_SNAPSHOT", "SYSTEM.STATS");
+
+        setupCluster(createBlockUpgradeMutex, "snapshots47.tar.gz", "snapshots4_7/", snapshotsToLoad, "true");
+    }
+
+    public static synchronized void setupCluster(boolean createBlockUpgradeMutex, String tarName, String snapshotDir, HashMap<String, String> snapshotsToLoad, String nameSpaceMapping) throws Exception {
+        SNAPSHOT_DIR = snapshotDir;
+        SNAPSHOTS_TO_LOAD = snapshotsToLoad;
         Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
         serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
-        serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+        serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping);
         Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
-        clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+        clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping);
 
         //Start minicluster without Phoenix first
         checkClusterInitialized(new ReadOnlyProps(serverProps.entrySet().iterator()));
@@ -122,7 +126,7 @@ public static synchronized void setupCluster(boolean createBlockUpgradeMutex) th
                 .getResource(SNAPSHOT_DIR);
 
         // extract the tar
-        File archive = new File(folderUrl.getFile() + "snapshots47.tar.gz");
+        File archive = new File(folderUrl.getFile() + tarName);
         File destination = new File(folderUrl.getFile());
 
         decompress(archive.toString(), destination);

phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java

@@ -64,6 +64,11 @@
 @Category(NeedsOwnMiniClusterTest.class)
 public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
 
+    private static final Set<String> HBASE_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
+        "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL"));
+    private static final Set<String> HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
+            Arrays.asList("SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL"));
+
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
             "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM",
@@ -191,7 +196,7 @@ public Void run() throws Exception {
         });
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() + HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
 
         try {
@@ -208,7 +213,7 @@ public Void run() throws Exception {
         }
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() + HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
     }
 
@@ -226,7 +231,7 @@ public Void run() throws Exception {
         });
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
 
         user2.doAs(new PrivilegedExceptionAction<Void>() {
@@ -239,7 +244,7 @@ public Void run() throws Exception {
         });
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
 
         try {
@@ -258,7 +263,7 @@ public Void run() throws Exception {
         }
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
 
         user2.doAs(new PrivilegedExceptionAction<Void>() {
@@ -271,7 +276,7 @@ public Void run() throws Exception {
         });
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
 
         user3.doAs(new PrivilegedExceptionAction<Void>() {
@@ -283,7 +288,7 @@ public Void run() throws Exception {
         });
 
         hbaseTables = getHBaseTables();
-        assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
+        assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() + HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
         assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
     }
 

phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java

@@ -78,12 +78,15 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.junit.runners.MethodSorters;
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
 @Category(NeedsOwnMiniClusterTest.class)
 public class SystemTablesCreationOnConnectionIT {
 
@@ -112,12 +115,12 @@ public class SystemTablesCreationOnConnectionIT {
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
             "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK","SYSTEM.TRANSFORM",
-            "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+            "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL"));
 
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
             Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
                     "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM",
-                    "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM"));
+                    "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL"));
 
     private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
 
@@ -345,7 +348,7 @@ public void testUpgradeNotAllowed() throws Exception {
     // Expected: We will migrate all SYSTEM\..* tables to the SYSTEM namespace and also upgrade
     // SYSTEM:CATALOG
     @Test
-    public void testMigrateToSystemNamespaceAndUpgradeSysCat() throws Exception {
+    public void testABCFirstMigrateToSystemNamespaceAndUpgradeSysCat() throws Exception {
         setOldTimestampToInduceUpgrade = true;
         PhoenixSysCatCreationTestingDriver driver =
                 firstConnNSMappingServerEnabledClientEnabledMappingDisabled();

phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+
+import org.apache.phoenix.mapreduce.index.IndexToolTableUtil;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Before;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.phoenix.mapreduce.index.IndexToolTableUtil.RESULT_TABLE_NAME;
+import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
+import static org.junit.Assert.*;
+
+public class UpgradeIndexToolTablesIT extends LoadSystemTableSnapshotBase {
+    protected String nameSpaceMapping = "true";
+
+    @Before
+    public synchronized void doSetup() throws Exception {
+        setupCluster(nameSpaceMapping);
+    }
+
+    public synchronized void setupCluster(String nameSpaceMappingEnabled) throws Exception {
+        HashMap<String, String> snapshotsToLoad = new HashMap<>();
+        snapshotsToLoad.put("phoenixtoolresultsnapshot", "PHOENIX_INDEX_TOOL_RESULT");
+        setupCluster(false, "indexToolsnapshot.tar.gz", "indexToolResultSnapshot/",  snapshotsToLoad, nameSpaceMappingEnabled);
+    }
+
+    @Test
+    public void testPhoenixUpgradeIndexToolTables() throws Exception {
+        try (Admin admin = utility.getAdmin()) {
+            // we load the  RESULT_TABLE_NAME from snapshot
+            assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_NAME)));
+            assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME)));
+            // we don't load the OUTPUT_TABLE_NAME
+            assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_NAME)));
+            assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME)));
+        }
+
+        Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+        serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+        serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping);
+        Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+        clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping);
+
+
+        //Now we can start Phoenix
+        setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet()
+                .iterator()));
+        assertTrue(true);
+
+
+        // Check the IndexTool Tables after upgrade
+        try (Admin admin = utility.getAdmin()) {
+            assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_NAME)));
+            assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_NAME)));
+            assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME)));
+            assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME)));
+        }
+
+        String tableName = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME);
+        if (nameSpaceMapping.equals("true")) {
+            assertEquals(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME, tableName.replace(QueryConstants.NAME_SEPARATOR,
+                    QueryConstants.NAMESPACE_SEPARATOR));
+        } else {
+            assertEquals(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME, tableName);
+        }
+
+    }
+
+}

phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.junit.Before;
+
+
+public class UpgradeIndexToolTablesNameSpaceMappingDisabledIT extends UpgradeIndexToolTablesIT {
+
+    @Override
+    @Before
+    public synchronized void doSetup() throws Exception {
+        nameSpaceMapping = "false";
+        setupCluster(nameSpaceMapping);
+    }
+}