IMPALA-8373: Test impala-shell with python3

Sets up a python3 virtualenv, installs impala-shell into it, and runs
tests.

Change-Id: I8e123aecd53a7ded44a7da7eb8c8b853cebbfc56
Reviewed-on: http://gerrit.cloudera.org:8080/18588
Reviewed-by: Joe McDonnell <[email protected]>
Tested-by: Joe McDonnell <[email protected]>

Author: MikaelSmith

CMakeLists.txt

@@ -467,8 +467,10 @@ add_custom_target(impala_python ALL
   COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
 )
 
+add_custom_target(impala_shell_pypi ALL DEPENDS shell_python2_install shell_python3_install)
+
 add_custom_target(notests_independent_targets DEPENDS
-  java cscope tarballs impala_python shell_python2_install
+  java cscope tarballs impala_python impala_shell_pypi
 )
 add_custom_target(notests_regular_targets DEPENDS
   impalad statestored catalogd admissiond fesupport loggingsupport ImpalaUdf udasample udfsample impala-profile-tool

bin/bootstrap_build.sh

@@ -32,7 +32,7 @@ set -euxo pipefail
 # Install non-java dependencies:
 sudo apt-get update
 sudo apt-get --yes install g++ gcc git libsasl2-dev libssl-dev make \
-    python-dev python-setuptools libffi-dev libkrb5-dev
+    python-dev python-setuptools python3-dev libffi-dev libkrb5-dev
 
 
 source /etc/lsb-release

bin/bootstrap_system.sh

@@ -218,7 +218,7 @@ ubuntu apt-get update
 ubuntu apt-get --yes install ccache curl gawk g++ gcc libffi-dev \
         libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \
         libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \
-        python-dev python-setuptools postgresql ssh wget vim-common psmisc \
+        python-dev python-setuptools python3-dev postgresql ssh wget vim-common psmisc \
         lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg apt-utils git ant
 # Required by Kudu in the minicluster
 ubuntu20 apt-get --yes install libtinfo5
@@ -266,7 +266,7 @@ redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \
         postgresql postgresql-server \
         wget vim-common nscd cmake fuse-devel zlib-devel \
         psmisc lsof openssh-server redhat-lsb java-1.8.0-openjdk-devel \
-        java-1.8.0-openjdk-src
+        java-1.8.0-openjdk-src python3-devel
 
 # Enable the Powertools repo for snappy-devel on RedHat 8
 redhat8 sudo yum install -y dnf-plugins-core

shell/CMakeLists.txt

@@ -35,11 +35,20 @@ add_custom_target(shell_pypi_test_package DEPENDS shell_tarball
 )
 
 set(PYTHON2_VENV "${CMAKE_SOURCE_DIR}/shell/build/py2_venv")
+set(PYTHON3_VENV "${CMAKE_SOURCE_DIR}/shell/build/py3_venv")
 
 add_custom_command(OUTPUT "${PYTHON2_VENV}" DEPENDS impala_python
   COMMAND impala-virtualenv --python python2 "${PYTHON2_VENV}"
 )
 
-add_custom_target(shell_python2_install ALL DEPENDS "${PYTHON2_VENV}" shell_pypi_test_package
+add_custom_target(shell_python2_install DEPENDS "${PYTHON2_VENV}" shell_pypi_test_package
   COMMAND "${PYTHON2_VENV}/bin/pip" install "${SHELL_TEST_PKG}"
 )
+
+add_custom_command(OUTPUT "${PYTHON3_VENV}" DEPENDS impala_python
+  COMMAND impala-virtualenv --python python3 "${PYTHON3_VENV}"
+)
+
+add_custom_target(shell_python3_install DEPENDS "${PYTHON3_VENV}" shell_pypi_test_package
+  COMMAND "${PYTHON3_VENV}/bin/pip" install "${SHELL_TEST_PKG}"
+)

tests/shell/test_shell_commandline.py

@@ -145,7 +145,7 @@ def add_test_dimensions(cls):
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
     cls.ImpalaTestMatrix.add_constraint(lambda v:
         v.get_value('protocol') != 'beeswax' or not v.get_value('strict_hs2_protocol'))
-    # Test with python2 and the raw tarball
+    # Test with python2, python3, and the raw tarball
     cls.ImpalaTestMatrix.add_dimension(create_impala_shell_executable_dimension())
 
   def test_no_args(self, vector):

tests/shell/test_shell_interactive.py

@@ -176,7 +176,7 @@ def add_test_dimensions(cls):
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
     cls.ImpalaTestMatrix.add_constraint(lambda v:
         v.get_value('protocol') != 'beeswax' or not v.get_value('strict_hs2_protocol'))
-    # Test with python2 and the raw tarball
+    # Test with python2, python3, and the raw tarball
     cls.ImpalaTestMatrix.add_dimension(create_impala_shell_executable_dimension())
 
   def _expect_with_cmd(self, proc, cmd, vector, expectations=(), db="default"):

tests/shell/util.py

@@ -324,7 +324,7 @@ def get_dev_impala_shell_executable():
 def create_impala_shell_executable_dimension():
   _, include_pypi = get_dev_impala_shell_executable()
   if include_pypi:
-    return ImpalaTestDimension('impala_shell', 'dev', 'python2')
+    return ImpalaTestDimension('impala_shell', 'dev', 'python2', 'python3')
   else:
     return ImpalaTestDimension('impala_shell', 'dev')
 
@@ -335,5 +335,6 @@ def get_impala_shell_executable(vector):
   impala_shell_executable, _ = get_dev_impala_shell_executable()
   return {
     'dev': impala_shell_executable,
-    'python2': os.path.join(IMPALA_HOME, 'shell/build/py2_venv/bin/impala-shell')
+    'python2': os.path.join(IMPALA_HOME, 'shell/build/py2_venv/bin/impala-shell'),
+    'python3': os.path.join(IMPALA_HOME, 'shell/build/py3_venv/bin/impala-shell')
   }[vector.get_value_with_default('impala_shell', 'dev')]