latest script generated from templates

Author: cjac

dask/dask.sh

@@ -14,15 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#
-#
-# Google Cloud Dataproc Initialization Actions v0.0.1
 #
 # This initialization action is generated from
 # initialization-actions/templates/dask/dask.sh.in
 #
-# Modifications made directly to generated files will be lost when the
-# templates are next evaluated.
+# Modifications made directly to the generated file will be lost when
+# the template is re-evaluated
 
 #
 # This initialization action script will install Dask and other relevant
@@ -363,25 +360,6 @@ function check_os() {
       exit 1
   fi
 
-  SPARK_VERSION="$(spark-submit --version 2>&1 | sed -n 's/.*version[[:blank:]]\+\([0-9]\+\.[0-9]\).*/\1/p' | head -n1)"
-  readonly SPARK_VERSION
-  if version_lt "${SPARK_VERSION}" "3.1" || \
-     version_ge "${SPARK_VERSION}" "4.0" ; then
-    echo "Error: Your Spark version is not supported. Please upgrade Spark to one of the supported versions."
-    exit 1
-  fi
-
-  # Detect dataproc image version
-  if (! test -v DATAPROC_IMAGE_VERSION) ; then
-    if test -v DATAPROC_VERSION ; then
-      DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
-    else
-      if   version_lt "${SPARK_VERSION}" "3.2" ; then DATAPROC_IMAGE_VERSION="2.0"
-      elif version_lt "${SPARK_VERSION}" "3.4" ; then DATAPROC_IMAGE_VERSION="2.1"
-      elif version_lt "${SPARK_VERSION}" "3.6" ; then DATAPROC_IMAGE_VERSION="2.2"
-      else echo "Unknown dataproc image version" ; exit 1 ; fi
-    fi
-  fi
 }
 
 function configure_dkms_certs() {
@@ -544,42 +522,30 @@ function prepare_conda_env() {
 }
 
 function prepare_common_env() {
-  define_os_comparison_functions
-
   # Verify OS compatability and Secure boot state
   check_os
   check_secure_boot
 
-  readonly _shortname="$(os_id)$(os_version|perl -pe 's/(\d+).*/$1/')"
-
-  # Dataproc configurations
-  readonly HADOOP_CONF_DIR='/etc/hadoop/conf'
-  readonly HIVE_CONF_DIR='/etc/hive/conf'
-  readonly SPARK_CONF_DIR='/etc/spark/conf'
-
+  # read-only configuration variables
+  _shortname="$(os_id)$(os_version|perl -pe 's/(\d+).*/$1/')"
+  HADOOP_CONF_DIR='/etc/hadoop/conf'
+  HIVE_CONF_DIR='/etc/hive/conf'
   OS_NAME="$(lsb_release -is | tr '[:upper:]' '[:lower:]')"
-  readonly OS_NAME
-
-  # node role
   ROLE="$(get_metadata_attribute dataproc-role)"
-  readonly ROLE
-
-  # master node
   MASTER="$(get_metadata_attribute dataproc-master)"
-  readonly MASTER
-
   workdir=/opt/install-dpgce
-  tmpdir=/tmp/
   temp_bucket="$(get_metadata_attribute dataproc-temp-bucket)"
-  readonly temp_bucket
-  readonly pkg_bucket="gs://${temp_bucket}/dpgce-packages"
+  pkg_bucket="gs://${temp_bucket}/dpgce-packages"
   uname_r=$(uname -r)
-  readonly uname_r
-  readonly bdcfg="/usr/local/bin/bdconfig"
-  export DEBIAN_FRONTEND=noninteractive
+  bdcfg="/usr/local/bin/bdconfig"
+  KNOX_HOME=/usr/lib/knox
 
-  # Knox config
-  readonly KNOX_HOME=/usr/lib/knox
+  readonly HADOOP_CONF_DIR HIVE_CONF_DIR OS_NAME ROLE MASTER workdir
+  readonly temp_bucket pkg_bucket uname_r bdconfig KNOX_HOME
+
+  tmpdir=/tmp/
+
+  export DEBIAN_FRONTEND=noninteractive
 
   mkdir -p "${workdir}/complete"
   set_proxy
@@ -720,6 +686,8 @@ print( "    samples-taken: ", scalar @siz, $/,
   echo "exit_handler has completed"
 }
 
+define_os_comparison_functions
+
 
 function configure_dask_yarn() {
   readonly DASK_YARN_CONFIG_DIR=/etc/dask/