Added dev_pkg spack environment, changed buildcache generation so it no longer uses upstream spack instance and uses the dev_pkg spack environment, all this is necessary because we not longer directly change the spack instance with uberenv, updated spheral spack package to allow for spack to correctly do an install

Author: ldowen

.gitlab-ci.yml

@@ -10,6 +10,7 @@ variables:
   SPHERAL_MODULE: "Spheral"
   SPACK_NAME: spheral
   SPACK_PKG_NAME: spheral
+  DEV_PKG_SPEC: "spheral@develop%gcc+mpi~caliper~network"
 
 .spheral_rev_str:
   before_script:

.gitlab/scripts.yml

@@ -145,7 +145,7 @@
     - INSTALL_DIR=/usr/gapps/Spheral/$SYS_TYPE/spheral-$SPHERAL_REV_STR
     - DEV_PKG_NAME=$SYS_TYPE-spheral-dev-pkg-$SPHERAL_REV_STR
 
-    - env SPHERAL_REV_STR=$SPHERAL_REV_STR STAGE_DIR=$INSTALL_DIR SPEC=$SPEC SPACK_PKG_NAME=$SPACK_PKG_NAME SCRIPT_DIR=$SCRIPT_DIR
+    - env SPHERAL_REV_STR=$SPHERAL_REV_STR STAGE_DIR=$INSTALL_DIR SPACK_PKG_NAME=$SPACK_PKG_NAME SCRIPT_DIR=$SCRIPT_DIR
       bash ./$SCRIPT_DIR/lc/generate-buildcache.sh
 
     - echo $INSTALL_DIR &> install-dir.txt
@@ -169,7 +169,7 @@
     - tar -xzf $DEV_PKG_NAME.tar.gz
     - cd $DEV_PKG_NAME
 
-    - env INSTALL_DIR=$INSTALL_DIR SPEC=$SPEC SPACK_PKG_NAME=$SPACK_PKG_NAME BUILD_ALLOC="" SCRIPT_DIR=$SCRIPT_DIR
+    - env INSTALL_DIR=$INSTALL_DIR DEV_PKG_SPEC=$DEV_PKG_SPEC SPACK_PKG_NAME=$SPACK_PKG_NAME BUILD_ALLOC="" SCRIPT_DIR=$SCRIPT_DIR
       bash ./$SCRIPT_DIR/lc/install-from-dev-pkg.sh
 
   artifacts:

scripts/devtools/tpl-manager.py

@@ -67,6 +67,8 @@ def parse_args(self):
                             help="Use to do everything but actually install. For testing purposes.")
         parser.add_argument("--id", type=str, default=None,
                             help="ID string to postfix an initconfig file.")
+        parser.add_argument("--dev-pkg", action="store_true",
+                            help="Tells tpl-manager to use the dev_pkg environment.")
 
         self.args = parser.parse_args()
 
@@ -223,6 +225,8 @@ def activate_spack_env(self):
         config_env_dir = os.path.join(get_config_dir(base_dir), "environments")
         # Check if we are on an LC machine and the environment exists
         default_env = os.getenv("SYS_TYPE")
+        if (self.args.dev_pkg):
+            default_env = "dev_pkg"
         if default_env and os.path.exists(os.path.join(config_env_dir, default_env)):
             # For LC systems
             env_dir = os.path.join(config_env_dir, default_env)

scripts/lc/generate-buildcache.sh

@@ -17,18 +17,12 @@ trap 'echo "# $BASH_COMMAND"' DEBUG
 # The expected spack package name for what we are packing up.
 SPACK_PKG_NAME=${SPACK_PKG_NAME:-spheral}
 
-# What spec are we targetting.
-SPEC=${SPEC:-gcc@10.3.1}
-
 # What is the version of spehral.
 SPHERAL_REV_STR=${SPHERAL_REV_STR:-undefined}
 
 # Where will we be staging the package as it is being compiled together.
 STAGE_DIR=${STAGE_DIR:-$PWD/../$SYS_TYPE/spheral-cache}
 
-# Where is the spack upstream located.
-UPSTREAM_DIR=${UPSTREAM_DIR:-/usr/WS2/sduser/Spheral/spack_upstream/0.22}
-
 # Where does the spheral pip_cache dir live.
 SPHERAL_PIP_CACHE_DIR=${SPHERAL_PIP_CACHE_DIR:-~/.cache/spheral_pip}
 
@@ -49,15 +43,11 @@ DEV_PKG_NAME=$SYS_TYPE-spheral-dev-pkg-$SPHERAL_REV_STR
 # Full path of what the package directory will look like as we compiler the dev pkg.
 DEV_PKG_DIR=$STAGE_DIR/$DEV_PKG_NAME
 
-# Full Spack spec.
-SPHERAL_SPEC=$SPACK_PKG_NAME@develop$SPEC
-
 # RESOURCE_DIR is a directory created internally to maintain spack & pip
 # resources required for building and running Spheral
 RESOURCE_DIR=$DEV_PKG_DIR/resources
 
 # Print for sanity check.
-echo $SPHERAL_SPEC
 echo $RESOURCE_DIR
 echo $STAGE_DIR
 echo $DEV_PKG_DIR
@@ -72,23 +62,20 @@ mkdir -p $RESOURCE_DIR && cp -a $CI_PROJECT_DIR/. $DEV_PKG_DIR
 mkdir -p $RESOURCE_DIR/pip_cache
 cp -a $SPHERAL_PIP_CACHE_DIR/. $RESOURCE_DIR/pip_cache
 
-# Initialize the upstream spack repo.
-./$SCRIPT_DIR/devtools/tpl-manager.py --init-only --spack-dir=$UPSTREAM_DIR
-source $UPSTREAM_DIR/spack/share/spack/setup-env.sh
-
-# Delete any semblance of a spack env in the STAGE_DIR.
-spack env rm -y -f $STAGE_DIR
+# tpl-manager --dev-pkg does the following:
+# Creates a local Spack repo
+# Activates and concretizes the dev_pkg Spheral Spack environment
+# Installs the Spheral dependencies
+./$SCRIPT_DIR/devtools/tpl-manager.py --dev-pkg
 
-# Create a spack env in STAGE_DIR and activate it.
-spack env create -d $STAGE_DIR
-spack env activate $STAGE_DIR
+# Source Spack for the current terminal
+source ../spheral-spack-tpls/spack/share/spack/setup-env.sh
 
-# Concretize our targetted SPHERAL_SPEC.
-spack add $SPHERAL_SPEC
-spack concretize -f --fresh --deprecated
+# Activate our dev spack environment
+spack env activate ./$SCRIPT_DIR/spack/environments/dev_pkg
 
 # Create a mirror of all tpl specs in our environment
-# (should only be our deps for SPHERAL_SPEC in the env).
+# (should only be our deps for a single spec in the env).
 spack mirror create -a -d $RESOURCE_DIR/mirror --exclude-specs "llnlspheral spheral"
 
 # Use spack to list all specs in the mirror and push them to the buildcache.

scripts/lc/install-from-dev-pkg.sh

@@ -6,8 +6,8 @@ BUILD_ALLOC=${BUILD_ALLOC}
 SCRIPT_DIR=${SCRIPT_DIR:-'scripts'}
 SPHERAL_PIP_CACHE_DIR=${SPHERAL_PIP_CACHE_DIR:-~/.cache/spheral_pip}
 
-if [[ -z "${SPEC}" ]]; then
-  echo "SPEC var must be set."
+if [[ -z "${DEV_PKG_SPEC}" ]]; then
+  echo "DEV_PKG_SPEC var must be set."
   exit 1
 fi
 
@@ -17,7 +17,7 @@ if [[ -z "${INSTALL_DIR}" ]]; then
 fi
 
 echo $SPACK_PKG_NAME
-echo $SPEC
+echo $DEV_PKG_SPEC
 echo $SPACK_URL
 echo $INSTALL_DIR
 echo $SCRIPT_DIR
@@ -39,9 +39,9 @@ spack mirror add --unsigned spheral-mirror $PWD/resources/mirror
 spack mirror add --unsigned spheral-cache $PWD/resources
 spack buildcache update-index $PWD/resources/mirror
 
-$BUILD_ALLOC spack install --fresh --deprecated --no-check-signature --only dependencies $SPACK_PKG_NAME@develop%$SPEC
+$BUILD_ALLOC spack install --fresh --deprecated --no-check-signature --only dependencies $DEV_PKG_SPEC
 
-$BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --spack-url $SPACK_URL --no-upstream --spack-dir $INSTALL_DIR/spheral-spack-tpls --spec "${SPACK_PKG_NAME}${SPEC}" --skip-init
+$BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --no-upstream --spack-dir $INSTALL_DIR/spheral-spack-tpls --spec $DEV_PKG_SPEC --skip-init --dev-pkg
 
 HOST_CONFIG_FILE=$(ls -t | grep -E "*\.cmake" | head -1)
 $BUILD_ALLOC ./$SCRIPT_DIR/devtools/host-config-build.py --host-config $HOST_CONFIG_FILE -i $INSTALL_DIR --build --no-clean -DSPHERAL_PIP_CACHE_DIR=$SPHERAL_PIP_CACHE_DIR -DSPHERAL_NETWORK_CONNECTED=Off

scripts/spack/environments/dev_pkg/spack.yaml

@@ -0,0 +1,18 @@
+spack:
+  specs:
+    - "spheral@develop%gcc+mpi~caliper~network"
+    - "spheral@develop%gcc+mpi~caliper+network"
+  view: false
+  concretizer:
+    unify: false
+  develop:
+    spheral:
+      path: ../../../../
+      spec: spheral@=develop
+  include:
+    - ../../configs/toss_4_x86_64_ib/packages.yaml
+    - ../../configs/toss_4_x86_64_ib/compilers.yaml
+    - ../../configs/config.yaml
+    - ../../configs/upstreams.yaml
+  repos:
+    - ../../

scripts/spack/packages/spheral/package.py

@@ -31,6 +31,9 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage):
     variant('docs', default=False, description='Enable building Docs.')
     variant('shared', default=True, description='Build C++ libs as shared.')
     variant('python', default=True, description='Build Python Dependencies.')
+    variant('caliper', default=True, description='Enable Caliper timers.')
+    variant('opensubdiv', default=True, description='Enable use of opensubdiv to do refinement.')
+    variant('network', default=True, description='Disable to build Spheral from a local buildcache.')
 
     # -------------------------------------------------------------------------
     # DEPENDS
@@ -59,16 +62,19 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on('axom +shared', when='~cuda', type='build')
     depends_on('axom ~shared', when='+cuda', type='build')
 
-    depends_on('[email protected] ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build')
+    with when("+caliper"):
+        depends_on("[email protected] ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic", type="build")
+        depends_on("caliper+mpi", type="build", when="+mpi")
+        depends_on("caliper~mpi", type="build", when="~mpi")
 
     depends_on("[email protected]", type="build")
 
-    depends_on('[email protected]+pic', type='build')
+    depends_on('[email protected]+pic', type='build', when="+opensubdiv")
 
-    depends_on('polytope +python', type='build', when='+python')
+    depends_on('polytope +python', type='build')
 
     # Forward MPI Variants
-    mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"]
+    mpi_tpl_list = ["hdf5", "conduit", "axom", "adiak~shared"]
     for ctpl in mpi_tpl_list:
         for mpiv in ["+mpi", "~mpi"]:
             depends_on(f"{ctpl} {mpiv}", type='build', when=f"{mpiv}")
@@ -174,15 +180,17 @@ def initconfig_package_entries(self):
         spec = self.spec
         entries = []
 
-        entries.append(cmake_cache_option('ENABLE_CXXONLY', False))
+        entries.append(cmake_cache_option('ENABLE_CXXONLY', not spec.satisfies("+python")))
         entries.append(cmake_cache_option('TPL_VERBOSE', False))
         entries.append(cmake_cache_option('BUILD_TPL', True))
 
         entries.append(cmake_cache_string('SPHERAL_SYS_ARCH', self._get_sys_type(spec)))
         entries.append(cmake_cache_string('SPHERAL_CONFIGURATION', self._get_config_name(spec)))
 
         # TPL locations
-        entries.append(cmake_cache_path('caliper_DIR', spec['caliper'].prefix))
+        if (spec.satisfies("+caliper")):
+            entries.append(cmake_cache_path('caliper_DIR', spec['caliper'].prefix))
+            entries.append(cmake_cache_option('ENABLE_TIMER', True))
 
         entries.append(cmake_cache_path('adiak_DIR', spec['adiak'].prefix))
 
@@ -207,7 +215,12 @@ def initconfig_package_entries(self):
         entries.append(cmake_cache_path('eigen_DIR', spec['eigen'].prefix))
         entries.append(cmake_cache_path('eigen_INCLUDES',spec['eigen'].prefix.include.eigen3))
 
-        entries.append(cmake_cache_path('opensubdiv_DIR', spec['opensubdiv'].prefix))
+        if (spec.satisfies("+opensubdiv")):
+            entries.append(cmake_cache_path('opensubdiv_DIR', spec['opensubdiv'].prefix))
+            entries.append(cmake_cache_path('ENABLE_OPENSUBDIV', True))
+
+        if (spec.satisfies("~network")):
+            entries.append(cmake_cache_path('SPHERAL_NETWORK_CONNECTED', False))
 
         entries.append(cmake_cache_path('polytope_DIR', spec['polytope'].prefix))