Introducing Spack environments

.gitignore

@@ -20,10 +20,7 @@ job.out*
 runit
 spam
 *.a
-job.out
 job.err
-out.job
-err.job
 runit
 dumps*
 src/BUILD
@@ -37,60 +34,12 @@ lc_*
 build_*
 !build_guide*
 *.core
-spack-build*
+spackbuild*
+.spack-env
+spack.lock
 *.logs
 rz*
-rz*
 toss_*.cmake
 blueos_*.cmake
 
-
-# These are files automatically generated in our configure process.
-src/aclocal.m4
-src/autom4te.cache
-src/config.log
-src/config.status
-src/configure
-src/helpers/mpic++
-src/helpers/mpicc
-src/PBGWraps/SpheralModulesBindings.py
-
-src/thirdPartyLibs/tau-2.21.4
-src/thirdPartyLibs/cmake*
-src/FractalGravity/fftw++.cc
-
-# The pybindgen autogenerated files.
-src/PBGWraps/SpheralModules.C
-src/PBGWraps/SpheralModules.hh
-src/PBGWraps/SpheralModules_ArtificialViscosity.C
-src/PBGWraps/SpheralModules_Boundary.C
-src/PBGWraps/SpheralModules_CSPH.C
-src/PBGWraps/SpheralModules_CXXTypes.C
-src/PBGWraps/SpheralModules_DataBase.C
-src/PBGWraps/SpheralModules_DataOutput.C
-src/PBGWraps/SpheralModules_Distributed.C
-src/PBGWraps/SpheralModules_ExternalForce.C
-src/PBGWraps/SpheralModules_Field.C
-src/PBGWraps/SpheralModules_FieldOperations.C
-src/PBGWraps/SpheralModules_FileIO.C
-src/PBGWraps/SpheralModules_Geometry.C
-src/PBGWraps/SpheralModules_Gravity.C
-src/PBGWraps/SpheralModules_Hydro.C
-src/PBGWraps/SpheralModules_Integrator.C
-src/PBGWraps/SpheralModules_Kernel.C
-src/PBGWraps/SpheralModules_Material.C
-src/PBGWraps/SpheralModules_Mesh.C
-src/PBGWraps/SpheralModules_Neighbor.C
-src/PBGWraps/SpheralModules_NodeGenerators.C
-src/PBGWraps/SpheralModules_NodeList.C
-src/PBGWraps/SpheralModules_Physics.C
-src/PBGWraps/SpheralModules_SPH.C
-src/PBGWraps/SpheralModules_Silo.C
-src/PBGWraps/SpheralModules_Utilities.C
-src/PBGWraps/SpheralModules_WildMagic.C
-
 /build-*
-src/*/*cc.2.cc
-src/*/*cc.3.cc
-src/*/*/*cc.2.cc
-src/*/*/*cc.3.cc

.gitlab/scripts.yml

@@ -24,7 +24,7 @@
     - cd $CI_BUILD_DIR && cat job-name.txt
     - echo $SPEC
 
-    - ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --tpls-only
+    - ./$SCRIPT_DIR/devtools/tpl-manager.py --spec="${SPACK_PKG_NAME}${SPEC}" --ci-run
   artifacts:
     paths:
       - ci-dir.txt
@@ -113,7 +113,7 @@
 .update_tpls:
   stage: update_tpls
   script:
-    - ./$SCRIPT_DIR/devtools/tpl-manager.py --no-upstream --spec-list="$SCRIPT_DIR/devtools/spec-list.json" --spheral-spack-dir=$UPSTREAM_DIR
+    - ./$SCRIPT_DIR/devtools/tpl-manager.py --no-upstream --spack-dir=$UPSTREAM_DIR
 
 .toss_update_permissions:
   stage: update_permissions

.gitlab/specs.yml

@@ -3,65 +3,60 @@
 
 .gcc_mvapich2_cxxonly:
   variables:
-    SPEC: 'gcc@$GCC_VERSION^mvapich2'
+    SPEC: '%gcc@$GCC_VERSION+mpi'
     EXTRA_CMAKE_ARGS: '-DENABLE_CXXONLY=On -DENABLE_WARNINGS_AS_ERRORS=On'
 
 .gcc_mvapich2:
   variables:
-    SPEC: 'gcc@$GCC_VERSION^mvapich2'
+    SPEC: '%gcc@$GCC_VERSION+mpi'
     EXTRA_CMAKE_ARGS: '-DENABLE_DOCS=On -DENABLE_WARNINGS_AS_ERRORS=On'
 
 .gcc_~mpi:
   variables:
-    SPEC: 'gcc@$GCC_VERSION~mpi'
+    SPEC: '%gcc@$GCC_VERSION~mpi'
     EXTRA_CMAKE_ARGS: '-DENABLE_DOCS=On -DENABLE_WARNINGS_AS_ERRORS=On'
 
 .gcc_~mpi_Debug:
   variables:
-    SPEC: 'gcc@$GCC_VERSION~mpi'
+    SPEC: '%gcc@$GCC_VERSION~mpi'
     EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On'
 
-.gcc_spectrum:
-  variables:
-    SPEC: 'gcc@$GCC_VERSION^spectrum-mpi'
-
-
 
 .clang_mvapich2:
   variables:
-    SPEC: 'clang@$CLANG_VERSION^mvapich2'
+    SPEC: '%clang@$CLANG_VERSION+mpi'
     EXTRA_CMAKE_ARGS: '-DENABLE_WARNINGS_AS_ERRORS=On -DENABLE_DEV_BUILD=On'
 
 
 
 .cuda_11_gcc_~mpi:
   variables:
-    SPEC: 'gcc@$GCC_VERSION+cuda~mpi cuda_arch=70'
+    SPEC: '%gcc@$GCC_VERSION~mpi+cuda cuda_arch=70'
 
 .cuda_11_gcc_spectrum:
   variables:
-    SPEC: 'gcc@$GCC_VERSION+cuda cuda_arch=70'
+    SPEC: '%gcc@$GCC_VERSION+mpi+cuda cuda_arch=70'
     EXTRA_CMAKE_ARGS: ''
 
 
 
 .rocm_mpich:
   variables:
-    SPEC: 'rocmcc@$ROCMCC_VERSION'
+    SPEC: '%rocmcc@$ROCMCC_VERSION+mpi~rocm'
 
 .rocm_~mpi:
   variables:
-    SPEC: 'rocmcc@$ROCMCC_VERSION~mpi'
+    SPEC: '%rocmcc@$ROCMCC_VERSION~mpi~rocm'
 
 .rocm_~mpi_Debug:
   variables:
-    SPEC: 'rocmcc@$ROCMCC_VERSION~mpi'
+    SPEC: '%rocmcc@$ROCMCC_VERSION~mpi~rocm'
     EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On'
 
 .hip_rocm_mpich:
   variables:
-    SPEC: 'rocmcc@$ROCMCC_VERSION+rocm amdgpu_target=gfx942'
+    SPEC: '%rocmcc@$ROCMCC_VERSION+mpi+rocm amdgpu_target=gfx942'
 
 .hip_rocm_~mpi:
   variables:
-    SPEC: 'rocmcc@$ROCMCC_VERSION~mpi+rocm amdgpu_target=gfx942'
+    SPEC: '%rocmcc@$ROCMCC_VERSION~mpi+rocm amdgpu_target=gfx942'

.gitmodules

@@ -2,10 +2,6 @@
 [submodule "blt"]
 	path = cmake/blt
 	url = https://github.com/llnl/blt
-[submodule "scripts/devtools/uberenv"]
-	path = scripts/devtools/uberenv
-	url = https://github.com/llnl/uberenv
-	branch = origin/master
 [submodule "extern/PYB11Generator"]
 	path = extern/PYB11Generator
 	url = https://github.com/LLNL/PYB11Generator

Dockerfile

@@ -31,7 +31,7 @@ ARG HOST_CONFIG=docker-$SPEC
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update -y
 RUN apt-get upgrade -y
-RUN apt-get install -y build-essential git gfortran mpich autotools-dev autoconf sqlite pkg-config uuid gettext cmake libncurses-dev libgdbm-dev libffi-dev libssl-dev libexpat-dev libreadline-dev libbz2-dev locales python python3 unzip libtool wget curl tk-dev
+RUN apt-get install -y build-essential git gfortran mpich autotools-dev autoconf sqlite pkg-config uuid gettext cmake libncurses-dev libgdbm-dev libffi-dev libssl-dev libexpat-dev libreadline-dev libbz2-dev locales python python3 unzip libtool wget curl libcurl4-openssl-dev tk-dev
 RUN apt-get install -y python3-dev python3-venv python3-pip
 RUN apt-get install -y iputils-ping
 
@@ -41,9 +41,9 @@ RUN locale-gen en_US.UTF-8
 # Set up TPLs for SPEC
 WORKDIR /home/spheral/workspace/
 COPY scripts scripts
-COPY .uberenv_config.json .
 
-RUN python3 scripts/devtools/tpl-manager.py --spec $SPEC --spheral-spack-dir /home
+
+RUN python3 scripts/devtools/tpl-manager.py --spec spheral%$SPEC --spack-dir /home
 
 COPY . .
 
@@ -78,7 +78,7 @@ WORKDIR /home/spheral/workspace/
 
 # Copy Spheral source and generate host config from tpl-manager (all dependencies should already be installed).
 COPY . .
-RUN python3 scripts/devtools/tpl-manager.py --spec $SPEC --upstream-dir /home/spack/opt/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder_ --spack-url /home/spack
+RUN python3 scripts/devtools/tpl-manager.py --spec spheral%$SPEC --spack-dir /home
 
 # Configure Spheral with SPEC TPLs.
 RUN mv *.cmake $HOST_CONFIG.cmake

docs/build_guide/include/appendecies/custom_spack_install_notes.rst.inc

@@ -5,5 +5,3 @@ Building Spheral TPLs with your own Spack installation will require deeper knowl
 
  - Point your spack instances `repo <https://spack.readthedocs.io/en/latest/repositories.html?highlight=repo#spack-repo>`_ at the ``scripts/spack/packages/`` dir. This contains all of our changes to spack packages that have not yet made it to upstream Spack.
  - You will want to model your ``compiler.yaml`` and ``packages.yaml`` files off of those found in ``scripts/spack/configs/`` (`Spack Configuration Files <https://spack.readthedocs.io/en/latest/configuration.html#configuration>`_).
-
-Further notes on setting up Spack and how it is used with the Spheral dev-tools scripts can be found in `Development Documentation: Spheral Spack / Uberenv <Development_Documentation.html#spheral-spack-uberenv>`_.

docs/build_guide/include/cloning.rst.inc

@@ -3,7 +3,7 @@
    ----------------------------------------
 
 [git_clone-section-start]
-If you use git to clone the Spheral source be aware Spheral includes git submodules: `BLT <https://github.com/LLNL/blt>`_ and `Uberenv <https://github.com/LLNL/uberenv>`_.  In order to ensure such submodules are properly downloaded when cloning Spheral be sure to use the ``--recursive`` git option:
+If you use git to clone the Spheral source be aware Spheral includes the `BLT <https://github.com/LLNL/blt>`_ git submodule.  In order to ensure such submodules are properly downloaded when cloning Spheral be sure to use the ``--recursive`` git option:
 
 ::
 

docs/build_guide/include/configure.rst.inc

@@ -8,7 +8,7 @@ Configuring Spheral
 
 After running ``tpl-manager`` you will see a file in your Spheral root directory following the format ``<sys_type>-<spec>.cmake``. 
 
-For example if you run ``tpl-manager`` with only a single ``--spec`` e.g. ``gcc~mpi``, you will only see:
+For example if you run ``tpl-manager`` with only a single ``--spec`` e.g. ``spheral%gcc~mpi``, you will only see:
 
 ::
 
@@ -22,19 +22,14 @@ Configuring Spheral
 
 After running ``tpl-manager`` you will see a file in your Spheral root directory following the format ``<sys_type>-<spec>.cmake``. 
 
-For example if you run ``tpl-manager`` with ``spec-list.json`` on a ``toss_3_x86_64_ib`` system you will see:
+For example if you run ``tpl-manager`` on a ``toss_4_x86_64_ib`` system with ``--spec spheral%clang+mpi``, you will see:
 
 ::
 
-  [email protected]
-  [email protected]
-  [email protected]
+  [email protected]
 
-However if you ran ``tpl-manager`` with only a single ``--spec`` e.g. ``[email protected]~mpi``, you will only see:
-
-::
-
-  [email protected]~mpi.cmake
+If no ``--spec`` is provided to ``tpl-manager``, the TPLs for all specs in an environment will be built, but no host config file will be generated.
+A ``--spec`` must be provided to generate a host config file.
 
 [lc_intro-end]
 

docs/build_guide/include/quickstart.rst.inc

@@ -36,7 +36,7 @@ Create a directory structure and clone spheral.
 Build our TPL dependencies from source with the Spheral tpl-management tool (``tpl-manager.py``).
 ::
 
-  python3 scripts/devtools/tpl-manager.py --spec gcc
+  python3 scripts/devtools/tpl-manager.py --spec spheral%gcc+mpi
 
 .. note::
    This command sequence assumes ``gcc`` is installed and will use the version in your system path. If you wish to use a different compiler (such as ``clang``) ensure it is in your path and replace ``gcc`` with the compiler name of your choice (e.g. ``clang``).
@@ -56,7 +56,7 @@ Build our TPL dependencies from source with the Spheral tpl-management tool (``t
 Build our TPL dependencies from source with the Spheral tpl-management tool (``tpl-manager.py``).
 ::
 
-  ./scripts/devtools/tpl-manager.py --spec [email protected]
+  ./scripts/devtools/tpl-manager.py --spec spheral%[email protected]+mpi
 
 .. warning::
   This command needs to be run under an allocation as any TPLs that need to be built will be built in parallel.

docs/build_guide/include/tpls.rst.inc

@@ -17,7 +17,7 @@ Spheral provides a tool (``tpl-manager.py``) in an attempt to simplify the spack
  - Set up a local ``Spack`` instance for Spheral.
  - Generate a dependency tree of third party libraries (relative to the provided configuration).
  - Build and install all dependent libraries in the local ``Spack`` instance.
- - Generate a `CMake host-config <https://llnl-blt.readthedocs.io/en/develop/tutorial/host_configs.html>`_  file for configuring Spheral builds.
+ - Generate a `CMake host-config <https://llnl-blt.readthedocs.io/en/develop/tutorial/host_configs.html>`_  file for configuring Spheral builds (if a spec is given).
 
 ``tpl-manager`` is located at ``scripts/devtools/tpl-manager.py``. ``tpl-manager`` can be used in two ways:
 
@@ -29,7 +29,7 @@ Spheral provides a tool (``tpl-manager.py``) in an attempt to simplify the spack
 [ex_tpl_manager_notes-start]
 
  .. note:
-    The External User Guide does not cover building the full range of tpl configurations. Please see the LC User Guide for more informtaion.
+    The External User Guide does not cover building the full range of tpl configurations. Please see the LC User Guide for more information.
 
 [ex_tpl_manager_notes-end]
 [lc_tpl_manager_notes-start]
@@ -47,56 +47,31 @@ Running tpl-manager
 
 ``tpl-manager`` requires ``python3`` and should be run from the root Spheral directory.
 
-``tpl-manager`` takes a ``--spec`` argument to determine what compiler to use and what configuration we want to build Spheral in.
-
-::
-
-  python3 scripts/devtools/tpl-manager.py --spec gcc
+``tpl-manager`` takes a ``--spec`` argument to determine what compiler to use and what configuration we want to build Spheral in. For example,
 
-This will install the local Spheral Spack instance into the adjacent directory of your Spheral root dir. You can use ``--spheral-spack-dir`` if you would like to setup the spack instance somewhere else. 
-
-Above we are telling ``tpl-manager`` to build our TPLs with the ``gcc`` that is in our path. By default this will build with ``+mpi`` support, however we can disable ``mpi`` support for the TPLs and Spheral by appending ``~mpi`` to our spec.
 ::
 
-  python3 scripts/devtools/tpl-manager.py --spec gcc~mpi
-
-.. note::
-   By default we have ``python`` bindings enabled (``+python``) and docs disabled (``~docs``). Therefore the spec ``gcc+mpi+python~docs`` will build the same TPL set as just ``gcc``.For more information on ``spec`` syntax please see the spack documentation on `specs-dependencies <https://spack.readthedocs.io/en/latest/basic_usage.html#specs-dependencies>`_.
+  python3 scripts/devtools/tpl-manager.py --spec spheral%gcc
 
-.. note::
-   Spheral minimally requires a C++14 compliant compiler.
+This will download a new Spack instance in a directory adjacent to your Spheral root directory called ``spheral-spack-tpls/spack``. You can use ``--spack-dir`` if you would like to setup the Spack instance somewhere else. By default, Spheral is built with MPI support unless the spec is appended with ``~mpi``. The ``%`` symbol denotes the compiler to use.
 
-ERROR: invalid spack config dir
-===============================
+When run for the first time, ``tpl-manager`` will do a few things:
 
-If you are trying to run ``tpl-manager.py`` on an operating system other than Ubuntu20.04, you will
-see an error to the effect of:
-``[ERROR: invalid spack config dir: /<path>/scripts/spack/configs/<OperatingSystem><Version> ]``
+#. Create and activate a Spack environment in ``spheral/scripts/spack/environments`` based on the output of ``spack arch``. All subsequent Spack commands will modify this environment by adding specs, compilers, and external packages to it.
 
-We define configuration files for Ubuntu20.04, as well as our common LLNL operating systems.
-You will need to create a set of files for your own system.
+#. Run ``spack compiler find`` to find system compilers. Spack searches the ``$PATH`` environment variable for compilers and packages. Add any paths to ``$PATH`` to ensure Spack will find them before running the ``tpl-manager``.
 
-The configuration files tell spack where the packages installed in :ref:`Required System Packages`
-are located and what version they are. We have provided a ``generic`` set of config files to 
-help in setting this up for you.
+#. Run ``spack external find --not-buildable`` It will run Spack commands to try to find existing system installs for things like CMake, Git, Python, and MPICH (when the spec has ``+mpi``).
 
-#. Copy the directory ``scripts/spack/configs/generic`` to ``scripts/spack/config/<OperatingSystem><Version>`` (you want to match the name of the directory to the one ``tpl-manager.py`` expects to find).
+#. Add the current spec to the environment with ``spack add <spec>`` and concretize using ``spack concretize -U``.
 
+#. Install the TPLs and create the host config file for the given spec with ``spack install -u initconfig <spec>``.
 
-#. For each package within the ``packages.py`` file of your new folder edit the version number to be the same as
-   what is installed on your system. There are a number of ways to retrieve versions for a given package:
-
-   * Most version numbers should be searchable through your package manager, there are however, some system libraries that may not be managed by your package manager.
-
-   * If the package has an executable, often you can run with some form of ``-V`` or ``--version``. e.g. for mpich:``mpiexec --version`` will report the MPI version.
-
-   * For packages that only provide libraries, often the system library will be symlinked to one with the version as the extension. e.g. ``ls -lha /usr/lib/x86_64_gnu-linux/libreadlines.so`` will show it is symlinked to ``libreadlines.so.8.1``.
-
-
-#. Each package requires the ``prefix:`` of the installation be provided. In most cases ``/usr`` is sufficient. Typically packages installed with a package manager will place files in: ``/usr/bin``, ``/usr/share``, ``/usr/lib``, ``/usr/lib64``, sometimes ``/usr/lib/x86_64.../``. Here the common prefix is ``/usr``. 
-
-   * If you are building Spheral on a system where you don't have permissions to run package manager and install to ``/usr``, then you might have installed the system packages somewhere else. In that case, replace the ``prefix:`` path for those given packages as necessary.
+.. note::
+   By default we have ``python`` bindings enabled (``+python``) and docs disabled (``~docs``). Therefore the spec ``spheral%gcc+mpi+python~docs`` will build the same TPL set as just ``spheral%gcc``.For more information on ``spec`` syntax please see the spack documentation on `specs-dependencies <https://spack.readthedocs.io/en/latest/basic_usage.html#specs-dependencies>`_.
 
+.. note::
+   Spheral minimally requires a C++14 compliant compiler.
 
 [ex_running_tpl_manager-end]
 
@@ -110,9 +85,9 @@ Running tpl-manager
 
 ::
 
-  ./scripts/devtools/tpl-manager.py --spec [email protected]
+  ./scripts/devtools/tpl-manager.py --spec spheral%[email protected]
 
-This will install the local Spheral Spack instance into the adjacent directory of your Spheral root dir. You can use ``--spheral-spack-dir`` if you would like to setup the spack instance somewhere else. 
+This will install the local Spheral Spack instance into the adjacent directory of your Spheral root dir. You can use ``--spack-dir`` if you would like to setup the spack instance somewhere else. 
 
 .. warning::
    If running on LC be sure to launch ``tpl-manager`` in a resource allocation, as ``tpl-manager`` will take advantage of parallel builds when compiling libraries.  An example appropriate for use on rzgenie would be::
@@ -135,7 +110,7 @@ CUDA Support
 
 To build Spheral TPLs and configure Spheral to build with CUDA on LC systems we need to add the spack variant ``+cuda`` to our spec. We also need to define the cuda hardware architecture we are targetting Spheral to run on. This can be done with ``cuda_arch=XX``. On most LC NVIDIA systems we will be targetting ``sm_70`` architecture. Therefore a gcc spec with cuda support will look as such.::
 
-  ./scripts/devtools/tpl-manager.py --spec "[email protected]+cuda cuda_arch=70"
+  ./scripts/devtools/tpl-manager.py --spec "spheral%[email protected]+cuda cuda_arch=70"
 
 .. note::
    We need to use "" here as spack requires spacing to define ``cuda_arch``. This ensures tpl-manager doesn't register the spec as separate arguments.