Pip Runtime Dependencies & CRAY AMD Support

.gitignore

@@ -89,7 +89,7 @@ src/PBGWraps/SpheralModules_Silo.C
 src/PBGWraps/SpheralModules_Utilities.C
 src/PBGWraps/SpheralModules_WildMagic.C
 
-build-*
+/build-*
 src/*/*cc.2.cc
 src/*/*cc.3.cc
 src/*/*/*cc.2.cc

.gitlab-ci.yml

@@ -17,6 +17,7 @@ variables:
     - SPHERAL_REV_STR="$SPHERAL_REV"
 
 stages:
+  - init
   - tpls
   - build_and_install
   - run_ats

.gitlab/jobs-mpi.yml

@@ -1,8 +1,12 @@
 # ------------------------------------------------------------------------------
 # BUILD JOBS
 
+toss_gcc_mvapich2_cxxonly_init:
+  extends: [.toss_shell1, .gcc_mvapich2_cxxonly, .init]
+
 toss_gcc_mvapich2_cxxonly_tpls:
   extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls]
+  needs: [toss_gcc_mvapich2_cxxonly_init]
 
 toss_gcc_mvapich2_cxxonly_build:
   extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build]
@@ -14,8 +18,12 @@ toss_gcc_mvapich2_cxxonly_cleanup:
 
 
 
+toss_gcc_mvapich2_init:
+  extends: [.toss_shell1, .gcc_mvapich2, .init]
+
 toss_gcc_mvapich2_tpls:
   extends: [.toss_resource2, .gcc_mvapich2, .tpls]
+  needs: [toss_gcc_mvapich2_init]
 
 toss_gcc_mvapich2_build:
   extends: [.toss_resource2, .gcc_mvapich2, .build_and_test]
@@ -25,17 +33,21 @@ toss_gcc_mvapich2_test:
   extends: [.toss_resource2, .gcc_mvapich2, .run_ats]
   needs: [toss_gcc_mvapich2_build]
 
+toss_gcc_mvapich2_perf:
+  extends: [.toss_shell2, .gcc_mvapich2, .run_perf, .merge_pr_rule]
+  needs: [toss_gcc_mvapich2_build]
+
 toss_gcc_mvapich2_cleanup:
   extends: [.toss_resource2, .gcc_mvapich2, .cleanup_dir]
 
-toss_gcc_mvapich2_perf:
-  extends: [.toss_shell1, .gcc_mvapich2, .run_perf, .merge_pr_rule]
-  needs: [toss_gcc_mvapich2_build]
 
 
+toss_clang_mvapich2_init:
+  extends: [.toss_shell1, .clang_mvapich2, .init]
 
 toss_clang_mvapich2_tpls:
   extends: [.toss_resource3, .clang_mvapich2, .tpls]
+  needs: [toss_clang_mvapich2_init]
 
 toss_clang_mvapich2_build:
   extends: [.toss_resource3, .clang_mvapich2, .build_and_test]
@@ -45,54 +57,61 @@ toss_clang_mvapich2_test:
   extends: [.toss_resource3, .clang_mvapich2, .run_ats]
   needs: [toss_clang_mvapich2_build]
 
-toss_clang_mvapich2_cleanup:
-  extends: [.toss_resource3, .clang_mvapich2, .cleanup_dir]
-
 toss_clang_mvapich2_perf:
-  extends: [.toss_shell1, .clang_mvapich2, .run_perf, .merge_pr_rule]
+  extends: [.toss_shell2, .clang_mvapich2, .run_perf, .merge_pr_rule]
   needs: [toss_clang_mvapich2_build]
 
+toss_clang_mvapich2_cleanup:
+  extends: [.toss_resource3, .clang_mvapich2, .cleanup_dir]
+
 
 
 #toss_oneapi_2022_1_mvapich2:
 #  extends: [.oneapi_2022_1_mvapich2, .build_and_test, .toss_resource]
 
 
+cray_rocm_mpich_init:
+  extends: [.cray_shell1, .rocm_mpich, .init]
+
+cray_rocm_mpich_tpls:
+  extends: [.cray_resource1, .rocm_mpich, .tpls]
+  needs: [cray_rocm_mpich_init]
 
-blueos_gcc_spectrum_tpls:
-  extends: [.blueos_resource1, .gcc_spectrum, .tpls]
+cray_rocm_mpich_build:
+  extends: [.cray_resource1, .rocm_mpich, .build_and_test]
+  needs: [cray_rocm_mpich_tpls]
 
-blueos_gcc_spectrum_build:
-  extends: [.blueos_resource1, .gcc_spectrum, .build_and_test]
-  needs: [blueos_gcc_spectrum_tpls]
+cray_rocm_mpich_test:
+  extends: [.cray_resource1, .rocm_mpich, .run_ats]
+  needs: [cray_rocm_mpich_build]
 
-blueos_gcc_spectrum_test:
-  extends: [.blueos_resource1, .gcc_spectrum, .run_ats]
-  needs: [blueos_gcc_spectrum_build]
+cray_rocm_mpich_perf:
+  extends: [.cray_shell2, .rocm_mpich, .run_perf, .merge_pr_rule]
+  needs: [cray_rocm_mpich_build]
 
-blueos_gcc_spectrum_cleanup:
-  extends: [.blueos_resource1, .gcc_spectrum, .cleanup_dir]
+cray_rocm_mpich_cleanup:
+  extends: [.cray_resource1, .rocm_mpich, .cleanup_dir]
 
-blueos_gcc_spectrum_perf:
-  extends: [.blueos_shell1, .gcc_spectrum, .run_perf, .merge_pr_rule]
-  needs: [blueos_gcc_spectrum_build]
 
 
+cray_hip_rocm_mpich_init:
+  extends: [.cray_shell1, .hip_rocm_mpich, .init]
 
-blueos_cuda_11_gcc_spectrum_tpls:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls]
+cray_hip_rocm_mpich_tpls:
+  extends: [.cray_resource2, .hip_rocm_mpich, .tpls]
+  needs: [cray_hip_rocm_mpich_init]
 
-blueos_cuda_11_gcc_spectrum_build:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test]
-  needs: [blueos_cuda_11_gcc_spectrum_tpls]
+cray_hip_rocm_mpich_build:
+  extends: [.cray_resource2, .hip_rocm_mpich, .build_and_test]
+  needs: [cray_hip_rocm_mpich_tpls]
 
-blueos_cuda_11_gcc_spectrum_test:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats]
-  needs: [blueos_cuda_11_gcc_spectrum_build]
+cray_hip_rocm_mpich_test:
+  extends: [.cray_resource2, .hip_rocm_mpich, .run_ats]
+  needs: [cray_hip_rocm_mpich_build]
 
-blueos_cuda_11_gcc_spectrum_cleanup:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .cleanup_dir]
+cray_hip_rocm_mpich_perf:
+  extends: [.cray_shell2, .hip_rocm_mpich, .run_perf, .merge_pr_rule]
+  needs: [cray_hip_rocm_mpich_build]
 
-blueos_cuda_11_gcc_spectrum_perf:
-  extends: [.blueos_shell2, .cuda_11_gcc_spectrum, .run_perf, .merge_pr_rule]
-  needs: [blueos_cuda_11_gcc_spectrum_build]
+cray_hip_rocm_mpich_cleanup:
+  extends: [.cray_resource2, .hip_rocm_mpich, .cleanup_dir]

.gitlab/jobs-prod.yml

@@ -4,8 +4,8 @@
 toss_update_tpls:
   extends: [.toss_resource2, .update_tpls, .merge_pr_rule]
 
-blueos_update_tpls:
-  extends: [.blueos_resource2, .update_tpls, .merge_pr_rule]
+cray_update_tpls:
+  extends: [.cray_resource2, .update_tpls, .merge_pr_rule]
   needs: [toss_update_tpls]
 
 # ------------------------------------------------------------------------------
@@ -37,6 +37,6 @@ toss_release_permissions:
 cleanup_old_dirs_toss:
   extends: [.toss_resource_general, .clean_old_dirs]
 
-cleanup_old_dirs_blueos:
-  extends: [.blueos_resource_general, .clean_old_dirs]
+cleanup_old_dirs_cray:
+  extends: [.cray_resource_general, .clean_old_dirs]
 

.gitlab/jobs-seq.yml

@@ -1,13 +1,17 @@
 # ------------------------------------------------------------------------------
 # BUILD JOBS
 
+toss_gcc_~mpi_init:
+  extends: [.gcc_~mpi, .init, .toss_shell1]
+
 toss_gcc_~mpi_tpls:
   extends: [.gcc_~mpi, .tpls, .toss_resource1]
-
+  needs: [toss_gcc_~mpi_init]
+
 toss_gcc_~mpi_build:
   extends: [.gcc_~mpi, .build_and_test, .toss_resource1]
   needs: [toss_gcc_~mpi_tpls]
-  
+
 toss_gcc_~mpi_test:
   extends: [.gcc_~mpi, .run_ats, .toss_resource1]
   needs: [toss_gcc_~mpi_build]
@@ -17,33 +21,43 @@ toss_gcc_~mpi_cleanup:
   needs: [toss_gcc_~mpi_test]
 
 
-blueos_cuda_11_gcc_~mpi_tpls:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls]
 
-blueos_cuda_11_gcc_~mpi_build:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test]
-  needs: [blueos_cuda_11_gcc_~mpi_tpls]
+cray_hip_rocm_~mpi_init:
+  extends: [.cray_shell1, .hip_rocm_~mpi, .init]
+
+cray_hip_rocm_~mpi_tpls:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .tpls]
+  needs: [cray_hip_rocm_~mpi_init]
+
+cray_hip_rocm_~mpi_build:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .build_and_test]
+  needs: [cray_hip_rocm_~mpi_tpls]
+
+cray_hip_rocm_~mpi_test:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .run_ats]
+  needs: [cray_hip_rocm_~mpi_build]
+
+cray_hip_rocm_~mpi_cleanup:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .cleanup_dir]
+  needs: [cray_hip_rocm_~mpi_test]
 
-blueos_cuda_11_gcc_~mpi_test:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats]
-  needs: [blueos_cuda_11_gcc_~mpi_build]
 
-blueos_cuda_11_gcc_~mpi_cleanup:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .cleanup_dir]
-  needs: [blueos_cuda_11_gcc_~mpi_test]
 
+cray_rocm_~mpi_Debug_init:
+  extends: [.cray_shell1, .rocm_~mpi_Debug, .init]
 
-blueos_gcc_~mpi_Debug_tpls:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls]
+cray_rocm_~mpi_Debug_tpls:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .tpls]
+  needs: [cray_rocm_~mpi_Debug_init]
 
-blueos_gcc_~mpi_Debug_build:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test]
-  needs: [blueos_gcc_~mpi_Debug_tpls]
+cray_rocm_~mpi_Debug_build:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .build_and_test]
+  needs: [cray_rocm_~mpi_Debug_tpls]
 
-blueos_gcc_~mpi_Debug_test:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats]
-  needs: [blueos_gcc_~mpi_Debug_build]
+cray_rocm_~mpi_Debug_test:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .run_ats]
+  needs: [cray_rocm_~mpi_Debug_build]
 
-blueos_gcc_~mpi_Debug_cleanup:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .cleanup_dir]
-  needs: [blueos_gcc_~mpi_Debug_test]
+cray_rocm_~mpi_Debug_cleanup:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .cleanup_dir]
+  needs: [cray_rocm_~mpi_Debug_test]

.gitlab/machines.yml

@@ -1,6 +1,26 @@
 # ------------------------------------------------------------------------------
 # MACHINE TEMPLATES
 
+.on_tioga:
+  tags:
+    - tioga
+    - flux
+  variables:
+    SCHEDULER_ACTION: alloc
+    SCHEDULER_PARAMETERS: "--exclusive -N 2 -t 120"
+    NPROC: 128
+    HOSTNAME: 'tioga'
+  timeout: 120 minutes
+  extends: [.on_toss_4_x86_cray]
+
+.tioga_shell:
+  tags:
+    - tioga
+    - shell
+  variables:
+    ENVIRONMENT: "INTERACTIVE"
+  extends: [.on_toss_4_x86_cray]
+
 .on_ruby:
   tags:
     - ruby
@@ -21,46 +41,28 @@
     ENVIRONMENT: "INTERACTIVE"
   extends: [.on_toss_4_x86]
 
-.on_lassen:
-  tags:
-    - lassen
-    - batch
-  variables:
-    SCHEDULER_PARAMETERS: "-nnodes 1 -W 150 -q pci -alloc_flags atsdisable"
-    LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD"
-    NPROC: 40
-    ENVIRONMENT: "BATCH"
-    HOSTNAME: 'lassen'
-    LC_MODULES: "cuda/11.1.0"
-  timeout: 150 minutes
-  extends: [.on_blueos_3_ppc64]
-
-.lassen_shell:
-  tags:
-    - lassen
-    - shell
-  variables:
-    ENVIRONMENT: "INTERACTIVE"
-    HOSTNAME: 'lassen'
-  extends: [.on_blueos_3_ppc64]
-
 # ------------------------------------------------------------------------------
 
 .toss_resource_general:
   extends: [.on_ruby]
 
-.blueos_resource_general:
-  extends: [.on_lassen]
+.cray_resource_general:
+  extends: [.on_tioga]
 
 .toss_shell1:
+  extends: [.ruby_shell]
+
+# Use for performance tests
+.toss_shell2:
   resource_group: toss2
   extends: [.ruby_shell]
 
-.blueos_shell1:
-  extends: [.lassen_shell]
+.cray_shell1:
+  extends: [.tioga_shell]
+
+.cray_shell2:
+  extends: [.tioga_shell]
 
-.blueos_shell2:
-  extends: [.lassen_shell]
 # ------------------------------------------------------------------------------
 #
 .toss_resource1:
@@ -75,11 +77,10 @@
   resource_group: toss3
   extends: [.toss_resource_general]
 
-.blueos_resource1:
-  #resource_group: blueos1
-  extends: [.blueos_resource_general]
-
-.blueos_resource2:
-  #resource_group: blueos2
-  extends: [.blueos_resource_general]
+.cray_resource1:
+  #resource_group: cray1
+  extends: [.cray_resource_general]
 
+.cray_resource2:
+  #resource_group: cray2
+  extends: [.cray_resource_general]

.gitlab/os.yml

@@ -13,13 +13,13 @@
     GCC_VERSION: '10.3.1'
     CLANG_VERSION: '14.0.6'
     SPHERAL_BUILDS_DIR: /p/lustre1/sphapp/spheral-ci-builds
+    RUN_CMD: 'srun'
   extends: [.sys_config]
 
-.on_blueos_3_ppc64:
+.on_toss_4_x86_cray:
   variables:
-    ARCH: 'blueos_3_ppc64le_ib_p9'
-    GCC_VERSION: '10.2.1'
-    CLANG_VERSION: '9.0.0'
-    SPHERAL_BUILDS_DIR: /p/gpfs1/sphapp/spheral-ci-builds
+    ARCH: 'toss_4_x86_64_ib_cray'
+    ROCMCC_VERSION: '6.2.0'
+    SPHERAL_BUILDS_DIR: /p/lustre1/sphapp/spheral-ci-builds
+    RUN_CMD: 'flux run'
   extends: [.sys_config]
-