From d36680c4b376398c854f2e03d678d5106d145e5b Mon Sep 17 00:00:00 2001
From: Alexander Smith <soulalex@gmail.com>
Date: Sun, 23 Feb 2025 23:39:49 +0000
Subject: [PATCH 1/2] Adding new modules and options

---
 job_scripts/polaris/polaris.submit | 38 ++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/job_scripts/polaris/polaris.submit b/job_scripts/polaris/polaris.submit
index 29df446..420df8b 100644
--- a/job_scripts/polaris/polaris.submit
+++ b/job_scripts/polaris/polaris.submit
@@ -1,31 +1,39 @@
 #!/bin/sh
-#PBS -l select=2:system=polaris
+#PBS -l select=64:system=polaris
+#PBS -l filesystems=home:eagle:grand
 #PBS -l place=scatter
-#PBS -l walltime=6:00:00
+#PBS -l walltime=00:30:00
 #PBS -q prod
 #PBS -A AstroExplosions
-#PBS -j eo
 
-EXEC=./Castro2d.gnu.MPI.CUDA.SMPLSDC.ex
-INPUTS=inputs_2d.N14.coarse
-
-module swap PrgEnv-nvhpc PrgEnv-gnu
-module load nvhpc-mixed
+EXEC=./Castro3d.cray.MPI.CUDA.ex
+INPUTS=inputs_nova_t7
 
 # Enable GPU-MPI (if supported by application)
 ##export MPICH_GPU_SUPPORT_ENABLED=1
 
+module use /soft/modulefiles
+module load PrgEnv-gnu
+module load nvhpc-mixed
+module load craype-accel-nvidia80
+module unload nvhpc-mixed
+module load cray-libsci
+module load craype-x86-milan
+module load PrgEnv-gnu
+module load nvhpc-mixed
+
 # Change to working directory
 cd ${PBS_O_WORKDIR}
 
 # MPI and OpenMP settings
 NNODES=`wc -l < $PBS_NODEFILE`
-NRANKS_PER_NODE=4
+NRANKS_PER_NODE=$(nvidia-smi -L | wc -l)
 NDEPTH=8
 NTHREADS=1
-
 NTOTRANKS=$(( NNODES * NRANKS_PER_NODE ))
 
+# If you need to start from the latest chk file, please uncomment the function
+# and the lines below:
 
 function find_chk_file {
     # find_chk_file takes a single argument -- the wildcard pattern
@@ -38,7 +46,8 @@ function find_chk_file {
     restartFile=""
     for f in ${temp_files}
     do
-        # the Header is the last thing written -- if it's there, update the restart file
+        # the Header is the last thing written -- check if it's there, otherwise,
+        # fall back to the second-to-last check file written
         if [ -f ${f}/Header ]; then
             restartFile="${f}"
         fi
@@ -66,6 +75,11 @@ else
     restartString="amr.restart=${restartFile}"
 fi
 
+# If you need to restart from an specific chk file, please uncomment
+# the two lines below:
+
+# restartFile="chk2740664"
+# restartString="amr.restart=${restartFile}"
 
 # For applications that need mpiexec to bind MPI ranks to GPUs
-mpiexec -n ${NTOTRANKS} --ppn ${NRANKS_PER_NODE} --depth=${NDEPTH} --cpu-bind depth --env OMP_NUM_THREADS=${NTHREADS} -env OMP_PLACES=threads ./set_affinity_gpu_polaris.sh ${EXEC} ${INPUTS} ${restartString}
+mpiexec -n ${NTOTRANKS} --ppn ${NRANKS_PER_NODE} --depth=${NDEPTH} --cpu-bind depth --env OMP_NUM_THREADS=${NTHREADS} -env OMP_PLACES=threads ./set_affinity_gpu_polaris.sh ${EXEC} ${INPUTS} ${restartString} amrex.the_arena_init_size = 0

From 92750914d87cb12bb450336baec8610256e7a7a5 Mon Sep 17 00:00:00 2001
From: Alexander Smith <soulalex@gmail.com>
Date: Sun, 23 Feb 2025 23:41:01 +0000
Subject: [PATCH 2/2] Adding parallel.submit

---
 job_scripts/polaris/parallel.submit | 31 +++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 job_scripts/polaris/parallel.submit

diff --git a/job_scripts/polaris/parallel.submit b/job_scripts/polaris/parallel.submit
new file mode 100644
index 0000000..220becb
--- /dev/null
+++ b/job_scripts/polaris/parallel.submit
@@ -0,0 +1,31 @@
+#!/bin/sh
+#PBS -l select=2:system=polaris
+#PBS -l filesystems=home:grand
+#PBS -l place=scatter
+#PBS -l walltime=01:00:00
+#PBS -q debug
+#PBS -A AstroExplosions
+
+EXEC="python z_series.py"
+#EXEC="python cei_energy.py"
+#EXEC="python test_grad_04.py"
+#EXEC="python plt_slice_vars.py"
+#EXEC="python velocity.py"
+# EXEC="python series_variable.py"
+# EXEC="python plot_compare.py"
+
+module use /soft/modulefiles
+module load conda
+conda activate
+CONDA_NAME=$(echo ${CONDA_PREFIX} | tr '\/' '\t' | sed -E 's/mconda3|\/base//g' | awk '{print $NF}')
+VENV_DIR="$(pwd)/venvs/${CONDA_NAME}"
+source "${VENV_DIR}/bin/activate"
+
+cd ${PBS_O_WORKDIR}
+
+NNODES=`wc -l < $PBS_NODEFILE`
+NRANKS_PER_NODE=32
+NTOTRANKS=$(( NNODES * NRANKS_PER_NODE ))
+
+# ${EXEC}
+mpiexec -n ${NTOTRANKS} -ppn ${NRANKS_PER_NODE} ${EXEC}
\ No newline at end of file