Skip to content

Commit 2a4ec60

Browse files
allaffaallaffa
committed
Add OPF fine-tuning campaign scripts and paper figure generation
- train_opf_finetune.py: FT1/FT3 training entry point with all regime options - plot_finetune_paper_figures.py: generates all 7 paper figures (fig09-fig14) - plot_ft1_bce_epoch_curves.py: BCE convergence curves for FT1 - job-frontier-FT{1,3}-single-method.sh: single-run Slurm job scripts - job-frontier-FT{1,3}-bundled.sh: bundled multi-run job scripts - submit_ft{1,3}_data_efficiency_sweep.sh: data-efficiency sweep launchers - submit_ft1_ft3_jobs.sh, submit_ft_{bundled,single_method}.sh: campaign launchers - submit_ft_debug_{chain,one}.sh: debug submission helpers - watch_and_chain_{per_n,bundles,debug}.sh: job chaining watchers
1 parent e38ae4f commit 2a4ec60

17 files changed

Lines changed: 1803 additions & 11 deletions

‎examples/opf/finetune/job-frontier-FT1-bundled.sh‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#!/bin/bash
2+
# =============================================================================
3+
# Frontier Slurm job — FT1 Feasibility Classification (bundled data-efficiency)
4+
#
5+
# Runs all sample sizes sequentially; within each sample size the 4 training
6+
# methods (full / partial / head_only / scratch) run concurrently with srun.
7+
#
8+
# Required env vars (set via sbatch --export or environment):
9+
# FT_ARCH HeteroSAGE | HeteroHEAT
10+
#
11+
# Optional env vars:
12+
# N_PER_RUN nodes per method (default: 8)
13+
# PRETRAINED_MODEL override pretrained model name
14+
# SAMPLE_SIZES space-separated list (default: 100 500 1000 2500 5000
15+
# 10000 25000 50000 100000)
16+
#
17+
# Usage (from examples/opf/finetune/):
18+
# sbatch --export=ALL,FT_ARCH=HeteroSAGE \
19+
# job-frontier-FT1-bundled.sh
20+
# =============================================================================
21+
#SBATCH -A eng164
22+
#SBATCH -J FT1-bundled
23+
#SBATCH -o /lustre/orion/lrn078/proj-shared/HydraGNN/FT1-bundled-%j.out
24+
#SBATCH -e /lustre/orion/lrn078/proj-shared/HydraGNN/FT1-bundled-%j.out
25+
#SBATCH -t 12:00:00
26+
#SBATCH -p batch
27+
#SBATCH -N 32
28+
29+
# -----------------------------------------------------------------------------
30+
# Configuration
31+
# -----------------------------------------------------------------------------
32+
FT_ARCH=${FT_ARCH:-HeteroSAGE}
33+
N_PER_RUN=${N_PER_RUN:-8}
34+
PRETRAINED_MODEL=${PRETRAINED_MODEL:-${FT_ARCH}_best}
35+
read -ra SAMPLE_SIZES <<< "${SAMPLE_SIZES:-100 500 1000 2500 5000 10000 25000 50000 100000}"
36+
37+
# -----------------------------------------------------------------------------
38+
# Environment
39+
# -----------------------------------------------------------------------------
40+
export all_proxy=socks://proxy.ccs.ornl.gov:3128/
41+
export ftp_proxy=ftp://proxy.ccs.ornl.gov:3128/
42+
export http_proxy=http://proxy.ccs.ornl.gov:3128/
43+
export https_proxy=http://proxy.ccs.ornl.gov:3128/
44+
export no_proxy='localhost,127.0.0.0/8,*.ccs.ornl.gov'
45+
46+
HYDRAGNN_ROOT=/lustre/orion/lrn078/proj-shared/HydraGNN
47+
48+
source /lustre/orion/lrn070/world-shared/mlupopa/module-to-load-frontier-rocm711.sh
49+
source activate /lustre/orion/lrn078/proj-shared/HydraGNN/installation_DOE_supercomputers/HydraGNN-Installation-Frontier/hydragnn_venv
50+
51+
export PYTHONPATH=$HYDRAGNN_ROOT:${PYTHONPATH:-}
52+
export PYTHONPATH=/lustre/orion/lrn078/proj-shared/HydraGNN/installation_DOE_supercomputers/HydraGNN-Installation-Frontier/hydragnn_venv/lib/python3.11/site-packages/:${PYTHONPATH:-}
53+
54+
module unload darshan-runtime
55+
56+
export PLUGIN_PATH=/ccs/sw/crusher/amdsw/aws-ofi-nccl/aws-ofi-nccl
57+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PLUGIN_PATH}/lib
58+
export FI_MR_CACHE_MONITOR=kdreg2
59+
export FI_CXI_DEFAULT_CQ_SIZE=131072
60+
export FI_CXI_DEFAULT_TX_SIZE=2048
61+
export FI_CXI_RX_MATCH_MODE=hybrid
62+
export FI_CXI_RDV_PROTO=alt_read
63+
export FI_CXI_DISABLE_HOST_REGISTER=1
64+
export NCCL_NET_PLUGIN=${PLUGIN_PATH}/lib/librccl-net.so
65+
export NCCL_NET_GDR_LEVEL="PHB"
66+
export NCCL_CROSS_NIC=1
67+
export NCCL_SOCKET_IFNAME=hsn0
68+
export NCCL_NET="AWS Libfabric"
69+
export TORCH_NCCL_HIGH_PRIORITY=1
70+
export GPU_MAX_HW_QUEUES=2
71+
export HSA_FORCE_FINE_GRAIN_PCIE=1
72+
export FI_CXI_RDZV_EAGER_SIZE=0
73+
export FI_CXI_RDZV_GET_MIN=0
74+
export FI_CXI_RDZV_THRESHOLD=0
75+
76+
# -----------------------------------------------------------------------------
77+
# Derived
78+
# -----------------------------------------------------------------------------
79+
FT_DIR=$HYDRAGNN_ROOT/examples/opf/finetune
80+
DATA_ROOT=$FT_DIR/../dataset
81+
82+
echo "============================================================"
83+
echo " FT1 Feasibility Classification — bundled data-efficiency"
84+
echo " Arch : $FT_ARCH (pretrained: $PRETRAINED_MODEL)"
85+
echo " Methods : full / partial / head_only / scratch (concurrent)"
86+
echo " Nodes per run : $N_PER_RUN (total: $((N_PER_RUN * 4)))"
87+
echo " Sample sizes : ${SAMPLE_SIZES[*]}"
88+
echo " Job ID : $SLURM_JOB_ID"
89+
echo "============================================================"
90+
91+
cd $FT_DIR
92+
93+
# Helper: launch one srun in the background for a given regime and sample size
94+
_launch() {
95+
local REGIME="$1"
96+
local MAX_TRAIN_SAMPLES="$2"
97+
local SCRATCH_FLAG="${3:-}"
98+
local _scratch_tag=""
99+
[[ -n "$SCRATCH_FLAG" ]] && _scratch_tag="_scratch"
100+
local LOG_NAME="FT1_feasibility_${FT_ARCH}_${REGIME}${_scratch_tag}_n${MAX_TRAIN_SAMPLES}"
101+
local LOGFILE="$HYDRAGNN_ROOT/${LOG_NAME}-${SLURM_JOB_ID}.out"
102+
103+
echo " Launching $LOG_NAME → $LOGFILE"
104+
105+
srun --exact -N${N_PER_RUN} -n$((N_PER_RUN * 8)) -c7 \
106+
--gpus-per-task=1 --gpu-bind=closest \
107+
python -u train_opf_ft1_classify.py \
108+
--inputfile FT1_feasibility_classification/config_${FT_ARCH}_${REGIME}.json \
109+
--modelname "$LOG_NAME" \
110+
--data_root $DATA_ROOT \
111+
--pretrained_model_dir $HYDRAGNN_ROOT/examples/opf/pretrained_models \
112+
--pretrained_model_name $PRETRAINED_MODEL \
113+
--finetune_regime $REGIME \
114+
$SCRATCH_FLAG \
115+
--max_train_samples $MAX_TRAIN_SAMPLES \
116+
> "$LOGFILE" 2>&1 &
117+
}
118+
119+
# -----------------------------------------------------------------------------
120+
# Main loop: iterate sample sizes sequentially, methods concurrently
121+
# -----------------------------------------------------------------------------
122+
ROUND=0
123+
for N in "${SAMPLE_SIZES[@]}"; do
124+
ROUND=$((ROUND + 1))
125+
TOTAL=$((N * 2)) # balanced: N feasible + N infeasible
126+
echo ""
127+
echo "── Round $ROUND / ${#SAMPLE_SIZES[@]}: N=$N (total train samples=$TOTAL) ──"
128+
129+
_launch full "$TOTAL"
130+
_launch partial "$TOTAL"
131+
_launch head_only "$TOTAL"
132+
_launch full "$TOTAL" "--no_pretrained" # scratch baseline
133+
134+
echo " Waiting for round $ROUND to finish..."
135+
wait
136+
echo " Round $ROUND done."
137+
done
138+
139+
echo ""
140+
echo "All rounds finished. Job ID: $SLURM_JOB_ID"

‎examples/opf/finetune/job-frontier-FT1-single-method.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
# sbatch --export=ALL,FT_ARCH=HeteroSAGE,FT_METHOD=full \
2020
# job-frontier-FT1-single-method.sh
2121
# =============================================================================
22-
#SBATCH -A mat746
22+
#SBATCH -A eng164
2323
#SBATCH -J FT1-single
2424
#SBATCH -o /lustre/orion/lrn078/proj-shared/HydraGNN/FT1-single-%j.out
2525
#SBATCH -e /lustre/orion/lrn078/proj-shared/HydraGNN/FT1-single-%j.out

‎examples/opf/finetune/job-frontier-FT3-bundled.sh‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
#!/bin/bash
2+
# =============================================================================
3+
# Frontier Slurm job — FT3 N-1 Contingency OPF Regression (bundled data-efficiency)
4+
#
5+
# Runs all sample sizes sequentially; within each sample size the 4 training
6+
# methods (full / partial / head_only / scratch) run concurrently with srun.
7+
#
8+
# Required env vars (set via sbatch --export or environment):
9+
# FT_ARCH HeteroSAGE | HeteroHEAT
10+
#
11+
# Optional env vars:
12+
# N_PER_RUN nodes per method (default: 8)
13+
# PRETRAINED_MODEL override pretrained model name
14+
# SAMPLE_SIZES space-separated list (default: 100 500 1000 2500 5000
15+
# 10000 25000 50000 100000 270000)
16+
#
17+
# Usage (from examples/opf/finetune/):
18+
# sbatch --export=ALL,FT_ARCH=HeteroSAGE \
19+
# job-frontier-FT3-bundled.sh
20+
# =============================================================================
21+
#SBATCH -A eng164
22+
#SBATCH -J FT3-bundled
23+
#SBATCH -o /lustre/orion/lrn078/proj-shared/HydraGNN/FT3-bundled-%j.out
24+
#SBATCH -e /lustre/orion/lrn078/proj-shared/HydraGNN/FT3-bundled-%j.out
25+
#SBATCH -t 12:00:00
26+
#SBATCH -p batch
27+
#SBATCH -N 32
28+
29+
# -----------------------------------------------------------------------------
30+
# Configuration
31+
# -----------------------------------------------------------------------------
32+
FT_ARCH=${FT_ARCH:-HeteroSAGE}
33+
N_PER_RUN=${N_PER_RUN:-8}
34+
PRETRAINED_MODEL=${PRETRAINED_MODEL:-${FT_ARCH}_best}
35+
FT_STRATEGY=FT3_contingency
36+
read -ra SAMPLE_SIZES <<< "${SAMPLE_SIZES:-100 500 1000 2500 5000 10000 25000 50000 100000 270000}"
37+
38+
# -----------------------------------------------------------------------------
39+
# Environment
40+
# -----------------------------------------------------------------------------
41+
export all_proxy=socks://proxy.ccs.ornl.gov:3128/
42+
export ftp_proxy=ftp://proxy.ccs.ornl.gov:3128/
43+
export http_proxy=http://proxy.ccs.ornl.gov:3128/
44+
export https_proxy=http://proxy.ccs.ornl.gov:3128/
45+
export no_proxy='localhost,127.0.0.0/8,*.ccs.ornl.gov'
46+
47+
HYDRAGNN_ROOT=/lustre/orion/lrn078/proj-shared/HydraGNN
48+
49+
source /lustre/orion/lrn070/world-shared/mlupopa/module-to-load-frontier-rocm711.sh
50+
source activate /lustre/orion/lrn078/proj-shared/HydraGNN/installation_DOE_supercomputers/HydraGNN-Installation-Frontier/hydragnn_venv
51+
52+
export PYTHONPATH=$HYDRAGNN_ROOT:${PYTHONPATH:-}
53+
export PYTHONPATH=/lustre/orion/lrn078/proj-shared/HydraGNN/installation_DOE_supercomputers/HydraGNN-Installation-Frontier/hydragnn_venv/lib/python3.11/site-packages/:${PYTHONPATH:-}
54+
55+
module unload darshan-runtime
56+
57+
export PLUGIN_PATH=/ccs/sw/crusher/amdsw/aws-ofi-nccl/aws-ofi-nccl
58+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PLUGIN_PATH}/lib
59+
export FI_MR_CACHE_MONITOR=kdreg2
60+
export FI_CXI_DEFAULT_CQ_SIZE=131072
61+
export FI_CXI_DEFAULT_TX_SIZE=2048
62+
export FI_CXI_RX_MATCH_MODE=hybrid
63+
export FI_CXI_RDV_PROTO=alt_read
64+
export FI_CXI_DISABLE_HOST_REGISTER=1
65+
export NCCL_NET_PLUGIN=${PLUGIN_PATH}/lib/librccl-net.so
66+
export NCCL_NET_GDR_LEVEL="PHB"
67+
export NCCL_CROSS_NIC=1
68+
export NCCL_SOCKET_IFNAME=hsn0
69+
export NCCL_NET="AWS Libfabric"
70+
export TORCH_NCCL_HIGH_PRIORITY=1
71+
export GPU_MAX_HW_QUEUES=2
72+
export HSA_FORCE_FINE_GRAIN_PCIE=1
73+
export FI_CXI_RDZV_EAGER_SIZE=0
74+
export FI_CXI_RDZV_GET_MIN=0
75+
export FI_CXI_RDZV_THRESHOLD=0
76+
77+
# -----------------------------------------------------------------------------
78+
# Derived
79+
# -----------------------------------------------------------------------------
80+
FT_DIR=$HYDRAGNN_ROOT/examples/opf/finetune
81+
DATA_ROOT=$FT_DIR/../dataset
82+
83+
echo "============================================================"
84+
echo " FT3 N-1 Contingency OPF — bundled data-efficiency"
85+
echo " Arch : $FT_ARCH (pretrained: $PRETRAINED_MODEL)"
86+
echo " Methods : full / partial / head_only / scratch (concurrent)"
87+
echo " Nodes per run : $N_PER_RUN (total: $((N_PER_RUN * 4)))"
88+
echo " Sample sizes : ${SAMPLE_SIZES[*]}"
89+
echo " Job ID : $SLURM_JOB_ID"
90+
echo "============================================================"
91+
92+
cd $FT_DIR
93+
94+
# Helper: launch one srun in the background for a given regime and sample size
95+
_launch() {
96+
local REGIME="$1"
97+
local MAX_TRAIN_SAMPLES="$2"
98+
local SCRATCH_FLAG="${3:-}"
99+
local _scratch_tag=""
100+
[[ -n "$SCRATCH_FLAG" ]] && _scratch_tag="_scratch"
101+
local LOG_NAME="finetune_${FT_STRATEGY}_${FT_ARCH}_${REGIME}${_scratch_tag}_n${MAX_TRAIN_SAMPLES}"
102+
local LOGFILE="$HYDRAGNN_ROOT/${LOG_NAME}-${SLURM_JOB_ID}.out"
103+
104+
echo " Launching $LOG_NAME → $LOGFILE"
105+
106+
srun --exact -N${N_PER_RUN} -n$((N_PER_RUN * 8)) -c7 \
107+
--gpus-per-task=1 --gpu-bind=closest \
108+
python -u train_opf_finetune.py \
109+
--inputfile ${FT_STRATEGY}/config_${FT_ARCH}_${REGIME}.json \
110+
--hdf5 \
111+
--modelname "$LOG_NAME" \
112+
--resume_if_exists \
113+
--data_root $DATA_ROOT \
114+
--pretrained_model_dir $HYDRAGNN_ROOT/examples/opf/pretrained_models \
115+
--pretrained_model_name $PRETRAINED_MODEL \
116+
--finetune_regime $REGIME \
117+
$SCRATCH_FLAG \
118+
--max_train_samples $MAX_TRAIN_SAMPLES \
119+
> "$LOGFILE" 2>&1 &
120+
}
121+
122+
# -----------------------------------------------------------------------------
123+
# Main loop: iterate sample sizes sequentially, methods concurrently
124+
# -----------------------------------------------------------------------------
125+
ROUND=0
126+
for N in "${SAMPLE_SIZES[@]}"; do
127+
ROUND=$((ROUND + 1))
128+
echo ""
129+
echo "── Round $ROUND / ${#SAMPLE_SIZES[@]}: N=$N training samples ──"
130+
131+
_launch full "$N"
132+
_launch partial "$N"
133+
_launch head_only "$N"
134+
_launch full "$N" "--no_pretrained" # scratch baseline
135+
136+
echo " Waiting for round $ROUND to finish..."
137+
wait
138+
echo " Round $ROUND done."
139+
done
140+
141+
echo ""
142+
echo "All rounds finished. Job ID: $SLURM_JOB_ID"

‎examples/opf/finetune/job-frontier-FT3-single-method.sh‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/bin/bash
2+
set -euo pipefail
23
# =============================================================================
34
# Frontier Slurm job — FT3 N-1 Contingency OPF Regression (single method)
45
#
@@ -20,7 +21,7 @@
2021
# sbatch --export=ALL,FT_ARCH=HeteroSAGE,FT_METHOD=full \
2122
# job-frontier-FT3-single-method.sh
2223
# =============================================================================
23-
#SBATCH -A mat746
24+
#SBATCH -A eng164
2425
#SBATCH -J FT3-single
2526
#SBATCH -o /lustre/orion/lrn078/proj-shared/HydraGNN/FT3-single-%j.out
2627
#SBATCH -e /lustre/orion/lrn078/proj-shared/HydraGNN/FT3-single-%j.out
@@ -135,6 +136,7 @@ for N in "${SAMPLE_SIZES[@]}"; do
135136
--inputfile ${FT_STRATEGY}/config_${FT_ARCH}_${REGIME}.json \
136137
--hdf5 \
137138
--modelname "$LOG_NAME" \
139+
--resume_if_exists \
138140
--data_root $DATA_ROOT \
139141
--pretrained_model_dir $HYDRAGNN_ROOT/examples/opf/pretrained_models \
140142
--pretrained_model_name $PRETRAINED_MODEL \
@@ -143,6 +145,13 @@ for N in "${SAMPLE_SIZES[@]}"; do
143145
--max_train_samples $N \
144146
> "$LOGFILE" 2>&1
145147

148+
RESULT_JSON="$FT_DIR/logs/$LOG_NAME/results.json"
149+
if [[ ! -f "$RESULT_JSON" ]]; then
150+
echo "ERROR: Missing expected result artifact: $RESULT_JSON"
151+
echo "See round log: $LOGFILE"
152+
exit 1
153+
fi
154+
146155
echo " Round $ROUND done."
147156
done
148157

0 commit comments

Comments
 (0)