ppml-workshop/train_attack_NN_model.sh at master · JosephMPedersen/ppml-workshop · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#!/bin/bash
#SBATCH --gres=gpu:1
#SBATCH --job-name=trAttack
#SBATCH --partition=all
#SBATCH --qos=default
#SBATCH --cpus-per-task=3
#SBATCH --mem=64g
#SBATCH --open-mode=append
#SBATCH --signal=B:USR1@120
#SBATCH --requeue
#SBATCH --output=logs/%A_%a.stdout
#SBATCH --error=logs/%A_%a.stderr
#SBATCH --array=0
#SBATCH --exclude=n[1-5,51-55]


SECONDS=0

restart(){
    echo "Calling restart"
    scontrol requeue $SLURM_JOB_ID
    echo "Scheduled job for restart"
}

ignore(){
    echo "Ignored SIGTERM"
}
trap restart USR1
trap ignore TERM

date


args=()


args+=("train_attack_NN_model.py --zip")


echo "Starting python ${args[${SLURM_ARRAY_TASK_ID}]}"

srun python ${args[${SLURM_ARRAY_TASK_ID}]}

echo "End python ${args[${SLURM_ARRAY_TASK_ID}]}"


DURATION=$SECONDS

echo "End of the program! $(($DURATION / 60)) minutes and $(($DURATION % 60)) seconds elapsed."