Move a3u load to a working cluster (#602)

* change cluster name * change cluster name
GoogleCloudPlatform · Feb 18, 2025 · 2ebb947 · 2ebb947
1 parent de647ec
commit 2ebb947
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 8 deletions.
diff --git a/dags/map_reproducibility/a3ultra_mixtral_8_7b_nemo.py b/dags/map_reproducibility/a3ultra_mixtral_8_7b_nemo.py
@@ -55,8 +55,8 @@
 VALUE_YAML_PATH = (
     f"training/{HYPERCOMPUTER}/{MODEL_ID}/nemo-pretraining-gke/values.yaml"
 )
-CLUSTER = "a3ultra-benchmark"
-CLUSTER_REGION = "us-west1"
+CLUSTER = "a3ultra-bmark72"
+CLUSTER_REGION = "europe-west1"
 SOFTWARE_ID = "pytorch_nemo"
 IMAGE_VERSION = "nemo24.07"
 DOCKER_IMAGE = f"us-central1-docker.pkg.dev/deeplearning-images/reproducibility/pytorch-gpu-nemo-nccl:{IMAGE_VERSION}-gib1.0.3-A3U"

diff --git a/dags/map_reproducibility/utils/common_utils.py b/dags/map_reproducibility/utils/common_utils.py
@@ -187,12 +187,7 @@ def get_nemo_metrics_cmds(
 
 
 def cleanup_cmds():
-  cleanup = (
-      "helm uninstall $JOB_NAME",
-      "kubectl get pods "
-      "--no-headers=true | awk '{print $1}' "
-      "| grep $JOB_NAME | xargs kubectl delete pods",
-  )
+  cleanup = ("helm uninstall $JOB_NAME",)
   return cleanup