fix

Author: mnoukhov

mason.py

@@ -477,11 +477,11 @@ def make_internal_command(command: list[str], args: argparse.Namespace, whoami:
                 try:
                     model_arg_idx = filtered_command.index("--model_name_or_path")
                     model_name_idx = model_arg_idx + 1
-                    model_name_or_path = filtered_command[model_name_idx]
+                    model_name_or_path = filtered_command[model_name_idx].rstrip("/")
 
                     if model_name_or_path.startswith("gs://"):
                         model_name_hash = hashlib.md5(model_name_or_path.encode("utf-8")).hexdigest()[:8]
-                        local_cache_folder = f"{args.auto_output_dir_path}/{whoami}/tokenizer_{model_name_hash}"
+                        local_cache_folder = f"{args.auto_output_dir_path}/{whoami}/tokenizer_{model_name_hash}/"
 
                         if not os.path.exists(local_cache_folder):
                             download_from_gs_bucket(f"{model_name_or_path}/tokenizer*", local_cache_folder)

open_instruct/utils.py

@@ -1066,6 +1066,10 @@ def download_from_hf(model_name_or_path: str, revision: str) -> None:
 
 def download_from_gs_bucket(src_path: str, dest_path: str) -> None:
     cmd = [
+        "mkdir",
+        "-p",
+        dest_path,
+        "&&",
         "gsutil",
         "-o",
         "GSUtil:parallel_thread_count=1",