Version 2.6.0: added cometx copy --path

Author: Douglas Blank

README.md

@@ -180,6 +180,8 @@ cometx copy examples:
 ```
 cometx copy SOURCE DESTINATION
 cometx copy --symlink SOURCE DESTINATION
+cometx copy --path /base/path SOURCE DESTINATION
+cometx copy --path ~/Downloads SOURCE DESTINATION
 ```
 
 where SOURCE is:
@@ -232,6 +234,23 @@ Not all combinations are possible:
 * `--quiet` - If given, don't display update info
 * `--symlink` - Instead of copying, create a link to an experiment in a project
 * `--sync` - Check to see if experiment name has been created first; if so, skip
+* `--path PATH` - Path to prepend to workspace_src when accessing files (supports ~ for home directory)
+
+### Using --path
+
+The `--path` option allows you to specify a base directory where your workspace folders are located. This is useful when your downloaded experiments are stored in a specific directory structure.
+
+Examples:
+```bash
+# Copy from experiments in /data/experiments/workspace
+cometx copy --path /data/experiments workspace dest-workspace
+
+# Copy from experiments in your home directory
+cometx copy --path ~ workspace dest-workspace
+
+# Copy from experiments in Downloads folder
+cometx copy --path ~/Downloads workspace dest-workspace
+```
 
 For more information, `cometx copy --help`
 

cometx/_version.py

@@ -12,5 +12,5 @@
 #  the express permission of Comet ML Inc.
 # *******************************************************
 
-version_info = (2, 5, 3)
+version_info = (2, 6, 0)
 __version__ = ".".join(map(str, version_info))

cometx/cli/copy.py

@@ -190,13 +190,21 @@ def get_parser_arguments(parser):
         type=int,
         default=None,
     )
+    parser.add_argument(
+        "--path",
+        help="Path to prepend to workspace_src when accessing files (supports ~ for home directory)",
+        type=str,
+        default=None,
+    )
 
 
 def copy(parsed_args, remaining=None):
     # Called via `cometx copy ...`
     try:
         copy_manager = CopyManager(
-            max_concurrent_uploads=parsed_args.parallel, debug=parsed_args.debug
+            max_concurrent_uploads=parsed_args.parallel,
+            debug=parsed_args.debug,
+            path=parsed_args.path,
         )
         copy_manager.copy(
             parsed_args.COMET_SOURCE,
@@ -508,7 +516,7 @@ def _create_progress_bar(self, progress, width=20):
 
 
 class CopyManager:
-    def __init__(self, max_concurrent_uploads=None, debug=False):
+    def __init__(self, max_concurrent_uploads=None, debug=False, path=None):
         """
         | Destination:       | WORKSPACE            | WORKSPACE/PROJECT      |
         | Source (below)     |                      |                        |
@@ -522,6 +530,7 @@ def __init__(self, max_concurrent_uploads=None, debug=False):
 
         self.api = API()
         self.debug = debug
+        self.path = path
         # Calculate default number of workers based on CPU count, similar to download command
         self.max_concurrent_uploads = (
             min(32, os.cpu_count() + 4)
@@ -542,6 +551,13 @@ def __init__(self, max_concurrent_uploads=None, debug=False):
 
         self._start_upload_workers()
 
+    def _get_path(self, workspace_src, project_src, *items):
+        """Get the full path, prepending self.path if provided and joining all components"""
+        if self.path:
+            expanded_path = os.path.expanduser(self.path)
+            return os.path.join(expanded_path, workspace_src, project_src, *items)
+        return os.path.join(workspace_src, project_src, *items)
+
     def _start_upload_workers(self):
         """Start background worker threads for handling uploads"""
         for i in range(self.max_concurrent_uploads):
@@ -812,7 +828,7 @@ def copy(self, source, destination, symlink, ignore, debug, sync):
         if project_src == "panels":
             # experiment_src may be "*" or filename
             for filename in glob.glob(
-                os.path.join(workspace_src, project_src, experiment_src)
+                self._get_path(workspace_src, project_src, experiment_src)
             ):
                 print("Uploading panel zip: %r to %r..." % (filename, workspace_dst))
                 self.api.upload_panel_zip(workspace_dst, filename)
@@ -870,7 +886,7 @@ def copy(self, source, destination, symlink, ignore, debug, sync):
 
             # Next, check if the project_dst exists:
             if temp_project_dst not in projects:
-                project_metadata_path = os.path.join(
+                project_metadata_path = self._get_path(
                     workspace_src, project_src, "project_metadata.json"
                 )
                 if os.path.exists(project_metadata_path):
@@ -896,7 +912,11 @@ def copy(self, source, destination, symlink, ignore, debug, sync):
                 )
             elif "experiments" not in self.ignore:
                 self.copy_experiment_to(
-                    experiment_folder, workspace_dst, temp_project_dst
+                    experiment_folder,
+                    workspace_dst,
+                    temp_project_dst,
+                    workspace_src,
+                    folder_project,
                 )
 
     def create_experiment(self, workspace_dst, project_dst, offline=True):
@@ -953,15 +973,18 @@ def filtered_method(message):
         return experiment
 
     def get_experiment_folders(self, workspace_src, project_src, experiment_src):
-        for path in glob.iglob(f"{workspace_src}/{project_src}/{experiment_src}"):
+        full_path = self._get_path(workspace_src, project_src, experiment_src)
+        for path in glob.iglob(full_path):
             if any(
                 [path.endswith("~"), path.endswith(".json"), path.endswith(".jsonl")]
             ):
                 continue
             else:
                 yield path
 
-    def copy_experiment_to(self, experiment_folder, workspace_dst, project_dst):
+    def copy_experiment_to(
+        self, experiment_folder, workspace_dst, project_dst, workspace_src, project_src
+    ):
         title = experiment_folder
         experiment_name = None
         # See if there is a name:
@@ -981,10 +1004,7 @@ def copy_experiment_to(self, experiment_folder, workspace_dst, project_dst):
         # Copy other project-level items to an experiment:
         if "reports" not in self.ignore and not self.copied_reports:
             experiment = None
-            workspace_src, project_src, _ = experiment_folder.replace("\\", "/").split(
-                "/"
-            )
-            reports = os.path.join(workspace_src, project_src, "reports", "*")
+            reports = self._get_path(workspace_src, project_src, "reports", "*")
             for filename in glob.glob(reports):
                 if filename.endswith("reports_metadata.jsonl"):
                     continue