In case of error fetch the entire folder. (#294)

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.6.24"
+version = "0.6.25"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/run_remote/remote_db.py

@@ -48,6 +48,7 @@ def remote_tmpdir_prune(
         username,
         private_key,
         [
+            "mkdir -p {}".format(temporary_dir),
             "rm -rf {}/*.log".format(temporary_dir),
             "rm -rf {}/*.config".format(temporary_dir),
             "rm -rf {}/*.rdb".format(temporary_dir),
@@ -152,22 +153,24 @@ def remote_db_spin(
                 redis_conns.append(local_redis_conn)
         except redis.exceptions.ConnectionError as e:
             logging.error("A error occurred while spinning DB: {}".format(e.__str__()))
-            remote_file = "{}/{}".format(temporary_dir, logfiles[0])
+            logfile = logfiles[0]
+
+            remote_file = "{}/{}".format(temporary_dir, logfile)
             logging.error(
-                "Trying to fetch DB remote log {} into {}".format(
-                    remote_file, logfiles[0]
-                )
+                "Trying to fetch DB remote log {} into {}".format(remote_file, logfile)
             )
-            failed_remote_run_artifact_store(
-                True,
-                client_public_ip,
+            db_error_artifacts(
+                db_ssh_port,
                 dirname,
-                remote_file,
-                logfiles[0],
+                full_logfiles,
+                logname,
+                private_key,
                 s3_bucket_name,
                 s3_bucket_path,
+                server_public_ip,
+                temporary_dir,
+                True,
                 username,
-                private_key,
             )
 
     if setup_type == "oss-standalone":
@@ -196,22 +199,18 @@ def remote_db_spin(
             redis_conns.append(local_redis_conn)
         except redis.exceptions.ConnectionError as e:
             logging.error("A error occurred while spinning DB: {}".format(e.__str__()))
-            remote_file = full_logfile
-            logging.error(
-                "Trying to fetch DB remote log {} into {}".format(
-                    remote_file, full_logfile
-                )
-            )
-            failed_remote_run_artifact_store(
-                True,
-                client_public_ip,
-                "",
-                remote_file,
-                full_logfile[1:],
+            db_error_artifacts(
+                db_ssh_port,
+                dirname,
+                full_logfiles,
+                logname,
+                private_key,
                 s3_bucket_name,
                 s3_bucket_path,
+                server_public_ip,
+                temporary_dir,
+                True,
                 username,
-                private_key,
             )
 
     if cluster_enabled:
@@ -314,6 +313,19 @@ def remote_db_spin(
         logging.info("Redis available")
     else:
         logging.error("Remote redis is not available")
+        db_error_artifacts(
+            db_ssh_port,
+            dirname,
+            full_logfiles,
+            logname,
+            private_key,
+            s3_bucket_name,
+            s3_bucket_path,
+            server_public_ip,
+            temporary_dir,
+            True,
+            username,
+        )
         raise Exception("Remote redis is not available. Aborting...")
     dataset_load_duration_seconds = (
         dataset_load_end_time - dataset_load_start_time
@@ -338,3 +350,52 @@ def remote_db_spin(
         server_plaintext_port,
         ssh_tunnel,
     )
+
+
+def db_error_artifacts(
+    db_ssh_port,
+    dirname,
+    full_logfiles,
+    logname,
+    private_key,
+    s3_bucket_name,
+    s3_bucket_path,
+    server_public_ip,
+    temporary_dir,
+    upload_s3,
+    username,
+):
+    local_zipfile = "{}.zip".format(logname)
+    remote_zipfile = "/home/{}/{}".format(username, local_zipfile)
+    execute_remote_commands(
+        server_public_ip,
+        username,
+        private_key,
+        [
+            "zip -r {} {}".format(remote_zipfile, temporary_dir),
+        ],
+        db_ssh_port,
+    )
+    failed_remote_run_artifact_store(
+        upload_s3,
+        server_public_ip,
+        dirname,
+        remote_zipfile,
+        local_zipfile,
+        s3_bucket_name,
+        s3_bucket_path,
+        username,
+        private_key,
+    )
+    if len(full_logfiles) > 0:
+        failed_remote_run_artifact_store(
+            upload_s3,
+            server_public_ip,
+            dirname,
+            full_logfiles[0],
+            logname,
+            s3_bucket_name,
+            s3_bucket_path,
+            username,
+            private_key,
+        )

redisbench_admin/run_remote/remote_failures.py

@@ -44,4 +44,8 @@ def failed_remote_run_artifact_store(
                 )
             )
             artifacts = [local_file_fullpath]
-            upload_artifacts_to_s3(artifacts, s3_bucket_name, s3_bucket_path)
+            artifacts_map = upload_artifacts_to_s3(
+                artifacts, s3_bucket_name, s3_bucket_path
+            )
+            for artifact_name, url in artifacts_map.items():
+                logging.info("Artifact: {}. URL: {}".format(artifact_name, url))