Fixes per SPEC usage of api (#253)

* [fix] Fixed redis.exceptions.DataError and extended logging

* [fix] Fixes per SPEC usage of api

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.5.26"
+version = "0.5.27"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/run/common.py

@@ -231,77 +231,68 @@ def common_exporter_logic(
     testcase_metric_context_paths = []
     version_target_tables = None
     branch_target_tables = None
+    used_ts = datapoints_timestamp
 
-    if exporter_timemetric_path is not None and len(metrics) > 0:
-        if datapoints_timestamp is None:
-            # extract timestamp
-            datapoints_timestamp = parse_exporter_timemetric(
-                exporter_timemetric_path, results_dict
-            )
-            if datapoints_timestamp is None:
-                datapoints_timestamp = int(
-                    datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000.0
-                )
-                logging.warning(
-                    "Error while trying to parse datapoints timestamp. Using current system timestamp Error: {}".format(
-                        datapoints_timestamp
-                    )
-                )
-        if (
-            artifact_version is not None
-            and artifact_version != ""
-            and artifact_version != "N/A"
-        ):
-            # extract per-version datapoints
-            (
-                _,
-                per_version_time_series_dict,
-                version_target_tables,
-            ) = extract_perversion_timeseries_from_results(
-                datapoints_timestamp,
-                metrics,
-                results_dict,
-                artifact_version,
-                tf_github_org,
-                tf_github_repo,
-                deployment_name,
-                deployment_type,
-                test_name,
-                tf_triggering_env,
-                metadata_tags,
-                build_variant_name,
-                running_platform,
-                testcase_metric_context_paths,
-            )
-        if tf_github_branch is not None and tf_github_branch != "":
-            # extract per branch datapoints
-            (
-                _,
-                per_branch_time_series_dict,
-                branch_target_tables,
-            ) = extract_perbranch_timeseries_from_results(
-                datapoints_timestamp,
-                metrics,
-                results_dict,
-                str(tf_github_branch),
-                tf_github_org,
-                tf_github_repo,
-                deployment_name,
-                deployment_type,
-                test_name,
-                tf_triggering_env,
-                metadata_tags,
-                build_variant_name,
-                running_platform,
-                testcase_metric_context_paths,
-            )
-        else:
-            logging.warning(
-                "Requested to push data to RedisTimeSeries but no git"
-                " branch definition was found. git branch value {}".format(
-                    tf_github_branch
-                )
+    if exporter_timemetric_path is not None and used_ts is None:
+        # extract timestamp
+        used_ts = parse_exporter_timemetric(exporter_timemetric_path, results_dict)
+
+    if used_ts is None:
+        used_ts = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000.0)
+        logging.warning(
+            "Error while trying to parse datapoints timestamp. Using current system timestamp Error: {}".format(
+                used_ts
             )
+        )
+    assert used_ts != None
+    if (
+        artifact_version is not None
+        and artifact_version != ""
+        and artifact_version != "N/A"
+    ):
+        # extract per-version datapoints
+        (
+            _,
+            per_version_time_series_dict,
+            version_target_tables,
+        ) = extract_perversion_timeseries_from_results(
+            used_ts,
+            metrics,
+            results_dict,
+            artifact_version,
+            tf_github_org,
+            tf_github_repo,
+            deployment_name,
+            deployment_type,
+            test_name,
+            tf_triggering_env,
+            metadata_tags,
+            build_variant_name,
+            running_platform,
+            testcase_metric_context_paths,
+        )
+    if tf_github_branch is not None and tf_github_branch != "":
+        # extract per branch datapoints
+        (
+            _,
+            per_branch_time_series_dict,
+            branch_target_tables,
+        ) = extract_perbranch_timeseries_from_results(
+            used_ts,
+            metrics,
+            results_dict,
+            str(tf_github_branch),
+            tf_github_org,
+            tf_github_repo,
+            deployment_name,
+            deployment_type,
+            test_name,
+            tf_triggering_env,
+            metadata_tags,
+            build_variant_name,
+            running_platform,
+            testcase_metric_context_paths,
+        )
     else:
         logging.error(
             "Requested to push data to RedisTimeSeries but "

redisbench_admin/run/redistimeseries.py

@@ -303,11 +303,11 @@ def timeseries_test_sucess_flow(
             for (
                 branch_target_table_keyname,
                 branch_target_table_dict,
-            ) in version_target_tables.items():
+            ) in branch_target_tables.items():
 
                 logging.info(
                     "Setting target table by branch on key {}".format(
-                        version_target_table_keyname
+                        branch_target_table_keyname
                     )
                 )
                 if "contains-target" in branch_target_table_dict:

redisbench_admin/utils/benchmark_config.py

@@ -178,16 +178,26 @@ def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
     dbconfig_present = False
     if dbconfig_keyname in benchmark_config:
         dbconfig_present = True
-        for k in benchmark_config[dbconfig_keyname]:
-            if "configuration-parameters" in k:
-                cp = k["configuration-parameters"]
-                for item in cp:
-                    for k, v in item.items():
-                        redis_configuration_parameters[k] = v
-            if "dataset_load_timeout_secs" in k:
-                dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
-            if "dataset_name" in k:
-                dataset_name = k["dataset_name"]
+        if type(benchmark_config[dbconfig_keyname]) == list:
+            for k in benchmark_config[dbconfig_keyname]:
+                if "configuration-parameters" in k:
+                    cp = k["configuration-parameters"]
+                    for item in cp:
+                        for k, v in item.items():
+                            redis_configuration_parameters[k] = v
+                if "dataset_load_timeout_secs" in k:
+                    dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
+                if "dataset_name" in k:
+                    dataset_name = k["dataset_name"]
+        if type(benchmark_config[dbconfig_keyname]) == dict:
+            if "configuration-parameters" in benchmark_config[dbconfig_keyname]:
+                cp = benchmark_config[dbconfig_keyname]["configuration-parameters"]
+                for k, v in cp.items():
+                    redis_configuration_parameters[k] = v
+                if "dataset_load_timeout_secs" in cp:
+                    dataset_load_timeout_secs = cp["dataset_load_timeout_secs"]
+                if "dataset_name" in cp:
+                    dataset_name = cp["dataset_name"]
 
     return (
         dbconfig_present,

redisbench_admin/utils/remote.py

@@ -512,12 +512,20 @@ def push_data_to_redistimeseries(rts, time_series_dict: dict):
             exporter_create_ts(rts, time_series, timeseries_name)
             for timestamp, value in time_series["data"].items():
                 try:
-                    rts.add(
-                        timeseries_name,
-                        timestamp,
-                        value,
-                        duplicate_policy="last",
-                    )
+                    if timestamp is None:
+                        logging.warning("The provided timestamp is null. Using auto-ts")
+                        rts.add(
+                            timeseries_name,
+                            value,
+                            duplicate_policy="last",
+                        )
+                    else:
+                        rts.add(
+                            timeseries_name,
+                            timestamp,
+                            value,
+                            duplicate_policy="last",
+                        )
                     datapoint_inserts += 1
                 except redis.exceptions.DataError:
                     logging.warning(