Enabled TF_OVERRIDE_NAME. Updated ann-benchmark to the latest (#329)

* Added deployment_name zsets

* Enabled TF_OVERRIDE_NAME. Updated ann-benchmark to the latest

* Fixes per flake check

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.7.23"
+version = "0.7.24"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/compare/args.py

@@ -33,13 +33,15 @@ def create_compare_arguments(parser):
         "--test",
         type=str,
         default="",
-        help="specify a test to use for comparison. If none is specified by default will use all of them.",
+        help="specify a test (or a comma separated list of tests) to use for comparison. If none is specified by default will use all of them.",
     )
     parser.add_argument("--github_repo", type=str, default=GITHUB_REPO)
     parser.add_argument("--github_org", type=str, default=GITHUB_ORG)
     parser.add_argument("--triggering_env", type=str, default=TRIGGERING_ENV)
-    parser.add_argument("--deployment_type", type=str, default="oss-standalone")
     parser.add_argument("--deployment_name", type=str, default="oss-standalone")
+    parser.add_argument("--deployment_type", type=str, default="oss-standalone")
+    parser.add_argument("--baseline_deployment_name", type=str, default="")
+    parser.add_argument("--comparison_deployment_name", type=str, default="")
     parser.add_argument("--metric_name", type=str, default="Tests.Overall.rps")
     parser.add_argument(
         "--metric_mode",

redisbench_admin/compare/compare.py

@@ -39,12 +39,19 @@ def compare_command_logic(args, project_name, project_version):
     tf_github_org = args.github_org
     tf_github_repo = args.github_repo
     tf_triggering_env = args.triggering_env
-    deployment_type = args.deployment_type
-    deployment_name = args.deployment_name
+    if args.baseline_deployment_name != "":
+        baseline_deployment_name = args.baseline_deployment_name
+    else:
+        baseline_deployment_name = args.deployment_name
+    if args.comparison_deployment_name != "":
+        comparison_deployment_name = args.comparison_deployment_name
+    else:
+        comparison_deployment_name = args.deployment_name
+
     logging.info(
-        "Using deployment_type={} and deployment_name={} for the analysis".format(
-            deployment_type,
-            deployment_name,
+        "Using baseline deployment_name={} and comparison deployment_name={} for the analysis".format(
+            baseline_deployment_name,
+            comparison_deployment_name,
         )
     )
     from_ts_ms = args.from_timestamp
@@ -115,32 +122,35 @@ def compare_command_logic(args, project_name, project_version):
         used_key = testcases_metric_context_path_setname
 
     tags_regex_string = re.compile(args.testname_regex)
+    if args.test != "":
+        test_names = args.test.split(",")
+        logging.info("Using test name {}".format(test_names))
+    else:
+        try:
+            test_names = rts.smembers(used_key)
+            test_names = list(test_names)
+            test_names.sort()
+            final_test_names = []
+            for test_name in test_names:
+                test_name = test_name.decode()
+                match_obj = re.search(tags_regex_string, test_name)
+                if match_obj is not None:
+                    final_test_names.append(test_name)
+            test_names = final_test_names
 
-    try:
-        test_names = rts.smembers(used_key)
-        test_names = list(test_names)
-        test_names.sort()
-        final_test_names = []
-        for test_name in test_names:
-            test_name = test_name.decode()
-            match_obj = re.search(tags_regex_string, test_name)
-            if match_obj is not None:
-                final_test_names.append(test_name)
-        test_names = final_test_names
+        except redis.exceptions.ResponseError as e:
+            logging.warning(
+                "Error while trying to fetch test cases set (key={}) {}. ".format(
+                    used_key, e.__str__()
+                )
+            )
+            pass
 
-    except redis.exceptions.ResponseError as e:
         logging.warning(
-            "Error while trying to fetch test cases set (key={}) {}. ".format(
-                used_key, e.__str__()
+            "Based on test-cases set (key={}) we have {} comparison points. ".format(
+                used_key, len(test_names)
             )
         )
-        pass
-
-    logging.warning(
-        "Based on test-cases set (key={}) we have {} comparison points. ".format(
-            used_key, len(test_names)
-        )
-    )
     profilers_artifacts_matrix = []
     detected_regressions = []
     total_improvements = 0
@@ -154,23 +164,32 @@ def compare_command_logic(args, project_name, project_version):
             "{}={}".format(by_str, baseline_str),
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
-            "deployment_type={}".format(deployment_type),
-            "deployment_name={}".format(deployment_name),
+            "deployment_name={}".format(baseline_deployment_name),
+            "triggering_env={}".format(tf_triggering_env),
         ]
         filters_comparison = [
             "{}={}".format(by_str, comparison_str),
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
-            "deployment_type={}".format(deployment_type),
-            "deployment_name={}".format(deployment_name),
+            "deployment_name={}".format(comparison_deployment_name),
+            "triggering_env={}".format(tf_triggering_env),
         ]
         baseline_timeseries = rts.ts().queryindex(filters_baseline)
         comparison_timeseries = rts.ts().queryindex(filters_comparison)
+
+        # avoiding target time-series
+        comparison_timeseries = [x for x in comparison_timeseries if "target" not in x]
+        baseline_timeseries = [x for x in baseline_timeseries if "target" not in x]
         progress.update()
+        if args.verbose:
+            logging.info("Baseline timeseries {}".format(len(baseline_timeseries)))
+            logging.info("Comparison timeseries {}".format(len(comparison_timeseries)))
         if len(baseline_timeseries) != 1:
             if args.verbose:
                 logging.warning(
-                    "Baseline timeseries {}".format(len(baseline_timeseries))
+                    "Skipping this test given the value of timeseries !=1. Baseline timeseries {}".format(
+                        len(baseline_timeseries)
+                    )
                 )
             continue
         else:
@@ -326,8 +345,8 @@ def compare_command_logic(args, project_name, project_version):
         ),
         headers=[
             "Test Case",
-            "Baseline (median obs. +- std.dev)",
-            "Comparison (median obs. +- std.dev)",
+            "Baseline {} (median obs. +- std.dev)".format(baseline_branch),
+            "Comparison {} (median obs. +- std.dev)".format(comparison_branch),
             "% change ({})".format(metric_mode),
             "Note",
         ],

redisbench_admin/run/ann/pkg

@@ -17,6 +17,7 @@
 
 
 REMOTE_INVENTORY = os.getenv("INVENTORY", None)
+TF_OVERRIDE_NAME = os.getenv("TF_OVERRIDE_NAME", None)
 REMOTE_USER = os.getenv("REMOTE_USER", "ubuntu")
 
 

redisbench_admin/run_remote/args.py

@@ -25,6 +25,7 @@ def remote_env_setup(
     tf_setup_name_sufix,
     tf_triggering_env,
     tf_timeout_secs=7200,
+    tf_override_name=None,
 ):
     server_plaintext_port = 6379
     db_ssh_port = args.db_ssh_port
@@ -67,6 +68,7 @@ def remote_env_setup(
             tf_setup_name_sufix,
             tf_triggering_env,
             tf_timeout_secs,
+            tf_override_name,
         )
     return (
         client_public_ip,

redisbench_admin/run_remote/remote_env.py

@@ -39,6 +39,7 @@
 from redisbench_admin.run.run import define_benchmark_plan
 from redisbench_admin.run.s3 import get_test_s3_bucket_path
 from redisbench_admin.run.ssh import ssh_pem_check
+from redisbench_admin.run_remote.args import TF_OVERRIDE_NAME
 from redisbench_admin.run_remote.consts import min_recommended_benchmark_duration
 from redisbench_admin.run_remote.remote_client import run_remote_client_tool
 from redisbench_admin.run_remote.remote_db import (
@@ -331,6 +332,7 @@ def run_remote_command_logic(args, project_name, project_version):
                                     tf_setup_name_sufix,
                                     tf_triggering_env,
                                     tf_timeout_secs,
+                                    TF_OVERRIDE_NAME,
                                 )
 
                                 # after we've created the env, even on error we should always teardown

redisbench_admin/run_remote/run_remote.py

@@ -28,6 +28,7 @@ def terraform_spin_or_reuse_env(
     tf_setup_name_sufix,
     tf_triggering_env,
     tf_timeout_secs=7200,
+    tf_override_name=None,
 ):
     (
         remote_setup,
@@ -39,7 +40,10 @@ def terraform_spin_or_reuse_env(
             repetition, BENCHMARK_REPETITIONS, test_name, remote_setup
         )
     )
-    tf_setup_name = "{}{}".format(remote_setup, tf_setup_name_sufix)
+    if tf_override_name is None:
+        tf_setup_name = "{}{}".format(remote_setup, tf_setup_name_sufix)
+    else:
+        tf_setup_name = tf_override_name
     logging.info("Using full setup name: {}".format(tf_setup_name))
     if remote_id not in remote_envs:
         # check if terraform is present

redisbench_admin/run_remote/terraform.py

@@ -256,7 +256,6 @@ def setup_remote_environment(
     tf_triggering_env,
     tf_timeout_secs=7200,
 ):
-    # key    = "benchmarks/infrastructure/tf-oss-redisgraph-standalone-r5.tfstate"
     _, _, _ = tf.init(
         capture_output=True,
         backend_config={

redisbench_admin/utils/remote.py

@@ -111,7 +111,9 @@ def test_timeseries_test_sucess_flow():
             assert rts.zcard(testcases_zsetname)
             assert rts.zcard(testcases_zsetname_component_search)
             assert test_name.encode() in rts.zpopmin(testcases_zsetname)[0][0]
-            assert deployment_name.encode() in rts.zpopmin(deployment_name_setname)[0][0]
+            assert (
+                deployment_name.encode() in rts.zpopmin(deployment_name_setname)[0][0]
+            )
             assert rts.exists(testcases_zsetname_component_search)
             assert (
                 test_name.encode()