[fix] Fixed #78: exported redis-benchmark csv that contains commas is not being properly parsed (#80)

Author: filipecosta90

redisbench_admin/run/common.py

@@ -112,9 +112,14 @@ def prepare_benchmark_parameters(
                     entry,
                     current_workdir,
                 )
+    printed_command_str = command_str
+    printed_command_arr = command_arr
+    if len(command_str) > 200:
+        printed_command_str = command_str[:200] + "... (trimmed output) ..."
+        printed_command_arr = printed_command_arr[:1] + ["(...) trimmed output...."]
     logging.info(
         "Running the benchmark with the following parameters:\n\tArgs array: {}\n\tArgs str: {}".format(
-            command_arr, command_str
+            printed_command_arr, printed_command_str
         )
     )
     return command_arr, command_str

redisbench_admin/run/redis_benchmark/redis_benchmark.py

@@ -15,15 +15,19 @@ def redis_benchmark_from_stdout_csv_to_json(
         "StartTime": start_time_ms,
         "StartTimeHuman": start_time_str,
     }
-    csv_data = list(csv.reader(csv_data.splitlines(), delimiter=","))
-    header = csv_data[0]
-    for row in csv_data[1:]:
-        test_name = row[0]
-        if overload_test_name is not None:
-            test_name = overload_test_name
-        results_dict["Tests"][test_name] = {}
-        for pos, value in enumerate(row[1:]):
-            results_dict["Tests"][test_name][header[pos + 1]] = value
+    csv_data = csv_data.splitlines()
+    full_csv = list(csv.reader(csv_data, delimiter=",", quoting=csv.QUOTE_ALL))
+    if len(full_csv) >= 2:
+        header = full_csv[0]
+        for raw_row in csv_data[1:]:
+            row = raw_row.rsplit(",", len(header) - 1)
+            assert len(row) == len(header)
+            test_name = row[0][1:-1].split(" ")[0]
+            if overload_test_name is not None:
+                test_name = overload_test_name
+            results_dict["Tests"][test_name] = {}
+            for pos, value in enumerate(row[1:]):
+                results_dict["Tests"][test_name][header[pos + 1]] = value
     return results_dict
 
 
@@ -67,10 +71,6 @@ def prepare_redis_benchmark_command(
     if last_append is not None:
         command_arr.extend(last_append)
         command_str = command_str + " " + last_str
-    logging.info(
-        "Running the benchmark with the following parameters:"
-        "\n\tArgs array: {}\n\tArgs str: {}".format(command_arr, command_str)
-    )
     return command_arr, command_str
 
 

tests/test_data/redis-benchmark-6.2.0-csv.out.2

@@ -0,0 +1,2 @@
+"test","rps","avg_latency_ms","min_latency_ms","p50_latency_ms","p95_latency_ms","p99_latency_ms","max_latency_ms"
+"JSON.SET pass-100 . {"sclr":0,"str":"b","sub_doc":{"sclr":10,"str":"c","arr":[1,2,3,{"sclr":20,"str":"d"}]},"array_of_docs":[-1,{"sclr":11,"str":"e","arr":[4,5,6,{"sclr":21,"str":"f"}]},{"sclr":12,"str":"g","arr":[7,8,9,{"sclr":22,"str":"h"}]},-2]}","73391.80","0.206","0.024","0.199","0.327","0.423","16.463"

tests/test_data/redis-benchmark-6.2.0-csv.out.3

@@ -0,0 +1 @@
+"test","rps","avg_latency_ms","min_latency_ms","p50_latency_ms","p95_latency_ms","p99_latency_ms","max_latency_ms"

tests/test_redis_benchmark_csv_format.py

@@ -30,3 +30,25 @@ def test_redis_benchmark_export_logic():
         csv_data = csv_file.read()
         results_dict = redis_benchmark_from_stdout_csv_to_json(csv_data, 1, "1")
         redis_benchmark_export_logic(results_dict, [], None, {})
+        assert "SET" in results_dict["Tests"]
+        assert "GET" in results_dict["Tests"]
+
+    with open("./tests/test_data/redis-benchmark-6.2.0-csv.out.2", "r") as csv_file:
+        csv_data = csv_file.read()
+        results_dict = redis_benchmark_from_stdout_csv_to_json(csv_data, 1, "1")
+        redis_benchmark_export_logic(results_dict, [], None, {})
+        assert "JSON.SET" in results_dict["Tests"]
+
+    with open("./tests/test_data/redis-benchmark-6.2.0-csv.out.2", "r") as csv_file:
+        csv_data = csv_file.read()
+        results_dict = redis_benchmark_from_stdout_csv_to_json(
+            csv_data, 1, "1", "Overall"
+        )
+        redis_benchmark_export_logic(results_dict, [], None, {})
+        assert "Overall" in results_dict["Tests"]
+
+    with open("./tests/test_data/redis-benchmark-6.2.0-csv.out.3", "r") as csv_file:
+        csv_data = csv_file.read()
+        results_dict = redis_benchmark_from_stdout_csv_to_json(csv_data, 1, "1")
+        redis_benchmark_export_logic(results_dict, [], None, {})
+        assert len(results_dict["Tests"].keys()) == 0