diff --git a/fill_postgres_db.py b/fill_postgres_db.py index 963d68ffc..7872651cb 100644 --- a/fill_postgres_db.py +++ b/fill_postgres_db.py @@ -282,8 +282,12 @@ def load_aggregated_stats(output_csv_path: str) -> Dict[str, Dict[str, Any]]: 'p99_mgas_s': float(row['p99 (MGas/s)']) if row.get('p99 (MGas/s)') and row['p99 (MGas/s)'].strip() else None, 'min_mgas_s': float(row['Min (MGas/s)']) if row.get('Min (MGas/s)') and row['Min (MGas/s)'].strip() else None, 'n_samples': int(row['N']) if row.get('N') and row['N'].strip() else None, - # Add timestamp fields if they exist + # Add timestamp and duration fields if they exist 'start_time': row.get('Start Time') if row.get('Start Time') else None, + 'end_time': row.get('End Time') if row.get('End Time') else None, + 'test_duration': float(row['Duration (ms)']) if row.get('Duration (ms)') and row['Duration (ms)'].strip() else None, + 'fcu_duration': float(row['FCU time (ms)']) if row.get('FCU time (ms)') and row['FCU time (ms)'].strip() else None, + 'np_duration': float(row['NP time (ms)']) if row.get('NP time (ms)') and row['NP time (ms)'].strip() else None, 'test_description': row.get('Description') } except ValueError as ve: @@ -472,6 +476,15 @@ def populate_data_for_client( start_time = agg_stats.get('start_time') if start_time in (0, "0", "", None): start_time = None + + end_time = agg_stats.get('end_time') + if end_time in (0, "0", "", None): + end_time = None + + test_duration = agg_stats.get('test_duration') + fcu_duration = agg_stats.get('fcu_duration') + np_duration = agg_stats.get('np_duration') + record: Dict[str, Any] = { 'client_name': client_name, 'client_version': client_version, @@ -488,6 +501,10 @@ def populate_data_for_client( 'raw_run_mgas_s': raw_run_mgas_s, 'raw_run_description': raw_run_description, # This is from the raw data row 'start_time': start_time, + 'end_time': end_time, + 'test_duration': test_duration, + 'fcu_duration': fcu_duration, + 'np_duration': np_duration, **computer_specs } records_to_insert.append(record) diff --git a/generate_postgres_schema.py b/generate_postgres_schema.py index ebbad70e6..3d969602e 100644 --- a/generate_postgres_schema.py +++ b/generate_postgres_schema.py @@ -37,8 +37,12 @@ def get_sql_for_benchmark_table(table_name: str) -> str: raw_run_duration_ms REAL NULL, -- Execution duration in milliseconds for this specific run raw_run_description TEXT NULL, -- Description from the raw_*.csv row, potentially more specific - -- Test execution timestamps + -- Test execution timestamps and durations start_time TIMESTAMP WITH TIME ZONE NULL, -- Test start timestamp + end_time TIMESTAMP WITH TIME ZONE NULL, -- Test end timestamp + test_duration REAL NULL, -- Test duration in milliseconds + fcu_duration REAL NULL, -- FCU (engine_forkchoiceUpdatedV3) duration in milliseconds + np_duration REAL NULL, -- NP (engine_newPayloadV4) duration in milliseconds -- Computer Specifications (parsed from system info, repeated per row, all nullable) spec_processor_type TEXT NULL, @@ -90,6 +94,10 @@ def execute_sql_on_db(db_params: Dict[str, Any], table_name: str) -> None: ("client_version", "TEXT NULL"), ("start_time", "TIMESTAMP WITH TIME ZONE NULL"), ("raw_run_duration_ms", "REAL NULL"), + ("end_time", "TIMESTAMP WITH TIME ZONE NULL"), + ("test_duration", "REAL NULL"), + ("fcu_duration", "REAL NULL"), + ("np_duration", "REAL NULL"), # Add other columns here in the future for schema evolution # e.g., ("new_feature_flag", "BOOLEAN DEFAULT FALSE") ] diff --git a/report_html.py b/report_html.py index f1f90179a..ed082e3b0 100644 --- a/report_html.py +++ b/report_html.py @@ -74,6 +74,10 @@ def get_html_report(client_results, clients, results_paths, test_cases, methods, 'N\n' 'Description\n' 'Start Time\n' + 'End Time\n' + f'Duration (ms) ↑ ↓\n' + f'FCU time (ms) ↑ ↓\n' + f'NP time (ms) ↑ ↓\n' '\n' '\n' '\n') @@ -88,7 +92,11 @@ def get_html_report(client_results, clients, results_paths, test_cases, methods, f'{data[1]}\n' f'{data[6]}\n' f'{data[7]}\n' - f'{data[8]}\n\n') + f'{data[8]}\n' + f'{data[9]}\n' + f'{data[10]}\n' + f'{data[11]}\n' + f'{data[12]}\n\n') results_to_print += '\n' results_to_print += ('\n' '\n') @@ -153,7 +161,7 @@ def get_html_report(client_results, clients, results_paths, test_cases, methods, print(formatted_html) if not os.path.exists('reports'): os.mkdir('reports') - with open(f'reports/index.html', 'w') as file: + with open('reports/index.html', 'w') as file: file.write(formatted_html) for client, gas_table in csv_table.items(): @@ -162,9 +170,9 @@ def get_html_report(client_results, clients, results_paths, test_cases, methods, csvwriter = csv.writer(csvfile) csvwriter.writerow( ['Title', 'Max (MGas/s)', 'p50 (MGas/s)', 'p95 (MGas/s)', 'p99 (MGas/s)', 'Min (MGas/s)', 'N', - 'Description', "Start Time"]) + 'Description', "Start Time", "End Time", "Duration (ms)", "FCU time (ms)", "NP time (ms)"]) for test_case, data in gas_table.items(): - csvwriter.writerow([data[0], data[2], data[3], data[4], data[5], data[1], data[6], data[7], data[8]]) + csvwriter.writerow([data[0], data[2], data[3], data[4], data[5], data[1], data[6], data[7], data[8], data[9], data[10], data[11], data[12]]) def main(): @@ -213,16 +221,31 @@ def main(): client_results[client][test_case_name][gas][method] = [] failed_tests[client][test_case_name][gas][method] = [] for run in range(1, runs + 1): - responses, results, timestamp = utils.extract_response_and_result(results_paths, client, test_case_name, + responses, results, timestamp, duration, fcu_duration, np_duration = utils.extract_response_and_result(results_paths, client, test_case_name, gas, run, method, fields) client_results[client][test_case_name][gas][method].append(results) failed_tests[client][test_case_name][gas][method].append(not responses) # print(test_case_name + " : " + str(timestamp)) if str(timestamp) != "0": - client_results[client][test_case_name]["timestamp"] = utils.convert_dotnet_ticks_to_utc(timestamp) + # Store raw timestamp in ticks for calculation, not converted string + client_results[client][test_case_name]["timestamp_ticks"] = timestamp + # Only store duration if non-zero to avoid overwriting valid values + if duration != 0: + client_results[client][test_case_name]["duration"] = duration + if fcu_duration != 0: + client_results[client][test_case_name]["fcu_duration"] = fcu_duration + if np_duration != 0: + client_results[client][test_case_name]["np_duration"] = np_duration else: - if "timestamp" not in str(client_results[client][test_case_name]): - client_results[client][test_case_name]["timestamp"] = 0 + if "timestamp_ticks" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["timestamp_ticks"] = 0 + # Initialize duration to 0 only if not set yet + if "duration" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["duration"] = 0 + if "fcu_duration" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["fcu_duration"] = 0 + if "np_duration" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["np_duration"] = 0 gas_set = set() for test_case_name, test_case_gas in test_cases.items(): diff --git a/report_tables.py b/report_tables.py index 5c12c790a..9f9716993 100644 --- a/report_tables.py +++ b/report_tables.py @@ -23,7 +23,7 @@ def get_table_report(client_results, clients, results_paths, test_cases, methods image_to_print = el_images[client_without_tag] results_to_print += f'{client.capitalize()} - {image_to_print} - Benchmarking Report' + '\n' results_to_print += (center_string('Title', - 68) + '| Min (MGas/s) | Max (MGas/s) | p50 (MGas/s) | p95 (MGas/s) | p99 (MGas/s) | N | Description | Start time\n') + 68) + '| Min (MGas/s) | Max (MGas/s) | p50 (MGas/s) | p95 (MGas/s) | p99 (MGas/s) | N | Description | Start time | End time | Duration (ms) | FCU time (ms) | NP time (ms)\n') gas_table_norm = utils.get_gas_table(client_results, client, test_cases, gas_set, methods[0], metadata) for test_case, data in gas_table_norm.items(): results_to_print += (f'{align_left_string(data[0], 68)}|' @@ -34,13 +34,17 @@ def get_table_report(client_results, clients, results_paths, test_cases, methods f'{center_string(data[5], 14)}|' f'{center_string(data[6], 7)}|' f'{align_left_string(data[7], 50)}|' - f'{data[8]}\n') + f'{data[8]}|' + f'{data[9]}|' + f'{center_string(data[10], 14)}|' + f'{center_string(data[11], 15)}|' + f'{center_string(data[12], 14)}\n') results_to_print += '\n' print(results_to_print) if not os.path.exists('reports'): os.mkdir('reports') - with open(f'reports/tables_norm.txt', 'w') as file: + with open('reports/tables_norm.txt', 'w') as file: file.write(results_to_print) @@ -105,16 +109,31 @@ def main(): client_results[client][test_case_name][gas][method] = [] failed_tests[client][test_case_name][gas][method] = [] for run in range(1, runs + 1): - responses, results, timestamp = utils.extract_response_and_result(results_paths, client, test_case_name, + responses, results, timestamp, duration, fcu_duration, np_duration = utils.extract_response_and_result(results_paths, client, test_case_name, gas, run, method, fields) client_results[client][test_case_name][gas][method].append(results) failed_tests[client][test_case_name][gas][method].append(not responses) # print(test_case_name + " : " + str(timestamp)) if str(timestamp) != "0": - client_results[client][test_case_name]["timestamp"] = utils.convert_dotnet_ticks_to_utc(timestamp) + # Store raw timestamp in ticks for calculation, not converted string + client_results[client][test_case_name]["timestamp_ticks"] = timestamp + # Only store duration if non-zero to avoid overwriting valid values + if duration != 0: + client_results[client][test_case_name]["duration"] = duration + if fcu_duration != 0: + client_results[client][test_case_name]["fcu_duration"] = fcu_duration + if np_duration != 0: + client_results[client][test_case_name]["np_duration"] = np_duration else: - if "timestamp" not in client_results[client][test_case_name]: - client_results[client][test_case_name]["timestamp"] = 0 + if "timestamp_ticks" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["timestamp_ticks"] = 0 + # Initialize duration to 0 only if not set yet + if "duration" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["duration"] = 0 + if "fcu_duration" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["fcu_duration"] = 0 + if "np_duration" not in client_results[client][test_case_name]: + client_results[client][test_case_name]["np_duration"] = 0 gas_set = set() diff --git a/utils.py b/utils.py index 233eaa3c0..d4e32c515 100644 --- a/utils.py +++ b/utils.py @@ -27,7 +27,8 @@ def read_results(text): if full_lines.startswith(' TIMESTAMP:'): timestamp = int(full_lines.split(':')[1]) elif full_lines.startswith(' MEASUREMENT:'): - measurement = full_lines.split(' ')[3].strip() + # Take everything after "MEASUREMENT: " to handle multi-word measurements + measurement = full_lines.split('MEASUREMENT:')[1].split('\n')[0].strip() elif full_lines.startswith(' TAGS:'): for line in full_lines.split('\n')[1:]: if not line: @@ -56,34 +57,58 @@ def extract_response_and_result(results_path, client, test_case_name, gas_used, if not os.path.exists(result_file): # print("No result: " + result_file) print("No result") - return False, 0, 0 + return False, 0, 0, 0, 0, 0 if not os.path.exists(response_file): print("No repsonse") - return False, 0, 0 + return False, 0, 0, 0, 0, 0 # Get the responses from the files with open(response_file, 'r') as file: text = file.read() if len(text) == 0: print("text len 0") - return False, 0, 0 + return False, 0, 0, 0, 0, 0 # Get latest line for line in text.split('\n'): if len(line) < 1: continue if not check_sync_status(line): print("Invalid sync status") - return False, 0, 0 + return False, 0, 0, 0, 0, 0 # Get the results from the files with open(result_file, 'r') as file: sections = read_results(file.read()) - if method not in sections: - print(f"Method '{method}' not found in sections for file {result_file}. Available methods: {list(sections.keys())}") + # Add [Application] prefix to method name if not present + method_key = f'[Application] {method}' if not method.startswith('[Application]') else method + + if method_key not in sections: + print(f"Method '{method_key}' not found in sections for file {result_file}. Available methods: {list(sections.keys())}") # Get timestamp from first available section, or 0 if no sections exist timestamp = getattr(next(iter(sections.values())), 'timestamp', 0) if sections else 0 - return False, 0, timestamp - result = sections[method].fields[field] - timestamp = getattr(sections[method], 'timestamp', 0) - return response, float(result), timestamp + return False, 0, timestamp, 0, 0, 0 + result = sections[method_key].fields[field] + timestamp = getattr(sections[method_key], 'timestamp', 0) + # Extract total running time if available (in milliseconds) + total_running_time_ms = 0 + if '[Application] Total Running Time' in sections: + total_running_time_section = sections['[Application] Total Running Time'] + if 'sum' in total_running_time_section.fields: + total_running_time_ms = float(total_running_time_section.fields['sum']) + + # Extract FCU (engine_forkchoiceUpdatedV3) duration + fcu_duration_ms = 0 + if '[Application] engine_forkchoiceUpdatedV3' in sections: + fcu_section = sections['[Application] engine_forkchoiceUpdatedV3'] + if 'sum' in fcu_section.fields: + fcu_duration_ms = float(fcu_section.fields['sum']) + + # Extract NP (engine_newPayloadV4) duration + np_duration_ms = 0 + if '[Application] engine_newPayloadV4' in sections: + np_section = sections['[Application] engine_newPayloadV4'] + if 'sum' in np_section.fields: + np_duration_ms = float(np_section.fields['sum']) + + return response, float(result), timestamp, total_running_time_ms, fcu_duration_ms, np_duration_ms def get_gas_table(client_results, client, test_cases, gas_set, method, metadata): @@ -103,11 +128,35 @@ def get_gas_table(client_results, client, test_cases, gas_set, method, metadata) for test_case, _ in test_cases.items(): results_norm = results_per_test_case[test_case] - gas_table_norm[test_case] = ['' for _ in range(9)] + gas_table_norm[test_case] = ['' for _ in range(13)] # test_case_name, description, N, MGgas/s, mean, max, min. std, p50, p95, p99 - # (norm) title, description, N , max, min, p50, p95, p99 - timestamp = client_results[client][test_case]["timestamp"] if client_results[client][test_case] and "timestamp" in client_results[client][test_case] else 0 - gas_table_norm[test_case][8] = timestamp + # (norm) title, description, N , max, min, p50, p95, p99, start_time, end_time, duration_ms, fcu_duration_ms, np_duration_ms + timestamp_ticks = client_results[client][test_case]["timestamp_ticks"] if client_results[client][test_case] and "timestamp_ticks" in client_results[client][test_case] else 0 + duration_ms = client_results[client][test_case]["duration"] if client_results[client][test_case] and "duration" in client_results[client][test_case] else 0 + fcu_duration_ms = client_results[client][test_case]["fcu_duration"] if client_results[client][test_case] and "fcu_duration" in client_results[client][test_case] else 0 + np_duration_ms = client_results[client][test_case]["np_duration"] if client_results[client][test_case] and "np_duration" in client_results[client][test_case] else 0 + + # Convert start timestamp to formatted string + start_time_str = convert_dotnet_ticks_to_utc(timestamp_ticks) if timestamp_ticks != 0 else 0 + gas_table_norm[test_case][8] = start_time_str + + # Calculate end time using raw ticks + duration + # 1 ms = 10,000 ticks (since 1 tick = 100 nanoseconds) + if timestamp_ticks != 0 and duration_ms != 0: + duration_ticks = int(duration_ms * 10_000) + end_time_ticks = timestamp_ticks + duration_ticks + end_time_str = convert_dotnet_ticks_to_utc(end_time_ticks) + gas_table_norm[test_case][9] = end_time_str + else: + gas_table_norm[test_case][9] = 0 + + # Store duration in milliseconds + gas_table_norm[test_case][10] = f'{duration_ms:.2f}' if duration_ms != 0 else '0' + + # Store FCU and NP durations + gas_table_norm[test_case][11] = f'{fcu_duration_ms:.2f}' if fcu_duration_ms != 0 else '0' + gas_table_norm[test_case][12] = f'{np_duration_ms:.2f}' if np_duration_ms != 0 else '0' + if test_case in metadata: gas_table_norm[test_case][0] = metadata[test_case]['Title'] gas_table_norm[test_case][7] = metadata[test_case]['Description']