RQ4 Cleanup (gousiosg#28)

* Updated rq4.tex

* Removed unnecessary mph-table properties in genResults

* Updated rq4.tex

* Removed unnecessary mph-table properties in genResults

* Removed unnecessary mph-table properties in genResults

* Removed unnecessary mph-table properties in genResults

* Removed unnecessary mph-table properties in genResults

* Removed unnecessary mph-table properties in genResults

* Resolved bug in mph-table results gen

* Updated rq4 w/ proper index

Author: ameka4

artifacts/experiments/RQ4/generateResults.py

@@ -14,14 +14,7 @@
 CALC_NAMES = ['Vanilla', 'Improved', 'Overhead']
 
 propertyShortNames = {
-    "TestSmartByteSerializer#canRoundTripBytes": 'byte',
-    "TestSmartIntegerSerializer#canRoundTripIntegers": 'int',
     "TestSmartListSerializer#canRoundTripSerializableLists": 'list',
-    "TestSmartLongSerializer#canRoundTripLongs": 'long',
-    "TestSmartOptionalSerializer#canRoundTripPresentOptionals": 'optionals',
-    "TestSmartPairSerializer#canRoundTripPairs": 'pair',
-    "TestSmartShortSerializer#canRoundTripShort": 'short',
-    "TestSmartStringSerializer#canRoundTripStrings": 'string',
     "TestSmartListSerializer#canRoundTripSerializableListsWithGenerator": 'list*',
     "GenTestFormat#dataRoundTrip": 'data',
     "GenTestFormat#messageRoundTrip": 'message',
@@ -41,7 +34,6 @@
     "X509ResourceCertificateParentChildValidatorTest#validParentChildOverClaimingLooseValidation": 'loose'
 }
 
-row_count = 1
 
 def obtain_stats_directories(results_directory: str) -> list[str]:
     directory_tree = [x for x in os.walk(results_directory)] # os.walk returns a tuple with structure (directory, subdirectories, files)
@@ -71,17 +63,27 @@ def evaluate_directories(project_name: str, results_directory: str, directories:
         directory_path = results_directory + directory + "/"
         directory_tree = [x[2] for x in os.walk(directory_path)]
         valid_htmls = [x for x in directory_tree[0] if 'html' in x]
-        directory_stats = retrieve_time_elapsed(directory_path=directory_path, valid_htmls=valid_htmls)
+        directory_stats = retrieve_time_elapsed(project_name=project_name, directory_path=directory_path, valid_htmls=valid_htmls)
         project_iteration = project_name + " - " + str(iteration)
         final_stats[project_iteration] = directory_stats
         iteration += 1
     return final_stats
 
-def retrieve_time_elapsed(directory_path: str, valid_htmls: list[str]) -> dict[str, str]:
+
+def retrieve_time_elapsed(project_name: str, directory_path: str, valid_htmls: list[str]) -> dict[str, str]:
     times_elapsed_dict = {}
     for html_file in valid_htmls:
         property_name = html_file.replace(".html", "")
+        if property_name not in propertyShortNames:
+            continue
         property_short_name = propertyShortNames[property_name]
+        if property_short_name == 'list*' and project_name == 'mph-table-fixed':
+            property_short_name = 'list'
+        elif property_short_name == 'list' and project_name == 'mph-table-fixed':
+            continue
+        elif property_short_name == 'list*':
+            print(project_name)
+            continue
         file_path = directory_path + html_file
         with open(file_path) as f:
             contents = f.read()
@@ -122,7 +124,7 @@ def generate_project_report(project_name: str, final_stats: dict[str, str], fina
     return final_report_dict
 
 
-def generate_project_df(final_stats: dict[str, str], final_fixed_stats: dict[str, str]) -> pd.DataFrame():
+def generate_project_df(final_stats: dict[str, str], final_fixed_stats: dict[str, str], row_count: int) -> (pd.DataFrame(), int):
     vanilla_df = pd.DataFrame()
     vanilla_df['Property'] = [key for key in final_stats.keys()]
     vanilla_df['Vanilla'] = [val for val in final_stats.values()]
@@ -133,13 +135,14 @@ def generate_project_df(final_stats: dict[str, str], final_fixed_stats: dict[str
 
     merged_df = pd.merge(vanilla_df, improved_df, how='outer', on='Property')
     merged_df['N'] = pd.RangeIndex(start=row_count, stop=len(merged_df.index) + row_count)
-
+    row_count += len(merged_df.index)
     final_df = merged_df[['N', 'Property', 'Vanilla', 'Improved']]
-    return final_df
+    return final_df, row_count
 
 
 def main():
     final_dataset = {}
+    row_count = 1
     for project_name in PROJECTS:
         fixed_project_name = project_name + "-fixed"
         results_directory = BASE_RESULT_DIR + project_name + "/"
@@ -157,7 +160,7 @@ def main():
         # obtain mean/st dev
         final_stats = generate_report_stats(stat_values=raw_stats)
         final_fixed_stats = generate_report_stats(stat_values=fixed_raw_stats)
-        project_df = generate_project_df(final_stats=final_stats, final_fixed_stats=final_fixed_stats)
+        project_df, row_count = generate_project_df(final_stats=final_stats, final_fixed_stats=final_fixed_stats, row_count=row_count)
         final_dataset[project_name] = project_df
 
 

artifacts/output/rq4.tex

@@ -3,42 +3,34 @@
 \hline
 \multicolumn{5}{c}{convex} \\
 \hline
-1 & message & 45.22 ± 3.36 & 42.80 ± 2.70 & 0.95 \\
-2 & data & 53.02 ± 1.99 & 53.92 ± 0.00 & 1.02 \\
-3 & primitive & 48.52 ± 0.88 & 50.12 ± 0.00 & 1.03 \\
-\textbf{} & \textbf{Average} & \textbf{48.92} & \textbf{48.95} & \textbf{1.00} \\
+1 & data & 39.01 ± 0.58 & nan & nan \\
+2 & message & 33.25 ± 2.40 & 30.88 ± 0.64 & 0.93 \\
+3 & primitive & 35.77 ± 0.61 & nan & nan \\
+\textbf{} & \textbf{Average} & \textbf{36.01} & \textbf{30.88} & \textbf{0.93} \\
 \hline
 \multicolumn{5}{c}{jflex} \\
 \hline
-1 & remove & 41.50 ± 6.41 & 31.56 ± 5.16 & 0.76 \\
-2 & addSingleton & 44.89 ± 0.77 & 51.57 ± 0.79 & 1.15 \\
-3 & contains & 43.82 ± 2.12 & 42.61 ± 1.68 & 0.97 \\
-4 & addSet & 44.56 ± 0.94 & 52.82 ± 0.87 & 1.19 \\
-5 & add & 43.06 ± 1.84 & 43.10 ± 1.80 & 1.00 \\
-6 & addString & 44.93 ± 1.64 & 51.53 ± 1.59 & 1.15 \\
-7 & addSingle & 44.40 ± 2.21 & 53.14 ± 1.53 & 1.20 \\
-\textbf{} & \textbf{Average} & \textbf{43.88} & \textbf{46.62} & \textbf{1.06} \\
+4 & addSingleton & 34.35 ± 0.59 & 42.69 ± 0.67 & 1.24 \\
+5 & contains & 34.33 ± 0.62 & 34.36 ± 0.57 & 1.00 \\
+6 & addSet & 34.47 ± 0.48 & 44.64 ± 0.98 & 1.30 \\
+7 & add & 34.58 ± 0.48 & 34.20 ± 0.77 & 0.99 \\
+8 & remove & 34.50 ± 1.02 & 24.36 ± 0.40 & 0.71 \\
+9 & addString & 34.65 ± 0.88 & 43.03 ± 0.63 & 1.24 \\
+10 & addSingle & 34.42 ± 0.55 & 45.21 ± 0.70 & 1.31 \\
+\textbf{} & \textbf{Average} & \textbf{34.47} & \textbf{38.36} & \textbf{1.11} \\
 \hline
 \multicolumn{5}{c}{mph-table} \\
 \hline
-1 & list & 13.49 ± 0.98 & 13.19 ± 0.94 & 0.98 \\
-2 & optionals & 12.75 ± 0.76 & 13.17 ± 1.08 & 1.03 \\
-3 & short & 12.68 ± 0.67 & 13.16 ± 0.58 & 1.04 \\
-4 & int & 13.09 ± 0.81 & 12.79 ± 1.00 & 0.98 \\
-5 & pair & 12.76 ± 0.81 & 13.16 ± 1.09 & 1.03 \\
-6 & byte & 12.57 ± 0.79 & 12.85 ± 1.04 & 1.02 \\
-7 & long & 12.80 ± 0.75 & 13.04 ± 0.91 & 1.02 \\
-8 & string & 12.73 ± 0.60 & 12.99 ± 0.97 & 1.02 \\
-9 & list* & nan & 27.19 ± 3.60 & nan \\
-\textbf{} & \textbf{Average} & \textbf{12.86} & \textbf{14.62} & \textbf{1.01} \\
+11 & list & 10.15 ± 0.31 & 20.30 ± 2.36 & 2.00 \\
+\textbf{} & \textbf{Average} & \textbf{10.15} & \textbf{20.30} & \textbf{2.00} \\
 \hline
 \multicolumn{5}{c}{rpki-commons} \\
 \hline
-1 & claiming & 25.33 ± 2.37 & nan & nan \\
-2 & aspa & 23.42 ± 1.72 & nan & nan \\
-3 & resources & 24.18 ± 1.60 & 26.14 ± 0.28 & 1.08 \\
-4 & roa & 24.29 ± 3.24 & nan & nan \\
-5 & manifest & 23.41 ± 2.44 & nan & nan \\
-6 & loose & 25.48 ± 2.62 & nan & nan \\
-\textbf{} & \textbf{Average} & \textbf{24.35} & \textbf{26.14} & \textbf{1.08} \\
+12 & claiming & 24.73 ± 0.55 & nan & nan \\
+13 & aspa & 22.98 ± 0.46 & nan & nan \\
+14 & resources & 23.39 ± 0.44 & 26.14 ± 0.40 & 1.12 \\
+15 & roa & 22.61 ± 0.62 & nan & nan \\
+16 & manifest & 22.62 ± 0.53 & nan & nan \\
+17 & loose & 24.65 ± 0.50 & nan & nan \\
+\textbf{} & \textbf{Average} & \textbf{23.50} & \textbf{26.14} & \textbf{1.12} \\
 \end{tabular}