Fixes for preprocessing submission

Author: pgmpablo157321

tools/submission/preprocess_submission.py

@@ -247,8 +247,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
 
             compliance_is_valid = True
             if is_closed_or_network:
-                compliance_dir = change_folder_name_in_path(
-                    scenario_path, "results", "compliance")
+                compliance_dir = scenario_path
                 if not checker.check_compliance_dir(
                     compliance_dir,
                     mlperf_model,
@@ -266,8 +265,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                 scenario_path, "results", "measurements")
             if scenario in [
                     "Offline", "MultiStream"] and (not accuracy_is_valid or not perf_is_valid) or division == "open":  # they can be inferred
-                scenario_compliance_path = change_folder_name_in_path(
-                    scenario_path, "results", "compliance")
+                scenario_compliance_path = scenario_path
                 log.warning(
                     f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...")
                 if os.path.exists(scenario_path):
@@ -278,10 +276,8 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                     shutil.rmtree(scenario_compliance_path)
             elif division in ["closed", "network"]:
                 model_results_path = os.path.dirname(scenario_path)
-                model_measurements_path = change_folder_name_in_path(
-                    model_results_path, "results", "measurements")
-                model_compliance_path = change_folder_name_in_path(
-                    model_results_path, "results", "compliance")
+                model_measurements_path = model_results_path
+                model_compliance_path = model_results_path
                 model_code_path = os.path.join(
                     change_folder_name_in_path(
                         log_path, "results", "code"), model)
@@ -301,8 +297,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                             f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...")
                         if os.path.exists(scenario_path):
                             shutil.rmtree(scenario_path)
-                        scenario_measurements_path = change_folder_name_in_path(
-                            scenario_path, "results", "measurements")
+                        scenario_measurements_path = scenario_path
                         if os.path.exists(scenario_measurements_path):
                             shutil.rmtree(scenario_measurements_path)
                     if not os.path.exists(target_results_path):

tools/submission/submission_structure.md

@@ -42,9 +42,8 @@ The following diagram describes the submission structure
 │   │   │   │   │   │   │   │   │   ├── mlperf_log_detail.txt
 │   │   │   │   │   │   │   │   │   └── mlperf_log_summary.txt
 │   │   │   │   │   │   ├── measurements.json
-│   │   │   │   │   │   ├── mlperf.conf
-│   │   │   │   │   │   ├── user.conf
-│   │   │   │   │   │   └── accuracy
+│   │   │   │   │   │   ├── mlperf.conf (optional)
+│   │   │   │   │   │   └── user.conf
 │   │   │   ├── ...
 │   │   │   └── <system_desc_id_n>
 │   │   ├── systems

tools/submission/truncate_accuracy_log.py

@@ -142,9 +142,7 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                 continue
 
             # process results
-            required_dirs = [
-                "results", "compliance"] if division in [
-                "closed", "network"] else ["results"]
+            required_dirs = ["results"]
             for directory in required_dirs:
 
                 log_path = os.path.join(division, submitter, directory)
@@ -164,10 +162,6 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                 name = os.path.join(
                                     log_path, system_desc, model, scenario
                                 )
-                                if directory == "compliance":
-                                    name = os.path.join(
-                                        log_path, system_desc, model, scenario, test
-                                    )
 
                                 hash_val = None
                                 acc_path = os.path.join(name, "accuracy")
@@ -178,7 +172,7 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                     acc_path, "accuracy.txt")
 
                                 # only TEST01 has an accuracy log
-                                if directory == "compliance" and test != "TEST01":
+                                if str(test).startswith("TEST") and test != "TEST01":
                                     continue
                                 if not os.path.exists(acc_log):
                                     log.error("%s missing", acc_log)