Fixes for preprocessing submission

Author: pgmpablo157321

tools/submission/preprocess_submission.py

@@ -51,6 +51,9 @@ def get_args():
     parser.add_argument("--nodelete-failed",
                         help="do not delete failed results (submission checker will fail)",
                         default=False, action="store_true")
+    parser.add_argument("--keep-structure",
+                        help="keep folder structure (newer versions of submission checker might fail)",
+                        default=False, action="store_true")
 
     parser.add_argument(
         "--version",
@@ -96,7 +99,7 @@ def delete_empty_dirs(src):
     return False
 
 
-def copy_submission_dir(src, dst, filter_submitter):
+def copy_submission_dir(src, dst, filter_submitter, keep_structure = True):
     """
     Copies the submission tree to output directory for processing
     """
@@ -106,10 +109,25 @@ def copy_submission_dir(src, dst, filter_submitter):
         for submitter in next(os.walk(os.path.join(src, division)))[1]:
             if filter_submitter and submitter != filter_submitter:
                 continue
-            shutil.copytree(
-                os.path.join(src, division, submitter),
-                os.path.join(dst, division, submitter),
-            )
+            if keep_structure:
+                shutil.copytree(
+                    os.path.join(src, division, submitter),
+                    os.path.join(dst, division, submitter),
+                )
+            else:
+                for object in os.listdir(os.path.join(src, division, submitter)):
+                    if os.path.isfile(os.path.join(src, division, submitter, object)):
+                        shutil.copyfile(
+                            os.path.join(src, division, submitter, object),
+                            os.path.join(dst, division, submitter, object)
+                        )
+                    elif os.path.isdir(os.path.join(src, division, submitter, object)):
+                        target_dir = "results" if object in ["compliance", "measurements"] else object
+                        shutil.copytree(
+                            os.path.join(src, division, submitter, object),
+                            os.path.join(dst, division, submitter, target_dir),
+                            dirs_exist_ok = True
+                        )
 
 
 def change_first_directory_to_open(path):
@@ -247,8 +265,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
 
             compliance_is_valid = True
             if is_closed_or_network:
-                compliance_dir = change_folder_name_in_path(
-                    scenario_path, "results", "compliance")
+                compliance_dir = scenario_path
                 if not checker.check_compliance_dir(
                     compliance_dir,
                     mlperf_model,
@@ -262,12 +279,10 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
 
         is_valid = accuracy_is_valid and perf_is_valid and compliance_is_valid
         if not is_valid:  # Remove the scenario result
-            scenario_measurements_path = change_folder_name_in_path(
-                scenario_path, "results", "measurements")
+            scenario_measurements_path = scenario_path
             if scenario in [
                     "Offline", "MultiStream"] and (not accuracy_is_valid or not perf_is_valid) or division == "open":  # they can be inferred
-                scenario_compliance_path = change_folder_name_in_path(
-                    scenario_path, "results", "compliance")
+                scenario_compliance_path = scenario_path
                 log.warning(
                     f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...")
                 if os.path.exists(scenario_path):
@@ -278,10 +293,8 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                     shutil.rmtree(scenario_compliance_path)
             elif division in ["closed", "network"]:
                 model_results_path = os.path.dirname(scenario_path)
-                model_measurements_path = change_folder_name_in_path(
-                    model_results_path, "results", "measurements")
-                model_compliance_path = change_folder_name_in_path(
-                    model_results_path, "results", "compliance")
+                model_measurements_path = model_results_path
+                model_compliance_path = model_results_path
                 model_code_path = os.path.join(
                     change_folder_name_in_path(
                         log_path, "results", "code"), model)
@@ -301,8 +314,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                             f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...")
                         if os.path.exists(scenario_path):
                             shutil.rmtree(scenario_path)
-                        scenario_measurements_path = change_folder_name_in_path(
-                            scenario_path, "results", "measurements")
+                        scenario_measurements_path = scenario_path
                         if os.path.exists(scenario_measurements_path):
                             shutil.rmtree(scenario_measurements_path)
                     if not os.path.exists(target_results_path):
@@ -367,9 +379,7 @@ def infer_scenario_results(args, config):
                 continue
 
             # process results
-            for directory in ["results", "measurements"] + \
-                    (["compliance"] if division == "closed" else []):
-
+            for directory in ["results"]:
                 log_path = os.path.join(division, submitter, directory)
                 if not os.path.exists(log_path):
                     log.error("no submission in %s", log_path)
@@ -550,7 +560,7 @@ def main():
         log.error(f"output directory {args.output} already exists")
         sys.exit(1)
     os.makedirs(args.output)
-    copy_submission_dir(args.input, args.output, args.submitter)
+    copy_submission_dir(args.input, args.output, args.submitter, args.keep_structure)
     src_dir = args.output
 
     config = checker.Config(
@@ -574,3 +584,6 @@ def main():
 
 if __name__ == "__main__":
     sys.exit(main())
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/submission/submission_checker.py

@@ -1061,6 +1061,9 @@ def set_type(self, submission_type):
             self.optional = self.base["optional-scenarios-datacenter-edge"]
         else:
             raise ValueError("invalid system type")
+        
+    def skip_calibration(self):
+        return self.skip_calibration_check or self.version in ["v5.0"]
 
     def get_mlperf_model(self, model, extra_model_mapping=None):
         # preferred - user is already using the official name
@@ -2392,7 +2395,7 @@ def log_result(
                     results[os.path.join(results_path)] = None
 
             #  Check for calibration documentation
-            if not config.skip_calibration_check and division not in ["open"]:
+            if not config.skip_calibration() and division not in ["open"]:
                 calibration_path_root = os.path.join(
                     division, submitter, "calibration.md")
                 calibration_path_doc = os.path.join(
@@ -3026,9 +3029,16 @@ def check_measurement_dir(
                 log.error("%s is having empty %s", measurement_dir, i)
                 is_valid = False
 
-    logging.info("%s", files)
-    if "measurements.json" in files:
-        system_file = "measurements.json"
+    for i in files:
+        if i.startswith(system_desc) and i.endswith(
+                "_" + scenario + ".json"):
+            system_file = i
+            end = len("_" + scenario + ".json")
+            break
+        elif i.startswith(system_desc) and i.endswith(".json"):
+            system_file = i
+            end = len(".json")
+            break
 
 
     weight_data_types = None

tools/submission/submission_structure.md

@@ -42,9 +42,8 @@ The following diagram describes the submission structure
 │   │   │   │   │   │   │   │   │   ├── mlperf_log_detail.txt
 │   │   │   │   │   │   │   │   │   └── mlperf_log_summary.txt
 │   │   │   │   │   │   ├── measurements.json
-│   │   │   │   │   │   ├── mlperf.conf
-│   │   │   │   │   │   ├── user.conf
-│   │   │   │   │   │   └── accuracy
+│   │   │   │   │   │   ├── mlperf.conf (optional)
+│   │   │   │   │   │   └── user.conf
 │   │   │   ├── ...
 │   │   │   └── <system_desc_id_n>
 │   │   ├── systems

tools/submission/truncate_accuracy_log.py

@@ -142,9 +142,7 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                 continue
 
             # process results
-            required_dirs = [
-                "results", "compliance"] if division in [
-                "closed", "network"] else ["results"]
+            required_dirs = ["results"]
             for directory in required_dirs:
 
                 log_path = os.path.join(division, submitter, directory)
@@ -164,10 +162,6 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                 name = os.path.join(
                                     log_path, system_desc, model, scenario
                                 )
-                                if directory == "compliance":
-                                    name = os.path.join(
-                                        log_path, system_desc, model, scenario, test
-                                    )
 
                                 hash_val = None
                                 acc_path = os.path.join(name, "accuracy")
@@ -178,7 +172,7 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                     acc_path, "accuracy.txt")
 
                                 # only TEST01 has an accuracy log
-                                if directory == "compliance" and test != "TEST01":
+                                if str(test).startswith("TEST") and test != "TEST01":
                                     continue
                                 if not os.path.exists(acc_log):
                                     log.error("%s missing", acc_log)