Update endpoints structure

tools/submission/submission_checker/constants.py

@@ -2036,11 +2036,12 @@
     "deepseek-r1"
 ]
 
-PERFORMANCE_ENDPOINTS_DIR = {
-    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
+ENDPOINTS_SCENARIO_DIR = {
+    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
 }
 
 ACCURACY_LOG_PATH = {
@@ -2067,14 +2068,6 @@
     "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json",
 }
 
-ACCURACY_ENDPOINTS_DIR = {
-    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-}
-
-
 POWER_DIR_PATH = {
     "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power",
     "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power",

tools/submission/submission_checker/loader.py

@@ -86,12 +86,9 @@ def __init__(self, root, version, config: Config) -> None:
         self.acc_json_path = os.path.join(
             self.root, ACCURACY_JSON_PATH.get(
                 version, ACCURACY_JSON_PATH["default"]))
-        self.perf_endpoints_dir = os.path.join(
-            self.root, PERFORMANCE_ENDPOINTS_DIR.get(
-                version, PERFORMANCE_ENDPOINTS_DIR["default"]))
-        self.acc_endpoints_dir = os.path.join(
-            self.root, ACCURACY_ENDPOINTS_DIR.get(
-                version, ACCURACY_ENDPOINTS_DIR["default"]))
+        self.endpoints_scenario_dir = os.path.join(
+            self.root, ENDPOINTS_SCENARIO_DIR.get(
+                version, ENDPOINTS_SCENARIO_DIR["default"]))
         self.system_log_path = os.path.join(
             self.root, SYSTEM_PATH.get(
                 version, SYSTEM_PATH["default"]))
@@ -234,21 +231,15 @@ def load_single_log(self, path, log_type: Literal["Performance", "Accuracy",
                 path)
         return log
 
-    def load_endpoints_logs(self, perf_dir, acc_dir):
-        perf_log = None
-        acc_log = None
-        if os.path.exists(acc_dir) and os.path.exists(perf_dir):
-            acc_log = EndpointsParser(acc_dir)
-            perf_log = EndpointsParser(perf_dir)
-        elif os.path.exists(perf_dir):
-            acc_log = EndpointsParser(perf_dir)
-            perf_log = EndpointsParser(perf_dir)
-        else:
-            self.logger.info(
-                "Could not load endpoints log from %s, path does not exist",
-                perf_dir
-            )
-        return perf_log, acc_log
+    def load_endpoints_logs(self, scenario_dir):
+        if os.path.exists(scenario_dir):
+            parser = EndpointsParser(scenario_dir)
+            return parser, parser
+        self.logger.info(
+            "Could not load endpoints log from %s, path does not exist",
+            scenario_dir
+        )
+        return None, None
 
     def check_scenarios(self, benchmark, model_mapping,
                         system_type, scenarios):
@@ -362,13 +353,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]:
                                 system=system,
                                 benchmark=benchmark,
                                 scenario=scenario)
-                            perf_endpoints_dir = self.perf_endpoints_dir.format(
-                                division=division,
-                                submitter=submitter,
-                                system=system,
-                                benchmark=benchmark,
-                                scenario=scenario)
-                            acc_endpoints_dir = self.acc_endpoints_dir.format(
+                            endpoints_scenario_dir = self.endpoints_scenario_dir.format(
                                 division=division,
                                 submitter=submitter,
                                 system=system,
@@ -483,7 +468,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]:
                             if perf_log is None and acc_log is None:
                                 is_endpoints_submission = True
                                 perf_log, acc_log = self.load_endpoints_logs(
-                                    perf_endpoints_dir, acc_endpoints_dir
+                                    endpoints_scenario_dir
                                 )
 
                             # Load test logs

tools/submission/submission_checker/parsers/endpoints_parser.py

@@ -24,7 +24,9 @@
     "sample_logs")
 
 _RESULT_SUMMARY_FILE = "results_summary.json"
-_RESULTS_FILE = "results.json"
+_ACCURACY_RESULTS_FILE = "accuracy_results.json"
+_PERF_SUBDIR = os.path.join("performance", "run_1")
+_ACC_SUBDIR = "accuracy"
 _CONFIG_FILES = ("config.yaml", "config.yml")
 
 
@@ -94,22 +96,23 @@ def _resolve_value(stripped, summary_data, results_data, yaml_data):
 
 
 class EndpointsParser(BaseParser):
-    def __init__(self, run_dir):
+    def __init__(self, scenario_dir):
         """
-        run_dir: path to the run directory containing:
-          - result_summary.json  (highest priority)
-          - results.json
-          - config.yaml / config.yml  (lowest priority)
+        scenario_dir: path to the scenario directory containing:
+          - config.yaml                              (scenario root)
+          - performance/run_1/results_summary.json   (performance metrics)
+          - accuracy/accuracy_results.json           (accuracy metrics)
         """
-        super().__init__(run_dir)
+        super().__init__(scenario_dir)
 
         self.logger = logging.getLogger("MLPerfLog")
         self.messages = {}
 
         summary_data = self._load_json(
-            os.path.join(run_dir, _RESULT_SUMMARY_FILE))
-        results_data = self._load_json(os.path.join(run_dir, _RESULTS_FILE))
-        yaml_data = self._load_yaml(run_dir)
+            os.path.join(scenario_dir, _PERF_SUBDIR, _RESULT_SUMMARY_FILE))
+        results_data = self._load_json(
+            os.path.join(scenario_dir, _ACC_SUBDIR, _ACCURACY_RESULTS_FILE))
+        yaml_data = self._load_yaml(scenario_dir)
 
         for endpoints_key, loadgen_key in ENDPOINTS_MAPPINGS.items():
             stripped = endpoints_key.strip()
@@ -153,7 +156,9 @@ def __init__(self, run_dir):
                 )
 
         self.keys = set(self.messages.keys())
-        self.logger.info("Successfully loaded endpoints log from %s.", run_dir)
+        self.logger.info(
+            "Successfully loaded endpoints log from %s.",
+            scenario_dir)
 
     def _load_json(self, path):
         try:
@@ -162,7 +167,6 @@ def _load_json(self, path):
         except BaseException:
             self.logger.error("Could not load json file from %s", path)
             return {}
-        return {}
 
     def _load_yaml(self, run_dir):
         for name in _CONFIG_FILES:
@@ -212,18 +216,20 @@ def main():
 
     backwards_map = _load_field_map("backwards.json")
 
-    # Collect all run directories (those containing at least one JSON and one
-    # YAML)
+    # Collect scenario-level directories (those containing config.yaml and
+    # the performance/run_1 subdirectory with results_summary.json)
     run_dirs = []
-    for root, _dirs, files in os.walk(_SAMPLE_LOGS_DIR):
-        has_json = any(f.endswith(".json") for f in files)
-        has_yaml = any(f.endswith(".yaml") or f.endswith(".yml")
-                       for f in files)
-        if has_json and has_yaml:
+    for root, dirs, files in os.walk(_SAMPLE_LOGS_DIR):
+        has_yaml = any(f in _CONFIG_FILES for f in files)
+        has_perf = os.path.exists(
+            os.path.join(root, _PERF_SUBDIR, _RESULT_SUMMARY_FILE))
+        if has_yaml and has_perf:
             run_dirs.append(root)
 
     if not run_dirs:
-        logger.error("No run directories found under %s.", _SAMPLE_LOGS_DIR)
+        logger.error(
+            "No scenario directories found under %s.",
+            _SAMPLE_LOGS_DIR)
         return 1
 
     for run_dir in sorted(run_dirs):

tools/submission/submission_structure.md

@@ -65,7 +65,7 @@ The following diagram describes the standard submission structure.
 
 ## Endpoints submission structure
 
-For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. You can provide a performance+accuracy run in the performance folder or one performance run and one accuracy run
+For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. The `config.yaml` is placed at the scenario root, `results_summary.json` contains performance metrics, and `accuracy_results.json` contains accuracy metrics.
 
 ```
 ...
@@ -77,15 +77,12 @@ For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structur
 │   │   │   ├── <system_desc_id_1>
 │   │   │   │   ├── <benchmark_name>
 │   │   │   │   │   ├── <scenario>
-│   │   │   │   │   │   ├── accuracy (optional)
-│   │   │   │   │   │   │   ├── config.yaml
-│   │   │   │   │   │   │   ├── results.json
-│   │   │   │   │   │   │   └── results_summary.json
+│   │   │   │   │   │   ├── config.yaml
+│   │   │   │   │   │   ├── accuracy
+│   │   │   │   │   │   │   └── accuracy_results.json
 │   │   │   │   │   │   ├── performance
-│   │   │   │   │   │   │   ├── run_1
-│   │   │   │   │   │   │   │   ├── config.yaml
-│   │   │   │   │   │   │   │   ├── results.json
-│   │   │   │   │   │   │   │   └── results_summary.json
+│   │   │   │   │   │   │   └── run_1
+│   │   │   │   │   │   │       └── results_summary.json
 │   │   │   │   │   │   ├── <TEST0X>
 │   │   │   │   │   │   │   ├── accuracy
 │   │   │   │   │   │   │   │   └── accuracy.txt

tools/submission/truncate_accuracy_log.py

@@ -18,7 +18,6 @@
 
 MAX_ACCURACY_LOG_SIZE = 10 * 1024
 VIEWABLE_SIZE = 4096
-RESPONSES_LIMIT = 10 * 1024  # cap on truncated responses bytes
 
 HELP_TEXT = """
 You can run this tool in 2 ways:
@@ -124,84 +123,18 @@ def copy_submission_dir(src, dst, filter_submitter):
             )
 
 
-def _truncate_endpoints_results(results_path, acc_path, backup):
-    """Truncate the ``responses`` field in an endpoints accuracy results.json.
+def _is_endpoints_submission(scenario_dir):
+    """Detect the endpoints submission layout via its scenario-root config.yaml.
 
-    Mirrors the behaviour of truncate_accuracy_log.py for traditional
-    submissions: backs up the original file (when --backup is given),
-    computes a SHA-256 hash of the original, appends ``hash=<hex>`` to
-    accuracy.txt, then writes back a copy with responses capped at
-    RESPONSES_LIMIT bytes.
+    Endpoints submissions have no mlperf_log_accuracy.json; the accuracy
+    artifact they produce (accuracy/accuracy_results.json) carries no large
+    per-sample payload to truncate, so callers should skip truncation
+    entirely rather than look for a file to shrink.
     """
-    try:
-        with open(results_path, "r", encoding="utf-8") as f:
-            original_bytes = f.read().encode()
-        data = json.loads(original_bytes)
-    except Exception as exc:
-        log.error("Could not read %s: %s", results_path, exc)
-        return
-
-    # Back up before any modification.
-    if backup:
-        backup_dir = os.path.join(backup, acc_path)
-        os.makedirs(backup_dir, exist_ok=True)
-        dst = os.path.join(backup_dir, "results.json")
-        if os.path.exists(dst):
-            log.error(
-                "not processing %s because %s already exists",
-                results_path,
-                dst,
-            )
-            return
-        shutil.copy(results_path, dst)
-
-    # Hash the original file for audit purposes (mirrors accuracy.txt hash
-    # written for traditional mlperf_log_accuracy.json truncation).
-    hash_val = hashlib.sha256(original_bytes).hexdigest()
-    acc_txt = os.path.join(acc_path, "accuracy.txt")
-    with open(acc_txt, "a", encoding="utf-8") as f:
-        f.write("\nhash={0}\n".format(hash_val))
-
-    # Truncate the responses collection so the file stays small.
-    responses = data.get("responses")
-    if responses is None or (isinstance(
-            responses, (list, dict)) and not responses):
-        log.info("%s has no responses to truncate", results_path)
-        return
-
-    if isinstance(responses, list):
-        total = 2  # "[]"
-        idx = 0
-        for i, r in enumerate(responses):
-            total += len(json.dumps(r).encode()) + (2 if i > 0 else 0)
-            if total > RESPONSES_LIMIT:
-                break
-            idx = i + 1
-        data["responses"] = responses[:idx]
-    elif isinstance(responses, dict):
-        total = 2  # "{}"
-        kept = {}
-        for i, (k, v) in enumerate(responses.items()):
-            entry = len(json.dumps(k).encode()) + \
-                len(json.dumps(v).encode()) + 2
-            if i > 0:
-                entry += 2
-            if total + entry > RESPONSES_LIMIT:
-                break
-            total += entry
-            kept[k] = v
-        data["responses"] = kept
-    else:
-        log.error(
-            "%s: responses has unexpected type %s; skipping truncation",
-            results_path,
-            type(responses).__name__,
-        )
-        return
-
-    with open(results_path, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2)
-    log.info("%s responses truncated", results_path)
+    return any(
+        os.path.exists(os.path.join(scenario_dir, name))
+        for name in ("config.yaml", "config.yml")
+    )
 
 
 def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
@@ -264,14 +197,10 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                 acc_txt = os.path.join(
                                     acc_path, "accuracy.txt")
                                 if not os.path.exists(acc_log):
-                                    # Endpoints submissions have results.json
-                                    # instead of mlperf_log_accuracy.json
-                                    endpoints_results = os.path.join(
-                                        acc_path, "results.json"
-                                    )
-                                    if os.path.exists(endpoints_results):
-                                        _truncate_endpoints_results(
-                                            endpoints_results, acc_path, backup
+                                    if _is_endpoints_submission(name):
+                                        log.info(
+                                            "%s is an endpoints submission; nothing to truncate",
+                                            name,
                                         )
                                     else:
                                         log.error("%s missing", acc_log)