From 50d4a29286624ee2e991ba0416bc2186a0f9c7eb Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 13 Jul 2026 22:45:11 -0500 Subject: [PATCH 1/7] Update folder structure for endpoints submissions --- .../submission_checker/constants.py | 10 ++++ tools/submission/submission_checker/loader.py | 43 ++++++----------- .../parsers/endpoints_parser.py | 46 +++++++++++-------- tools/submission/submission_structure.md | 15 +++--- 4 files changed, 56 insertions(+), 58 deletions(-) diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 8278fe77a8..05ec443be4 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -2040,9 +2040,18 @@ "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", } +ENDPOINTS_SCENARIO_DIR = { + "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", +} + ACCURACY_LOG_PATH = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt", @@ -2071,6 +2080,7 @@ "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", } diff --git a/tools/submission/submission_checker/loader.py b/tools/submission/submission_checker/loader.py index 13c531bd93..f58876e007 100644 --- a/tools/submission/submission_checker/loader.py +++ b/tools/submission/submission_checker/loader.py @@ -86,12 +86,9 @@ def __init__(self, root, version, config: Config) -> None: self.acc_json_path = os.path.join( self.root, ACCURACY_JSON_PATH.get( version, ACCURACY_JSON_PATH["default"])) - self.perf_endpoints_dir = os.path.join( - self.root, PERFORMANCE_ENDPOINTS_DIR.get( - version, PERFORMANCE_ENDPOINTS_DIR["default"])) - self.acc_endpoints_dir = os.path.join( - self.root, ACCURACY_ENDPOINTS_DIR.get( - version, ACCURACY_ENDPOINTS_DIR["default"])) + self.endpoints_scenario_dir = os.path.join( + self.root, ENDPOINTS_SCENARIO_DIR.get( + version, ENDPOINTS_SCENARIO_DIR["default"])) self.system_log_path = os.path.join( self.root, SYSTEM_PATH.get( version, SYSTEM_PATH["default"])) @@ -234,21 +231,15 @@ def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", path) return log - def load_endpoints_logs(self, perf_dir, acc_dir): - perf_log = None - acc_log = None - if os.path.exists(acc_dir) and os.path.exists(perf_dir): - acc_log = EndpointsParser(acc_dir) - perf_log = EndpointsParser(perf_dir) - elif os.path.exists(perf_dir): - acc_log = EndpointsParser(perf_dir) - perf_log = EndpointsParser(perf_dir) - else: - self.logger.info( - "Could not load endpoints log from %s, path does not exist", - perf_dir - ) - return perf_log, acc_log + def load_endpoints_logs(self, scenario_dir): + if os.path.exists(scenario_dir): + parser = EndpointsParser(scenario_dir) + return parser, parser + self.logger.info( + "Could not load endpoints log from %s, path does not exist", + scenario_dir + ) + return None, None def check_scenarios(self, benchmark, model_mapping, system_type, scenarios): @@ -362,13 +353,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: system=system, benchmark=benchmark, scenario=scenario) - perf_endpoints_dir = self.perf_endpoints_dir.format( - division=division, - submitter=submitter, - system=system, - benchmark=benchmark, - scenario=scenario) - acc_endpoints_dir = self.acc_endpoints_dir.format( + endpoints_scenario_dir = self.endpoints_scenario_dir.format( division=division, submitter=submitter, system=system, @@ -483,7 +468,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: if perf_log is None and acc_log is None: is_endpoints_submission = True perf_log, acc_log = self.load_endpoints_logs( - perf_endpoints_dir, acc_endpoints_dir + endpoints_scenario_dir ) # Load test logs diff --git a/tools/submission/submission_checker/parsers/endpoints_parser.py b/tools/submission/submission_checker/parsers/endpoints_parser.py index bb4b09f552..9b1a5a2e53 100644 --- a/tools/submission/submission_checker/parsers/endpoints_parser.py +++ b/tools/submission/submission_checker/parsers/endpoints_parser.py @@ -24,7 +24,9 @@ "sample_logs") _RESULT_SUMMARY_FILE = "results_summary.json" -_RESULTS_FILE = "results.json" +_ACCURACY_RESULTS_FILE = "accuracy_results.json" +_PERF_SUBDIR = os.path.join("performance", "run_1") +_ACC_SUBDIR = "accuracy" _CONFIG_FILES = ("config.yaml", "config.yml") @@ -94,22 +96,23 @@ def _resolve_value(stripped, summary_data, results_data, yaml_data): class EndpointsParser(BaseParser): - def __init__(self, run_dir): + def __init__(self, scenario_dir): """ - run_dir: path to the run directory containing: - - result_summary.json (highest priority) - - results.json - - config.yaml / config.yml (lowest priority) + scenario_dir: path to the scenario directory containing: + - config.yaml (scenario root) + - performance/run_1/results_summary.json (performance metrics) + - accuracy/accuracy_results.json (accuracy metrics) """ - super().__init__(run_dir) + super().__init__(scenario_dir) self.logger = logging.getLogger("MLPerfLog") self.messages = {} summary_data = self._load_json( - os.path.join(run_dir, _RESULT_SUMMARY_FILE)) - results_data = self._load_json(os.path.join(run_dir, _RESULTS_FILE)) - yaml_data = self._load_yaml(run_dir) + os.path.join(scenario_dir, _PERF_SUBDIR, _RESULT_SUMMARY_FILE)) + results_data = self._load_json( + os.path.join(scenario_dir, _ACC_SUBDIR, _ACCURACY_RESULTS_FILE)) + yaml_data = self._load_yaml(scenario_dir) for endpoints_key, loadgen_key in ENDPOINTS_MAPPINGS.items(): stripped = endpoints_key.strip() @@ -153,7 +156,9 @@ def __init__(self, run_dir): ) self.keys = set(self.messages.keys()) - self.logger.info("Successfully loaded endpoints log from %s.", run_dir) + self.logger.info( + "Successfully loaded endpoints log from %s.", + scenario_dir) def _load_json(self, path): try: @@ -162,7 +167,6 @@ def _load_json(self, path): except BaseException: self.logger.error("Could not load json file from %s", path) return {} - return {} def _load_yaml(self, run_dir): for name in _CONFIG_FILES: @@ -212,18 +216,20 @@ def main(): backwards_map = _load_field_map("backwards.json") - # Collect all run directories (those containing at least one JSON and one - # YAML) + # Collect scenario-level directories (those containing config.yaml and + # the performance/run_1 subdirectory with results_summary.json) run_dirs = [] - for root, _dirs, files in os.walk(_SAMPLE_LOGS_DIR): - has_json = any(f.endswith(".json") for f in files) - has_yaml = any(f.endswith(".yaml") or f.endswith(".yml") - for f in files) - if has_json and has_yaml: + for root, dirs, files in os.walk(_SAMPLE_LOGS_DIR): + has_yaml = any(f in _CONFIG_FILES for f in files) + has_perf = os.path.exists( + os.path.join(root, _PERF_SUBDIR, _RESULT_SUMMARY_FILE)) + if has_yaml and has_perf: run_dirs.append(root) if not run_dirs: - logger.error("No run directories found under %s.", _SAMPLE_LOGS_DIR) + logger.error( + "No scenario directories found under %s.", + _SAMPLE_LOGS_DIR) return 1 for run_dir in sorted(run_dirs): diff --git a/tools/submission/submission_structure.md b/tools/submission/submission_structure.md index ac9e81ba7a..febe1d0f4a 100644 --- a/tools/submission/submission_structure.md +++ b/tools/submission/submission_structure.md @@ -65,7 +65,7 @@ The following diagram describes the standard submission structure. ## Endpoints submission structure -For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. You can provide a performance+accuracy run in the performance folder or one performance run and one accuracy run +For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. The `config.yaml` is placed at the scenario root, `results_summary.json` contains performance metrics, and `accuracy_results.json` contains accuracy metrics. ``` ... @@ -77,15 +77,12 @@ For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structur │ │ │ ├── │ │ │ │ ├── │ │ │ │ │ ├── -│ │ │ │ │ │ ├── accuracy (optional) -│ │ │ │ │ │ │ ├── config.yaml -│ │ │ │ │ │ │ ├── results.json -│ │ │ │ │ │ │ └── results_summary.json +│ │ │ │ │ │ ├── config.yaml +│ │ │ │ │ │ ├── accuracy +│ │ │ │ │ │ │ └── accuracy_results.json │ │ │ │ │ │ ├── performance -│ │ │ │ │ │ │ ├── run_1 -│ │ │ │ │ │ │ │ ├── config.yaml -│ │ │ │ │ │ │ │ ├── results.json -│ │ │ │ │ │ │ │ └── results_summary.json +│ │ │ │ │ │ │ └── run_1 +│ │ │ │ │ │ │ └── results_summary.json │ │ │ │ │ │ ├── │ │ │ │ │ │ │ ├── accuracy │ │ │ │ │ │ │ │ └── accuracy.txt From 7d8ae1a190a71d61b2e6d8677bea7f732d1512da Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 13 Jul 2026 22:46:07 -0500 Subject: [PATCH 2/7] Remove unused code --- tools/submission/submission_checker/utils.py | 84 -------------------- 1 file changed, 84 deletions(-) diff --git a/tools/submission/submission_checker/utils.py b/tools/submission/submission_checker/utils.py index e9e6d0d443..60aae0878e 100644 --- a/tools/submission/submission_checker/utils.py +++ b/tools/submission/submission_checker/utils.py @@ -128,90 +128,6 @@ def contains_list(l1, l2): return missing, len(missing) == 0 -def get_performance_metric( - config, model, path, scenario_fixed): - # Assumes new logging format - version = config.version - - fname = os.path.join(path, "mlperf_log_detail.txt") - mlperf_log = LoadgenParser(fname) - if ( - "result_validity" in mlperf_log.get_keys() - and mlperf_log["result_validity"] == "VALID" - ): - is_valid = True - scenario = mlperf_log["effective_scenario"] - - res = float(mlperf_log[RESULT_FIELD_NEW[version][scenario]]) - if ( - version in RESULT_FIELD_BENCHMARK_OVERWRITE - and model in RESULT_FIELD_BENCHMARK_OVERWRITE[version] - and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model] - ): - res = float( - mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version] - [model][scenario]] - ) - - inferred = False - if scenario_fixed != scenario: - inferred, res, _ = get_inferred_result( - scenario_fixed, scenario, res, mlperf_log, config, False - ) - - return res - - -def get_inferred_result( - scenario_fixed, scenario, res, mlperf_log, config, log_error=False -): - - inferred = False - is_valid = True - # Check if current scenario (and version) uses early stopping - uses_early_stopping = config.uses_early_stopping(scenario) - - latency_mean = mlperf_log["result_mean_latency_ns"] - if scenario in ["MultiStream"]: - latency_99_percentile = mlperf_log[ - "result_99.00_percentile_per_query_latency_ns" - ] - latency_mean = mlperf_log["result_mean_query_latency_ns"] - samples_per_query = mlperf_log["effective_samples_per_query"] - if scenario == "SingleStream": - # qps_wo_loadgen_overhead is only used for inferring Offline from - # SingleStream; only for old submissions - qps_wo_loadgen_overhead = mlperf_log["result_qps_without_loadgen_overhead"] - - # special case for results inferred from different scenario - if scenario_fixed in ["Offline"] and scenario in ["SingleStream"]: - inferred = True - res = qps_wo_loadgen_overhead - - if (scenario_fixed in ["Offline"]) and scenario in ["MultiStream"]: - inferred = True - res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS) - - if (scenario_fixed in ["MultiStream"]) and scenario in ["SingleStream"]: - inferred = True - # samples_per_query does not match with the one reported in the logs - # when inferring MultiStream from SingleStream - samples_per_query = 8 - if uses_early_stopping: - early_stopping_latency_ms = mlperf_log["early_stopping_latency_ms"] - if early_stopping_latency_ms == 0 and log_error: - log.error( - "Not enough samples were processed for early stopping to make an estimate" - ) - is_valid = False - res = (early_stopping_latency_ms * samples_per_query) / MS_TO_NS - else: - res = (latency_99_percentile * samples_per_query) / MS_TO_NS - if (scenario_fixed in ["Interactive"]) and scenario not in ["Server"]: - is_valid = False - return inferred, res, is_valid - - def check_compliance_perf_dir(test_dir): is_valid = False import logging From 5ae81f766e70cdf4cef32deb026caf1fcf20dba4 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Tue, 14 Jul 2026 09:54:43 -0500 Subject: [PATCH 3/7] Revert "Remove unused code" This reverts commit 7d8ae1a190a71d61b2e6d8677bea7f732d1512da. --- tools/submission/submission_checker/utils.py | 84 ++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tools/submission/submission_checker/utils.py b/tools/submission/submission_checker/utils.py index 60aae0878e..e9e6d0d443 100644 --- a/tools/submission/submission_checker/utils.py +++ b/tools/submission/submission_checker/utils.py @@ -128,6 +128,90 @@ def contains_list(l1, l2): return missing, len(missing) == 0 +def get_performance_metric( + config, model, path, scenario_fixed): + # Assumes new logging format + version = config.version + + fname = os.path.join(path, "mlperf_log_detail.txt") + mlperf_log = LoadgenParser(fname) + if ( + "result_validity" in mlperf_log.get_keys() + and mlperf_log["result_validity"] == "VALID" + ): + is_valid = True + scenario = mlperf_log["effective_scenario"] + + res = float(mlperf_log[RESULT_FIELD_NEW[version][scenario]]) + if ( + version in RESULT_FIELD_BENCHMARK_OVERWRITE + and model in RESULT_FIELD_BENCHMARK_OVERWRITE[version] + and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model] + ): + res = float( + mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version] + [model][scenario]] + ) + + inferred = False + if scenario_fixed != scenario: + inferred, res, _ = get_inferred_result( + scenario_fixed, scenario, res, mlperf_log, config, False + ) + + return res + + +def get_inferred_result( + scenario_fixed, scenario, res, mlperf_log, config, log_error=False +): + + inferred = False + is_valid = True + # Check if current scenario (and version) uses early stopping + uses_early_stopping = config.uses_early_stopping(scenario) + + latency_mean = mlperf_log["result_mean_latency_ns"] + if scenario in ["MultiStream"]: + latency_99_percentile = mlperf_log[ + "result_99.00_percentile_per_query_latency_ns" + ] + latency_mean = mlperf_log["result_mean_query_latency_ns"] + samples_per_query = mlperf_log["effective_samples_per_query"] + if scenario == "SingleStream": + # qps_wo_loadgen_overhead is only used for inferring Offline from + # SingleStream; only for old submissions + qps_wo_loadgen_overhead = mlperf_log["result_qps_without_loadgen_overhead"] + + # special case for results inferred from different scenario + if scenario_fixed in ["Offline"] and scenario in ["SingleStream"]: + inferred = True + res = qps_wo_loadgen_overhead + + if (scenario_fixed in ["Offline"]) and scenario in ["MultiStream"]: + inferred = True + res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS) + + if (scenario_fixed in ["MultiStream"]) and scenario in ["SingleStream"]: + inferred = True + # samples_per_query does not match with the one reported in the logs + # when inferring MultiStream from SingleStream + samples_per_query = 8 + if uses_early_stopping: + early_stopping_latency_ms = mlperf_log["early_stopping_latency_ms"] + if early_stopping_latency_ms == 0 and log_error: + log.error( + "Not enough samples were processed for early stopping to make an estimate" + ) + is_valid = False + res = (early_stopping_latency_ms * samples_per_query) / MS_TO_NS + else: + res = (latency_99_percentile * samples_per_query) / MS_TO_NS + if (scenario_fixed in ["Interactive"]) and scenario not in ["Server"]: + is_valid = False + return inferred, res, is_valid + + def check_compliance_perf_dir(test_dir): is_valid = False import logging From eaa98a73a2122ede6822a149349422c497ea61ea Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 15 Jul 2026 23:35:16 +0530 Subject: [PATCH 4/7] fix: skip truncation for endpoints submissions instead of targeting stale results.json (#2631) The endpoints accuracy artifact was renamed from results.json (with a responses field) to accuracy/accuracy_results.json in mlcommons/endpoints#400, which also dropped the responses field entirely (per-sample text now lives in events.jsonl, not part of the submission). truncate_accuracy_log.py still looked for the old results.json/responses shape and would only log "missing" for every endpoints submission. Detect the endpoints layout via the scenario-root config.yaml and skip truncation, and remove the now-dead PERFORMANCE_ENDPOINTS_DIR/ACCURACY_ENDPOINTS_DIR constants left over from the loader.py refactor to ENDPOINTS_SCENARIO_DIR. Co-authored-by: Claude Sonnet 5 --- .../submission_checker/constants.py | 17 ---- tools/submission/truncate_accuracy_log.py | 99 +++---------------- 2 files changed, 14 insertions(+), 102 deletions(-) diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 05ec443be4..2be1a5bba8 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -2036,14 +2036,6 @@ "deepseek-r1" ] -PERFORMANCE_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", -} - ENDPOINTS_SCENARIO_DIR = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", @@ -2076,15 +2068,6 @@ "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json", } -ACCURACY_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", -} - - POWER_DIR_PATH = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", diff --git a/tools/submission/truncate_accuracy_log.py b/tools/submission/truncate_accuracy_log.py index 62d8616927..b9d3d686a4 100755 --- a/tools/submission/truncate_accuracy_log.py +++ b/tools/submission/truncate_accuracy_log.py @@ -18,7 +18,6 @@ MAX_ACCURACY_LOG_SIZE = 10 * 1024 VIEWABLE_SIZE = 4096 -RESPONSES_LIMIT = 10 * 1024 # cap on truncated responses bytes HELP_TEXT = """ You can run this tool in 2 ways: @@ -124,84 +123,18 @@ def copy_submission_dir(src, dst, filter_submitter): ) -def _truncate_endpoints_results(results_path, acc_path, backup): - """Truncate the ``responses`` field in an endpoints accuracy results.json. +def _is_endpoints_submission(scenario_dir): + """Detect the endpoints submission layout via its scenario-root config.yaml. - Mirrors the behaviour of truncate_accuracy_log.py for traditional - submissions: backs up the original file (when --backup is given), - computes a SHA-256 hash of the original, appends ``hash=`` to - accuracy.txt, then writes back a copy with responses capped at - RESPONSES_LIMIT bytes. + Endpoints submissions have no mlperf_log_accuracy.json; the accuracy + artifact they produce (accuracy/accuracy_results.json) carries no large + per-sample payload to truncate, so callers should skip truncation + entirely rather than look for a file to shrink. """ - try: - with open(results_path, "r", encoding="utf-8") as f: - original_bytes = f.read().encode() - data = json.loads(original_bytes) - except Exception as exc: - log.error("Could not read %s: %s", results_path, exc) - return - - # Back up before any modification. - if backup: - backup_dir = os.path.join(backup, acc_path) - os.makedirs(backup_dir, exist_ok=True) - dst = os.path.join(backup_dir, "results.json") - if os.path.exists(dst): - log.error( - "not processing %s because %s already exists", - results_path, - dst, - ) - return - shutil.copy(results_path, dst) - - # Hash the original file for audit purposes (mirrors accuracy.txt hash - # written for traditional mlperf_log_accuracy.json truncation). - hash_val = hashlib.sha256(original_bytes).hexdigest() - acc_txt = os.path.join(acc_path, "accuracy.txt") - with open(acc_txt, "a", encoding="utf-8") as f: - f.write("\nhash={0}\n".format(hash_val)) - - # Truncate the responses collection so the file stays small. - responses = data.get("responses") - if responses is None or (isinstance( - responses, (list, dict)) and not responses): - log.info("%s has no responses to truncate", results_path) - return - - if isinstance(responses, list): - total = 2 # "[]" - idx = 0 - for i, r in enumerate(responses): - total += len(json.dumps(r).encode()) + (2 if i > 0 else 0) - if total > RESPONSES_LIMIT: - break - idx = i + 1 - data["responses"] = responses[:idx] - elif isinstance(responses, dict): - total = 2 # "{}" - kept = {} - for i, (k, v) in enumerate(responses.items()): - entry = len(json.dumps(k).encode()) + \ - len(json.dumps(v).encode()) + 2 - if i > 0: - entry += 2 - if total + entry > RESPONSES_LIMIT: - break - total += entry - kept[k] = v - data["responses"] = kept - else: - log.error( - "%s: responses has unexpected type %s; skipping truncation", - results_path, - type(responses).__name__, - ) - return - - with open(results_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - log.info("%s responses truncated", results_path) + return any( + os.path.exists(os.path.join(scenario_dir, name)) + for name in ("config.yaml", "config.yml") + ) def truncate_results_dir(filter_submitter, backup, scenarios_to_skip): @@ -264,14 +197,10 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip): acc_txt = os.path.join( acc_path, "accuracy.txt") if not os.path.exists(acc_log): - # Endpoints submissions have results.json - # instead of mlperf_log_accuracy.json - endpoints_results = os.path.join( - acc_path, "results.json" - ) - if os.path.exists(endpoints_results): - _truncate_endpoints_results( - endpoints_results, acc_path, backup + if _is_endpoints_submission(name): + log.info( + "%s is an endpoints submission; nothing to truncate", + name, ) else: log.error("%s missing", acc_log) From 3e27e6ff1b684b13e0588b2b29f77b729aef6b39 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Fri, 17 Jul 2026 11:12:30 -0500 Subject: [PATCH 5/7] Add compliance checking for endpoints submission (#2633) --- .../checks/compliance_check.py | 73 +++++++++++++++++-- .../submission_checker/constants.py | 12 +++ 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/tools/submission/submission_checker/checks/compliance_check.py b/tools/submission/submission_checker/checks/compliance_check.py index e110af9c40..15ad2e5706 100644 --- a/tools/submission/submission_checker/checks/compliance_check.py +++ b/tools/submission/submission_checker/checks/compliance_check.py @@ -8,6 +8,7 @@ from ..utils import * import re import os +import json class ComplianceCheck(BaseCheck): @@ -41,6 +42,7 @@ def __init__(self, log, path, config: Config, self.submission_logs = submission_logs self.config = config self.name = "compliance check" + self.perf_log = self.submission_logs.performance_log self.model = self.submission_logs.loader_data.get("benchmark", "") self.model_mapping = self.submission_logs.loader_data.get( "model_mapping", {}) @@ -60,14 +62,15 @@ def setup_checks(self): Appends the per-submission validation callables to `self.checks` in the order they should be executed by the checking framework. """ - self.checks.append(self.dir_exists_check) - self.checks.append(self.performance_check) - self.checks.append(self.accuracy_check) - self.checks.append(self.compliance_performance_check) - self.apply_checks = set(self.checks) - # No compliance tests for endpoints for now - if self.is_endpoints: - self.apply_checks = set() + if not self.is_endpoints: + self.checks.append(self.dir_exists_check) + self.checks.append(self.performance_check) + self.checks.append(self.accuracy_check) + self.checks.append(self.compliance_performance_check) + self.apply_checks = set(self.checks) + else: + self.checks.append(self.compliance_endpoints_check) + self.apply_checks = set(self.checks) def get_test_list(self, model): """Return the list of compliance tests applicable to `model`. @@ -494,3 +497,57 @@ def compliance_performance_check(self): diff) is_valid = False return is_valid + + def compliance_endpoints_check(self): + test_dir = os.path.join(self.compliance_dir, "audit") + is_valid = True + for test in self.test_list: + if test in ENDPOINTS_COMPLIANCE_MAPPING: + test_name = ENDPOINTS_COMPLIANCE_MAPPING.get(test, test) + fname = os.path.join(test_dir, f"audit_{test_name}.json") + if not os.path.exists(fname): + self.log.error("%s is missing in %s", fname, test_dir) + is_valid = False + else: + try: + with open(fname) as f: + test_file = json.load(f) + + if test_file["test"] != test_name: + self.log.error( + "%s compliance test is incorrect %s. " + + "Expected name=%s" + + "Actual name=%s", + fname, + test_dir, + test_name, + test_file["test"] + ) + is_valid = False + elif not test_file["passed"]: + self.log.error( + "%s compliance test can not be loaded in %s", fname, test_dir) + is_valid = False + else: + self.log.info( + "%s compliance test %s passed for %s", fname, test_name, test_dir) + + except BaseException: + self.log.error( + "%s compliance test can not be loaded in %s", fname, test_dir) + is_valid = False + + if test in ["TEST09"]: + mean_output_tokens = self.perf_log["mean_output_tokens"] + if mean_output_tokens > TEST09_HIGH or mean_output_tokens < TEST09_LOW: + self.log.error("TEST 09 (Verify Output Token Length in Performance Mode)" + + " failed for %s. Mean output tokens need to be in the range" + + " [%s, %s]. Found %s", + self.compliance_dir, + TEST09_LOW, + TEST09_HIGH, + mean_output_tokens + ) + is_valid = False + + return is_valid diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 2be1a5bba8..7d3e1d4f69 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -1501,6 +1501,9 @@ "compliance_accuracy.txt", ] +TEST09_LOW = 1150.38 +TEST09_HIGH = 1406.02 + OFFLINE_MIN_SPQ_SINCE_V4 = { "resnet": 24576, "retinanet": 24576, @@ -2233,6 +2236,7 @@ "tps": "result_completed_tokens_per_second", "result.total": "result_query_count", "result.failed": "num_errors", + "output_sequence_lengths.avg": "mean_output_tokens", } @@ -2267,3 +2271,11 @@ ENDPOINTS_INFERRED_FIELDS = { "effective_accuracy_sample_count": "result_query_count" } + +ENDPOINTS_COMPLIANCE_MAPPING = { + "TEST01": "accuracy_in_performance_test", + "TEST04": "output_caching_test", + "TEST06": "consistency_output_test", + "TEST07": "accuracy_in_performance_full_test", + "TEST08": "accuracy_in_performance_dlrmv3_test", +} From 75cf49509b1ec1467f7fe4947f772d821595fead Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Fri, 17 Jul 2026 13:00:02 -0500 Subject: [PATCH 6/7] Quick fixes for endpoints parser --- .../submission_checker/parsers/endpoints_parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/submission/submission_checker/parsers/endpoints_parser.py b/tools/submission/submission_checker/parsers/endpoints_parser.py index 9b1a5a2e53..2bdb341ecd 100644 --- a/tools/submission/submission_checker/parsers/endpoints_parser.py +++ b/tools/submission/submission_checker/parsers/endpoints_parser.py @@ -23,9 +23,9 @@ "helper", "sample_logs") -_RESULT_SUMMARY_FILE = "results_summary.json" +_RESULT_SUMMARY_FILE = "result_summary.json" _ACCURACY_RESULTS_FILE = "accuracy_results.json" -_PERF_SUBDIR = os.path.join("performance", "run_1") +_PERF_SUBDIR = "performance" _ACC_SUBDIR = "accuracy" _CONFIG_FILES = ("config.yaml", "config.yml") @@ -100,7 +100,7 @@ def __init__(self, scenario_dir): """ scenario_dir: path to the scenario directory containing: - config.yaml (scenario root) - - performance/run_1/results_summary.json (performance metrics) + - performance/result_summary.json (performance metrics) - accuracy/accuracy_results.json (accuracy metrics) """ super().__init__(scenario_dir) @@ -217,7 +217,7 @@ def main(): backwards_map = _load_field_map("backwards.json") # Collect scenario-level directories (those containing config.yaml and - # the performance/run_1 subdirectory with results_summary.json) + # the performance subdirectory with result_summary.json) run_dirs = [] for root, dirs, files in os.walk(_SAMPLE_LOGS_DIR): has_yaml = any(f in _CONFIG_FILES for f in files) From 03db2adf8bd438a9b8f4cdd372a747791a3cbb51 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 13 Jul 2026 18:36:19 -0500 Subject: [PATCH 7/7] Add qwen3.6-27b benchmark to submission checker --- .../checks/performance_check.py | 32 +++++++----- .../submission_checker/constants.py | 50 +++++++++++++------ 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/tools/submission/submission_checker/checks/performance_check.py b/tools/submission/submission_checker/checks/performance_check.py index ed8ad18bec..036dacbaeb 100644 --- a/tools/submission/submission_checker/checks/performance_check.py +++ b/tools/submission/submission_checker/checks/performance_check.py @@ -56,10 +56,18 @@ def __init__(self, log, path, config: Config, self.is_endpoints = self.submission_logs.loader_data.get( "is_endpoints_submission", False) if self.is_endpoints: - if self.scenario.lower() == "online": - self.scenario = "Server" + self.scenario = self.map_endpoints_scenario( + self.scenario, self.scenario_fixed + ) self.setup_checks() + def map_endpoints_scenario(self, scenario, scenario_fixed): + if scenario_fixed.lower() == "singlestream": + return "SingleStream" + if scenario.lower() == "online": + return "Server" + return scenario + def setup_checks(self): """Register individual performance-related checks. @@ -492,7 +500,7 @@ def inferred_check(self): ("singlestream", "offline") ] if (self.scenario.lower(), self.scenario_fixed.lower() - ) not in list_inferred: + ) not in list_inferred: self.log.error( "Result for scenario %s can not be inferred from %s for: %s", self.scenario_fixed, @@ -519,13 +527,13 @@ def get_performance_metric_check(self): and self.mlperf_log["result_validity"] == "VALID" ): is_valid = True - scenario = self.mlperf_log["effective_scenario"] + scenario = self.scenario + res = None if self.is_endpoints: - if scenario.lower() == "online": - scenario = "Server" - scenario = scenario.capitalize() - - res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]]) + res = float( + self.mlperf_log[RESULT_FIELD_ENDPOINTS[version][scenario.lower()]]) + else: + res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]]) if ( not self.is_endpoints and version in RESULT_FIELD_BENCHMARK_OVERWRITE @@ -606,12 +614,12 @@ def get_inferred_result(self, res): res = qps_wo_loadgen_overhead if (scenario_fixed in ["Offline"] - ) and scenario in ["MultiStream"]: + ) and scenario in ["MultiStream"]: inferred = True res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS) if (scenario_fixed in ["MultiStream"] - ) and scenario in ["SingleStream"]: + ) and scenario in ["SingleStream"]: inferred = True # samples_per_query does not match with the one reported in the logs # when inferring MultiStream from SingleStream @@ -628,6 +636,6 @@ def get_inferred_result(self, res): else: res = (latency_99_percentile * samples_per_query) / MS_TO_NS if (scenario_fixed in ["Interactive"] - ) and scenario not in ["Server"]: + ) and scenario not in ["Server"]: is_valid = False return res, is_valid diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 7d3e1d4f69..6e2697db97 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -17,6 +17,7 @@ "gpt-oss-120b", "wan-2.2-t2v-a14b", "qwen3-vl-235b-a22b", + "qwen3.6-27b", "dlrm-v3", "yolo-95", "yolo-99", @@ -58,6 +59,7 @@ "whisper": ["Offline"], "yolo-95": ["SingleStream", "MultiStream", "Offline"], "yolo-99": ["SingleStream", "MultiStream", "Offline"], + "qwen3.6-27b": ["SingleStream"], }, "optional-scenarios-edge": {}, "required-scenarios-datacenter-edge": { @@ -79,6 +81,7 @@ "dlrm-v3": ["Offline", "Server"], "yolo-95": ["SingleStream", "MultiStream", "Offline"], "yolo-99": ["SingleStream", "MultiStream", "Offline"], + "qwen3.6-27b": ["SingleStream"], }, "optional-scenarios-datacenter-edge": { "llama2-70b-99": ["Interactive", "Server"], @@ -164,6 +167,7 @@ # established "e2e": ("E2E_ACCURACY", ""), "e2e_vectorDB": ("E2E_ACCURACY", ""), + "qwen3.6-27b": ("mAP", 86.23 * 0.99), }, "accuracy-upper-limit": { "stable-diffusion-xl": ( @@ -203,11 +207,13 @@ "yolo-95": 64, "yolo-99": 64, "e2e": 824, - "e2e_vectorDB": 824 + "e2e_vectorDB": 824, + "qwen3.6-27b": 995, }, "accuracy-sample-count": { "gpt-oss-120b": 4395, "wan-2.2-t2v-a14b": 248, + "qwen3.6-27b": 995, }, "dataset-size": { "resnet": 50000, @@ -231,6 +237,7 @@ "yolo-99": 1525, "e2e": 824, "e2e_vectorDB": 824, + "qwen3.6-27b": 995, }, "model_mapping": { "ssd-resnet34": "retinanet", @@ -291,6 +298,7 @@ "yolo-99": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1}, "e2e": {"Offline": 824}, "e2e_vectorDB": {"Offline": 824}, + "qwen3.6-27b": {"SingleStream": 995}, }, "models_TEST01": [ "resnet", @@ -1580,6 +1588,16 @@ }, } +RESULT_FIELD_ENDPOINTS = { + "v6.1": { + "offline": "result_samples_per_second", + "singlestream": "result_mean_latency_ns", + "multistream": "result_mean_latency_ns", + "server": "result_completed_samples_per_sec", + "interactive": "result_completed_samples_per_sec", + }, +} + RESULT_FIELD_BENCHMARK_OVERWRITE = { "v5.0": { "llama2-70b-99": { @@ -2033,6 +2051,7 @@ ENDPOINTS_ALLOWED_MODELS = [ "wan-2.2-t2v-a14b", "qwen3-vl-235b-a22b", + "qwen3.6-27b", "llama3.1-8b", "llama3.1-8b-edge", "gpt-oss-120b", @@ -2203,7 +2222,7 @@ "effective_sample_concatenate_permutation": "effective_sample_concatenate_permutation", "effective_samples_per_query": "effective_samples_per_query", "generated_query_count": "generated_query_count", - "generated_query_duration": "generated_query_duration", + "duration_ns": "generated_query_duration", "target_qps": "effective_target_qps", "result_scheduled_samples_per_sec": "result_scheduled_samples_per_sec", "qps": "result_completed_samples_per_sec", @@ -2212,23 +2231,23 @@ "latency.min": "result_min_latency_ns", "latency.max": "result_max_latency_ns", "latency.avg": "result_mean_latency_ns", - "latency.percentiles.50.0": "result_50.00_percentile_latency_ns", - "latency.percentiles.90.0": "result_90.00_percentile_latency_ns", - "latency.percentiles.95.0": "result_95.00_percentile_latency_ns", - "latency.percentiles.99.0": "result_99.00_percentile_latency_ns", + "latency.percentiles.50": "result_50.00_percentile_latency_ns", + "latency.percentiles.90": "result_90.00_percentile_latency_ns", + "latency.percentiles.95": "result_95.00_percentile_latency_ns", + "latency.percentiles.99": "result_99.00_percentile_latency_ns", "latency.percentiles.99.9": "result_99.90_percentile_latency_ns", "ttft.min": "result_first_token_min_latency_ns", "ttft.max": "result_first_token_max_latency_ns", "ttft.avg": "result_first_token_mean_latency_ns", - "ttft.percentiles.50.0": "result_first_token_50.00_percentile_latency_ns", - "ttft.percentiles.90.0": "result_first_token_90.00_percentile_latency_ns", - "ttft.percentiles.95.0": "result_first_token_95.00_percentile_latency_ns", - "ttft.percentiles.99.0": "result_first_token_99.00_percentile_latency_ns", + "ttft.percentiles.50": "result_first_token_50.00_percentile_latency_ns", + "ttft.percentiles.90": "result_first_token_90.00_percentile_latency_ns", + "ttft.percentiles.95": "result_first_token_95.00_percentile_latency_ns", + "ttft.percentiles.99": "result_first_token_99.00_percentile_latency_ns", "ttft.percentiles.99.9": "result_first_token_99.90_percentile_latency_ns", - "tpot.percentiles.50.0": "result_time_per_output_token_50.00_percentile_ns", - "tpot.percentiles.90.0": "result_time_per_output_token_90.00_percentile_ns", - "tpot.percentiles.95.0": "result_time_per_output_token_95.00_percentile_ns", - "tpot.percentiles.99.0": "result_time_per_output_token_99.00_percentile_ns", + "tpot.percentiles.50": "result_time_per_output_token_50.00_percentile_ns", + "tpot.percentiles.90": "result_time_per_output_token_90.00_percentile_ns", + "tpot.percentiles.95": "result_time_per_output_token_95.00_percentile_ns", + "tpot.percentiles.99": "result_time_per_output_token_99.00_percentile_ns", "tpot.percentiles.99.9": "result_time_per_output_token_99.90_percentile_ns", "tpot.min": "result_time_per_output_token_min", "tpot.max": "result_time_per_output_token_max", @@ -2269,7 +2288,8 @@ } ENDPOINTS_INFERRED_FIELDS = { - "effective_accuracy_sample_count": "result_query_count" + "effective_accuracy_sample_count": "result_query_count", + "generated_query_count": "qsl_reported_total_count", } ENDPOINTS_COMPLIANCE_MAPPING = {