From 50d4a29286624ee2e991ba0416bc2186a0f9c7eb Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 13 Jul 2026 22:45:11 -0500 Subject: [PATCH 1/4] Update folder structure for endpoints submissions --- .../submission_checker/constants.py | 10 ++++ tools/submission/submission_checker/loader.py | 43 ++++++----------- .../parsers/endpoints_parser.py | 46 +++++++++++-------- tools/submission/submission_structure.md | 15 +++--- 4 files changed, 56 insertions(+), 58 deletions(-) diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 8278fe77a8..05ec443be4 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -2040,9 +2040,18 @@ "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", } +ENDPOINTS_SCENARIO_DIR = { + "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", +} + ACCURACY_LOG_PATH = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt", @@ -2071,6 +2080,7 @@ "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", } diff --git a/tools/submission/submission_checker/loader.py b/tools/submission/submission_checker/loader.py index 13c531bd93..f58876e007 100644 --- a/tools/submission/submission_checker/loader.py +++ b/tools/submission/submission_checker/loader.py @@ -86,12 +86,9 @@ def __init__(self, root, version, config: Config) -> None: self.acc_json_path = os.path.join( self.root, ACCURACY_JSON_PATH.get( version, ACCURACY_JSON_PATH["default"])) - self.perf_endpoints_dir = os.path.join( - self.root, PERFORMANCE_ENDPOINTS_DIR.get( - version, PERFORMANCE_ENDPOINTS_DIR["default"])) - self.acc_endpoints_dir = os.path.join( - self.root, ACCURACY_ENDPOINTS_DIR.get( - version, ACCURACY_ENDPOINTS_DIR["default"])) + self.endpoints_scenario_dir = os.path.join( + self.root, ENDPOINTS_SCENARIO_DIR.get( + version, ENDPOINTS_SCENARIO_DIR["default"])) self.system_log_path = os.path.join( self.root, SYSTEM_PATH.get( version, SYSTEM_PATH["default"])) @@ -234,21 +231,15 @@ def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", path) return log - def load_endpoints_logs(self, perf_dir, acc_dir): - perf_log = None - acc_log = None - if os.path.exists(acc_dir) and os.path.exists(perf_dir): - acc_log = EndpointsParser(acc_dir) - perf_log = EndpointsParser(perf_dir) - elif os.path.exists(perf_dir): - acc_log = EndpointsParser(perf_dir) - perf_log = EndpointsParser(perf_dir) - else: - self.logger.info( - "Could not load endpoints log from %s, path does not exist", - perf_dir - ) - return perf_log, acc_log + def load_endpoints_logs(self, scenario_dir): + if os.path.exists(scenario_dir): + parser = EndpointsParser(scenario_dir) + return parser, parser + self.logger.info( + "Could not load endpoints log from %s, path does not exist", + scenario_dir + ) + return None, None def check_scenarios(self, benchmark, model_mapping, system_type, scenarios): @@ -362,13 +353,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: system=system, benchmark=benchmark, scenario=scenario) - perf_endpoints_dir = self.perf_endpoints_dir.format( - division=division, - submitter=submitter, - system=system, - benchmark=benchmark, - scenario=scenario) - acc_endpoints_dir = self.acc_endpoints_dir.format( + endpoints_scenario_dir = self.endpoints_scenario_dir.format( division=division, submitter=submitter, system=system, @@ -483,7 +468,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: if perf_log is None and acc_log is None: is_endpoints_submission = True perf_log, acc_log = self.load_endpoints_logs( - perf_endpoints_dir, acc_endpoints_dir + endpoints_scenario_dir ) # Load test logs diff --git a/tools/submission/submission_checker/parsers/endpoints_parser.py b/tools/submission/submission_checker/parsers/endpoints_parser.py index bb4b09f552..9b1a5a2e53 100644 --- a/tools/submission/submission_checker/parsers/endpoints_parser.py +++ b/tools/submission/submission_checker/parsers/endpoints_parser.py @@ -24,7 +24,9 @@ "sample_logs") _RESULT_SUMMARY_FILE = "results_summary.json" -_RESULTS_FILE = "results.json" +_ACCURACY_RESULTS_FILE = "accuracy_results.json" +_PERF_SUBDIR = os.path.join("performance", "run_1") +_ACC_SUBDIR = "accuracy" _CONFIG_FILES = ("config.yaml", "config.yml") @@ -94,22 +96,23 @@ def _resolve_value(stripped, summary_data, results_data, yaml_data): class EndpointsParser(BaseParser): - def __init__(self, run_dir): + def __init__(self, scenario_dir): """ - run_dir: path to the run directory containing: - - result_summary.json (highest priority) - - results.json - - config.yaml / config.yml (lowest priority) + scenario_dir: path to the scenario directory containing: + - config.yaml (scenario root) + - performance/run_1/results_summary.json (performance metrics) + - accuracy/accuracy_results.json (accuracy metrics) """ - super().__init__(run_dir) + super().__init__(scenario_dir) self.logger = logging.getLogger("MLPerfLog") self.messages = {} summary_data = self._load_json( - os.path.join(run_dir, _RESULT_SUMMARY_FILE)) - results_data = self._load_json(os.path.join(run_dir, _RESULTS_FILE)) - yaml_data = self._load_yaml(run_dir) + os.path.join(scenario_dir, _PERF_SUBDIR, _RESULT_SUMMARY_FILE)) + results_data = self._load_json( + os.path.join(scenario_dir, _ACC_SUBDIR, _ACCURACY_RESULTS_FILE)) + yaml_data = self._load_yaml(scenario_dir) for endpoints_key, loadgen_key in ENDPOINTS_MAPPINGS.items(): stripped = endpoints_key.strip() @@ -153,7 +156,9 @@ def __init__(self, run_dir): ) self.keys = set(self.messages.keys()) - self.logger.info("Successfully loaded endpoints log from %s.", run_dir) + self.logger.info( + "Successfully loaded endpoints log from %s.", + scenario_dir) def _load_json(self, path): try: @@ -162,7 +167,6 @@ def _load_json(self, path): except BaseException: self.logger.error("Could not load json file from %s", path) return {} - return {} def _load_yaml(self, run_dir): for name in _CONFIG_FILES: @@ -212,18 +216,20 @@ def main(): backwards_map = _load_field_map("backwards.json") - # Collect all run directories (those containing at least one JSON and one - # YAML) + # Collect scenario-level directories (those containing config.yaml and + # the performance/run_1 subdirectory with results_summary.json) run_dirs = [] - for root, _dirs, files in os.walk(_SAMPLE_LOGS_DIR): - has_json = any(f.endswith(".json") for f in files) - has_yaml = any(f.endswith(".yaml") or f.endswith(".yml") - for f in files) - if has_json and has_yaml: + for root, dirs, files in os.walk(_SAMPLE_LOGS_DIR): + has_yaml = any(f in _CONFIG_FILES for f in files) + has_perf = os.path.exists( + os.path.join(root, _PERF_SUBDIR, _RESULT_SUMMARY_FILE)) + if has_yaml and has_perf: run_dirs.append(root) if not run_dirs: - logger.error("No run directories found under %s.", _SAMPLE_LOGS_DIR) + logger.error( + "No scenario directories found under %s.", + _SAMPLE_LOGS_DIR) return 1 for run_dir in sorted(run_dirs): diff --git a/tools/submission/submission_structure.md b/tools/submission/submission_structure.md index ac9e81ba7a..febe1d0f4a 100644 --- a/tools/submission/submission_structure.md +++ b/tools/submission/submission_structure.md @@ -65,7 +65,7 @@ The following diagram describes the standard submission structure. ## Endpoints submission structure -For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. You can provide a performance+accuracy run in the performance folder or one performance run and one accuracy run +For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. The `config.yaml` is placed at the scenario root, `results_summary.json` contains performance metrics, and `accuracy_results.json` contains accuracy metrics. ``` ... @@ -77,15 +77,12 @@ For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structur │ │ │ ├── │ │ │ │ ├── │ │ │ │ │ ├── -│ │ │ │ │ │ ├── accuracy (optional) -│ │ │ │ │ │ │ ├── config.yaml -│ │ │ │ │ │ │ ├── results.json -│ │ │ │ │ │ │ └── results_summary.json +│ │ │ │ │ │ ├── config.yaml +│ │ │ │ │ │ ├── accuracy +│ │ │ │ │ │ │ └── accuracy_results.json │ │ │ │ │ │ ├── performance -│ │ │ │ │ │ │ ├── run_1 -│ │ │ │ │ │ │ │ ├── config.yaml -│ │ │ │ │ │ │ │ ├── results.json -│ │ │ │ │ │ │ │ └── results_summary.json +│ │ │ │ │ │ │ └── run_1 +│ │ │ │ │ │ │ └── results_summary.json │ │ │ │ │ │ ├── │ │ │ │ │ │ │ ├── accuracy │ │ │ │ │ │ │ │ └── accuracy.txt From 7d8ae1a190a71d61b2e6d8677bea7f732d1512da Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 13 Jul 2026 22:46:07 -0500 Subject: [PATCH 2/4] Remove unused code --- tools/submission/submission_checker/utils.py | 84 -------------------- 1 file changed, 84 deletions(-) diff --git a/tools/submission/submission_checker/utils.py b/tools/submission/submission_checker/utils.py index e9e6d0d443..60aae0878e 100644 --- a/tools/submission/submission_checker/utils.py +++ b/tools/submission/submission_checker/utils.py @@ -128,90 +128,6 @@ def contains_list(l1, l2): return missing, len(missing) == 0 -def get_performance_metric( - config, model, path, scenario_fixed): - # Assumes new logging format - version = config.version - - fname = os.path.join(path, "mlperf_log_detail.txt") - mlperf_log = LoadgenParser(fname) - if ( - "result_validity" in mlperf_log.get_keys() - and mlperf_log["result_validity"] == "VALID" - ): - is_valid = True - scenario = mlperf_log["effective_scenario"] - - res = float(mlperf_log[RESULT_FIELD_NEW[version][scenario]]) - if ( - version in RESULT_FIELD_BENCHMARK_OVERWRITE - and model in RESULT_FIELD_BENCHMARK_OVERWRITE[version] - and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model] - ): - res = float( - mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version] - [model][scenario]] - ) - - inferred = False - if scenario_fixed != scenario: - inferred, res, _ = get_inferred_result( - scenario_fixed, scenario, res, mlperf_log, config, False - ) - - return res - - -def get_inferred_result( - scenario_fixed, scenario, res, mlperf_log, config, log_error=False -): - - inferred = False - is_valid = True - # Check if current scenario (and version) uses early stopping - uses_early_stopping = config.uses_early_stopping(scenario) - - latency_mean = mlperf_log["result_mean_latency_ns"] - if scenario in ["MultiStream"]: - latency_99_percentile = mlperf_log[ - "result_99.00_percentile_per_query_latency_ns" - ] - latency_mean = mlperf_log["result_mean_query_latency_ns"] - samples_per_query = mlperf_log["effective_samples_per_query"] - if scenario == "SingleStream": - # qps_wo_loadgen_overhead is only used for inferring Offline from - # SingleStream; only for old submissions - qps_wo_loadgen_overhead = mlperf_log["result_qps_without_loadgen_overhead"] - - # special case for results inferred from different scenario - if scenario_fixed in ["Offline"] and scenario in ["SingleStream"]: - inferred = True - res = qps_wo_loadgen_overhead - - if (scenario_fixed in ["Offline"]) and scenario in ["MultiStream"]: - inferred = True - res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS) - - if (scenario_fixed in ["MultiStream"]) and scenario in ["SingleStream"]: - inferred = True - # samples_per_query does not match with the one reported in the logs - # when inferring MultiStream from SingleStream - samples_per_query = 8 - if uses_early_stopping: - early_stopping_latency_ms = mlperf_log["early_stopping_latency_ms"] - if early_stopping_latency_ms == 0 and log_error: - log.error( - "Not enough samples were processed for early stopping to make an estimate" - ) - is_valid = False - res = (early_stopping_latency_ms * samples_per_query) / MS_TO_NS - else: - res = (latency_99_percentile * samples_per_query) / MS_TO_NS - if (scenario_fixed in ["Interactive"]) and scenario not in ["Server"]: - is_valid = False - return inferred, res, is_valid - - def check_compliance_perf_dir(test_dir): is_valid = False import logging From 5ae81f766e70cdf4cef32deb026caf1fcf20dba4 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Tue, 14 Jul 2026 09:54:43 -0500 Subject: [PATCH 3/4] Revert "Remove unused code" This reverts commit 7d8ae1a190a71d61b2e6d8677bea7f732d1512da. --- tools/submission/submission_checker/utils.py | 84 ++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tools/submission/submission_checker/utils.py b/tools/submission/submission_checker/utils.py index 60aae0878e..e9e6d0d443 100644 --- a/tools/submission/submission_checker/utils.py +++ b/tools/submission/submission_checker/utils.py @@ -128,6 +128,90 @@ def contains_list(l1, l2): return missing, len(missing) == 0 +def get_performance_metric( + config, model, path, scenario_fixed): + # Assumes new logging format + version = config.version + + fname = os.path.join(path, "mlperf_log_detail.txt") + mlperf_log = LoadgenParser(fname) + if ( + "result_validity" in mlperf_log.get_keys() + and mlperf_log["result_validity"] == "VALID" + ): + is_valid = True + scenario = mlperf_log["effective_scenario"] + + res = float(mlperf_log[RESULT_FIELD_NEW[version][scenario]]) + if ( + version in RESULT_FIELD_BENCHMARK_OVERWRITE + and model in RESULT_FIELD_BENCHMARK_OVERWRITE[version] + and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model] + ): + res = float( + mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version] + [model][scenario]] + ) + + inferred = False + if scenario_fixed != scenario: + inferred, res, _ = get_inferred_result( + scenario_fixed, scenario, res, mlperf_log, config, False + ) + + return res + + +def get_inferred_result( + scenario_fixed, scenario, res, mlperf_log, config, log_error=False +): + + inferred = False + is_valid = True + # Check if current scenario (and version) uses early stopping + uses_early_stopping = config.uses_early_stopping(scenario) + + latency_mean = mlperf_log["result_mean_latency_ns"] + if scenario in ["MultiStream"]: + latency_99_percentile = mlperf_log[ + "result_99.00_percentile_per_query_latency_ns" + ] + latency_mean = mlperf_log["result_mean_query_latency_ns"] + samples_per_query = mlperf_log["effective_samples_per_query"] + if scenario == "SingleStream": + # qps_wo_loadgen_overhead is only used for inferring Offline from + # SingleStream; only for old submissions + qps_wo_loadgen_overhead = mlperf_log["result_qps_without_loadgen_overhead"] + + # special case for results inferred from different scenario + if scenario_fixed in ["Offline"] and scenario in ["SingleStream"]: + inferred = True + res = qps_wo_loadgen_overhead + + if (scenario_fixed in ["Offline"]) and scenario in ["MultiStream"]: + inferred = True + res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS) + + if (scenario_fixed in ["MultiStream"]) and scenario in ["SingleStream"]: + inferred = True + # samples_per_query does not match with the one reported in the logs + # when inferring MultiStream from SingleStream + samples_per_query = 8 + if uses_early_stopping: + early_stopping_latency_ms = mlperf_log["early_stopping_latency_ms"] + if early_stopping_latency_ms == 0 and log_error: + log.error( + "Not enough samples were processed for early stopping to make an estimate" + ) + is_valid = False + res = (early_stopping_latency_ms * samples_per_query) / MS_TO_NS + else: + res = (latency_99_percentile * samples_per_query) / MS_TO_NS + if (scenario_fixed in ["Interactive"]) and scenario not in ["Server"]: + is_valid = False + return inferred, res, is_valid + + def check_compliance_perf_dir(test_dir): is_valid = False import logging From eaa98a73a2122ede6822a149349422c497ea61ea Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 15 Jul 2026 23:35:16 +0530 Subject: [PATCH 4/4] fix: skip truncation for endpoints submissions instead of targeting stale results.json (#2631) The endpoints accuracy artifact was renamed from results.json (with a responses field) to accuracy/accuracy_results.json in mlcommons/endpoints#400, which also dropped the responses field entirely (per-sample text now lives in events.jsonl, not part of the submission). truncate_accuracy_log.py still looked for the old results.json/responses shape and would only log "missing" for every endpoints submission. Detect the endpoints layout via the scenario-root config.yaml and skip truncation, and remove the now-dead PERFORMANCE_ENDPOINTS_DIR/ACCURACY_ENDPOINTS_DIR constants left over from the loader.py refactor to ENDPOINTS_SCENARIO_DIR. Co-authored-by: Claude Sonnet 5 --- .../submission_checker/constants.py | 17 ---- tools/submission/truncate_accuracy_log.py | 99 +++---------------- 2 files changed, 14 insertions(+), 102 deletions(-) diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 05ec443be4..2be1a5bba8 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -2036,14 +2036,6 @@ "deepseek-r1" ] -PERFORMANCE_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", -} - ENDPOINTS_SCENARIO_DIR = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", @@ -2076,15 +2068,6 @@ "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json", } -ACCURACY_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", -} - - POWER_DIR_PATH = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", diff --git a/tools/submission/truncate_accuracy_log.py b/tools/submission/truncate_accuracy_log.py index 62d8616927..b9d3d686a4 100755 --- a/tools/submission/truncate_accuracy_log.py +++ b/tools/submission/truncate_accuracy_log.py @@ -18,7 +18,6 @@ MAX_ACCURACY_LOG_SIZE = 10 * 1024 VIEWABLE_SIZE = 4096 -RESPONSES_LIMIT = 10 * 1024 # cap on truncated responses bytes HELP_TEXT = """ You can run this tool in 2 ways: @@ -124,84 +123,18 @@ def copy_submission_dir(src, dst, filter_submitter): ) -def _truncate_endpoints_results(results_path, acc_path, backup): - """Truncate the ``responses`` field in an endpoints accuracy results.json. +def _is_endpoints_submission(scenario_dir): + """Detect the endpoints submission layout via its scenario-root config.yaml. - Mirrors the behaviour of truncate_accuracy_log.py for traditional - submissions: backs up the original file (when --backup is given), - computes a SHA-256 hash of the original, appends ``hash=`` to - accuracy.txt, then writes back a copy with responses capped at - RESPONSES_LIMIT bytes. + Endpoints submissions have no mlperf_log_accuracy.json; the accuracy + artifact they produce (accuracy/accuracy_results.json) carries no large + per-sample payload to truncate, so callers should skip truncation + entirely rather than look for a file to shrink. """ - try: - with open(results_path, "r", encoding="utf-8") as f: - original_bytes = f.read().encode() - data = json.loads(original_bytes) - except Exception as exc: - log.error("Could not read %s: %s", results_path, exc) - return - - # Back up before any modification. - if backup: - backup_dir = os.path.join(backup, acc_path) - os.makedirs(backup_dir, exist_ok=True) - dst = os.path.join(backup_dir, "results.json") - if os.path.exists(dst): - log.error( - "not processing %s because %s already exists", - results_path, - dst, - ) - return - shutil.copy(results_path, dst) - - # Hash the original file for audit purposes (mirrors accuracy.txt hash - # written for traditional mlperf_log_accuracy.json truncation). - hash_val = hashlib.sha256(original_bytes).hexdigest() - acc_txt = os.path.join(acc_path, "accuracy.txt") - with open(acc_txt, "a", encoding="utf-8") as f: - f.write("\nhash={0}\n".format(hash_val)) - - # Truncate the responses collection so the file stays small. - responses = data.get("responses") - if responses is None or (isinstance( - responses, (list, dict)) and not responses): - log.info("%s has no responses to truncate", results_path) - return - - if isinstance(responses, list): - total = 2 # "[]" - idx = 0 - for i, r in enumerate(responses): - total += len(json.dumps(r).encode()) + (2 if i > 0 else 0) - if total > RESPONSES_LIMIT: - break - idx = i + 1 - data["responses"] = responses[:idx] - elif isinstance(responses, dict): - total = 2 # "{}" - kept = {} - for i, (k, v) in enumerate(responses.items()): - entry = len(json.dumps(k).encode()) + \ - len(json.dumps(v).encode()) + 2 - if i > 0: - entry += 2 - if total + entry > RESPONSES_LIMIT: - break - total += entry - kept[k] = v - data["responses"] = kept - else: - log.error( - "%s: responses has unexpected type %s; skipping truncation", - results_path, - type(responses).__name__, - ) - return - - with open(results_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - log.info("%s responses truncated", results_path) + return any( + os.path.exists(os.path.join(scenario_dir, name)) + for name in ("config.yaml", "config.yml") + ) def truncate_results_dir(filter_submitter, backup, scenarios_to_skip): @@ -264,14 +197,10 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip): acc_txt = os.path.join( acc_path, "accuracy.txt") if not os.path.exists(acc_log): - # Endpoints submissions have results.json - # instead of mlperf_log_accuracy.json - endpoints_results = os.path.join( - acc_path, "results.json" - ) - if os.path.exists(endpoints_results): - _truncate_endpoints_results( - endpoints_results, acc_path, backup + if _is_endpoints_submission(name): + log.info( + "%s is an endpoints submission; nothing to truncate", + name, ) else: log.error("%s missing", acc_log)