Update endpoints structure

tools/submission/submission_checker/checks/accuracy_check.py

@@ -242,7 +242,7 @@ def dataset_check(self):
             )
             return True
         expected_qsl_total_count = self.config.get_accuracy_sample_count(
-            self.model)
+            self.model, self.scenario_fixed)
         if "effective_accuracy_sample_count" in self.mlperf_log.get_keys():
             qsl_total_count = self.mlperf_log["effective_accuracy_sample_count"]
         else:

tools/submission/submission_checker/checks/compliance_check.py

@@ -8,6 +8,7 @@
 from ..utils import *
 import re
 import os
+import json
 
 
 class ComplianceCheck(BaseCheck):
@@ -41,6 +42,7 @@ def __init__(self, log, path, config: Config,
         self.submission_logs = submission_logs
         self.config = config
         self.name = "compliance check"
+        self.perf_log = self.submission_logs.performance_log
         self.model = self.submission_logs.loader_data.get("benchmark", "")
         self.model_mapping = self.submission_logs.loader_data.get(
             "model_mapping", {})
@@ -60,14 +62,15 @@ def setup_checks(self):
         Appends the per-submission validation callables to `self.checks` in
         the order they should be executed by the checking framework.
         """
-        self.checks.append(self.dir_exists_check)
-        self.checks.append(self.performance_check)
-        self.checks.append(self.accuracy_check)
-        self.checks.append(self.compliance_performance_check)
-        self.apply_checks = set(self.checks)
-        # No compliance tests for endpoints for now
-        if self.is_endpoints:
-            self.apply_checks = set()
+        if not self.is_endpoints:
+            self.checks.append(self.dir_exists_check)
+            self.checks.append(self.performance_check)
+            self.checks.append(self.accuracy_check)
+            self.checks.append(self.compliance_performance_check)
+            self.apply_checks = set(self.checks)
+        else:
+            self.checks.append(self.compliance_endpoints_check)
+            self.apply_checks = set(self.checks)
 
     def get_test_list(self, model):
         """Return the list of compliance tests applicable to `model`.
@@ -494,3 +497,71 @@ def compliance_performance_check(self):
                                 diff)
                             is_valid = False
         return is_valid
+
+    def compliance_endpoints_check(self):
+        test_dir = os.path.join(self.compliance_dir, "audit")
+        is_valid = True
+        for test in self.test_list:
+            if test in ENDPOINTS_COMPLIANCE_MAPPING:
+                test_name = ENDPOINTS_COMPLIANCE_MAPPING.get(test, test)
+                fname = os.path.join(test_dir, f"audit_{test_name}.json")
+                if not os.path.exists(fname):
+                    if self.endpoint_audit_file_required(test):
+                        # if audit file is required, but missing, error out
+                        self.log.error("%s is missing in %s", fname, test_dir)
+                        is_valid = False
+                    else:
+                        # if audit file is not required, skip
+                        continue
+                else:
+                    try:
+                        with open(fname) as f:
+                            test_file = json.load(f)
+
+                        if test_file["test"] != test_name:
+                            self.log.error(
+                                "%s compliance test is incorrect %s. " +
+                                "Expected name=%s" +
+                                "Actual name=%s",
+                                fname,
+                                test_dir,
+                                test_name,
+                                test_file["test"]
+                            )
+                            is_valid = False
+                        elif not test_file["passed"]:
+                            self.log.error(
+                                "%s compliance test can not be loaded in %s", fname, test_dir)
+                            is_valid = False
+                        else:
+                            self.log.info(
+                                "%s compliance test %s passed for %s", fname, test_name, test_dir)
+
+                    except BaseException:
+                        self.log.error(
+                            "%s compliance test can not be loaded in %s", fname, test_dir)
+                        is_valid = False
+
+            if test in ["TEST09"]:
+                mean_output_tokens = self.perf_log["mean_output_tokens"]
+                if mean_output_tokens > TEST09_HIGH or mean_output_tokens < TEST09_LOW:
+                    self.log.error("TEST 09 (Verify Output Token Length in Performance Mode)" +
+                                   " failed for %s. Mean output tokens need to be in the range" +
+                                   " [%s, %s]. Found %s",
+                                   self.compliance_dir,
+                                   TEST09_LOW,
+                                   TEST09_HIGH,
+                                   mean_output_tokens
+                                   )
+                    is_valid = False
+
+        return is_valid
+
+    def endpoint_audit_file_required(self, test):
+        # currently, only test04 is enabled in endpoints. and Wan is the only
+        # model that requires test04
+        return (
+            test == "TEST04"
+            and self.model in ENDPOINTS_ALLOWED_MODELS
+            and self.model in self.config.base.get("models_TEST04", [])
+        )

tools/submission/submission_checker/checks/performance_check.py

@@ -56,10 +56,18 @@ def __init__(self, log, path, config: Config,
         self.is_endpoints = self.submission_logs.loader_data.get(
             "is_endpoints_submission", False)
         if self.is_endpoints:
-            if self.scenario.lower() == "online":
-                self.scenario = "Server"
+            self.scenario = self.map_endpoints_scenario(
+                self.scenario, self.scenario_fixed
+            )
         self.setup_checks()
 
+    def map_endpoints_scenario(self, scenario, scenario_fixed):
+        if scenario_fixed.lower() == "singlestream":
+            return "SingleStream"
+        if scenario.lower() == "online":
+            return "Server"
+        return scenario
+
     def setup_checks(self):
         """Register individual performance-related checks.
 
@@ -251,6 +259,7 @@ def latency_check(self):
             bool: True if latency constraints are satisfied, False otherwise.
         """
         uses_early_stopping = self.config.uses_early_stopping(self.scenario)
+        # traditional loadgen log path:
         if uses_early_stopping and not self.is_endpoints:
             # check if early_stopping condition was met
             if not self.mlperf_log["early_stopping_met"]:
@@ -282,7 +291,14 @@ def latency_check(self):
                     )
                     return False
         else:
-            # check if the benchmark meets latency constraint
+            if self.is_endpoints and self.model in self.config.get_llm_models():
+                # Endpoint LLM latency is enforced by llm_check using
+                # TTFT/TPOT.
+                return True
+
+            # check if the benchmark meets latency constraint, works for both Endpoint and non-Endpoint based logs
+            # Qwen3VL falls in this category, it has scenario specific e2e
+            # latency constraints
             latency_99_percentile = self.mlperf_log["result_99.00_percentile_latency_ns"]
             target_latency = self.config.latency_constraint.get(
                 self.model, dict()).get(self.scenario)
@@ -418,54 +434,63 @@ def llm_check(self):
             bool: True if LLM checks pass or model is not an LLM,
                 False otherwise.
         """
-        if self.model in self.config.get_llm_models():
-            if self.is_endpoints:
-                # Endpoints don't use the loadgen use_token_latencies flag;
-                # check TTFT/TPOT directly from the endpoints result JSON.
-                if self.scenario not in ["Server", "Interactive"]:
-                    return True
-                limits = LLM_LATENCY_LIMITS[self.model][self.scenario]
-                ttft = self.mlperf_log["result_first_token_99.00_percentile_latency_ns"]
-                tpot = self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"]
-                if ttft is None or tpot is None:
-                    self.log.warning(
-                        "%s TTFT or TPOT percentile data missing for endpoints LLM check",
-                        self.path)
-                    return True
-                if ttft < limits["ttft"] and tpot < limits["tpot"]:
-                    return True
-                self.log.error(
-                    'Failed extra check for TTFT and TPOT. Obtained: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f. Required: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f',
-                    ttft, tpot, limits["ttft"], limits["tpot"])
-                return False
-            if self.mlperf_log["requested_use_token_latencies"]:
-                if self.scenario not in ["Server", "Interactive"]:
-                    # For offline, singlestream and multistream no further checks are
-                    # necessary
-                    return True
-                else:
-                    limits = LLM_LATENCY_LIMITS[self.model][self.scenario]
-                    if (
-                        self.mlperf_log["result_first_token_99.00_percentile_latency_ns"]
-                        < limits["ttft"]
-                        and self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"]
-                        < limits["tpot"]
-                    ):
-                        return True
-            else:
-                self.log.error(
-                    f"use_token_latencies flag needs to be enabled for Llama2 benchmark")
-                return False
+        if self.model not in self.config.get_llm_models():
+            return True
+
+        # Endpoint logs do not carry LoadGen's requested_use_token_latencies
+        # setting, but the endpoint parser maps token latency fields to the
+        # same LoadGen-style result keys used below.
+        requested_use_token_latencies = (
+            self.is_endpoints
+            or self.mlperf_log["requested_use_token_latencies"]
+        )
+        if not requested_use_token_latencies:
+            self.log.error(
+                "use_token_latencies flag needs to be enabled for LLM benchmark")
+            return False
+
+        if self.scenario not in ["Server", "Interactive"]:
+            # For offline, singlestream and multistream no further checks are
+            # necessary.
+            return True
 
+        limits = LLM_LATENCY_LIMITS.get(self.model, {}).get(self.scenario)
+        if limits is None:
             self.log.error(
-                'Failed extra check for TTFT and TPOT. Obtained: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f. Required: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f',
-                self.mlperf_log["result_first_token_99.00_percentile_latency_ns"],
-                self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"],
-                limits["ttft"],
-                limits["tpot"]
+                "%s TTFT/TPOT latency limits missing for model=%s scenario=%s",
+                self.path,
+                self.model,
+                self.scenario,
             )
             return False
-        return True
+
+        ttft = self.mlperf_log["result_first_token_99.00_percentile_latency_ns"]
+        tpot = self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"]
+        if ttft is None or tpot is None:
+            self.log.error(
+                "%s TTFT or TPOT percentile data missing for LLM check",
+                self.path)
+            return False
+
+        self.log.info(
+            "TTFT target: %s, TTFT: %s, TPOT target: %s, TPOT: %s, Scenario: %s",
+            limits["ttft"],
+            ttft,
+            limits["tpot"],
+            tpot,
+            self.scenario,
+        )
+        if ttft < limits["ttft"] and tpot < limits["tpot"]:
+            return True
+
+        self.log.error(
+            'Failed extra check for TTFT and TPOT. Obtained: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f. Required: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f',
+            ttft,
+            tpot,
+            limits["ttft"],
+            limits["tpot"]
+        )
+        return False
 
     def inferred_check(self):
         """Validate rules for inferring results across scenarios.
@@ -519,13 +544,13 @@ def get_performance_metric_check(self):
             and self.mlperf_log["result_validity"] == "VALID"
         ):
             is_valid = True
-        scenario = self.mlperf_log["effective_scenario"]
+        scenario = self.scenario
+        res = None
         if self.is_endpoints:
-            if scenario.lower() == "online":
-                scenario = "Server"
-            scenario = scenario.capitalize()
-
-        res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]])
+            res = float(
+                self.mlperf_log[RESULT_FIELD_ENDPOINTS[version][scenario.lower()]])
+        else:
+            res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]])
         if (
             not self.is_endpoints
             and version in RESULT_FIELD_BENCHMARK_OVERWRITE

tools/submission/submission_checker/configuration/configuration.py

@@ -158,11 +158,20 @@ def get_performance_sample_count(self, model):
             raise ValueError("model not known: " + model)
         return self.performance_sample_count[model]
 
-    def get_accuracy_sample_count(self, model):
+    def get_accuracy_sample_count(self, model, scenario=None):
+        # get expected accuracy sample count from config, qwen has scenario
+        # specific sample counts as special case
         model = self.get_mlperf_model(model)
         if model not in self.accuracy_sample_count:
             return self.get_dataset_size(model)
-        return self.accuracy_sample_count[model]
+        sample_count = self.accuracy_sample_count[model]
+        # handle Qwen's scenario specific sample counts
+        if isinstance(sample_count, dict):
+            if scenario in sample_count:
+                return sample_count[scenario]
+            if scenario is not None and scenario.lower() in sample_count:
+                return sample_count[scenario.lower()]
+        return sample_count
 
     def ignore_errors(self, line):
         for error in self.base["ignore_errors"]:
@@ -231,5 +240,6 @@ def get_llm_models(self):
             "llama3.1-405b",
             "llama3.1-8b",
             "llama3.1-8b-edge",
-            "deepseek-r1"
+            "deepseek-r1",
+            "gpt-oss-120b"
         ]