Add qwen3.6-27b benchmark to submission checker

tools/submission/submission_checker/checks/compliance_check.py

@@ -8,6 +8,7 @@
 from ..utils import *
 import re
 import os
+import json
 
 
 class ComplianceCheck(BaseCheck):
@@ -41,6 +42,7 @@ def __init__(self, log, path, config: Config,
         self.submission_logs = submission_logs
         self.config = config
         self.name = "compliance check"
+        self.perf_log = self.submission_logs.performance_log
         self.model = self.submission_logs.loader_data.get("benchmark", "")
         self.model_mapping = self.submission_logs.loader_data.get(
             "model_mapping", {})
@@ -60,14 +62,15 @@ def setup_checks(self):
         Appends the per-submission validation callables to `self.checks` in
         the order they should be executed by the checking framework.
         """
-        self.checks.append(self.dir_exists_check)
-        self.checks.append(self.performance_check)
-        self.checks.append(self.accuracy_check)
-        self.checks.append(self.compliance_performance_check)
-        self.apply_checks = set(self.checks)
-        # No compliance tests for endpoints for now
-        if self.is_endpoints:
-            self.apply_checks = set()
+        if not self.is_endpoints:
+            self.checks.append(self.dir_exists_check)
+            self.checks.append(self.performance_check)
+            self.checks.append(self.accuracy_check)
+            self.checks.append(self.compliance_performance_check)
+            self.apply_checks = set(self.checks)
+        else:
+            self.checks.append(self.compliance_endpoints_check)
+            self.apply_checks = set(self.checks)
 
     def get_test_list(self, model):
         """Return the list of compliance tests applicable to `model`.
@@ -494,3 +497,57 @@ def compliance_performance_check(self):
                                 diff)
                             is_valid = False
         return is_valid
+
+    def compliance_endpoints_check(self):
+        test_dir = os.path.join(self.compliance_dir, "audit")
+        is_valid = True
+        for test in self.test_list:
+            if test in ENDPOINTS_COMPLIANCE_MAPPING:
+                test_name = ENDPOINTS_COMPLIANCE_MAPPING.get(test, test)
+                fname = os.path.join(test_dir, f"audit_{test_name}.json")
+                if not os.path.exists(fname):
+                    self.log.error("%s is missing in %s", fname, test_dir)
+                    is_valid = False
+                else:
+                    try:
+                        with open(fname) as f:
+                            test_file = json.load(f)
+
+                        if test_file["test"] != test_name:
+                            self.log.error(
+                                "%s compliance test is incorrect %s. " +
+                                "Expected name=%s" +
+                                "Actual name=%s",
+                                fname,
+                                test_dir,
+                                test_name,
+                                test_file["test"]
+                            )
+                            is_valid = False
+                        elif not test_file["passed"]:
+                            self.log.error(
+                                "%s compliance test can not be loaded in %s", fname, test_dir)
+                            is_valid = False
+                        else:
+                            self.log.info(
+                                "%s compliance test %s passed for %s", fname, test_name, test_dir)
+
+                    except BaseException:
+                        self.log.error(
+                            "%s compliance test can not be loaded in %s", fname, test_dir)
+                        is_valid = False
+
+            if test in ["TEST09"]:
+                mean_output_tokens = self.perf_log["mean_output_tokens"]
+                if mean_output_tokens > TEST09_HIGH or mean_output_tokens < TEST09_LOW:
+                    self.log.error("TEST 09 (Verify Output Token Length in Performance Mode)" +
+                                   " failed for %s. Mean output tokens need to be in the range" +
+                                   " [%s, %s]. Found %s",
+                                   self.compliance_dir,
+                                   TEST09_LOW,
+                                   TEST09_HIGH,
+                                   mean_output_tokens
+                                   )
+                    is_valid = False
+
+        return is_valid

tools/submission/submission_checker/checks/performance_check.py

@@ -56,10 +56,18 @@ def __init__(self, log, path, config: Config,
         self.is_endpoints = self.submission_logs.loader_data.get(
             "is_endpoints_submission", False)
         if self.is_endpoints:
-            if self.scenario.lower() == "online":
-                self.scenario = "Server"
+            self.scenario = self.map_endpoints_scenario(
+                self.scenario, self.scenario_fixed
+            )
         self.setup_checks()
 
+    def map_endpoints_scenario(self, scenario, scenario_fixed):
+        if scenario_fixed.lower() == "singlestream":
+            return "SingleStream"
+        if scenario.lower() == "online":
+            return "Server"
+        return scenario
+
     def setup_checks(self):
         """Register individual performance-related checks.
 
@@ -492,7 +500,7 @@ def inferred_check(self):
                 ("singlestream", "offline")
             ]
             if (self.scenario.lower(), self.scenario_fixed.lower()
-                ) not in list_inferred:
+                    ) not in list_inferred:
                 self.log.error(
                     "Result for scenario %s can not be inferred from %s for: %s",
                     self.scenario_fixed,
@@ -519,13 +527,13 @@ def get_performance_metric_check(self):
             and self.mlperf_log["result_validity"] == "VALID"
         ):
             is_valid = True
-        scenario = self.mlperf_log["effective_scenario"]
+        scenario = self.scenario
+        res = None
         if self.is_endpoints:
-            if scenario.lower() == "online":
-                scenario = "Server"
-            scenario = scenario.capitalize()
-
-        res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]])
+            res = float(
+                self.mlperf_log[RESULT_FIELD_ENDPOINTS[version][scenario.lower()]])
+        else:
+            res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]])
         if (
             not self.is_endpoints
             and version in RESULT_FIELD_BENCHMARK_OVERWRITE
@@ -606,12 +614,12 @@ def get_inferred_result(self, res):
             res = qps_wo_loadgen_overhead
 
         if (scenario_fixed in ["Offline"]
-                ) and scenario in ["MultiStream"]:
+            ) and scenario in ["MultiStream"]:
             inferred = True
             res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS)
 
         if (scenario_fixed in ["MultiStream"]
-                ) and scenario in ["SingleStream"]:
+            ) and scenario in ["SingleStream"]:
             inferred = True
             # samples_per_query does not match with the one reported in the logs
             # when inferring MultiStream from SingleStream
@@ -628,6 +636,6 @@ def get_inferred_result(self, res):
             else:
                 res = (latency_99_percentile * samples_per_query) / MS_TO_NS
         if (scenario_fixed in ["Interactive"]
-                ) and scenario not in ["Server"]:
+            ) and scenario not in ["Server"]:
             is_valid = False
         return res, is_valid

tools/submission/submission_checker/constants.py

@@ -17,6 +17,7 @@
             "gpt-oss-120b",
             "wan-2.2-t2v-a14b",
             "qwen3-vl-235b-a22b",
+            "qwen3.6-27b",
             "dlrm-v3",
             "yolo-95",
             "yolo-99",
@@ -58,6 +59,7 @@
             "whisper": ["Offline"],
             "yolo-95": ["SingleStream", "MultiStream", "Offline"],
             "yolo-99": ["SingleStream", "MultiStream", "Offline"],
+            "qwen3.6-27b": ["SingleStream"],
         },
         "optional-scenarios-edge": {},
         "required-scenarios-datacenter-edge": {
@@ -79,6 +81,7 @@
             "dlrm-v3": ["Offline", "Server"],
             "yolo-95": ["SingleStream", "MultiStream", "Offline"],
             "yolo-99": ["SingleStream", "MultiStream", "Offline"],
+            "qwen3.6-27b": ["SingleStream"],
         },
         "optional-scenarios-datacenter-edge": {
             "llama2-70b-99": ["Interactive", "Server"],
@@ -164,6 +167,7 @@
             # established
             "e2e": ("E2E_ACCURACY", ""),
             "e2e_vectorDB": ("E2E_ACCURACY", ""),
+            "qwen3.6-27b": ("mAP", 86.23 * 0.99),
         },
         "accuracy-upper-limit": {
             "stable-diffusion-xl": (
@@ -203,11 +207,13 @@
             "yolo-95": 64,
             "yolo-99": 64,
             "e2e": 824,
-            "e2e_vectorDB": 824
+            "e2e_vectorDB": 824,
+            "qwen3.6-27b": 995,
         },
         "accuracy-sample-count": {
             "gpt-oss-120b": 4395,
             "wan-2.2-t2v-a14b": 248,
+            "qwen3.6-27b": 995,
         },
         "dataset-size": {
             "resnet": 50000,
@@ -231,6 +237,7 @@
             "yolo-99": 1525,
             "e2e": 824,
             "e2e_vectorDB": 824,
+            "qwen3.6-27b": 995,
         },
         "model_mapping": {
             "ssd-resnet34": "retinanet",
@@ -291,6 +298,7 @@
             "yolo-99": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1},
             "e2e": {"Offline": 824},
             "e2e_vectorDB": {"Offline": 824},
+            "qwen3.6-27b": {"SingleStream": 995},
         },
         "models_TEST01": [
             "resnet",
@@ -1501,6 +1509,9 @@
     "compliance_accuracy.txt",
 ]
 
+TEST09_LOW = 1150.38
+TEST09_HIGH = 1406.02
+
 OFFLINE_MIN_SPQ_SINCE_V4 = {
     "resnet": 24576,
     "retinanet": 24576,
@@ -1577,6 +1588,16 @@
     },
 }
 
+RESULT_FIELD_ENDPOINTS = {
+    "v6.1": {
+        "offline": "result_samples_per_second",
+        "singlestream": "result_mean_latency_ns",
+        "multistream": "result_mean_latency_ns",
+        "server": "result_completed_samples_per_sec",
+        "interactive": "result_completed_samples_per_sec",
+    },
+}
+
 RESULT_FIELD_BENCHMARK_OVERWRITE = {
     "v5.0": {
         "llama2-70b-99": {
@@ -2030,17 +2051,19 @@
 ENDPOINTS_ALLOWED_MODELS = [
     "wan-2.2-t2v-a14b",
     "qwen3-vl-235b-a22b",
+    "qwen3.6-27b",
     "llama3.1-8b",
     "llama3.1-8b-edge",
     "gpt-oss-120b",
     "deepseek-r1"
 ]
 
-PERFORMANCE_ENDPOINTS_DIR = {
-    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
+ENDPOINTS_SCENARIO_DIR = {
+    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
+    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
 }
 
 ACCURACY_LOG_PATH = {
@@ -2067,14 +2090,6 @@
     "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json",
 }
 
-ACCURACY_ENDPOINTS_DIR = {
-    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-}
-
-
 POWER_DIR_PATH = {
     "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power",
     "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power",
@@ -2207,7 +2222,7 @@
     "effective_sample_concatenate_permutation": "effective_sample_concatenate_permutation",
     "effective_samples_per_query": "effective_samples_per_query",
     "generated_query_count": "generated_query_count",
-    "generated_query_duration": "generated_query_duration",
+    "duration_ns": "generated_query_duration",
     "target_qps": "effective_target_qps",
     "result_scheduled_samples_per_sec": "result_scheduled_samples_per_sec",
     "qps": "result_completed_samples_per_sec",
@@ -2216,30 +2231,31 @@
     "latency.min": "result_min_latency_ns",
     "latency.max": "result_max_latency_ns",
     "latency.avg": "result_mean_latency_ns",
-    "latency.percentiles.50.0": "result_50.00_percentile_latency_ns",
-    "latency.percentiles.90.0": "result_90.00_percentile_latency_ns",
-    "latency.percentiles.95.0": "result_95.00_percentile_latency_ns",
-    "latency.percentiles.99.0": "result_99.00_percentile_latency_ns",
+    "latency.percentiles.50": "result_50.00_percentile_latency_ns",
+    "latency.percentiles.90": "result_90.00_percentile_latency_ns",
+    "latency.percentiles.95": "result_95.00_percentile_latency_ns",
+    "latency.percentiles.99": "result_99.00_percentile_latency_ns",
     "latency.percentiles.99.9": "result_99.90_percentile_latency_ns",
     "ttft.min": "result_first_token_min_latency_ns",
     "ttft.max": "result_first_token_max_latency_ns",
     "ttft.avg": "result_first_token_mean_latency_ns",
-    "ttft.percentiles.50.0": "result_first_token_50.00_percentile_latency_ns",
-    "ttft.percentiles.90.0": "result_first_token_90.00_percentile_latency_ns",
-    "ttft.percentiles.95.0": "result_first_token_95.00_percentile_latency_ns",
-    "ttft.percentiles.99.0": "result_first_token_99.00_percentile_latency_ns",
+    "ttft.percentiles.50": "result_first_token_50.00_percentile_latency_ns",
+    "ttft.percentiles.90": "result_first_token_90.00_percentile_latency_ns",
+    "ttft.percentiles.95": "result_first_token_95.00_percentile_latency_ns",
+    "ttft.percentiles.99": "result_first_token_99.00_percentile_latency_ns",
     "ttft.percentiles.99.9": "result_first_token_99.90_percentile_latency_ns",
-    "tpot.percentiles.50.0": "result_time_per_output_token_50.00_percentile_ns",
-    "tpot.percentiles.90.0": "result_time_per_output_token_90.00_percentile_ns",
-    "tpot.percentiles.95.0": "result_time_per_output_token_95.00_percentile_ns",
-    "tpot.percentiles.99.0": "result_time_per_output_token_99.00_percentile_ns",
+    "tpot.percentiles.50": "result_time_per_output_token_50.00_percentile_ns",
+    "tpot.percentiles.90": "result_time_per_output_token_90.00_percentile_ns",
+    "tpot.percentiles.95": "result_time_per_output_token_95.00_percentile_ns",
+    "tpot.percentiles.99": "result_time_per_output_token_99.00_percentile_ns",
     "tpot.percentiles.99.9": "result_time_per_output_token_99.90_percentile_ns",
     "tpot.min": "result_time_per_output_token_min",
     "tpot.max": "result_time_per_output_token_max",
     "tpot.avg": "result_time_per_output_token_mean",
     "tps": "result_completed_tokens_per_second",
     "result.total": "result_query_count",
     "result.failed": "num_errors",
+    "output_sequence_lengths.avg": "mean_output_tokens",
 }
 
 
@@ -2272,5 +2288,14 @@
 }
 
 ENDPOINTS_INFERRED_FIELDS = {
-    "effective_accuracy_sample_count": "result_query_count"
+    "effective_accuracy_sample_count": "result_query_count",
+    "generated_query_count": "qsl_reported_total_count",
+}
+
+ENDPOINTS_COMPLIANCE_MAPPING = {
+    "TEST01": "accuracy_in_performance_test",
+    "TEST04": "output_caching_test",
+    "TEST06": "consistency_output_test",
+    "TEST07": "accuracy_in_performance_full_test",
+    "TEST08": "accuracy_in_performance_dlrmv3_test",
 }