Initial PR with codecov report logging.

.coveragerc

@@ -0,0 +1,8 @@
+[run]
+# Coverage is scoped to the submission checker (the only code under test).
+# The CI step also passes --cov=submission_checker; keeping it here means a
+# bare `pytest --cov` / `coverage` run is scoped correctly too.
+source = submission_checker
+omit =
+    */site-packages/*
+    */dist-packages/*

.github/workflows/codecov.yml

@@ -0,0 +1,74 @@
+name: Tests and Coverage
+
+on:
+  push:
+    branches: ["arav-codecov-impl"]
+    paths:
+      - "tools/submission/**"
+      - ".github/workflows/codecov.yml"
+  pull_request:
+    branches: ["arav-codecov-impl"]
+    paths:
+      - "tools/submission/**"
+      - ".github/workflows/codecov.yml"
+
+# Keep the pinned inputs in sync with tools/submission/tests/test_snapshot.py
+env:
+  SUBMISSION_REPO: https://github.com/mlcommons/inference_results_v6.0
+  PINNED_SHA: 4d3916ac9cf474b679cdfcf492d43a0559418ad1
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install test dependencies
+        run: pip install pytest pytest-cov syrupy pyyaml
+
+      - name: Fetch pinned submissions repo
+        run: |
+          set -euo pipefail
+          # Partial clone: fetch only the blobs we check out. The sparse-checkout
+          # excludes large artifacts the checker never reads (accuracy/trace
+          # dumps, dataset JSONs, model caches, vendored binaries).
+          git clone --filter=blob:none --no-checkout "$SUBMISSION_REPO" submissions
+          cd submissions
+          git sparse-checkout set --no-cone \
+            '/*' \
+            '!openimages-mlperf-v21.json' \
+            '!mlperf_log_trace.json' \
+            '!mlc-deps.png' \
+            '!environment.pickle' \
+            '!unet.cache' \
+            '!libamdhip64.so.6' \
+            '!val_map.txt'
+          git checkout "$PINNED_SHA"
+
+      - name: Run tests with coverage
+        # The snapshot test runs the checker as a subprocess; pytest-cov
+        # instruments it automatically, so coverage of submission_checker is
+        # captured end-to-end.
+        env:
+          MLPERF_SUBMISSION_DIR: ${{ github.workspace }}/submissions
+          # Resolve `submission_checker` as an importable package so coverage
+          # can map the subprocess data to it (a path-based --cov would not
+          # match, because the checker subprocess runs from tools/submission).
+          PYTHONPATH: tools/submission
+        run: >
+          pytest tools/submission/tests/
+          --cov=submission_checker
+          --cov-report=xml
+          --cov-report=term-missing
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          files: coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/test-submission-checker-snapshot.yml

@@ -0,0 +1,64 @@
+# Snapshot (golden-file) test for the MLPerf inference submission checker.
+# Runs the checker end-to-end against a pinned commit of a real submission
+# repository and diffs its summary.csv against a committed golden snapshot.
+# See tools/submission/tests/test_snapshot.py for details and update instructions.
+
+name: Snapshot test for MLPerf inference submission checker
+
+on:
+  pull_request:
+    branches: [ "master", "dev" ]
+    paths:
+      - tools/submission/**
+      - .github/workflows/test-submission-checker-snapshot.yml
+      - '!**.md'
+
+# Keep these in sync with tools/submission/tests/test_snapshot.py
+env:
+  SUBMISSION_REPO: https://github.com/mlcommons/inference_results_v6.0
+  PINNED_SHA: 4d3916ac9cf474b679cdfcf492d43a0559418ad1
+
+jobs:
+  snapshot:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.10", "3.12" ]
+
+    steps:
+    - name: Checkout inference repo
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: python3 -m pip install pytest syrupy pyyaml
+
+    - name: Fetch pinned submissions repo
+      run: |
+        set -euo pipefail
+        # Partial clone: fetch only the blobs we actually check out. Combined
+        # with the sparse-checkout excludes below, this skips large artifacts
+        # the checker never reads (accuracy/trace dumps, dataset JSONs, model
+        # caches, vendored binaries), trimming the ~2.2 GB tree to ~1.6 GB.
+        git clone --filter=blob:none --no-checkout "$SUBMISSION_REPO" submissions
+        cd submissions
+        git sparse-checkout set --no-cone \
+          '/*' \
+          '!openimages-mlperf-v21.json' \
+          '!mlperf_log_trace.json' \
+          '!mlc-deps.png' \
+          '!environment.pickle' \
+          '!unet.cache' \
+          '!libamdhip64.so.6' \
+          '!val_map.txt'
+        git checkout "$PINNED_SHA"
+
+    - name: Run submission checker snapshot test
+      env:
+        MLPERF_SUBMISSION_DIR: ${{ github.workspace }}/submissions
+      run: python3 -m pytest tools/submission/tests/test_snapshot.py -q

e2e-rag/QSL.py

@@ -85,7 +85,8 @@ def __init__(self, dataset_path, perf_count=None, skip_qsl=False):
 
         print(f"Dataset loaded: {self.count} queries")
         if perf_count is not None:
-            print(f"  (limited to first {perf_count} queries for performance testing)")
+            print(
+                f"  (limited to first {perf_count} queries for performance testing)")
 
     def load_query_samples(self, sample_list):
         """
@@ -161,15 +162,12 @@ def __init__(self, dataset_path, perf_count=None):
 # limitations under the License.
 # =============================================================================
 
+
 """
 Query Sample Library for RAG-QnA workload.
 Loads queries from frames_dataset.tsv and provides them to MLPerf Loadgen.
 """
 
-import os
-import pandas as pd
-import mlperf_loadgen as lg
-
 
 class E2EQSL:
     """Query Sample Library for RAG-QnA multi-hop RAG benchmark."""
@@ -233,7 +231,8 @@ def __init__(self, dataset_path, perf_count=None, skip_qsl=False):
 
         print(f"Dataset loaded: {self.count} queries")
         if perf_count is not None:
-            print(f"  (limited to first {perf_count} queries for performance testing)")
+            print(
+                f"  (limited to first {perf_count} queries for performance testing)")
 
     def load_query_samples(self, sample_list):
         """

e2e-rag/accuracy_eval.py

@@ -34,9 +34,10 @@
 # OpenRouter configuration
 DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions"
 DEFAULT_JUDGE_MODEL = "gpt-oss-20b"
-# Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter)
+# Masked API key (set OPENROUTER_API_KEY environment variable to use
+# OpenRouter)
 OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY',
-    'sk-or-v1-****')
+                                    'sk-or-v1-****')
 
 
 JUDGE_PROMPT = """You are an expert evaluator comparing LLM-generated answers to ground truth answers.
@@ -83,7 +84,11 @@ def call_judge(question: str, ground_truth: str, llm_answer: str,
     }
 
     try:
-        response = requests.post(service_url, json=payload, headers=headers, timeout=60)
+        response = requests.post(
+            service_url,
+            json=payload,
+            headers=headers,
+            timeout=60)
         response.raise_for_status()
         result = response.json()
 
@@ -105,7 +110,8 @@ def call_judge(question: str, ground_truth: str, llm_answer: str,
         return {"correct": False, "reasoning": f"Judge error: {e}"}
 
 
-def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[str]) -> Dict:
+def calculate_retrieval_metrics(
+        retrieved_urls: List[str], expected_urls: List[str]) -> Dict:
     """Calculate precision, recall, F1 for retrieval."""
 
     retrieved_set = set(retrieved_urls)
@@ -118,7 +124,8 @@ def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[s
 
     precision = len(correct) / len(retrieved_set) if retrieved_set else 0.0
     recall = len(correct) / len(expected_set) if expected_set else 0.0
-    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+    f1 = 2 * precision * recall / \
+        (precision + recall) if (precision + recall) > 0 else 0.0
 
     return {
         "precision": precision,
@@ -128,8 +135,8 @@ def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[s
 
 
 def evaluate_results(results: Dict, dataset_path: str, num_workers: int = 4,
-                    judge_service_url: str = DEFAULT_JUDGE_URL,
-                    judge_model: str = DEFAULT_JUDGE_MODEL) -> Dict:
+                     judge_service_url: str = DEFAULT_JUDGE_URL,
+                     judge_model: str = DEFAULT_JUDGE_MODEL) -> Dict:
     """
     Evaluate loadgen results.
 
@@ -190,12 +197,13 @@ def evaluate_single_query(query_id, result):
         expected_urls = gt_data['expected_urls']
 
         # Calculate retrieval metrics
-        retrieval_metrics = calculate_retrieval_metrics(retrieved_urls, expected_urls)
+        retrieval_metrics = calculate_retrieval_metrics(
+            retrieved_urls, expected_urls)
 
         # Judge answer correctness
         judge_result = call_judge(query, ground_truth, llm_answer,
-                                 service_url=judge_service_url,
-                                 model_name=judge_model)
+                                  service_url=judge_service_url,
+                                  model_name=judge_model)
         answer_correct = judge_result.get('correct', False)
 
         return {
@@ -229,7 +237,8 @@ def evaluate_single_query(query_id, result):
                     total_queries += 1
 
                     if total_queries % 10 == 0:
-                        print(f"  Evaluated {total_queries}/{len(results)} queries...")
+                        print(
+                            f"  Evaluated {total_queries}/{len(results)} queries...")
             except Exception as e:
                 print(f"Error evaluating query: {e}")
 
@@ -257,14 +266,37 @@ def evaluate_single_query(query_id, result):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Evaluate RAG-QnA loadgen accuracy")
-    parser.add_argument('--log_dir', required=True, help='Loadgen log directory')
-    parser.add_argument('--results_file', required=True, help='SUT results JSON file')
-    parser.add_argument('--dataset_path', required=True, help='Path to frames_dataset.tsv')
-    parser.add_argument('--num_workers', type=int, default=4, help='Number of parallel judge workers')
-    parser.add_argument('--output', default='accuracy_results.json', help='Output file for detailed results')
-    parser.add_argument('--judge_service_url', default=DEFAULT_JUDGE_URL, help='Judge LLM service URL')
-    parser.add_argument('--judge_model', default=DEFAULT_JUDGE_MODEL, help='Judge LLM model name')
+    parser = argparse.ArgumentParser(
+        description="Evaluate RAG-QnA loadgen accuracy")
+    parser.add_argument(
+        '--log_dir',
+        required=True,
+        help='Loadgen log directory')
+    parser.add_argument(
+        '--results_file',
+        required=True,
+        help='SUT results JSON file')
+    parser.add_argument(
+        '--dataset_path',
+        required=True,
+        help='Path to frames_dataset.tsv')
+    parser.add_argument(
+        '--num_workers',
+        type=int,
+        default=4,
+        help='Number of parallel judge workers')
+    parser.add_argument(
+        '--output',
+        default='accuracy_results.json',
+        help='Output file for detailed results')
+    parser.add_argument(
+        '--judge_service_url',
+        default=DEFAULT_JUDGE_URL,
+        help='Judge LLM service URL')
+    parser.add_argument(
+        '--judge_model',
+        default=DEFAULT_JUDGE_MODEL,
+        help='Judge LLM model name')
     args = parser.parse_args()
 
     # Load results
@@ -280,17 +312,17 @@ def main():
                                judge_model=args.judge_model)
 
     # Print summary
-    print("\n" + "="*80)
+    print("\n" + "=" * 80)
     print("ACCURACY EVALUATION RESULTS")
-    print("="*80)
+    print("=" * 80)
     print(f"Total Queries:        {metrics['total_queries']}")
     print(f"\nRetrieval Metrics:")
     print(f"  Precision@N:        {metrics['retrieval_precision']:.3f}")
     print(f"  Recall@N:           {metrics['retrieval_recall']:.3f}")
     print(f"  F1@N:               {metrics['retrieval_f1']:.3f}")
     print(f"\nAnswer Quality:")
     print(f"  LLM Judge Accuracy: {metrics['answer_accuracy']:.3f}")
-    print("="*80 + "\n")
+    print("=" * 80 + "\n")
 
     # Save detailed results
     with open(args.output, 'w') as f: