DO_NOT_MERGE : Cherry-pick PR403

examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py

@@ -28,6 +28,8 @@
 
 import argparse
 import json
+import os
+import shutil
 import sys
 import traceback
 from importlib.resources import files as _pkg_files
@@ -36,6 +38,12 @@
 import vbench as _vbench_pkg
 from vbench import VBench
 
+# VBench's reference checkpoints are full pickles and fail under torch>=2.6's
+# weights_only=True default. This escape hatch is read at torch.load call time,
+# only applies when the callsite does not set weights_only explicitly, and is
+# scoped to this subprocess.
+os.environ.setdefault("TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD", "1")
+
 
 def _emit_error(exc: BaseException) -> None:
     """Print a structured JSON error line on stderr for the parent to surface."""
@@ -85,6 +93,29 @@ def main() -> int:
     )
     args = parser.parse_args()
 
+    # vbench downloads weights via wget/unzip subprocesses on a cold cache;
+    # fail fast instead of a FileNotFoundError mid-evaluation.
+    missing_tools = [t for t in ("wget", "unzip") if shutil.which(t) is None]
+    if missing_tools:
+        print(
+            json.dumps(
+                {
+                    "status": "error",
+                    "type": "MissingSystemDependency",
+                    "message": (
+                        f"Required system tool(s) not found: {', '.join(missing_tools)}. "
+                        "VBench downloads its per-dimension model weights via "
+                        "wget/unzip on first use. Install them in the client "
+                        "environment (e.g. `apt-get install wget unzip`) or "
+                        "pre-populate the VBench cache directory."
+                    ),
+                }
+            ),
+            file=sys.stderr,
+            flush=True,
+        )
+        return 2
+
     if not torch.cuda.is_available() and not args.allow_cpu:
         print(
             json.dumps(

scripts/Dockerfile.dev

@@ -2,6 +2,11 @@
 # From project root:
 # docker build -f scripts/Dockerfile.dev --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
 # docker run -v $(pwd):/mnt/inference-endpoint -it --shm-size=512m inference-endpoint-dev bash
+#
+# VBench (WAN 2.2) accuracy scorer is ON by default (consistent with the DeepSeek-R1
+# evaluator, PROVISION_DSR1=1). It lives under examples/, not src/, so it is COPYed in
+# explicitly. To skip it (leaner image, no wan22 accuracy), build with PROVISION_VBENCH=0:
+# docker build -f scripts/Dockerfile.dev --build-arg PROVISION_VBENCH=0 --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
 
 FROM python:3.12.11-slim
 
@@ -19,8 +24,11 @@ ENV PYTHONUNBUFFERED=1 \
 # git + curl + ca-certificates support the legacy_mlperf_deepseek_r1 evaluator's setup_eval.sh
 # (it git-clones prm800k / LiveCodeBench at pinned commits) in the provisioning RUN below;
 # without them that step fails and mlperf_eval/ is never baked. (eval_accuracy.py is vendored.)
+# libgl1 + libglib2.0-0: opencv (cv2), pulled in by the VBench accuracy scorer's video
+# reader, needs libGL.so.1 / libgthread at import time (only used when PROVISION_VBENCH=1).
 RUN apt-get update && \
     apt-get install -y --no-install-recommends build-essential procps git curl ca-certificates \
+        libgl1 libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /mnt/inference-endpoint
@@ -38,7 +46,8 @@ RUN if ! getent group ${GROUP_ID}; then \
     fi && \
     useradd -u ${USER_ID} -g ${GROUP_ID} --create-home --shell /bin/bash appuser --no-log-init && \
     chown -R ${USER_ID}:${GROUP_ID} /mnt/inference-endpoint && \
-    mkdir -p /opt/venv && chown ${USER_ID}:${GROUP_ID} /opt/venv
+    mkdir -p /opt/venv /opt/uv-python /opt/vbench_accuracy && \
+    chown ${USER_ID}:${GROUP_ID} /opt/venv /opt/uv-python /opt/vbench_accuracy
 USER appuser
 ENV PATH="/opt/venv/bin:/home/appuser/.local/bin:$PATH"
 
@@ -77,3 +86,24 @@ RUN if [ "${PROVISION_DSR1}" = "1" ]; then \
     else \
         echo "PROVISION_DSR1=${PROVISION_DSR1}: skipping DeepSeek-R1 evaluator provisioning" ; \
     fi
+
+# VBench (WAN 2.2) accuracy scorer, gated by PROVISION_VBENCH (default 1, like PROVISION_DSR1).
+# It lives under examples/ (not src/), so COPY it in and build its own .venv at
+# /opt/vbench_accuracy; accuracy runs reuse it via UV_NO_SYNC=1. Isolated like DSR1: pinned to
+# Python 3.11 (tokenizers 0.13.3 has no cp312 wheel) with UV_PROJECT_ENVIRONMENT set per call so
+# its old deps stay out of /opt/venv. pyproject patches (requires-python, decord->decord2) keep
+# the install wheel-only on aarch64. PROVISION_VBENCH=0 skips it.
+ARG PROVISION_VBENCH=1
+ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python
+COPY --chown=${USER_ID}:${GROUP_ID} examples/09_Wan22_VideoGen_Example/accuracy /opt/vbench_accuracy
+RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \
+        cd /opt/vbench_accuracy && \
+        sed -i 's#requires-python = ">=3.12"#requires-python = ">=3.10"#' pyproject.toml && \
+        sed -i 's#    "vbench==0.1.5",#    "vbench==0.1.5",\n    "decord2>=3.4.0",#' pyproject.toml && \
+        sed -i "s#^package = false#package = false\noverride-dependencies = [\"decord; sys_platform == 'never'\"]#" pyproject.toml && \
+        UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv lock --python 3.11 && \
+        UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv sync --python 3.11 && \
+        UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv run python -c "import torch, decord, tokenizers, vbench; print('vbench', vbench.__file__, '| decord', decord.__version__, '| tokenizers', tokenizers.__version__, '| torch', torch.__version__)" ; \
+    else \
+        echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \
+    fi

src/inference_endpoint/evaluation/scoring.py

@@ -1549,7 +1549,9 @@ def _stage_videos(
             # strict=True surfaces missing/unmounted sources here, not as an
             # opaque decord read failure inside VBench 30 minutes later.
             resolved_src = src.resolve(strict=True)
-            dst = staged_dir / f"{safe_prompt}-{idx}{src.suffix or '.mp4'}"
+            # Always .mp4: VBench dispatches on extension (non-mp4 raises
+            # NotImplementedError); decord detects the container by content.
+            dst = staged_dir / f"{safe_prompt}-{idx}.mp4"
             dst.symlink_to(resolved_src)
 
     def _run_vbench_subprocess(

tests/unit/evaluation/test_scoring.py

@@ -775,6 +775,26 @@ def test_stage_clears_stale_files_from_prior_run(
         assert names == ["a cat-0.mp4", "a dog-0.mp4", "a tree-0.mp4"]
         assert not zombie.exists()
 
+    def test_stage_videos_renames_non_mp4_sources_to_mp4(
+        self, dataset, staged, vbench_project, tmp_path
+    ):
+        """Non-mp4 sources (e.g. .avi) stage under .mp4 names for VBench."""
+        report_dir, _ = staged
+        avi = tmp_path / "video_x.avi"
+        avi.write_bytes(b"")
+        scorer = VBenchScorer(
+            dataset_name="vid_acc",
+            dataset=dataset,
+            report_dir=report_dir,
+            ground_truth_column="prompt",
+            vbench_project_path=vbench_project,
+        )
+        staged_dir = report_dir / "vbench_videos"
+        scorer._stage_videos(staged_dir, [str(avi)], ["a cat"])
+        (staged_file,) = staged_dir.iterdir()
+        assert staged_file.name == "a cat-0.mp4"
+        assert staged_file.resolve() == avi.resolve()
+
     def test_subprocess_failure_includes_stderr_tail(
         self, dataset, staged, vbench_project, monkeypatch, tmp_path
     ):