diff --git a/.config/nextest.toml b/.config/nextest.toml index ef238deecd..a8ef16d7af 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -23,6 +23,24 @@ # https://nexte.st/docs/configuration/per-test-overrides/#override-precedence # https://nexte.st/docs/configuration/?h=hierar#hierarchical-configuration +# virtio-villain tests each boot a dedicated VM that runs one fault-injection +# test and powers off. A test that never halts indicates a genuinely wedged +# guest or device model (e.g. a malformed-descriptor host hang that OpenVMM +# cannot recover from or even tear down) -- a real product bug. Terminate such +# a test so it is reported as a failure rather than hanging the run. Require +# two threads per test to bound concurrency and avoid vCPU starvation that +# would otherwise cause spurious slow boots. (The package name does not contain +# "vmm_tests", so the package(~vmm_tests) overrides below do not apply here.) +[[profile.default.overrides]] +filter = 'package(virtio_villain_tests)' +slow-timeout = { period = "20s", terminate-after = 3 } +threads-required = 2 + +[[profile.ci.overrides]] +filter = 'package(virtio_villain_tests)' +slow-timeout = { period = "20s", terminate-after = 3 } +threads-required = 2 + [[profile.ci.overrides]] # use fuzzy-matching for the package() to allow out-of-tree tests to use the # same profile diff --git a/.github/workflows/openvmm-ci.yaml b/.github/workflows/openvmm-ci.yaml index d4219fa8d1..96cb308fb1 100644 --- a/.github/workflows/openvmm-ci.yaml +++ b/.github/workflows/openvmm-ci.yaml @@ -3992,6 +3992,7 @@ jobs: shell: bash - name: creating new test content dir run: |- + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 15 flowey.exe e 20 flowey_core::pipeline::artifact::resolve 13 flowey.exe e 20 flowey_core::pipeline::artifact::resolve 3 flowey.exe e 20 flowey_core::pipeline::artifact::resolve 11 @@ -4006,7 +4007,6 @@ jobs: flowey.exe e 20 flowey_core::pipeline::artifact::resolve 9 flowey.exe e 20 flowey_core::pipeline::artifact::resolve 12 flowey.exe e 20 flowey_core::pipeline::artifact::resolve 14 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 15 flowey.exe e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: create gh-release-download cache dir @@ -6260,72 +6260,34 @@ jobs: run: flowey e 27 flowey_lib_common::cache 11 shell: bash job28: - name: test flowey local backend - runs-on: ubuntu-latest + name: run vmm-tests [x64-linux-kvm-virtio-villain] + runs-on: + - self-hosted + - 1ES.Pool=openvmm-gh-amd-westus3-v7 + - 1ES.ImageOverride=ubuntu2404-amd64 + - JobId=job28-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} permissions: contents: read id-token: write + needs: + - job9 if: github.event.pull_request.draft == false steps: - - run: | - set -x - i=0; while [ $i -lt 5 ] && ! sudo apt-get update; do let "i=i+1"; sleep 1; done; - sudo apt-get -o DPkg::Lock::Timeout=60 install gcc -y - curl --fail --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=1.95.0 -y - . "$HOME/.cargo/env" - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - rustup show - if: runner.os == 'Linux' - name: rustup (Linux) - shell: bash - - run: | - set -x - curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init - ./rustup-init.exe -y --default-toolchain=1.95.0 - echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH - if: runner.os == 'Windows' && runner.arch == 'X64' - name: rustup (Windows X64) - shell: bash - - run: | - set -x - curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init - ./rustup-init.exe -y --default-toolchain=1.95.0 - echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH - if: runner.os == 'Windows' && runner.arch == 'ARM64' - name: rustup (Windows ARM64) - shell: bash - - uses: actions/checkout@v6 + - name: πŸŒΌπŸ“¦ Download artifacts + uses: actions/download-artifact@v8 with: - path: flowey_bootstrap - - name: Build flowey - run: | - set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-unknown-linux-gnu --profile flowey-ci - OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') - mkdir -p "$OutDirNormal" - mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" - mv target/x86_64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" - working-directory: flowey_bootstrap - shell: bash - - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH + pattern: '{_internal-flowey-bootstrap-x86_64-linux-uid-11,x64-linux-openvmm,x64-linux-virtio-villain-tests-archive}' + path: ${{ runner.temp }}/used_artifacts/ + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-11" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - - name: πŸŒΌπŸ”Ž Self-check YAML - run: |- - ESCAPED_AGENT_TEMPDIR=$( - cat <<'EOF' | sed 's/\\/\\\\/g' - ${{ runner.temp }} - EOF - ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci - shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-11/flowey echo '"debug"' | flowey v 28 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey v 28 '_internal_WORKING_DIR' --is-raw-string update @@ -6333,11 +6295,86 @@ jobs: cat <<'EOF' | flowey v 28 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 28 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-virtio-villain-tests-archive" | flowey v 28 'artifact_use_from_x64-linux-virtio-villain-tests-archive' --is-raw-string update + shell: bash + - name: ensure hypervisor device is accessible + run: flowey e 28 flowey_lib_hvlite::test_virtio_villain 0 + shell: bash + - name: create cargo-nextest cache dir + run: |- + flowey e 28 flowey_lib_common::download_cargo_nextest 0 + flowey e 28 flowey_lib_common::download_cargo_nextest 1 + flowey e 28 flowey_lib_common::download_cargo_nextest 2 + flowey e 28 flowey_lib_common::download_cargo_nextest 3 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey e 28 flowey_lib_common::cache 0 + flowey v 28 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar4 + flowey v 28 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v5 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: cargo-nextest' + - name: downloading cargo-nextest + run: |- + flowey v 28 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.95.0 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.95.0 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v6 with: - pattern: '{_internal-flowey-bootstrap-x86_64-linux-uid-12,aarch64-linux-vmgstool,aarch64-windows-vmgstool,x64-linux-vmgstool,x64-windows-vmgstool}' - path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-12" >> $GITHUB_PATH + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" + mv target/x86_64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-12/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey echo '"debug"' | flowey v 29 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey v 29 '_internal_WORKING_DIR' --is-raw-string update @@ -6435,47 +6521,6 @@ jobs: cat <<'EOF' | flowey v 29 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "$AgentTempDirNormal/used_artifacts/aarch64-linux-vmgstool" | flowey v 29 'artifact_use_from_aarch64-linux-vmgstool' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/aarch64-windows-vmgstool" | flowey v 29 'artifact_use_from_aarch64-windows-vmgstool' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmgstool" | flowey v 29 'artifact_use_from_x64-linux-vmgstool' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-windows-vmgstool" | flowey v 29 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - shell: bash - - name: create gh cache dir - run: flowey e 29 flowey_lib_common::download_gh_cli 0 - shell: bash - - name: Pre-processing cache vars - run: |- - flowey e 29 flowey_lib_common::cache 0 - flowey v 29 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar2 - flowey v 29 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar3 - shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v5 - with: - key: ${{ env.floweyvar2 }} - path: ${{ env.floweyvar3 }} - name: 'Restore cache: gh-cli' - - name: installing gh - run: |- - flowey v 29 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-12/flowey + + echo '"debug"' | flowey v 30 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 30 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey v 30 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "$AgentTempDirNormal/used_artifacts/aarch64-linux-vmgstool" | flowey v 30 'artifact_use_from_aarch64-linux-vmgstool' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/aarch64-windows-vmgstool" | flowey v 30 'artifact_use_from_aarch64-windows-vmgstool' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmgstool" | flowey v 30 'artifact_use_from_x64-linux-vmgstool' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-windows-vmgstool" | flowey v 30 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + shell: bash + - name: create gh cache dir + run: flowey e 30 flowey_lib_common::download_gh_cli 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey e 30 flowey_lib_common::cache 0 + flowey v 30 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar2 + flowey v 30 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar3 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v5 + with: + key: ${{ env.floweyvar2 }} + path: ${{ env.floweyvar3 }} + name: 'Restore cache: gh-cli' + - name: installing gh + run: |- + flowey v 30 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH @@ -6070,11 +6075,86 @@ jobs: cat <<'EOF' | flowey v 28 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 28 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-virtio-villain-tests-archive" | flowey v 28 'artifact_use_from_x64-linux-virtio-villain-tests-archive' --is-raw-string update + shell: bash + - name: ensure hypervisor device is accessible + run: flowey e 28 flowey_lib_hvlite::test_virtio_villain 0 + shell: bash + - name: create cargo-nextest cache dir + run: |- + flowey e 28 flowey_lib_common::download_cargo_nextest 0 + flowey e 28 flowey_lib_common::download_cargo_nextest 1 + flowey e 28 flowey_lib_common::download_cargo_nextest 2 + flowey e 28 flowey_lib_common::download_cargo_nextest 3 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey e 28 flowey_lib_common::cache 0 + flowey v 28 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar4 + flowey v 28 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v5 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: cargo-nextest' + - name: downloading cargo-nextest + run: |- + flowey v 28 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-1/flowey + + echo '"debug"' | flowey v 29 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 29 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey v 29 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + shell: bash + - name: check if openvmm needs to be cloned + run: |- + flowey e 29 flowey_lib_common::git_checkout 0 + flowey v 29 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs write-to-env github floweyvar1 + flowey v 29 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs' write-to-env github FLOWEY_CONDITION + shell: bash + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v6 + with: + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar1 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories + run: |- + flowey v 29 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH @@ -6834,11 +6839,86 @@ jobs: cat <<'EOF' | flowey v 30 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 30 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-virtio-villain-tests-archive" | flowey v 30 'artifact_use_from_x64-linux-virtio-villain-tests-archive' --is-raw-string update + shell: bash + - name: ensure hypervisor device is accessible + run: flowey e 30 flowey_lib_hvlite::test_virtio_villain 0 + shell: bash + - name: create cargo-nextest cache dir + run: |- + flowey e 30 flowey_lib_common::download_cargo_nextest 0 + flowey e 30 flowey_lib_common::download_cargo_nextest 1 + flowey e 30 flowey_lib_common::download_cargo_nextest 2 + flowey e 30 flowey_lib_common::download_cargo_nextest 3 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey e 30 flowey_lib_common::cache 0 + flowey v 30 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar4 + flowey v 30 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v5 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: cargo-nextest' + - name: downloading cargo-nextest + run: |- + flowey v 30 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-1/flowey + + echo '"debug"' | flowey v 31 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 31 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey v 31 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + shell: bash + - name: check if openvmm needs to be cloned + run: |- + flowey e 31 flowey_lib_common::git_checkout 0 + flowey v 31 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs write-to-env github floweyvar1 + flowey v 31 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs' write-to-env github FLOWEY_CONDITION + shell: bash + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v6 + with: + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar1 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories + run: |- + flowey v 31 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs' --is-raw-string update --env-source github.workspace <` on the kernel +command line. Villain runs that test, prints a `[TAG] ` verdict to the serial +console, and powers off. The harness reads the verdict from petri's serial log and +maps it to a result: + +| villain verdict | outcome | +|-----------------|---------| +| `[PASS]` and other good verdicts | pass | +| `[FAIL]` and other bad verdicts | fail | +| `[SKIP]` | fail (see below) | +| no verdict marker (guest wedged or never booted) | fail | + +The test manifest is villain's `tests.tsv`, which has one row per test (name, +description, spec section, target `device_id`, and so on). The guest kernel is the +existing OpenVMM linux-direct test `vmlinux`. Villain's `initramfs.cpio.gz` and +`tests.tsv` come from the `openvmm-deps` release artifact, or locally from +`--villain-initramfs` / `--villain-tsv` (or the `VILLAIN_INITRAMFS` / +`VILLAIN_TSV` environment variables). + +### Why a `[SKIP]` is a failure + +A `[SKIP]` means the guest did not exercise anything, because the target device +was absent or a precondition was not met. On the kitchen-sink VM, which attaches +every device we support, that usually indicates a harness problem: a device we +intended to attach was not attached (for example, when a `#[cfg]` guard once +dropped the vsock device). Reporting such a case as a pass would hide the bug, so +the harness treats every `[SKIP]` as a failure. + +Tests for devices we do not attach are ignored up front (see below), so they do +not reach this rule during a normal run. + +## When a test is ignored + +A villain test does not gate for one of two reasons, and both are reported as +`ignored` in nextest output rather than as a pass. Both use libtest's `#[ignore]`: +skipped by default, and runnable with `--run-ignored`. + +### Unsupported device (`supported_devices.rs`) + +Villain has roughly 1400 tests but only a handful of device classes. Rather than +list every test to skip, we list the devices the kitchen-sink VM attaches, in +`DEVICE_CAPS` (keyed by device ID, which follows the virtio convention `0x1040 + +virtio_device_type`): + +| device_id | device | +|-----------|--------| +| `0x1041` | network | +| `0x1042` | block | +| `0x1043` | console | +| `0x1044` | entropy (rng) | +| `0x1053` | vsock | +| `0x105a` | virtio-fs | +| `0x105b` | pmem | + +A test whose target device is not in this set is ignored when its trial is +constructed, so it never boots a VM. Device-agnostic MMIO tests (`device_id == +0x0000`) run against whatever device villain's `virtio_mmio_find` locates. Their +PCI counterparts are ignored: villain's `virtio_pci_find(0)` looks for a literal +PCI device `0x0000`, which no virtio device is, so they can only `[SKIP]`. + +This avoids booting a VM for roughly 270 tests covering devices OpenVMM does not +yet emulate (IOMMU, memory balloon, virtio-mem, watchdog, RTC). To enable a +device's tests, add an entry to `DEVICE_CAPS` and attach the device in +`attach_kitchen_sink` (`run.rs`); keep the two in sync. + +Because these tests are ignored rather than run, `--run-ignored` will run them and +report their absent-device `[SKIP]` as a failure. + +### Known OpenVMM failure (`known_failures.rs`) + +Tests that OpenVMM currently fails are listed in `KNOWN_FAILURES` and ignored, so +the gate stays green while the underlying bugs are triaged separately. Each entry +records the test name and a reason, ideally referencing a tracking issue. When a +bug is fixed, remove its entry so the test gates again. + +These are ignored rather than inverted (XFAIL) because several are unrecoverable +host hangs: OpenVMM's virtio worker spins on a malformed descriptor chain and the +VM never powers off. Such a test can only be ended by the nextest timeout, so an +in-harness expected-failure inversion is not possible. + +## Running locally (flowey) + +`cargo xflowey virtio-villain-run` builds OpenVMM, downloads the villain artifact, +stages the guest kernel, and runs the suite: + +```bash +cargo xflowey virtio-villain-run +``` + +Use `--filter` with a [nextest filter](https://nexte.st/docs/filtersets/) to run +a subset: + +```bash +cargo xflowey virtio-villain-run --filter "test(B0001)" +``` + +Use `--run-ignored` to also run the ignored tests, for example when developing a +fix for a known failure. Because this also un-ignores unsupported-device tests +(which then fail on an absent-device `[SKIP]`), pair it with a `--filter` that +targets the specific test: + +```bash +cargo xflowey virtio-villain-run --run-ignored --filter "test(B0002)" +``` + +## Running locally (manual nextest) + +If you already have the villain payload, invoke nextest directly and point it at +the manifest and initramfs: + +```bash +VILLAIN_TSV=/path/to/tests.tsv \ +VILLAIN_INITRAMFS=/path/to/initramfs.cpio.gz \ +cargo nextest run -p virtio_villain_tests --filter-expr 'test(B0001)' +``` + +`--run-ignored ignored-only` runs only the ignored tests; `--run-ignored all` +runs both ignored and normal tests. + +## In CI + +The `x64-linux-kvm-virtio-villain` job runs the suite under KVM as a separate, +parallel per-PR job. The work is split across two machines so the build does not +need KVM and the test run does not need a full toolchain: + +- The Linux build machine produces a `cargo nextest archive` + (`x64-linux-virtio-villain-tests-archive`) along with the OpenVMM binary it + drives. The archive does not run test binaries; enumeration happens at run time. +- The KVM test machine consumes that archive, injects `VILLAIN_TSV` and + `VILLAIN_INITRAMFS`, and runs nextest. Its result logs + (`x64-linux-kvm-virtio-villain-vmm-tests-logs`) match the `upload-petri-results` + glob, so villain results appear alongside the other VMM-test results. + +Villain boots linux-direct with its own initramfs, so its test environment sets +`stage_uefi_and_virtio_win = false` and skips the mu_msvm UEFI firmware and +Windows `virtio-win` driver staging used by the full VMM-tests environment. + +[virtio-villain]: https://github.com/weltling/virtio-villain diff --git a/ci-flowey/openvmm-pr.yaml b/ci-flowey/openvmm-pr.yaml index 6ffe68e7f2..c2d357c0bb 100644 --- a/ci-flowey/openvmm-pr.yaml +++ b/ci-flowey/openvmm-pr.yaml @@ -1934,6 +1934,8 @@ jobs: echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-openvmm_vhost" | $FLOWEY_BIN v 9 'artifact_publish_from_x64-linux-openvmm_vhost' --is-raw-string update mkdir -p "$(AgentTempDirNormal)/publish_artifacts/x64-linux-prep_steps" echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-prep_steps" | $FLOWEY_BIN v 9 'artifact_publish_from_x64-linux-prep_steps' --is-raw-string update + mkdir -p "$(AgentTempDirNormal)/publish_artifacts/x64-linux-virtio-villain-tests-archive" + echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-virtio-villain-tests-archive" | $FLOWEY_BIN v 9 'artifact_publish_from_x64-linux-virtio-villain-tests-archive' --is-raw-string update mkdir -p "$(AgentTempDirNormal)/publish_artifacts/x64-linux-vmgs_lib" echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-vmgs_lib" | $FLOWEY_BIN v 9 'artifact_publish_from_x64-linux-vmgs_lib' --is-raw-string update mkdir -p "$(AgentTempDirNormal)/publish_artifacts/x64-linux-vmgstool" @@ -2013,7 +2015,7 @@ jobs: - bash: |- set -e $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_openvmm_magicpath_protoc 0 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 1 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 2 displayName: symlink protoc - bash: |- set -e @@ -2026,7 +2028,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 5 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_openvmm 0 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 0 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 3 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 4 displayName: split debug symbols - bash: |- set -e @@ -2039,7 +2041,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 9 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_openvmm_vhost 0 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 1 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 6 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 7 displayName: split debug symbols - bash: |- set -e @@ -2052,7 +2054,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 18 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_vmgstool 0 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 2 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 5 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 6 displayName: split debug symbols - bash: |- set -e @@ -2065,7 +2067,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 16 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_vmgstool 1 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 3 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 4 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 5 displayName: split debug symbols - bash: |- set -e @@ -2078,7 +2080,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_and_test_vmgs_lib 1 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_and_test_vmgs_lib 2 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 4 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 2 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 3 displayName: test vmgs_lib - bash: |- set -e @@ -2091,7 +2093,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 1 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_igvmfilegen 0 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 5 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 7 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 8 displayName: split debug symbols - bash: |- set -e @@ -2110,7 +2112,7 @@ jobs: - bash: |- set -e $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_openvmm_magicpath_openhcl_sysroot 0 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 9 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 10 displayName: extract X86_64 sysroot.tar.gz - bash: |- set -e @@ -2123,7 +2125,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 7 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_openvmm 1 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 7 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 10 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 11 displayName: split debug symbols - bash: |- set -e @@ -2136,7 +2138,7 @@ jobs: $(FLOWEY_BIN) e 9 flowey_lib_hvlite::run_cargo_build 11 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_openvmm_vhost 1 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 8 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 11 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 12 displayName: split debug symbols - bash: |- set -e @@ -2183,21 +2185,28 @@ jobs: - bash: |- set -e $(FLOWEY_BIN) e 9 flowey_lib_common::install_cargo_nextest 0 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 0 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 1 displayName: installing cargo-nextest - bash: |- set -e - $(FLOWEY_BIN) e 9 flowey_lib_common::run_cargo_nextest_archive 0 + $(FLOWEY_BIN) e 9 flowey_lib_common::run_cargo_nextest_archive 1 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_nextest_vmm_tests 0 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 10 - $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 8 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 9 displayName: build + archive 'vmm_tests' nextests - bash: |- set -e - $(FLOWEY_BIN) e 9 flowey_lib_common::run_cargo_nextest_archive 1 + $(FLOWEY_BIN) e 9 flowey_lib_common::run_cargo_nextest_archive 2 $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_nextest_vmm_tests 1 $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 11 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::init_cross_build 0 displayName: build + archive 'vmm_tests' nextests + - bash: |- + set -e + $(FLOWEY_BIN) e 9 flowey_lib_common::run_cargo_nextest_archive 0 + $(FLOWEY_BIN) e 9 flowey_lib_hvlite::build_nextest_virtio_villain_tests 0 + $(FLOWEY_BIN) e 9 flowey_core::pipeline::artifact::publish 12 + displayName: build + archive 'virtio_villain_tests' nextests - bash: $(FLOWEY_BIN) e 9 flowey_lib_common::cache 3 displayName: 'validate cache entry: cargo-nextest' - bash: $(FLOWEY_BIN) e 9 flowey_lib_common::cache 7 @@ -2226,6 +2235,9 @@ jobs: - publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-prep_steps displayName: πŸŒΌπŸ“¦ Publish x64-linux-prep_steps artifact: x64-linux-prep_steps + - publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-virtio-villain-tests-archive + displayName: πŸŒΌπŸ“¦ Publish x64-linux-virtio-villain-tests-archive + artifact: x64-linux-virtio-villain-tests-archive - publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-vmgs_lib displayName: πŸŒΌπŸ“¦ Publish x64-linux-vmgs_lib artifact: x64-linux-vmgs_lib @@ -2238,6 +2250,190 @@ jobs: - publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/x64-linux-vmm-tests-archive displayName: πŸŒΌπŸ“¦ Publish x64-linux-vmm-tests-archive artifact: x64-linux-vmm-tests-archive +- job: job21 + displayName: run vmm-tests [x64-linux-kvm-virtio-villain] + pool: + demands: + - ImageOverride -equals ubuntu2404-amd64 + name: openvmm-ado-amd-westus2-v6 + dependsOn: + - job0 + - job9 + condition: and(succeeded(), not(canceled())) + variables: + - name: FLOWEY_TEMP_DIR + value: $(Build.StagingDirectory)/.flowey-internal + steps: + - task: DownloadPipelineArtifact@2 + displayName: 🌼πŸ₯Ύ Download bootstrapped flowey + inputs: + artifact: _internal-flowey-bootstrap-x86_64-linux-uid-1 + path: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey + - task: DownloadPipelineArtifact@2 + displayName: πŸŒΌπŸ“¦ Download x64-linux-openvmm + inputs: + artifact: x64-linux-openvmm + path: $(FLOWEY_TEMP_DIR)/used_artifacts/x64-linux-openvmm + - task: DownloadPipelineArtifact@2 + displayName: πŸŒΌπŸ“¦ Download x64-linux-virtio-villain-tests-archive + inputs: + artifact: x64-linux-virtio-villain-tests-archive + path: $(FLOWEY_TEMP_DIR)/used_artifacts/x64-linux-virtio-villain-tests-archive + - bash: | + set -e + AgentTempDirNormal="$(FLOWEY_TEMP_DIR)" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "##vso[task.setvariable variable=AgentTempDirNormal;]$AgentTempDirNormal" + + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + FLOWEY_BIN="$AgentTempDirNormal/bootstrapped-flowey/flowey" + echo "##vso[task.setvariable variable=FLOWEY_BIN;]$FLOWEY_BIN" + displayName: Set flowey path + - bash: | + set -e + echo '"debug"' | $FLOWEY_BIN v 21 'FLOWEY_LOG' update + echo "$(FLOWEY_TEMP_DIR)/work" | $FLOWEY_BIN v 21 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | tr '[:upper:]' '[:lower:]' | $FLOWEY_BIN v 21 'verbose' update + ${{ parameters.verbose }} + EOF + echo "$(FLOWEY_TEMP_DIR)/used_artifacts/x64-linux-openvmm" | $FLOWEY_BIN v 21 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$(FLOWEY_TEMP_DIR)/used_artifacts/x64-linux-virtio-villain-tests-archive" | $FLOWEY_BIN v 21 'artifact_use_from_x64-linux-virtio-villain-tests-archive' --is-raw-string update + displayName: πŸŒΌπŸ›« Initialize job + - bash: $(FLOWEY_BIN) e 21 flowey_lib_hvlite::test_virtio_villain 0 + displayName: ensure hypervisor device is accessible + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::download_cargo_nextest 0 + $(FLOWEY_BIN) e 21 flowey_lib_common::download_cargo_nextest 1 + $(FLOWEY_BIN) e 21 flowey_lib_common::download_cargo_nextest 2 + $(FLOWEY_BIN) e 21 flowey_lib_common::download_cargo_nextest 3 + displayName: create cargo-nextest cache dir + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::cache 0 + $FLOWEY_BIN v 21 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env ado floweyvar3 + $FLOWEY_BIN v 21 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env ado floweyvar4 + displayName: Pre-processing cache vars + - task: Cache@2 + inputs: + key: $(floweyvar4) + path: $(floweyvar3) + cacheHitVar: FLOWEY_CACHE_HITVAR + displayName: 'Restore cache: cargo-nextest' + - bash: |- + set -e + $FLOWEY_BIN v 21 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source FLOWEY_CACHE_HITVAR <<'EOF' + $(FLOWEY_CACHE_HITVAR) + EOF + $(FLOWEY_BIN) e 21 flowey_lib_common::cache 2 + displayName: map ADO hitvar to flowey + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::download_cargo_nextest 4 + $(FLOWEY_BIN) e 21 flowey_core::pipeline::artifact::resolve 1 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::_jobs::consume_and_test_nextest_virtio_villain_archive 1 + $(FLOWEY_BIN) e 21 flowey_core::pipeline::artifact::resolve 0 + displayName: downloading cargo-nextest + - bash: $(FLOWEY_BIN) e 21 flowey_lib_hvlite::_jobs::consume_and_test_nextest_virtio_villain_archive 0 + displayName: creating new test content dir + - bash: $(FLOWEY_BIN) e 21 flowey_lib_common::download_gh_release 0 + displayName: create gh-release-download cache dir + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::cache 4 + $FLOWEY_BIN v 21 'flowey_lib_common::cache:8:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env ado floweyvar5 + $FLOWEY_BIN v 21 'flowey_lib_common::cache:9:flowey_lib_common/src/cache.rs' --is-raw-string write-to-env ado floweyvar6 + displayName: Pre-processing cache vars + - task: Cache@2 + inputs: + key: $(floweyvar6) + path: $(floweyvar5) + cacheHitVar: FLOWEY_CACHE_HITVAR + displayName: 'Restore cache: gh-release-download' + - bash: |- + set -e + $FLOWEY_BIN v 21 'flowey_lib_common::cache:11:flowey_lib_common/src/cache.rs' --is-raw-string update --env-source FLOWEY_CACHE_HITVAR <<'EOF' + $(FLOWEY_CACHE_HITVAR) + EOF + $(FLOWEY_BIN) e 21 flowey_lib_common::cache 6 + displayName: map ADO hitvar to flowey + - bash: $(FLOWEY_BIN) e 21 flowey_lib_common::download_gh_release 1 + displayName: download artifacts from github releases + - bash: $(FLOWEY_BIN) e 21 flowey_lib_hvlite::resolve_openvmm_test_initrd 0 + displayName: unpack openvmm-test-initrd archives + - bash: $(FLOWEY_BIN) e 21 flowey_lib_hvlite::resolve_openvmm_test_linux_kernel 0 + displayName: unpack openvmm-test-linux archives + - bash: $(FLOWEY_BIN) e 21 flowey_lib_hvlite::resolve_virtio_villain 0 + displayName: unpack virtio-villain artifacts + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::init_vmm_tests_env 0 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::run_cargo_nextest_run 1 + displayName: setting up vmm_tests env + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::git_checkout 0 + $FLOWEY_BIN v 21 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs write-to-env ado floweyvar2 + $FLOWEY_BIN v 21 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs' write-to-env ado FLOWEY_CONDITION + displayName: check if openvmm needs to be cloned + - checkout: self + path: repo0 + fetchTags: false + fetchDepth: 1 + persistCredentials: $(floweyvar2) + submodules: recursive + displayName: checkout repo openvmm + condition: and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled())) + - bash: |- + set -e + $FLOWEY_BIN v 21 'flowey_lib_common::git_checkout:4:flowey_lib_common/src/git_checkout.rs' --is-raw-string update --env-source Pipeline.Workspace <<'EOF' + $(Pipeline.Workspace) + EOF + $(FLOWEY_BIN) e 21 flowey_lib_common::git_checkout 3 + displayName: report cloned repo directories + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::system_info 0 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::git_checkout_openvmm_repo 0 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::test_virtio_villain 1 + displayName: print system info + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::gen_cargo_nextest_run_cmd 0 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::run_cargo_nextest_run 0 + displayName: generate nextest command + - bash: $(FLOWEY_BIN) e 21 flowey_lib_hvlite::install_vmm_tests_deps 0 + displayName: install vmm tests deps (linux) + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_common::run_cargo_nextest_run 0 + $(FLOWEY_BIN) e 21 flowey_lib_common::run_cargo_nextest_run 1 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::test_virtio_villain 3 + $(FLOWEY_BIN) e 21 flowey_lib_common::publish_test_results 1 + $(FLOWEY_BIN) e 21 flowey_lib_common::ado_task_publish_test_results 0 + $(FLOWEY_BIN) e 21 flowey_lib_common::publish_test_results 0 + $FLOWEY_BIN v 21 'flowey_lib_common::ado_task_publish_test_results:0:flowey_lib_common/src/ado_task_publish_test_results.rs' --is-raw-string --condvar flowey_lib_common::publish_test_results:0:flowey_lib_common/src/publish_test_results.rs write-to-env ado floweyvar1 + $FLOWEY_BIN v 21 'flowey_lib_common::publish_test_results:0:flowey_lib_common/src/publish_test_results.rs' write-to-env ado FLOWEY_CONDITION + displayName: run 'virtio_villain_tests' nextest tests + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + testResultsFiles: $(floweyvar1) + testRunTitle: x64-linux-kvm-virtio-villain-vmm-tests + displayName: 'publish test results: x64-linux-kvm-virtio-villain-vmm-tests (JUnit XML)' + condition: and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled())) + - bash: |- + set -e + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::test_virtio_villain 2 + $(FLOWEY_BIN) e 21 flowey_lib_common::publish_test_results 2 + $(FLOWEY_BIN) e 21 flowey_lib_common::publish_test_results 3 + $(FLOWEY_BIN) e 21 flowey_lib_hvlite::test_virtio_villain 4 + displayName: report virtio-villain test results + - bash: $(FLOWEY_BIN) e 21 flowey_lib_common::cache 3 + displayName: 'validate cache entry: cargo-nextest' + - bash: $(FLOWEY_BIN) e 21 flowey_lib_common::cache 7 + displayName: 'validate cache entry: gh-release-download' - job: job8 displayName: build artifacts (for VMM tests) [aarch64-linux] pool: @@ -5036,6 +5232,7 @@ jobs: displayName: πŸŒΌπŸ›« Initialize job - bash: |- set -e + $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 3 $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 11 $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 4 $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 5 @@ -5050,7 +5247,6 @@ jobs: $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 14 $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 15 $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 13 - $(FLOWEY_BIN) e 17 flowey_core::pipeline::artifact::resolve 3 $(FLOWEY_BIN) e 17 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 displayName: creating new test content dir - bash: $(FLOWEY_BIN) e 17 flowey_lib_common::download_gh_release 0 diff --git a/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs b/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs index defb9d45ea..f853b9727d 100644 --- a/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs +++ b/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs @@ -203,6 +203,14 @@ impl IntoPipeline for CheckinGatesCli { let (pub_vmm_tests_archive_linux_aarch64, use_vmm_tests_archive_linux_aarch64) = pipeline.new_typed_artifact("aarch64-linux-vmm-tests-archive"); + // virtio-villain nextest archive (built on the linux build machine, + // consumed by the villain test job) + the OpenVMM binary it drives, + // reused from the linux build job below. + let (pub_villain_archive_linux_x86, use_villain_archive_linux_x86) = + pipeline.new_typed_artifact("x64-linux-virtio-villain-tests-archive"); + let mut pub_villain_archive_linux_x86 = Some(pub_villain_archive_linux_x86); + let mut use_openvmm_linux_x86 = None; + // wrap each publish handle in an option, so downstream code can // `.take()` the handle when emitting the corresponding job let mut pub_vmm_tests_archive_linux_x86 = Some(pub_vmm_tests_archive_linux_x86); @@ -803,6 +811,7 @@ impl IntoPipeline for CheckinGatesCli { match arch { CommonArch::X86_64 => { vmm_tests_artifacts_linux_x86.use_openvmm = Some(use_openvmm.clone()); + use_openvmm_linux_x86 = Some(use_openvmm.clone()); vmm_tests_artifacts_linux_x86.use_openvmm_vhost = Some(use_openvmm_vhost.clone()); vmm_tests_artifacts_linux_x86.use_prep_steps = Some(use_prep_steps.clone()); @@ -986,6 +995,23 @@ impl IntoPipeline for CheckinGatesCli { ), } }); + + // Also build the virtio-villain nextest archive on this + // build machine so the villain test job can consume it + // without a Rust toolchain. + let pub_villain_archive_linux_x86 = + pub_villain_archive_linux_x86.take().unwrap(); + job = job.publish(pub_villain_archive_linux_x86, |archive| { + flowey_lib_hvlite::build_nextest_virtio_villain_tests::Request { + target: CommonTriple::Common { + arch, + platform: CommonPlatform::LinuxGnu, + } + .as_triple(), + profile: CommonProfile::from_release(release), + archive, + } + }); } CommonArch::Aarch64 => { let pub_vmm_tests_archive_linux_aarch64 = @@ -1760,6 +1786,61 @@ impl IntoPipeline for CheckinGatesCli { } } + // virtio-villain conformance suite: a separate, parallel per-PR job that + // drives OpenVMM against the guest-side villain test matrix. + // + // It consumes the villain nextest archive + the OpenVMM binary built by + // the linux build job above, so this test job needs no Rust toolchain + // (matching the vmm_tests build/consume split). It runs alongside the + // vmm_tests jobs for parallelism. The `-vmm-tests` suffix on the label + // makes the published logs artifact + // (`x64-linux-kvm-virtio-villain-vmm-tests-logs`) match the + // `upload-petri-results` workflow glob, so villain results land on the + // logview website alongside the regular VMM test results. + // + // This job consumes the virtio-villain artifact shipped by openvmm-deps + // (bumped to a release that includes it via `OPENVMM_DEPS` in + // `cfg_versions.rs`), so it resolves and runs in PR CI now. Its results + // seed and maintain the known-failure list (see the + // `virtio_villain_tests` crate's `known_failures` module). + { + let label = "x64-linux-kvm-virtio-villain"; + let test_label = format!("{label}-vmm-tests"); + + let pub_villain_results = if matches!(backend_hint, PipelineBackendHint::Local) { + Some(pipeline.new_artifact(&test_label).0) + } else { + None + }; + + let use_openvmm = use_openvmm_linux_x86 + .clone() + .expect("linux x86 openvmm artifact built above"); + + let villain_job = pipeline + .new_job( + FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu), + FlowArch::X86_64, + format!("run vmm-tests [{label}]"), + ) + .gh_set_pool(gh_pools::linux_amd_v7_1es()) + .ado_set_pool(ado_pools::linux_amd_v6_1es()) + .dep_on(|ctx| { + flowey_lib_hvlite::_jobs::consume_and_test_nextest_virtio_villain_archive::Params { + junit_test_label: test_label, + nextest_villain_archive: ctx + .use_typed_artifact(&use_villain_archive_linux_x86), + openvmm: ctx.use_typed_artifact(&use_openvmm), + target: CommonTriple::X86_64_LINUX_GNU.as_triple(), + run_ignored: false, + artifact_dir: pub_villain_results.map(|x| ctx.publish_artifact(x)), + done: ctx.new_done_handle(), + } + }) + .finish(); + all_jobs.push(villain_job); + } + // test the flowey local backend by running cargo xflowey build-igvm on x64 { if matches!(backend_hint, PipelineBackendHint::Github) { diff --git a/flowey/flowey_hvlite/src/pipelines/vmm_tests_run.rs b/flowey/flowey_hvlite/src/pipelines/vmm_tests_run.rs index 874c913b46..9227bb0973 100644 --- a/flowey/flowey_hvlite/src/pipelines/vmm_tests_run.rs +++ b/flowey/flowey_hvlite/src/pipelines/vmm_tests_run.rs @@ -14,6 +14,7 @@ use flowey::node::prelude::ReadVar; use flowey::pipeline::prelude::*; use flowey_lib_hvlite::_jobs::local_build_and_run_nextest_vmm_tests::BuildSelections; use flowey_lib_hvlite::_jobs::local_build_and_run_nextest_vmm_tests::VmmTestSelections; +use flowey_lib_hvlite::common::CommonArch; use flowey_lib_hvlite::common::CommonPlatform; use flowey_lib_hvlite::common::CommonTriple; use flowey_lib_hvlite::install_vmm_tests_deps::VmmTestsDepSelections; @@ -102,6 +103,25 @@ pub struct VmmTestsRunCli { #[clap(long)] ci_profile: bool, + /// Which test suite to run. + /// + /// `vmm-tests` (the default) is the standard suite with automatic artifact + /// discovery. `virtio-villain` is the guest virtio conformance / + /// fault-injection suite; it runs on the Linux host and ignores the + /// vmm_tests-specific options (`--target`, `--incubator`, VHD/kernel + /// options), reusing only the shared options (`--dir`, `--filter`, + /// `--ci-profile`, `--run-ignored`, `--verbose`, `--install-missing-deps`). + #[clap(long, value_enum, default_value_t = TestSuiteCli::VmmTests)] + suite: TestSuiteCli, + + /// Also run tests marked `#[ignore]`. + /// + /// For `vmm-tests` this includes host-incompatible tests; for + /// `virtio-villain` this includes the known-failing tests that exercise + /// OpenVMM bugs (some wedge the VM), useful during fix development. + #[clap(long)] + run_ignored: bool, + /// Don't reuse prepped vhds, even if they already exist. /// Use when making changes to prep_steps #[clap(long)] @@ -182,11 +202,29 @@ impl IntoPipeline for VmmTestsRunCli { custom_kernel, custom_uefi_firmware, ci_profile, + suite, + run_ignored, no_reuse_prepped_vhds, disable_secure_avic, incubator, } = self; + // The virtio-villain suite is a separate nextest package whose test list + // lives in a versioned guest artifact, so it can't participate in the + // vmm_tests build-time artifact discovery below. Route it to its own job + // (which builds a fixed artifact set), reusing the shared options. + if matches!(suite, TestSuiteCli::VirtioVillain) { + return into_virtio_villain_pipeline(VirtioVillainArgs { + backend_hint, + dir, + filter, + ci_profile, + verbose, + run_ignored, + install_missing_deps, + }); + } + // When --incubator is set, --target must also be specified // to indicate the cross-compilation target for the incubator. if incubator.is_some() && target.is_none() { @@ -239,7 +277,7 @@ impl IntoPipeline for VmmTestsRunCli { // checks (the source of nextest's `#[ignore]` flag) would wrongly drop // incubator tests, so for the incubator path we enumerate ignored tests // too β€” their artifacts still need to be built. - let include_ignored = build_only || incubator_profile.is_some(); + let include_ignored = build_only || incubator_profile.is_some() || run_ignored; // Run artifact discovery inline at pipeline construction time since // flowey doesn't support conditional requests yet @@ -446,6 +484,7 @@ impl IntoPipeline for VmmTestsRunCli { flowey_lib_hvlite::run_cargo_nextest_run::NextestProfile::Default }, reuse_prepped_vhds: !no_reuse_prepped_vhds, + run_ignored, disable_secure_avic, incubator_profile, done: ctx.new_done_handle(), @@ -458,6 +497,106 @@ impl IntoPipeline for VmmTestsRunCli { } } +struct VirtioVillainArgs { + backend_hint: PipelineBackendHint, + dir: Option, + filter: String, + ci_profile: bool, + verbose: bool, + run_ignored: bool, + install_missing_deps: bool, +} + +/// Build OpenVMM and run the virtio-villain guest conformance / +/// fault-injection suite against it. +/// +/// virtio-villain's test list lives in a versioned `openvmm-deps` guest +/// artifact, so it can't participate in the vmm_tests build-time artifact +/// discovery above; it routes to its own job with a fixed artifact set. +fn into_virtio_villain_pipeline(args: VirtioVillainArgs) -> anyhow::Result { + let VirtioVillainArgs { + backend_hint, + dir, + filter, + ci_profile, + verbose, + run_ignored, + install_missing_deps, + } = args; + + let target = match ( + FlowArch::host(backend_hint), + FlowPlatform::host(backend_hint), + ) { + (FlowArch::X86_64, FlowPlatform::Linux(_)) => CommonTriple::Common { + arch: CommonArch::X86_64, + platform: CommonPlatform::LinuxGnu, + }, + (FlowArch::Aarch64, FlowPlatform::Linux(_)) => CommonTriple::Common { + arch: CommonArch::Aarch64, + platform: CommonPlatform::LinuxGnu, + }, + _ => anyhow::bail!("the virtio-villain suite currently requires a Linux host"), + }; + + // Stage test content + publish results here, mirroring the vmm_tests `--dir` + // (defaulting under `target/` rather than the internal flowey work dir). + let test_content_dir = + dir.unwrap_or_else(|| crate::repo_root().join("target").join("virtio_villain")); + std::fs::create_dir_all(&test_content_dir) + .context("failed to create virtio-villain output directory")?; + + // Shared with vmm_tests: filter defaults to `all()`; villain wants None. + let nextest_filter_expr = (filter != "all()").then_some(filter); + + let openvmm_repo = flowey_lib_common::git_checkout::RepoSource::ExistingClone( + ReadVar::from_static(crate::repo_root()), + ); + + let mut pipeline = Pipeline::new(); + + pipeline + .new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "virtio-villain: run tests", + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) + .dep_on( + |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo.clone(), + }, + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: install_missing_deps, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(verbose), + locked: false, + deny_warnings: false, + no_incremental: false, + }) + .dep_on( + |ctx| flowey_lib_hvlite::_jobs::run_virtio_villain_tests::Params { + target, + run_ignored, + nextest_filter_expr, + test_content_dir, + nextest_profile: if ci_profile { + flowey_lib_hvlite::run_cargo_nextest_run::NextestProfile::Ci + } else { + flowey_lib_hvlite::run_cargo_nextest_run::NextestProfile::Default + }, + done: ctx.new_done_handle(), + }, + ) + .finish(); + + Ok(pipeline) +} + /// Get test binaries and associated matching tests for a given nextest filter. // TODO: this function should really be a flowey node without automatic // dependency installation, but that would require conditional requests. @@ -616,6 +755,14 @@ fn query_test_binary_artifacts(suite: &RustSuite) -> anyhow::Result> Ok(artifacts) } +#[derive(clap::ValueEnum, Copy, Clone, PartialEq, Eq)] +enum TestSuiteCli { + /// The standard vmm_tests suite (with automatic artifact discovery). + VmmTests, + /// The virtio-villain guest conformance / fault-injection suite. + VirtioVillain, +} + #[derive(clap::ValueEnum, Copy, Clone)] enum VmmTestTargetCli { /// Windows Aarch64 diff --git a/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs b/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs index 4602ba5fe4..a58b3233a7 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs @@ -30,7 +30,7 @@ pub const NODEJS: &str = "24.x"; // None disables hcl-dev builds and tests; Some(version) enables them. pub const OPENHCL_KERNEL_DEV_VERSION: Option<&str> = None; pub const OPENHCL_KERNEL_STABLE_VERSION: &str = "6.18.37.2"; -pub const OPENVMM_DEPS: &str = "0.3.0-110"; +pub const OPENVMM_DEPS: &str = "0.3.0-112"; pub const PROTOC: &str = "27.1"; flowey_request! { @@ -64,6 +64,7 @@ impl FlowNode for Node { ctx.import::(); ctx.import::(); ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); ctx.import::(); @@ -223,6 +224,13 @@ impl FlowNode for Node { version: Some(OPENVMM_DEPS.into()), ..Default::default() }); + // Resolves the virtio-villain artifact from the pinned openvmm-deps + // release (see `OPENVMM_DEPS` above); the resolve node only downloads on + // demand, so this costs nothing for jobs that don't consume it. + ctx.config(crate::resolve_virtio_villain::Config { + version: Some(OPENVMM_DEPS.into()), + ..Default::default() + }); ctx.config(crate::resolve_openvmm_qemu::Config { version: Some(OPENVMM_DEPS.into()), ..Default::default() diff --git a/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_virtio_villain_archive.rs b/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_virtio_villain_archive.rs new file mode 100644 index 0000000000..d413fc6157 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_virtio_villain_archive.rs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Consume a pre-built virtio-villain nextest archive and run it against +//! OpenVMM. + +use crate::build_nextest_virtio_villain_tests::NextestVirtioVillainTestsArchive; +use crate::build_openvmm::OpenvmmOutput; +use crate::run_cargo_nextest_run::NextestProfile; +use flowey::node::prelude::*; +use flowey_lib_common::run_cargo_nextest_run::NextestRunKind; + +flowey_request! { + pub struct Params { + /// Artifact label prefix for the published results. To be picked up by + /// the `upload-petri-results` workflow (which globs `*-vmm-tests-logs`), + /// this MUST end in `-vmm-tests` so the logs attachment becomes + /// `-logs`. + pub junit_test_label: String, + /// Pre-built virtio-villain nextest archive. + pub nextest_villain_archive: ReadVar, + /// Pre-built OpenVMM binary (the villain crate launches this). + pub openvmm: ReadVar, + /// Target triple the virtio-villain tests were compiled for. + pub target: target_lexicon::Triple, + /// Also run known-failing (ignored) villain tests. + pub run_ignored: bool, + /// Local-backend only: copy JUnit + logs into this published artifact + /// dir. + pub artifact_dir: Option>, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + } + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + junit_test_label, + nextest_villain_archive, + openvmm, + target, + run_ignored, + artifact_dir, + done, + } = request; + + // Ad-hoc, step-local dir used as a staging ground for test content + // (OpenVMM binary + guest kernel), mirroring the vmm_tests consume job. + let test_content_dir = ctx.emit_rust_stepv("creating new test content dir", |_| { + |rt| Ok(rt.sh.current_dir()) + }); + + let nextest_archive = nextest_villain_archive.map(ctx, |x| x.archive_file); + + ctx.req(crate::test_virtio_villain::Request { + target, + openvmm, + run_kind: NextestRunKind::RunFromArchive { + archive_file: nextest_archive, + target: None, + nextest_bin: None, + }, + nextest_profile: NextestProfile::Ci, + nextest_filter_expr: None, + run_ignored, + test_content_dir, + junit_test_label, + artifact_dir, + // CI test machines need the runtime deps installed and /dev/kvm + // access granted before running. + install_deps: true, + disable_remote_artifacts: true, + done, + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs b/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs index a0d93b64c3..6a8b02f313 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs @@ -260,6 +260,9 @@ impl SimpleFlowNode for Node { use_relative_paths: false, disable_remote_artifacts: true, reuse_prepped_vhds: false, + stage_uefi: true, + stage_virtio_win: true, + stage_virtio_villain: false, }); // Start the test_igvm_agent_rpc_server before running tests (Windows only). diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_build_and_run_nextest_vmm_tests.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_build_and_run_nextest_vmm_tests.rs index 827505c608..4181dbd81a 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_build_and_run_nextest_vmm_tests.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_build_and_run_nextest_vmm_tests.rs @@ -90,6 +90,9 @@ flowey_request! { pub nextest_profile: crate::run_cargo_nextest_run::NextestProfile, + /// Also run tests marked `#[ignore]` (e.g. host-incompatible tests). + pub run_ignored: bool, + pub reuse_prepped_vhds: bool, pub disable_secure_avic: bool, @@ -152,6 +155,7 @@ impl SimpleFlowNode for Node { custom_kernel, skip_vhd_prompt, nextest_profile, + run_ignored, reuse_prepped_vhds, disable_secure_avic, incubator_profile, @@ -729,6 +733,9 @@ impl SimpleFlowNode for Node { use_relative_paths: build_only, disable_remote_artifacts: false, reuse_prepped_vhds, + stage_uefi: true, + stage_virtio_win: true, + stage_virtio_villain: false, }); let mut side_effects = Vec::new(); @@ -796,7 +803,7 @@ impl SimpleFlowNode for Node { tool_config_files: Vec::new(), nextest_profile: nextest_profile.as_str().to_owned(), nextest_filter_expr: Some(nextest_filter_expr.clone()), - run_ignored: false, + run_ignored, fail_fast: None, extra_env: Some(extra_env.clone()), extra_commands: register_prep_steps diff --git a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs index be79bee33d..8b4082c56f 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs @@ -21,6 +21,7 @@ pub mod check_openvmm_hcl_size; pub mod check_xtask_fmt; pub mod consolidate_and_publish_gh_pages; pub mod consume_and_test_nextest_unit_tests_archive; +pub mod consume_and_test_nextest_virtio_villain_archive; pub mod consume_and_test_nextest_vmm_tests_archive; pub mod local_build_and_run_nextest_vmm_tests; pub mod local_build_igvm; @@ -31,4 +32,5 @@ pub mod local_restore_packages; pub mod local_run_cca_test; pub mod local_update_cca_emu; pub mod publish_vmgstool_gh_release; +pub mod run_virtio_villain_tests; pub mod test_local_flowey_build_igvm; diff --git a/flowey/flowey_lib_hvlite/src/_jobs/run_virtio_villain_tests.rs b/flowey/flowey_lib_hvlite/src/_jobs/run_virtio_villain_tests.rs new file mode 100644 index 0000000000..135ef76373 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/run_virtio_villain_tests.rs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Local job: build OpenVMM, build the `virtio_villain_tests` nextest suite +//! from source, and run it against OpenVMM. + +use crate::common::CommonProfile; +use crate::common::CommonTriple; +use crate::run_cargo_nextest_run::NextestProfile; +use flowey::node::prelude::*; +use flowey_lib_common::run_cargo_build::CargoBuildProfile; +use flowey_lib_common::run_cargo_nextest_run::NextestRunKind; +use flowey_lib_common::run_cargo_nextest_run::build_params::NextestBuildParams; +use flowey_lib_common::run_cargo_nextest_run::build_params::TestPackages; + +flowey_request! { + pub struct Params { + /// Target triple to build and run the tests for. + pub target: CommonTriple, + /// Also run known-failing (ignored) villain tests. + pub run_ignored: bool, + /// Optional nextest filter expression to run only a subset of tests. + pub nextest_filter_expr: Option, + /// Directory to stage test content into and publish per-test logs + /// (JUnit + petri logs) under. + pub test_content_dir: PathBuf, + /// Nextest profile to run under (e.g. `ci` to emit JUnit). + pub nextest_profile: NextestProfile, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + ctx.import::(); + ctx.import::(); + } + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + target, + run_ignored, + nextest_filter_expr, + test_content_dir, + nextest_profile, + done, + } = request; + + let target_triple = target.as_triple(); + + let test_content_dir = test_content_dir.absolute()?; + + // Build the OpenVMM binary the villain crate launches. + let openvmm = ctx.reqv(|v| crate::build_openvmm::Request { + params: crate::build_openvmm::OpenvmmBuildParams { + target: target.clone(), + profile: CommonProfile::Debug, + features: Default::default(), + }, + version: None, + openvmm: v, + }); + + // Build env for the test binary compilation (native, so effectively a + // no-op, but keeps cross-build parity with the vmm_tests runner). + let build_env = ctx.reqv(|v| crate::init_cross_build::Request { + target: target_triple.clone(), + injected_env: v, + }); + + let build_params = NextestBuildParams { + packages: ReadVar::from_static(TestPackages::Crates { + crates: vec!["virtio_villain_tests".into()], + }), + features: Default::default(), + no_default_features: false, + target: target_triple.clone(), + profile: CargoBuildProfile::Debug, + extra_env: build_env, + }; + + ctx.req(crate::test_virtio_villain::Request { + target: target_triple, + openvmm, + run_kind: NextestRunKind::BuildAndRun(build_params), + nextest_profile, + nextest_filter_expr, + run_ignored, + test_content_dir: ReadVar::from_static(test_content_dir.clone()), + junit_test_label: "virtio-villain-tests".into(), + // Publish JUnit + logs into the same content dir (mirrors + // vmm-tests-run), so a local run leaves a self-contained results + // tree under `--dir` rather than in the internal flowey work dir. + artifact_dir: Some(ReadVar::from_static(test_content_dir)), + // Local dev machines are assumed already provisioned; don't install + // deps or chmod /dev/kvm out from under the developer. + install_deps: false, + disable_remote_artifacts: false, + done, + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/build_nextest_unit_tests.rs b/flowey/flowey_lib_hvlite/src/build_nextest_unit_tests.rs index a0c1d2c747..4084b6ceb3 100644 --- a/flowey/flowey_lib_hvlite/src/build_nextest_unit_tests.rs +++ b/flowey/flowey_lib_hvlite/src/build_nextest_unit_tests.rs @@ -103,6 +103,10 @@ impl FlowNode for Node { let mut exclude = [ // Skip VMM tests, they get run in a different step. "vmm_tests", + // Skip virtio-villain tests, they get run in a different + // step (they need a built OpenVMM + guest artifacts, and + // must not run under the plain unit-test harness). + "virtio_villain_tests", // Skip guest_test_uefi, as it's a no_std UEFI crate "guest_test_uefi", // Exclude various proc_macro crates, since they don't compile successfully diff --git a/flowey/flowey_lib_hvlite/src/build_nextest_virtio_villain_tests.rs b/flowey/flowey_lib_hvlite/src/build_nextest_virtio_villain_tests.rs new file mode 100644 index 0000000000..a2d6790226 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/build_nextest_virtio_villain_tests.rs @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Build and archive the `virtio_villain_tests` nextest suite. +//! +//! This is the virtio-villain analogue of [`crate::build_nextest_vmm_tests`]: +//! it emits a self-contained cargo-nextest archive for the +//! `virtio_villain_tests` package. + +use crate::common::CommonProfile; +use flowey::node::prelude::*; +use flowey_lib_common::run_cargo_build::CargoBuildProfile; +use flowey_lib_common::run_cargo_nextest_run::build_params::NextestBuildParams; +use flowey_lib_common::run_cargo_nextest_run::build_params::TestPackages; + +/// Type-safe wrapper around a built nextest archive containing the +/// virtio-villain tests. +#[derive(Serialize, Deserialize)] +pub struct NextestVirtioVillainTestsArchive { + #[serde(rename = "virtio_villain_tests.tar.zst")] + pub archive_file: PathBuf, +} + +impl Artifact for NextestVirtioVillainTestsArchive {} + +flowey_request! { + pub struct Request { + /// Target triple to build the tests for. + pub target: target_lexicon::Triple, + /// Cargo profile to build the tests with. + pub profile: CommonProfile, + /// Resulting nextest archive. + pub archive: WriteVar, + } +} + +new_flow_node!(struct Node); + +impl FlowNode for Node { + type Request = Request; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + ctx.import::(); + ctx.import::(); + ctx.import::(); + ctx.import::(); + } + + fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let mut ambient_deps = vec![ctx.reqv(crate::install_openvmm_rust_build_essential::Request)]; + + if matches!( + ctx.platform(), + FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu) + ) { + ambient_deps.push(ctx.reqv(|v| { + flowey_lib_common::install_dist_pkg::Request::Install { + package_names: vec![ + "libssl-dev".into(), + "pkg-config".into(), + "build-essential".into(), + ], + done: v, + } + })); + } + + for Request { + target, + profile, + archive, + } in requests + { + let injected_env = ctx.reqv(|v| crate::init_cross_build::Request { + target: target.clone(), + injected_env: v, + }); + + let build_params = NextestBuildParams { + packages: ReadVar::from_static(TestPackages::Crates { + crates: vec!["virtio_villain_tests".into()], + }), + features: Default::default(), + no_default_features: false, + target: target.clone(), + profile: match profile { + CommonProfile::Release => CargoBuildProfile::Release, + CommonProfile::Debug => CargoBuildProfile::Debug, + }, + extra_env: injected_env, + }; + + let openvmm_repo_path = ctx.reqv(crate::git_checkout_openvmm_repo::req::GetRepoDir); + + let archive_file = + ctx.reqv(|v| flowey_lib_common::run_cargo_nextest_archive::Request { + friendly_label: "virtio_villain_tests".into(), + working_dir: openvmm_repo_path, + build_params, + pre_run_deps: ambient_deps.clone(), + archive_file: v, + }); + + ctx.emit_minor_rust_step("report built virtio_villain_tests", |ctx| { + let archive_file = archive_file.claim(ctx); + let archive = archive.claim(ctx); + |rt| { + let archive_file = rt.read(archive_file); + rt.write(archive, &NextestVirtioVillainTestsArchive { archive_file }); + } + }); + } + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs b/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs index 7aaee4767a..f5e83f2006 100644 --- a/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs +++ b/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs @@ -68,6 +68,18 @@ flowey_request! { pub disable_remote_artifacts: bool, /// Whether to reuse VHDs created with prep_steps pub reuse_prepped_vhds: bool, + /// Whether to resolve and stage the UEFI firmware (`mu_msvm`) into the + /// content dir. + pub stage_uefi: bool, + /// Whether to resolve and stage the Windows `virtio-win` guest drivers + /// into the content dir. + pub stage_virtio_win: bool, + /// Whether to resolve and stage the virtio-villain guest artifact (the + /// initramfs and `tests.tsv`) into the content dir. + /// + /// Only the virtio-villain runner needs this, so its callers set this + /// true; the standard vmm_tests callers set it false. + pub stage_virtio_villain: bool, } } @@ -79,6 +91,7 @@ impl SimpleFlowNode for Node { fn imports(ctx: &mut ImportCtx<'_>) { ctx.import::(); ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); ctx.import::(); @@ -110,6 +123,9 @@ impl SimpleFlowNode for Node { use_relative_paths, disable_remote_artifacts, reuse_prepped_vhds, + stage_uefi, + stage_virtio_win, + stage_virtio_villain, } = request; let arch = CommonArch::from_architecture(vmm_tests_target.architecture)?; @@ -138,10 +154,15 @@ impl SimpleFlowNode for Node { }) }); - let uefi = - ctx.reqv(|v| crate::download_uefi_mu_msvm::Request::GetMsvmFd { arch, msvm_fd: v }); + let uefi = stage_uefi.then(|| { + ctx.reqv(|v| crate::download_uefi_mu_msvm::Request::GetMsvmFd { arch, msvm_fd: v }) + }); + + let virtio_win_dir = stage_virtio_win + .then(|| ctx.reqv(crate::resolve_openvmm_test_virtio_win::Request::Get)); - let virtio_win_dir = ctx.reqv(crate::resolve_openvmm_test_virtio_win::Request::Get); + let virtio_villain = stage_virtio_villain + .then(|| ctx.reqv(|v| crate::resolve_virtio_villain::Request::Get(arch, v))); // In CI, unstable test failures are non-gating and should be reported as // passing (with a warning). Outside of CI, unstable test failures are @@ -170,14 +191,15 @@ impl SimpleFlowNode for Node { let test_linux_initrd = test_linux_initrd.claim(ctx); let test_linux_kernel = test_linux_kernel.claim(ctx); let test_linux_bzimage = test_linux_bzimage.claim(ctx); - let uefi = uefi.claim(ctx); - let virtio_win_dir = virtio_win_dir.claim(ctx); + let uefi = uefi.map(|v| v.claim(ctx)); + let virtio_win_dir = virtio_win_dir.map(|v| v.claim(ctx)); + let virtio_villain = virtio_villain.map(|v| v.claim(ctx)); let release_igvm_files_dir = release_igvm_files.claim(ctx); move |rt| { let test_linux_initrd = rt.read(test_linux_initrd); let test_linux_kernel = rt.read(test_linux_kernel); let test_linux_bzimage = test_linux_bzimage.map(|v| rt.read(v)); - let uefi = rt.read(uefi); + let uefi = uefi.map(|v| rt.read(v)); let release_igvm_files_dir = rt.read(release_igvm_files_dir); let test_content_dir = rt.read(test_content_dir); @@ -470,18 +492,32 @@ impl SimpleFlowNode for Node { )?; } - let uefi_dir = test_content_dir.join(match arch { - CommonArch::Aarch64 => { - "hyperv.uefi.mscoreuefi.AARCH64.RELEASE/MsvmAARCH64/RELEASE_CLANGPDB/FV" - } - CommonArch::X86_64 => { - "hyperv.uefi.mscoreuefi.x64.RELEASE/MsvmX64/RELEASE_VS2022/FV" - } - }); - fs_err::create_dir_all(&uefi_dir)?; - fs_err::copy(uefi, uefi_dir.join("MSVM.fd"))?; + if let Some(virtio_villain) = virtio_villain { + let artifact = rt.read(virtio_villain); + fs_err::copy( + artifact.initramfs, + test_content_dir.join(arch_dir).join("villain-initrd"), + )?; + fs_err::copy( + artifact.tsv, + test_content_dir.join(arch_dir).join("villain-tests.tsv"), + )?; + } + + if let Some(uefi) = uefi { + let uefi_dir = test_content_dir.join(match arch { + CommonArch::Aarch64 => { + "hyperv.uefi.mscoreuefi.AARCH64.RELEASE/MsvmAARCH64/RELEASE_CLANGPDB/FV" + } + CommonArch::X86_64 => { + "hyperv.uefi.mscoreuefi.x64.RELEASE/MsvmX64/RELEASE_VS2022/FV" + } + }); + fs_err::create_dir_all(&uefi_dir)?; + fs_err::copy(uefi, uefi_dir.join("MSVM.fd"))?; + } - { + if let Some(virtio_win_dir) = virtio_win_dir { let src = rt.read(virtio_win_dir); let dst = test_content_dir.join("virtio-win"); let _ = fs_err::remove_dir_all(&dst); diff --git a/flowey/flowey_lib_hvlite/src/lib.rs b/flowey/flowey_lib_hvlite/src/lib.rs index b9d4afa76f..0e1e4ec739 100644 --- a/flowey/flowey_lib_hvlite/src/lib.rs +++ b/flowey/flowey_lib_hvlite/src/lib.rs @@ -15,6 +15,7 @@ pub mod build_hypestv; pub mod build_igvmfilegen; pub mod build_incubator; pub mod build_nextest_unit_tests; +pub mod build_nextest_virtio_villain_tests; pub mod build_nextest_vmm_tests; pub mod build_ohcldiag_dev; pub mod build_openhcl_boot; @@ -59,6 +60,7 @@ pub mod resolve_openvmm_qemu; pub mod resolve_openvmm_test_initrd; pub mod resolve_openvmm_test_linux_kernel; pub mod resolve_openvmm_test_virtio_win; +pub mod resolve_virtio_villain; pub mod run_cargo_build; pub mod run_cargo_nextest_run; pub mod run_igvmfilegen; @@ -68,4 +70,5 @@ pub mod run_test_igvm_agent_rpc_server; pub mod stop_test_igvm_agent_rpc_server; pub mod test_nextest_unit_tests_archive; pub mod test_nextest_vmm_tests_archive; +pub mod test_virtio_villain; pub mod write_incubator_target_runner; diff --git a/flowey/flowey_lib_hvlite/src/resolve_virtio_villain.rs b/flowey/flowey_lib_hvlite/src/resolve_virtio_villain.rs new file mode 100644 index 0000000000..6cb7d334ce --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/resolve_virtio_villain.rs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Download the virtio-villain guest test artifact from the `openvmm-deps` +//! GitHub release, or use a local directory if specified. +//! +//! virtio-villain ships as its own versioned artifact +//! (`openvmm-test-virtio-villain...tar.gz`) containing the guest +//! `initramfs.cpio.gz` (a static musl `init` that drives the virtio fault +//! injection) and `tests.tsv` (the enumerated test list). The +//! `virtio_villain_tests` crate consumes these two files via the +//! `VILLAIN_INITRAMFS` / `VILLAIN_TSV` env vars. +//! +//! By default this resolves the artifact from the `openvmm-deps` release pinned +//! by [`crate::_jobs::cfg_versions::OPENVMM_DEPS`]; pass a local path override to +//! use a locally built artifact instead. + +use crate::common::CommonArch; +use flowey::node::prelude::*; +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +/// Resolved paths to the two files that make up the virtio-villain artifact. +#[derive(Clone, Serialize, Deserialize)] +pub struct VirtioVillainArtifact { + /// Path to `initramfs.cpio.gz`. + pub initramfs: PathBuf, + /// Path to `tests.tsv`. + pub tsv: PathBuf, +} + +flowey_config! { + /// Config for the resolve_virtio_villain node. + pub struct Config { + /// Specify version of the github release to pull from. + pub version: Option, + /// Use a locally-downloaded virtio-villain artifact directory (which + /// must contain `initramfs.cpio.gz` and `tests.tsv`), keyed by + /// architecture. + pub local_paths: BTreeMap>, + } +} + +flowey_request! { + pub enum Request { + /// Get the resolved virtio-villain artifact for a given architecture. + Get(CommonArch, WriteVar), + } +} + +new_flow_node_with_config!(struct Node); + +impl FlowNodeWithConfig for Node { + type Request = Request; + type Config = Config; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + ctx.import::(); + } + + fn emit( + config: Config, + requests: Vec, + ctx: &mut NodeCtx<'_>, + ) -> anyhow::Result<()> { + let Config { + version, + local_paths, + } = config; + let mut deps: BTreeMap>> = BTreeMap::new(); + + for req in requests { + match req { + Request::Get(arch, var) => { + deps.entry(arch).or_default().push(var); + } + } + } + + if version.is_some() && !local_paths.is_empty() { + anyhow::bail!("cannot specify both `version` and `local_paths`"); + } + + if version.is_none() && local_paths.is_empty() { + anyhow::bail!("must specify either `version` or `local_paths`"); + } + + // -- end of req processing -- // + + if deps.is_empty() { + return Ok(()); + } + + if !local_paths.is_empty() { + ctx.emit_rust_step("use local virtio-villain artifact", |ctx| { + let deps = deps.claim(ctx); + let local_paths: BTreeMap<_, _> = local_paths + .into_iter() + .map(|(key, var)| (key, var.claim(ctx))) + .collect(); + move |rt| { + let resolved_paths: BTreeMap = local_paths + .into_iter() + .map(|(key, var)| (key, rt.read(var))) + .collect(); + + for (arch, vars) in deps { + let base_dir = resolved_paths.get(&arch).ok_or_else(|| { + anyhow::anyhow!("No local path specified for {:?}", arch) + })?; + let artifact = resolve_from_dir(base_dir)?; + rt.write_all(vars, &artifact) + } + + Ok(()) + } + }); + + return Ok(()); + } + + // The openvmm-test-virtio-villain...tar.gz archive contains + // `initramfs.cpio.gz` and `tests.tsv` at the archive root. Download one + // archive per requested architecture. + let needed_archives: BTreeSet = deps.keys().copied().collect(); + + let mut archives = BTreeMap::new(); + for arch in needed_archives { + let version = version.clone().expect("local requests handled above"); + let arch_str = match arch { + CommonArch::X86_64 => "x86_64", + CommonArch::Aarch64 => "aarch64", + }; + let archive = ctx.reqv(|v| flowey_lib_common::download_gh_release::Request { + repo_owner: "microsoft".into(), + repo_name: "openvmm-deps".into(), + needs_auth: false, + tag: version.clone(), + file_name: format!("openvmm-test-virtio-villain.{arch_str}.{version}.tar.gz"), + path: v, + }); + archives.insert(arch, archive); + } + + let persistent_dir = ctx.persistent_dir(); + + ctx.emit_rust_step("unpack virtio-villain artifacts", |ctx| { + let persistent_dir = persistent_dir.claim(ctx); + let archives = archives.claim(ctx); + let deps = deps.claim(ctx); + let version = version.clone().expect("local requests handled above"); + move |rt| { + let persistent_dir = persistent_dir.map(|d| rt.read(d)); + + let mut extract_dirs = BTreeMap::new(); + for (arch, archive) in archives { + let file = rt.read(archive); + let dir = flowey_lib_common::_util::extract::extract_tar_gz_if_new( + rt, + persistent_dir.as_deref(), + &file, + &version, + )?; + extract_dirs.insert(arch, dir); + } + + for (arch, vars) in deps { + let artifact = resolve_from_dir(&extract_dirs[&arch])?; + rt.write_all(vars, &artifact) + } + + Ok(()) + } + }); + + Ok(()) + } +} + +/// Resolve the two virtio-villain files from a directory, erroring if either is +/// missing. +fn resolve_from_dir(dir: &Path) -> anyhow::Result { + let initramfs = dir.join("initramfs.cpio.gz"); + let tsv = dir.join("tests.tsv"); + if !initramfs.exists() { + anyhow::bail!( + "virtio-villain initramfs.cpio.gz not found in {}", + dir.display() + ); + } + if !tsv.exists() { + anyhow::bail!("virtio-villain tests.tsv not found in {}", dir.display()); + } + Ok(VirtioVillainArtifact { initramfs, tsv }) +} diff --git a/flowey/flowey_lib_hvlite/src/test_virtio_villain.rs b/flowey/flowey_lib_hvlite/src/test_virtio_villain.rs new file mode 100644 index 0000000000..274fc9a112 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/test_virtio_villain.rs @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Shared core for running the `virtio_villain_tests` nextest suite against a +//! staged OpenVMM. +//! +//! This is the villain analogue of [`crate::test_nextest_vmm_tests_archive`]: +//! it owns everything both the local build-and-run job +//! ([`crate::_jobs::run_virtio_villain_tests`]) and the CI consume-archive job +//! ([`crate::_jobs::consume_and_test_nextest_virtio_villain_archive`]) have in +//! common, so those jobs stay thin and never drift apart. The only thing that +//! differs between them β€” build from source vs. run a prebuilt archive β€” is +//! expressed through the [`NextestRunKind`] passed in as [`Request::run_kind`]. +//! +//! Concretely, this node stages OpenVMM, the linux-direct guest kernel, and the +//! villain guest artifact (initramfs + `tests.tsv`) into a test content dir +//! (via [`crate::init_vmm_tests_env`], which also exports `TEST_OUTPUT_PATH` +//! for the publishable per-test petri logs), runs the nextest suite, and +//! publishes the JUnit + per-test logs so the `upload-petri-results` workflow +//! forwards them to the logview website. + +use crate::build_openvmm::OpenvmmOutput; +use crate::install_vmm_tests_deps::VmmTestsDepSelections; +use crate::run_cargo_nextest_run::NextestProfile; +use flowey::node::prelude::*; +use flowey_lib_common::run_cargo_nextest_run::NextestRunKind; +use std::collections::BTreeMap; + +flowey_request! { + pub struct Request { + /// Target triple the virtio-villain tests are built for. + pub target: target_lexicon::Triple, + /// The OpenVMM binary the villain crate launches (built from source by + /// the local job, or resolved from an artifact by the consume job). + pub openvmm: ReadVar, + /// How to run the suite: build it from source ([`NextestRunKind::BuildAndRun`]) + /// or run a prebuilt nextest archive ([`NextestRunKind::RunFromArchive`]). + pub run_kind: NextestRunKind, + /// Nextest profile to run under (e.g. `ci` to emit JUnit). + pub nextest_profile: NextestProfile, + /// Optional nextest filter expression to run only a subset of tests. + pub nextest_filter_expr: Option, + /// Also run known-failing (ignored) villain tests. + pub run_ignored: bool, + /// Directory to stage test content into and root the per-test logs at. + pub test_content_dir: ReadVar, + /// Artifact label prefix for the published results. To be picked up by + /// the `upload-petri-results` workflow (which globs `*-vmm-tests-logs`), + /// this MUST end in `-vmm-tests` so the logs attachment becomes + /// `-logs`. + pub junit_test_label: String, + /// Local-backend only: copy JUnit + logs into this published artifact + /// dir. + pub artifact_dir: Option>, + /// Install the vmm-tests runtime dependencies (and grant `/dev/kvm` + /// access on CI machines) before running. The local job leaves this to + /// the developer's already-provisioned machine. + pub install_deps: bool, + /// Forwarded to [`crate::init_vmm_tests_env`]: whether to forbid + /// downloading remote test artifacts (CI test machines provide + /// everything locally). + pub disable_remote_artifacts: bool, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Request; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + ctx.import::(); + ctx.import::(); + ctx.import::(); + ctx.import::(); + } + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Request { + target, + openvmm, + run_kind, + nextest_profile, + nextest_filter_expr, + run_ignored, + test_content_dir, + junit_test_label, + artifact_dir, + install_deps, + disable_remote_artifacts, + done, + } = request; + + // Stage OpenVMM, the linux-direct guest kernel, and the villain guest + // artifact (initramfs + tests.tsv) into the content dir and get the env + // the villain crate needs (VMM_TESTS_CONTENT_DIR points the known-paths + // resolver at all of them; TEST_OUTPUT_PATH is the publishable per-test + // log dir). All the other `register_*` inputs are vmm_tests-specific and + // villain does not use them. + let (test_log_path, get_test_log_path) = ctx.new_var(); + let extra_env = ctx.reqv(|get_env| crate::init_vmm_tests_env::Request { + test_content_dir: test_content_dir.clone(), + vmm_tests_target: target.clone(), + register_openvmm: Some(openvmm), + register_openvmm_vhost: None, + register_pipette_windows: None, + register_pipette_linux_musl: None, + register_guest_test_uefi: None, + register_tmks: None, + register_tmk_vmm: None, + register_tmk_vmm_linux_musl: None, + register_vmgstool: None, + register_vmgstool_dev: None, + register_tpm_guest_tests_windows: None, + register_tpm_guest_tests_linux: None, + register_test_igvm_agent_rpc_server: None, + disk_images_dir: None, + register_openhcl_igvm_files: Vec::new(), + get_test_log_path: Some(get_test_log_path), + get_env, + release_igvm_files: None, + use_relative_paths: false, + disable_remote_artifacts, + reuse_prepped_vhds: false, + // Linux-direct only: skip the UEFI firmware and Windows virtio-win + // driver downloads, which villain never uses. + stage_uefi: false, + stage_virtio_win: false, + stage_virtio_villain: true, + }); + + let mut pre_run_deps = Vec::new(); + if install_deps { + // Runtime dependencies for the test machine (no Rust toolchain + // needed to *run* an archive). + ctx.config(crate::install_vmm_tests_deps::Config { + selections: Some(VmmTestsDepSelections::Linux), + auto_install: None, + }); + pre_run_deps.push(ctx.reqv(crate::install_vmm_tests_deps::Request::Install)); + + // Make /dev/kvm accessible to the test (CI machines only). + if !matches!(ctx.backend(), FlowBackend::Local) { + pre_run_deps.push(ctx.emit_rust_step( + "ensure hypervisor device is accessible", + |_| { + |rt| { + if Path::new("/dev/kvm").exists() { + flowey::shell_cmd!(rt, "sudo chmod a+rw /dev/kvm").run()?; + } + Ok(()) + } + }, + )); + } + } + + // Use the repo's nextest config so the villain per-test slow-timeout + // overrides (`.config/nextest.toml`) apply. + let openvmm_repo_path = ctx.reqv(crate::git_checkout_openvmm_repo::req::GetRepoDir); + let nextest_config_file = openvmm_repo_path + .clone() + .map(ctx, |p| p.join(".config").join("nextest.toml")); + + let results = ctx.reqv(|results| crate::run_cargo_nextest_run::Request { + friendly_name: "virtio_villain_tests".into(), + run_kind, + nextest_profile, + nextest_filter_expr, + nextest_working_dir: Some(openvmm_repo_path), + nextest_config_file: Some(nextest_config_file), + run_ignored, + extra_env: Some(extra_env), + pre_run_deps, + results, + }); + + // Publish JUnit + per-test petri logs so the upload-petri-results + // workflow forwards them to logview. The publish step is claimed by the + // report step below, so logs upload even when tests fail. + let test_log_path = test_log_path.depending_on(ctx, &results); + let junit_xml = results.clone().map(ctx, |r| r.junit_xml); + let reported_results = ctx.reqv(|v| flowey_lib_common::publish_test_results::Request { + junit_xml, + test_label: junit_test_label, + attachments: BTreeMap::from([("logs".to_string(), (test_log_path, false))]), + output_dir: artifact_dir, + done: v, + }); + + ctx.emit_rust_step("report virtio-villain test results", |ctx| { + reported_results.claim(ctx); + done.claim(ctx); + let results = results.claim(ctx); + move |rt| { + let results = rt.read(results); + if !results.all_tests_passed { + anyhow::bail!("virtio-villain tests failed"); + } + Ok(()) + } + }); + + Ok(()) + } +} diff --git a/petri/src/tracing.rs b/petri/src/tracing.rs index bff3b943fc..cc538a6cb0 100644 --- a/petri/src/tracing.rs +++ b/petri/src/tracing.rs @@ -298,14 +298,11 @@ macro_rules! log { }; } -/// Initialize Petri tracing with the given output path for log files. +/// Initializes the global tracing subscriber and returns a [`PetriLogSource`] +/// rooted at `root_path`. /// -/// Events go to three places: -/// - `petri.jsonl`, in newline-separated JSON format. -/// - standard output, in human readable format. -/// - a log file, in human readable format. This file is `petri.log`, except -/// for events whose target ends in `.log`, which go to separate files named by -/// the target. +/// This installs the process-wide tracing subscriber and may only be called +/// once per process. pub fn try_init_tracing( root_path: &Path, default_level: LevelFilter, diff --git a/petri/src/vm/openvmm/construct.rs b/petri/src/vm/openvmm/construct.rs index ad93b75e33..b0adcdfb8c 100644 --- a/petri/src/vm/openvmm/construct.rs +++ b/petri/src/vm/openvmm/construct.rs @@ -172,20 +172,27 @@ impl PetriVmConfigOpenVmm { let mut load_mode = setup.load_firmware()?; - // If using pipette-as-init, replace the initrd with the pre-built - // one that has pipette injected. run_core() guarantees that - // prebuilt_initrd is set when uses_pipette_as_init is true. - if properties.uses_pipette_as_init { - if let LoadMode::Linux { initrd, .. } = &mut load_mode { - let prebuilt = properties - .prebuilt_initrd - .as_ref() - .expect("uses_pipette_as_init requires prebuilt_initrd"); - let file = std::fs::File::open(prebuilt).with_context(|| { - format!("failed to open prebuilt initrd at {}", prebuilt.display()) - })?; - *initrd = Some(file); - } + // If a prebuilt initrd was provided, use it in place of the initrd + // produced by `load_firmware()`. This covers both the pipette-as-init + // path (run_core() injects pipette into a prebuilt initrd) and callers + // that supply their own fully-formed initrd (e.g. the virtio-villain + // runner, whose initramfs is its own PID-1 `init`). + // + // Preserve the pipette-as-init invariant: run_core() guarantees a + // prebuilt initrd is set in that case, and without it pipette-as-init + // would silently fall back to the firmware initrd and hang confusingly. + anyhow::ensure!( + !properties.uses_pipette_as_init || properties.prebuilt_initrd.is_some(), + "uses_pipette_as_init requires a prebuilt initrd, but none was set" + ); + if let Some(prebuilt) = properties.prebuilt_initrd.as_ref() { + let LoadMode::Linux { initrd, .. } = &mut load_mode else { + unreachable!("prebuilt_initrd is only meaningful for Linux direct boot") + }; + let file = std::fs::File::open(prebuilt).with_context(|| { + format!("failed to open prebuilt initrd at {}", prebuilt.display()) + })?; + *initrd = Some(file); } let (emulated_serial_config, log_stream_tasks, linux_direct_serial_agent) = diff --git a/vmm_tests/petri_artifact_resolver_openvmm_known_paths/src/lib.rs b/vmm_tests/petri_artifact_resolver_openvmm_known_paths/src/lib.rs index 45830a6e29..c9a2bfb2b0 100644 --- a/vmm_tests/petri_artifact_resolver_openvmm_known_paths/src/lib.rs +++ b/vmm_tests/petri_artifact_resolver_openvmm_known_paths/src/lib.rs @@ -76,6 +76,11 @@ impl petri_artifacts_core::ResolveTestArtifact for OpenvmmKnownPathsTestArtifact _ if id == petritools::PETRITOOLS_EROFS_X64 => petritools_erofs_path(MachineArch::X86_64), _ if id == petritools::PETRITOOLS_EROFS_AARCH64 => petritools_erofs_path(MachineArch::Aarch64), + _ if id == virtio_villain::VIRTIO_VILLAIN_INITRD_X64 => virtio_villain_path(MachineArch::X86_64, VirtioVillainFile::Initrd), + _ if id == virtio_villain::VIRTIO_VILLAIN_INITRD_AARCH64 => virtio_villain_path(MachineArch::Aarch64, VirtioVillainFile::Initrd), + _ if id == virtio_villain::VIRTIO_VILLAIN_TSV_X64 => virtio_villain_path(MachineArch::X86_64, VirtioVillainFile::Tsv), + _ if id == virtio_villain::VIRTIO_VILLAIN_TSV_AARCH64 => virtio_villain_path(MachineArch::Aarch64, VirtioVillainFile::Tsv), + _ if id == loadable::PCAT_FIRMWARE_X64 => pcat_firmware_path(), _ if id == loadable::SVGA_FIRMWARE_X64 => svga_firmware_path(), _ if id == loadable::UEFI_FIRMWARE_X64 => uefi_firmware_path(MachineArch::X86_64), @@ -234,6 +239,10 @@ pub fn resolve_bundle_name(id: ErasedArtifactHandle) -> Option<&'static str> { _ if id == loadable::LINUX_DIRECT_TEST_INITRD_AARCH64 => Some("aarch64/initrd"), _ if id == petritools::PETRITOOLS_EROFS_X64 => Some("x64/petritools.erofs"), _ if id == petritools::PETRITOOLS_EROFS_AARCH64 => Some("aarch64/petritools.erofs"), + _ if id == virtio_villain::VIRTIO_VILLAIN_INITRD_X64 => Some("x64/villain-initrd"), + _ if id == virtio_villain::VIRTIO_VILLAIN_INITRD_AARCH64 => Some("aarch64/villain-initrd"), + _ if id == virtio_villain::VIRTIO_VILLAIN_TSV_X64 => Some("x64/villain-tests.tsv"), + _ if id == virtio_villain::VIRTIO_VILLAIN_TSV_AARCH64 => Some("aarch64/villain-tests.tsv"), _ if id == loadable::UEFI_FIRMWARE_X64 => { Some("hyperv.uefi.mscoreuefi.x64.RELEASE/MsvmX64/RELEASE_VS2022/FV/MSVM.fd") } @@ -275,6 +284,15 @@ enum OpenhclExtras { UmDbg, } +/// Which file of the virtio-villain guest artifact to resolve. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +enum VirtioVillainFile { + /// The guest `initramfs.cpio.gz`. + Initrd, + /// The `tests.tsv` metadata file. + Tsv, +} + /// The architecture specific fragment of the name of the directory used by rust when referring to specific targets. fn target_arch_path(arch: MachineArch) -> &'static str { match arch { @@ -662,6 +680,32 @@ fn petritools_erofs_path(arch: MachineArch) -> anyhow::Result { ) } +/// Path to our packaged virtio-villain guest artifact file (initramfs or tsv). +fn virtio_villain_path(arch: MachineArch, file: VirtioVillainFile) -> anyhow::Result { + use petri_artifacts_vmm_test::artifacts::virtio_villain; + let id = match (arch, file) { + (MachineArch::X86_64, VirtioVillainFile::Initrd) => { + virtio_villain::VIRTIO_VILLAIN_INITRD_X64.erase() + } + (MachineArch::Aarch64, VirtioVillainFile::Initrd) => { + virtio_villain::VIRTIO_VILLAIN_INITRD_AARCH64.erase() + } + (MachineArch::X86_64, VirtioVillainFile::Tsv) => { + virtio_villain::VIRTIO_VILLAIN_TSV_X64.erase() + } + (MachineArch::Aarch64, VirtioVillainFile::Tsv) => { + virtio_villain::VIRTIO_VILLAIN_TSV_AARCH64.erase() + } + }; + get_path( + ".packages/underhill-deps-private", + resolve_bundle_name(id).unwrap(), + MissingCommand::Restore { + description: "virtio-villain guest artifact", + }, + ) +} + /// Path to our packaged linux direct test kernel. fn linux_direct_arm_image_path() -> anyhow::Result { use petri_artifacts_vmm_test::artifacts::loadable; diff --git a/vmm_tests/petri_artifacts_vmm_test/src/lib.rs b/vmm_tests/petri_artifacts_vmm_test/src/lib.rs index f0d8e87f76..d79f6694d3 100644 --- a/vmm_tests/petri_artifacts_vmm_test/src/lib.rs +++ b/vmm_tests/petri_artifacts_vmm_test/src/lib.rs @@ -201,6 +201,29 @@ pub mod artifacts { } } + /// virtio-villain guest test artifacts (from OpenVMM deps). + /// + /// virtio-villain ships as its own versioned `openvmm-deps` release + /// artifact containing a guest `initramfs.cpio.gz` (a static PID-1 `init` + /// that drives the virtio fault injection) and a `tests.tsv` metadata file + /// enumerating the test list. Both are staged into the test content + /// directory and resolved via the known-paths resolver like any other + /// test artifact. + pub mod virtio_villain { + use petri_artifacts_core::declare_artifacts; + + declare_artifacts! { + /// virtio-villain guest initramfs (x64) + VIRTIO_VILLAIN_INITRD_X64, + /// virtio-villain guest initramfs (aarch64) + VIRTIO_VILLAIN_INITRD_AARCH64, + /// virtio-villain `tests.tsv` metadata (x64) + VIRTIO_VILLAIN_TSV_X64, + /// virtio-villain `tests.tsv` metadata (aarch64) + VIRTIO_VILLAIN_TSV_AARCH64, + } + } + /// OpenHCL IGVM artifacts pub mod openhcl_igvm { use petri_artifacts_common::tags::IsLoadable; diff --git a/vmm_tests/virtio_villain_tests/Cargo.toml b/vmm_tests/virtio_villain_tests/Cargo.toml new file mode 100644 index 0000000000..3a6d338de0 --- /dev/null +++ b/vmm_tests/virtio_villain_tests/Cargo.toml @@ -0,0 +1,43 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "virtio_villain_tests" +edition.workspace = true +rust-version.workspace = true + +[[test]] +name = "villain" +# Custom libtest-mimic harness: one trial is generated per villain test, so +# `cargo nextest run -p virtio_villain_tests` discovers and runs each villain +# test in its own process. Requires the villain image/TSV (see tests/villain.rs). +harness = false + +[dependencies] +petri.workspace = true +petri_artifacts_common.workspace = true +petri_artifacts_vmm_test.workspace = true +petri_artifact_resolver_openvmm_known_paths.workspace = true +openvmm_defs.workspace = true +virtio_resources.workspace = true +disk_backend_resources.workspace = true +net_backend_resources.workspace = true +serial_core.workspace = true +vm_resource.workspace = true +pal_async.workspace = true +virtio_spec.workspace = true + +anyhow.workspace = true +fs-err.workspace = true +libtest-mimic.workspace = true +tempfile.workspace = true +tracing.workspace = true +# `unix_socket` is cross-platform (provides `UnixListener` on Windows too), so +# it is an unconditional dependency: the vsock device is attached on every host. +unix_socket.workspace = true + +[dev-dependencies] +test_with_tracing.workspace = true + +[lints] +workspace = true diff --git a/vmm_tests/virtio_villain_tests/src/known_failures.rs b/vmm_tests/virtio_villain_tests/src/known_failures.rs new file mode 100644 index 0000000000..7d0bf40c64 --- /dev/null +++ b/vmm_tests/virtio_villain_tests/src/known_failures.rs @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! OpenVMM-specific list of villain tests that OpenVMM does not currently pass +//! and are therefore marked *ignored* in the libtest-mimic harness. +//! +//! This list lives here (in the OpenVMM tree), not upstream in villain, +//! because it describes OpenVMM-specific outcomes. Entries fall into three +//! kinds, distinguished by their `reason`: +//! - **OpenVMM bugs** β€” a device-model defect, linked to a filed +//! `microsoft/openvmm#NNNN` issue. Remove the entry when the bug is fixed so +//! the test runs (and gates) in CI again. +//! - **Not an OpenVMM bug** β€” a villain test defect, a precondition the test +//! fails to establish, or an assertion that is not spec-grounded. OpenVMM +//! behaves correctly, but the test still reports failure, so it stays ignored. +//! The `reason` starts with "Not an OpenVMM bug:" and explains why. +//! - **Accepted-by-design** β€” a behavioral difference OpenVMM will not change +//! (e.g. RNG0004); the `reason` captures the full rationale inline. +//! +//! Ignored tests behave like libtest `#[ignore]` tests: +//! - They are **skipped by default**, so CI (which runs with the default +//! `run-ignored` setting) does not run them. This keeps the gate green +//! against known product bugs and, importantly, avoids paying the per-test +//! timeout for cases that wedge OpenVMM (see below). +//! - They can still be run locally during fix development with +//! `cargo nextest run -p virtio_villain_tests --run-ignored all` +//! (or `--run-ignored ignored-only` to run *only* the known failures). +//! +//! Why ignore rather than invert (XFAIL)? Several of these are unrecoverable +//! host hangs: OpenVMM's virtio worker spins on a malformed descriptor chain +//! and the VM never powers off (and cannot even be torn down). Such a test can +//! only be ended by the external nextest timeout, so it can never reach an +//! in-harness "expected failure" inversion. Skipping is both correct and much +//! cheaper. + +/// A villain test that OpenVMM does not currently pass, and that we therefore +/// skip by default (mark ignored). +pub struct KnownFailure { + /// The villain test name (matches `tests.tsv` column 1 / `vv.test=`). + pub name: &'static str, + /// Human-readable reason. For OpenVMM bugs, reference the tracking issue, + /// e.g. `"… (microsoft/openvmm#4045)"`. For tests where OpenVMM behaves + /// correctly, start with `"Not an OpenVMM bug:"` and explain why the test + /// still fails. + pub reason: &'static str, +} + +/// The known-failure list. Keep sorted by name. +/// +/// The residual failures from the first full-suite CI run were root-caused +/// against OpenVMM and villain source. The genuine device-model bugs were filed +/// as `microsoft/openvmm#4045-4049` (grouped by root cause); the remainder are +/// villain test defects / spec-permitted behavior that OpenVMM handles correctly +/// but the test still flags. Keep the `reason` accurate to which kind each is. +pub const KNOWN_FAILURES: &[KnownFailure] = &[ + KnownFailure { + name: "B0002", + reason: "virtio-blk out-of-range read near sector UINT64_MAX: unchecked \ + LBA arithmetic wraps past the capacity check and returns \ + success with zeros (microsoft/openvmm#4046)", + }, + KnownFailure { + name: "B0081", + reason: "virtio-blk out-of-range write at sector UINT64_MAX: unchecked \ + LBA arithmetic wraps past the capacity check; the write is \ + accepted and stored (microsoft/openvmm#4046)", + }, + KnownFailure { + name: "B0082", + reason: "Not an OpenVMM bug: villain test defect. The test advertises \ + 32 KiB of physically-contiguous GPA from 8 anonymous mmap pages \ + but only translates the first page to a PFN; OpenVMM performs \ + finite, correct I/O over the GPAs as programmed.", + }, + KnownFailure { + name: "B0091", + reason: "virtio-blk out-of-range read at sector UINT64_MAX: unchecked \ + LBA arithmetic wraps past the capacity check and returns \ + success with zeros (microsoft/openvmm#4046)", + }, + KnownFailure { + name: "B0125", + reason: "virtio-blk request descriptor-chain loop: OpenVMM detects the \ + bad chain but re-parses the same head forever, wedging the \ + worker unrecoverably (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "B0139", + reason: "Not an OpenVMM bug: stale/currently-conformant. The discard \ + sector+num_sectors add is a latent overflow (same class as \ + microsoft/openvmm#4046), but for this test's operands the \ + request correctly returns S_IOERR.", + }, + KnownFailure { + name: "E0028", + reason: "virtio-pmem reports region size 0: the device maps a region but \ + neither offers VIRTIO_PMEM_F_SHMEM_REGION nor populates the \ + start/size config fields (microsoft/openvmm#4048)", + }, + KnownFailure { + name: "E0032", + reason: "virtio-pmem reports start/capacity 0: the device maps a region \ + but neither offers VIRTIO_PMEM_F_SHMEM_REGION nor populates the \ + start/size config fields (microsoft/openvmm#4048)", + }, + KnownFailure { + name: "M0024", + reason: "Not an OpenVMM bug: villain precondition defect. The test reads \ + QueueReset without negotiating VIRTIO_F_RING_RESET (which \ + OpenVMM does not offer); the register is only operative once \ + the feature is negotiated (spec 4.2.2.2).", + }, + KnownFailure { + name: "M0030", + reason: "Not an OpenVMM bug: unsubstantiated/stale. OpenVMM validates \ + the wrapped QueueDesc range before any ring access and fails \ + the queue-enable cleanly (state reset); no guest-kill \ + reproduces on the current code.", + }, + KnownFailure { + name: "P0003", + reason: "virtio-blk packed descriptor chain exceeding queue size: \ + detected but re-parsed forever, wedging the worker \ + (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "PCI0102", + reason: "Not an OpenVMM bug: villain assertion contradicts spec. The \ + test requires subsystem vendor 0x1AF4, but spec 4.1.2.1 permits \ + subsystem IDs to reflect the environment; OpenVMM reports the \ + Microsoft subsystem vendor 0x1414.", + }, + KnownFailure { + name: "PCI0114", + reason: "virtio-pci raises a spurious config-change interrupt (ISR bit \ + 1) at DRIVER_OK, so ISR reads 0x02 before any I/O \ + (microsoft/openvmm#4049)", + }, + KnownFailure { + name: "RNG0004", + reason: "virtio-rng non-atomic writable-descriptor handling. OpenVMM \ + streams entropy into an over-long writable descriptor (len past \ + end of RAM) and writes the backed portion of guest RAM (vring + \ + init) before erroring on the first unbacked page, killing the \ + guest before it emits its verdict. ACCEPTED BY DESIGN, not a \ + bug: this write-through behavior matches physical/vDPA hardware \ + (which faults per-access via the IOMMU after a partial write); \ + the software VMMs QEMU (map-first) and Cloud Hypervisor \ + (check_range) instead pre-validate the whole range and write \ + nothing. Not a host hang, not a spec violation (spec 2.7.5 puts \ + buffer validity on the driver), and no host-memory-safety issue. \ + OpenVMM is intentionally left as-is. Represents the villain \ + `*_huge_len_past_ram` (\"crosses end of RAM\") family", + }, + KnownFailure { + name: "S0048", + reason: "Not an OpenVMM bug: villain test-ordering defect. It writes \ + queue_size after the harness has already enabled the queue and \ + set DRIVER_OK; OpenVMM correctly ignores post-enable writes \ + (spec 4.1.4.3.2 requires configuring before enabling).", + }, + KnownFailure { + name: "T0001", + reason: "virtio-blk self-looping descriptor chain: detected but \ + re-parsed forever, wedging the worker unrecoverably \ + (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "T0002", + reason: "virtio-blk descriptor chain exceeding queue size: detected but \ + re-parsed forever, wedging the worker unrecoverably \ + (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "T0003", + reason: "virtio-blk out-of-bounds descriptor `next` index: detected but \ + re-parsed forever, wedging the worker unrecoverably \ + (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "T0008", + reason: "virtio-blk descriptor addr+len 64-bit wrap: unchecked GPA-range \ + arithmetic panics in checked builds (release returns S_IOERR) \ + (microsoft/openvmm#4047)", + }, + KnownFailure { + name: "T0022", + reason: "Not an OpenVMM bug: spec-permitted. Rejecting a driver-reoffered \ + in-flight head is a driver MUST-NOT (spec 2.7.6), not a device \ + requirement; OpenVMM re-parses each dequeue into fresh buffers \ + with no host memory-safety issue. Optional hardening only.", + }, + KnownFailure { + name: "T0025", + reason: "virtio-blk out-of-bounds available-ring entry: triggers the \ + wedging worker busy-loop (microsoft/openvmm#4045). Note: this \ + villain test always reports PASS, so it cannot detect a \ + regression of that bug.", + }, + KnownFailure { + name: "T0054", + reason: "virtio-blk descriptor chain length == queue_size + 1: detected \ + but re-parsed forever, wedging the worker \ + (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "T0073", + reason: "Not an OpenVMM bug: villain test defect. It hardcodes a \ + top-of-RAM address and points a writable descriptor at an \ + unowned page instead of discovering the real RAM top; OpenVMM's \ + I/O is finite and in-range.", + }, + KnownFailure { + name: "T0082", + reason: "virtio-blk descriptor with addr/len/flags/next all UINT_MAX: \ + the invalid indirect/OOB access is detected but re-parsed \ + forever, wedging the worker (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "T0084", + reason: "virtio-blk available ring full of out-of-bounds descriptor \ + indices: triggers the wedging worker busy-loop \ + (microsoft/openvmm#4045)", + }, + KnownFailure { + name: "Z0014", + reason: "Not an OpenVMM bug: villain test defect. OpenVMM advertises no \ + VIRTIO_BLK_F_ZONED and correctly returns S_UNSUPP; the test \ + never negotiates ZONED and waits for only 1 of 64 completions.", + }, +]; + +/// Look up the known-failure entry for `name`, if present. +pub fn lookup(name: &str) -> Option<&'static KnownFailure> { + KNOWN_FAILURES.iter().find(|e| e.name == name) +} diff --git a/vmm_tests/virtio_villain_tests/src/lib.rs b/vmm_tests/virtio_villain_tests/src/lib.rs new file mode 100644 index 0000000000..9c9ce15f21 --- /dev/null +++ b/vmm_tests/virtio_villain_tests/src/lib.rs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Library half of the virtio-villain OpenVMM test runner: villain test +//! enumeration, serial verdict parsing, the OpenVMM known-failure list, and the +//! per-test VM driver. +//! +//! The nextest harness entrypoint lives in the `tests/villain.rs` +//! `harness = false` `[[test]]` target, which enumerates the villain +//! `tests.tsv` into one libtest-mimic trial per test. + +pub mod known_failures; +pub mod run; +pub mod supported_devices; +pub mod villain; diff --git a/vmm_tests/virtio_villain_tests/src/run.rs b/vmm_tests/virtio_villain_tests/src/run.rs new file mode 100644 index 0000000000..e8a2d256df --- /dev/null +++ b/vmm_tests/virtio_villain_tests/src/run.rs @@ -0,0 +1,309 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Runs a single villain test in one OpenVMM VM and reads its serial verdict. + +use crate::villain::VerdictScan; +use anyhow::Context as _; +use petri_artifacts_common::tags::MachineArch; +use std::io::BufRead; +use std::path::PathBuf; + +/// Guest kernel initcalls to blacklist so the in-guest virtio drivers don't +/// claim the devices villain wants to drive itself. Mirrors villain's own +/// runner (`run`, `skip_initcalls`). +const INITCALL_BLACKLIST: &str = concat!( + "virtio_blk_init,", + "virtio_net_driver_init,", + "virtio_console_init,", + "virtio_balloon_driver_init,", + "virtio_rng_driver_init,", + "virtio_mem_driver_init,", + "virtio_pmem_driver_init,", + "virtio_iommu_drv_init,", + "virtio_fs_init,", + "virtio_vsock_init,", + "virtio_rtc_drv_init,", + "virtio_watchdog_driver_init,", + "virtio_input_driver_init,", + "ext4_init_fs,", + "fuse_init,", + "audit_init,", + "cpufreq_core_init", +); + +/// Static, per-run VM parameters shared by every villain test. +#[derive(Clone)] +pub struct VmParams { + /// Path to villain's `initramfs.cpio.gz` (bare PID-1 `init`). + pub initramfs: PathBuf, + /// Guest architecture. + pub arch: MachineArch, + /// Guest RAM in bytes. + pub mem_bytes: u64, +} + +/// Boot one VM for `test_name`, wait for it to halt, and scan the serial +/// console log for the verdict marker. +/// +/// `mmio` selects the virtio transport the kitchen-sink devices are attached +/// over: `true` attaches them on the virtio-MMIO bus (required by villain's +/// `TEST_FLAG_MMIO` / `M####` transport tests, which probe the MMIO register +/// interface and would otherwise self-`[SKIP]`), `false` uses PCIe. See +/// [`crate::villain::VillainTest::is_mmio`]. +/// +/// `log_source` must be rooted at this test's own directory so that its +/// `linux.log` (the teed serial0 console) is isolated from other tests. +pub fn run_one( + params: &VmParams, + artifacts: &petri::TestArtifacts, + log_source: &petri::PetriLogSource, + test_name: &str, + mmio: bool, +) -> anyhow::Result { + // Backing resources for the kitchen-sink devices. These must live until + // after the VM tears down, so they are owned here and only their paths + // are moved into the config closure. + let pmem_file = tempfile::NamedTempFile::new().context("failed to create pmem backing file")?; + pmem_file + .as_file() + .set_len(128 * 1024 * 1024) + .context("failed to size pmem backing file")?; + let pmem_path = pmem_file.path().to_string_lossy().into_owned(); + + let fs_dir = tempfile::tempdir().context("failed to create virtio-fs root")?; + let fs_path = fs_dir.path().to_string_lossy().into_owned(); + + let vsock_dir = tempfile::tempdir().context("failed to create vsock dir")?; + let vsock_socket = vsock_dir.path().join("vsock"); + // Bind the vsock listener up front so a failure is a hard error rather + // than a silently-omitted device (which would make vsock tests falsely + // self-SKIP). The socket lives in a fresh unique tempdir, so binding is + // expected to succeed on any functioning host. `unix_socket` provides + // `UnixListener` on all platforms, so this needs no target gating. + let vsock_listener = unix_socket::UnixListener::bind(&vsock_socket).with_context(|| { + format!( + "failed to bind vsock listener at {}", + vsock_socket.display() + ) + })?; + + let arch = params.arch; + let console = match arch { + MachineArch::X86_64 => "ttyS0", + MachineArch::Aarch64 => "ttyAMA0", + }; + let cmdline = + format!("console={console} initcall_blacklist={INITCALL_BLACKLIST} vv.test={test_name}"); + + let initramfs = params.initramfs.clone(); + let mem_bytes = params.mem_bytes; + + let log_path = log_source.output_dir().join("linux.log"); + + pal_async::DefaultPool::run_with(async |driver| -> anyhow::Result<()> { + let mut post_test_hooks = Vec::new(); + let petri_params = petri::PetriTestParams { + test_name, + logger: log_source, + post_test_hooks: &mut post_test_hooks, + }; + + let resolver = petri::ArtifactResolver::resolver(artifacts); + let firmware = petri::Firmware::linux_direct(&resolver, arch); + let vm_artifacts = petri::PetriVmArtifacts::::new( + &resolver, firmware, arch, false, + ) + .context("firmware/arch not compatible with OpenVMM backend")?; + + let builder = petri::PetriVmBuilder::minimal(petri_params, vm_artifacts, &driver)? + .with_serial_output() + .with_processor_topology(petri::ProcessorTopology { + vp_count: 1, + ..Default::default() + }) + .with_memory(petri::MemoryConfig { + startup_bytes: mem_bytes, + ..Default::default() + }) + .with_prebuilt_initrd(initramfs) + .modify_backend(move |b| { + attach_kitchen_sink( + b, + cmdline, + pmem_path, + fs_path, + vsock_socket, + vsock_listener, + mmio, + ) + }); + + let mut vm = builder + .run_without_agent() + .await + .context("failed to boot villain VM")?; + + // Villain runs the selected test, prints `[TAG] `, then powers + // off (reboot(RB_POWER_OFF)). Wait for that halt. + // + // A genuinely wedged device model (e.g. a malformed descriptor chain + // that sends OpenVMM's virtio worker into a non-terminating loop) will + // never power off and cannot even be torn down. We deliberately do NOT + // paper over that here: the nextest per-test `slow-timeout` + // (.config/nextest.toml) terminates such a test and reports it as a + // failure, which is the correct outcome for a real product bug. + let halt = vm + .wait_for_halt() + .await + .context("villain VM did not halt")?; + tracing::info!(?halt, "villain VM halted"); + vm.teardown().await.context("failed to tear down VM")?; + // This runner uses minimal() and attaches no device that registers a + // post-test hook, so none should have accumulated. If that ever + // changes, fail loudly rather than silently dropping the hook (e.g. + // guest crash-dump extraction) β€” wire up hook execution here. + assert!( + post_test_hooks.is_empty(), + "villain runner would silently drop {} post-test hook(s); \ + wire up hook execution before attaching hook-registering devices", + post_test_hooks.len(), + ); + Ok(()) + })?; + + let log = fs_err::File::open(&log_path) + .with_context(|| format!("failed to open serial log {}", log_path.display()))?; + let mut scanner = crate::villain::VerdictScanner::default(); + for line in std::io::BufReader::new(log).lines() { + let line = + line.with_context(|| format!("failed to read serial log {}", log_path.display()))?; + if let Some(scan) = scanner.scan_line(&line, test_name) { + return Ok(scan); + } + } + + // Keep backing resources alive until here. + drop(pmem_file); + drop(fs_dir); + drop(vsock_dir); + + Ok(scanner.finish()) +} + +/// Attach every virtio device villain can probe, plus the required cmdline. +/// Devices OpenVMM does not model are simply omitted; villain self-SKIPs them. +/// +/// `mmio` selects the transport: `true` attaches the devices on the virtio-MMIO +/// bus (OpenVMM advertises each as an `LNRO0005` ACPI device so villain's +/// `virtio_mmio_find` discovers it), `false` attaches them as PCIe devices, +/// each on its own root port. +fn attach_kitchen_sink( + b: petri::openvmm::PetriVmConfigOpenVmm, + cmdline: String, + pmem_path: String, + fs_path: String, + vsock_socket: PathBuf, + vsock_listener: unix_socket::UnixListener, + mmio: bool, +) -> petri::openvmm::PetriVmConfigOpenVmm { + use openvmm_defs::config::LoadMode; + use openvmm_defs::config::PcieDeviceConfig; + use openvmm_defs::config::VirtioBus; + use vm_resource::IntoResource; + use vm_resource::Resource; + use vm_resource::kind::VirtioDeviceHandle; + + // Collect every virtio device villain can probe as an inner + // `Resource`. Each is later wrapped in a + // `VirtioPciDeviceHandle` and attached to its own PCIe root port. Devices + // OpenVMM does not model are simply omitted; villain self-SKIPs them. + let mut inner: Vec> = vec![ + // rng + virtio_resources::rng::VirtioRngHandle.into_resource(), + // block (RAM-backed, writable) + virtio_resources::blk::VirtioBlkHandle { + disk: disk_backend_resources::LayeredDiskHandle::single_layer( + disk_backend_resources::layer::RamDiskLayerHandle { + len: Some(64 * 1024 * 1024), + sector_size: None, + }, + ) + .into_resource(), + read_only: false, + } + .into_resource(), + // console (serial backend goes nowhere) + virtio_resources::console::VirtioConsoleHandle { + backend: serial_core::resources::DisconnectedSerialBackendHandle.into_resource(), + } + .into_resource(), + // net (null endpoint: drops tx, never rx) + virtio_resources::net::VirtioNetHandle { + max_queues: None, + mac_address: net_backend_resources::mac_address::MacAddress::new([ + 0x00, 0x15, 0x5d, 0x00, 0x00, 0x01, + ]), + endpoint: net_backend_resources::null::NullHandle.into_resource(), + } + .into_resource(), + // virtio-fs (built-in HostFs backend, no external virtiofsd) + virtio_resources::fs::VirtioFsHandle { + tag: "villainfs".into(), + fs: virtio_resources::fs::VirtioFsBackend::HostFs { + root_path: fs_path, + mount_options: String::new(), + }, + } + .into_resource(), + // pmem (file-backed) + virtio_resources::pmem::VirtioPmemHandle { path: pmem_path }.into_resource(), + ]; + + // vsock (the listener was bound by the caller so a bind failure is a hard + // error, not a silently-dropped device). + inner.push( + virtio_resources::vsock::VirtioVsockHandle { + guest_cid: 3, + base_path: vsock_socket.to_string_lossy().into_owned(), + listener: vsock_listener, + } + .into_resource(), + ); + + // MMIO transport tests need the devices on the virtio-MMIO bus; every other + // test uses PCIe root ports. + if mmio { + b.with_custom_config(move |c| { + // Replace the kernel command line wholesale. + let LoadMode::Linux { cmdline: cl, .. } = &mut c.load_mode else { + unreachable!("the villain kitchen-sink VM is always linux-direct") + }; + *cl = cmdline; + + // Each device becomes a separate virtio-MMIO slot; OpenVMM emits an + // `LNRO0005` ACPI device per slot so the guest (and thus villain's + // sysfs `virtio_mmio_find`) enumerates them. + for handle in inner { + c.virtio_devices.push((VirtioBus::Mmio, handle)); + } + }) + } else { + // One PCIe root port per device (segment 0, root complex 0). + b.with_pcie_root_topology(1, 1, inner.len() as u64) + .with_custom_config(move |c| { + // Replace the kernel command line wholesale. + let LoadMode::Linux { cmdline: cl, .. } = &mut c.load_mode else { + unreachable!("the villain kitchen-sink VM is always linux-direct") + }; + *cl = cmdline; + + for (i, handle) in inner.into_iter().enumerate() { + c.pcie_devices.push(PcieDeviceConfig { + port_name: format!("s0rc0rp{i}"), + resource: virtio_resources::VirtioPciDeviceHandle(handle).into_resource(), + }); + } + }) + } +} diff --git a/vmm_tests/virtio_villain_tests/src/supported_devices.rs b/vmm_tests/virtio_villain_tests/src/supported_devices.rs new file mode 100644 index 0000000000..a3ad4650e5 --- /dev/null +++ b/vmm_tests/virtio_villain_tests/src/supported_devices.rs @@ -0,0 +1,717 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Decides which villain tests the "kitchen-sink" VM should `#[ignore]` because +//! they can only `[SKIP]` on our configuration β€” and, crucially, *never* ignores +//! a test that currently runs and passes. +//! +//! # Why ignore, and why never over-ignore +//! +//! Each villain test either exercises OpenVMM (PASS/FAIL/WEDGED/…) or self-skips +//! because a precondition it needs isn't met on this VM. On the kitchen-sink VM +//! [`crate::villain::evaluate`] treats a `[SKIP]` as a **failure**: a skip means +//! the test exercised nothing, which is how a `#[cfg]` mistake once silently +//! dropped the vsock device. To keep the gate meaningful we therefore *ignore* +//! (don't run) the tests we can predict will skip. +//! +//! The hard rule is **never ignore a test that currently passes**. If we ignored +//! a passing test and someone later removed the capability it exercises, the test +//! would silently start skipping and no one would notice β€” exactly the regression +//! this suite exists to catch. So [`expected_skip`] is built and validated to +//! ignore only tests that genuinely skip today. +//! +//! # How the decision is made: metadata by default, exceptions where it's wrong +//! +//! The default rule is a **metadata pre-check**. Villain's `tests.tsv` records, +//! per test, the virtio feature bits (`required_features`) and virtqueue count +//! (`min_queues`) it needs. A test is expected to skip when it targets a device +//! we attach but asks for a feature we don't offer or more queues than we expose +//! (see `DeviceCaps`). This is derived directly from the device models, so it +//! tracks reality as the models change. +//! +//! Villain's metadata is imperfect, so a small, explicit set of exceptions +//! overrides the default rule. Every exception is a documented deviation with a +//! `TODO` to follow up on *why* villain's metadata didn't match: +//! +//! * `FORCE_RUN` β€” the pre-check predicts a skip but the test actually **passes** +//! (its body only exercises the feature when negotiated, and passes vacuously +//! otherwise). These must keep running so a real regression surfaces. +//! * `FORCE_IGNORE` β€” the pre-check predicts the test can run but it self-skips +//! at runtime because villain's `tests.tsv` *understates* the precondition +//! (e.g. a multiqueue test registered with `min_queues = 0`). +//! +//! Two structural rules round it out: tests for a device we don't attach can only +//! skip, and PCI-transport tests registered with `device_id 0` can never resolve +//! a device in villain's harness (a villain bug β€” see [`expected_skip`]). +//! +//! Known *product* failures (real device-model bugs) are handled separately in +//! [`crate::known_failures`]; both are OR'd together at trial construction. + +/// Transport-common feature bits OpenVMM's virtio transport offers for **every** +/// device, on top of whatever the device model itself advertises. The transport +/// unconditionally turns these on (see +/// `vm/devices/virtio/virtio/src/transport/core.rs`, `with_version_1(true)` and +/// `with_access_platform(true)`), so they must be included when deciding whether +/// a required feature is offered. +const COMMON_FEATURES: u64 = virtio_spec::VirtioDeviceFeatures::new() + .with_version_1(true) + .with_access_platform(true) + .into_bits(); + +/// The virtio capabilities the kitchen-sink VM exposes for one device. +struct DeviceCaps { + /// Virtio PCI device id, derived from [`virtio_spec::pci`] and + /// [`virtio_spec::VirtioDeviceType`]. + device_id: u16, + /// Device-specific feature bits the model advertises via its `traits()` + /// (the transport-common bits in [`COMMON_FEATURES`] are added by + /// [`DeviceCaps::offered_features`], not stored here). + device_features: u64, + /// Number of virtqueues the device exposes. + num_queues: u16, +} + +impl DeviceCaps { + /// The full set of feature bits negotiable on this device, i.e. the device's + /// own bits plus the transport-common [`COMMON_FEATURES`]. + fn offered_features(&self) -> u64 { + self.device_features | COMMON_FEATURES + } +} + +/// Ring feature bits every OpenVMM virtio device model advertises. +const RING_FEATURES: virtio_spec::VirtioDeviceFeatures = virtio_spec::VirtioDeviceFeatures::new() + .with_ring_indirect_desc(true) + .with_ring_event_idx(true) + .with_ring_packed(true); + +/// The virtio PCI device id for a device type. +const fn device_id(device_type: virtio_spec::VirtioDeviceType) -> u16 { + virtio_spec::pci::VIRTIO_PCI_DEVICE_ID_BASE + device_type.0 +} + +/// Per-device capabilities exposed by [`crate::run`]'s `attach_kitchen_sink`. +/// +/// Keep in sync with the device models. The `device_features` masks below are +/// the device-specific bits each model advertises from its `traits()` +/// implementation (ring features included, transport-common bits excluded): +/// +/// * net `vm/devices/net/virtio_net/src/lib.rs` (`traits`) +/// * block `vm/devices/virtio/virtio_blk/src/lib.rs` (`traits`) +/// * console `vm/devices/virtio/virtio_console/src/lib.rs` (`traits`) +/// * rng `vm/devices/virtio/virtio_rng/src/lib.rs` (`traits`) +/// * vsock `vm/devices/virtio/virtio_vsock/src/lib.rs` (`traits`) +/// * fs `vm/devices/virtio/virtiofs/src/virtio.rs` (`traits`) +/// * pmem `vm/devices/virtio/virtio_pmem/src/lib.rs` (`traits`) +static DEVICE_CAPS: [DeviceCaps; 7] = [ + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::NET), + device_features: RING_FEATURES + .with_in_order(true) + .with_device_specific_low(0x0001_1823) + // VIRTIO_NET_F_HOST_USO, overall bit 56. + .with_device_specific_high(1 << 5) + .into_bits(), + num_queues: 2, + }, + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::BLK), + device_features: RING_FEATURES + .with_device_specific_low( + virtio_spec::blk::VIRTIO_BLK_F_SEG_MAX + | virtio_spec::blk::VIRTIO_BLK_F_BLK_SIZE + | virtio_spec::blk::VIRTIO_BLK_F_FLUSH + | virtio_spec::blk::VIRTIO_BLK_F_TOPOLOGY + | virtio_spec::blk::VIRTIO_BLK_F_DISCARD, + ) + .into_bits(), + num_queues: 1, + }, + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::CONSOLE), + device_features: RING_FEATURES + // VIRTIO_CONSOLE_F_SIZE. + .with_device_specific_low(1 << 0) + .into_bits(), + num_queues: 2, + }, + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::RNG), + device_features: RING_FEATURES.into_bits(), + num_queues: 1, + }, + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::VSOCK), + device_features: virtio_spec::VirtioDeviceFeatures::new() + // VIRTIO_VSOCK_F_STREAM and VIRTIO_VSOCK_F_SEQPACKET. + .with_device_specific_low((1 << 0) | (1 << 2)) + .into_bits(), + num_queues: 3, + }, + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::FS), + device_features: RING_FEATURES.into_bits(), + num_queues: 3, + }, + DeviceCaps { + device_id: device_id(virtio_spec::VirtioDeviceType::PMEM), + device_features: RING_FEATURES.into_bits(), + num_queues: 1, + }, +]; + +/// The capabilities for `device_id`, if the kitchen-sink VM attaches it. +fn device_caps(device_id: u16) -> Option<&'static DeviceCaps> { + DEVICE_CAPS.iter().find(|c| c.device_id == device_id) +} + +/// A villain test whose metadata pre-check verdict we deliberately override, +/// with the reason and a follow-up to reconcile villain's metadata. +struct Exception { + /// The villain test name (`tests.tsv` column 1 / `vv.test=`). + name: &'static str, + /// Why the pre-check is wrong for this test, and what to follow up on. + reason: &'static str, +} + +/// Tests the metadata pre-check predicts will skip, but which actually **pass**: +/// their body only exercises the declared feature when it is negotiated and +/// passes vacuously otherwise. They must keep running so a genuine regression +/// (the test starting to fail) is caught rather than hidden behind `#[ignore]`. +const FORCE_RUN: &[Exception] = &[Exception { + name: "N0161", + reason: "tests.tsv declares VIRTIO_NET_F_RSC_EXT (bit 61), which OpenVMM's \ + net device does not offer, so the pre-check would ignore it. But \ + the body only validates a coalescing report *if* the device sets \ + RSC_INFO; with RSC_EXT un-negotiated it never does and the test \ + passes. Keep running it. TODO(villain): mark RSC_EXT optional in \ + tests.tsv so the metadata matches the test's actual behavior.", +}]; + +/// Tests the metadata pre-check predicts can run, but which self-`[SKIP]` at +/// runtime because villain's `tests.tsv` understates the precondition they check +/// for. Ignored with the specific reason; each carries a `TODO(villain)` to fix +/// the upstream metadata so it can be dropped from this hand-list. +const FORCE_IGNORE: &[Exception] = &[ + // --- Multiqueue: self-SKIP unless the device exposes >= 2 (or 3) virtqueues, + // but tests.tsv records min_queues = 0. OpenVMM's block exposes 1 request + // queue and net exposes 2 (no control/RSS queues). TODO(villain): record the + // real min_queues for these. + Exception { + name: "B0044", + reason: "self-SKIPs unless block exposes >= 2 virtqueues (nq < 2); \ + tests.tsv min_queues = 0 understates it. TODO(villain): set \ + min_queues.", + }, + Exception { + name: "B0054", + reason: "self-SKIPs unless block exposes >= 2 virtqueues (nq < 2); \ + tests.tsv min_queues = 0 understates it. TODO(villain): set \ + min_queues.", + }, + Exception { + name: "B0060", + reason: "self-SKIPs unless block exposes >= 2 virtqueues (nq < 2); \ + tests.tsv min_queues = 0 understates it. TODO(villain): set \ + min_queues.", + }, + Exception { + name: "B0080", + reason: "self-SKIPs unless block exposes >= 2 virtqueues (nq < 2); \ + tests.tsv min_queues = 0 understates it. TODO(villain): set \ + min_queues.", + }, + Exception { + name: "B0114", + reason: "self-SKIPs unless block exposes >= 2 virtqueues (nq < 2); \ + tests.tsv min_queues = 0 understates it. TODO(villain): set \ + min_queues.", + }, + Exception { + name: "B0131", + reason: "self-SKIPs unless block exposes >= 2 virtqueues (nq < 2); \ + tests.tsv min_queues = 0 understates it. TODO(villain): set \ + min_queues.", + }, + Exception { + name: "T0081", + reason: "block transport test self-SKIPs unless >= 2 virtqueues \ + (nq < 2); tests.tsv min_queues = 0 understates it. \ + TODO(villain): set min_queues.", + }, + Exception { + name: "T0083", + reason: "block transport test self-SKIPs unless >= 2 virtqueues \ + (nq < 2); tests.tsv min_queues = 0 understates it. \ + TODO(villain): set min_queues.", + }, + Exception { + name: "N0051", + reason: "self-SKIPs without VIRTIO_NET_F_HASH_REPORT or VIRTIO_NET_F_RSS \ + and >= 2 queues; OpenVMM net offers neither feature. \ + TODO(villain): record required_features/min_queues.", + }, + Exception { + name: "N0064", + reason: "self-SKIPs without VIRTIO_NET_F_RSS and >= 3 queues; OpenVMM \ + net offers neither. TODO(villain): record \ + required_features/min_queues.", + }, + Exception { + name: "N0065", + reason: "self-SKIPs without VIRTIO_NET_F_RSS and >= 3 queues; OpenVMM \ + net offers neither. TODO(villain): record \ + required_features/min_queues.", + }, + // --- Hot-add / resize / config-change: self-SKIP when the device reports no + // newly added slot or unchanged capacity. OpenVMM does not hot-add or resize + // virtio devices at runtime. TODO(villain): gate these behind a capability + // flag in tests.tsv. + Exception { + name: "B0200", + reason: "hot-add test self-SKIPs when the device reports no new slot \ + (new_slot == 0); OpenVMM does not hot-add virtio devices. \ + TODO(villain): gate on a hot-plug capability.", + }, + Exception { + name: "B0201", + reason: "hot-add test self-SKIPs when the device reports no new slot \ + (new_slot == 0); OpenVMM does not hot-add virtio devices. \ + TODO(villain): gate on a hot-plug capability.", + }, + Exception { + name: "B0202", + reason: "block-resize test self-SKIPs when capacity is unchanged; \ + OpenVMM's backing capacity is fixed. TODO(villain): gate on a \ + resize capability.", + }, + Exception { + name: "E0200", + reason: "hot-add test self-SKIPs when the device reports no new slot \ + (new_slot == 0); OpenVMM does not hot-add virtio devices. \ + TODO(villain): gate on a hot-plug capability.", + }, + Exception { + name: "N0200", + reason: "hot-add test self-SKIPs when the device reports no new slot \ + (new_slot == 0); OpenVMM does not hot-add virtio devices. \ + TODO(villain): gate on a hot-plug capability.", + }, + Exception { + name: "V0200", + reason: "hot-add test self-SKIPs when the device reports no new slot \ + (new_slot == 0); OpenVMM does not hot-add virtio devices. \ + TODO(villain): gate on a hot-plug capability.", + }, + // --- Console MULTIPORT / EMERG_WRITE: self-SKIP without the console feature + // they exercise. OpenVMM's console is single-port and offers neither. + // TODO(villain): record required_features for these. + Exception { + name: "C0030", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_EMERG_WRITE, which OpenVMM's \ + console does not offer. TODO(villain): record required_features.", + }, + Exception { + name: "C0032", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + Exception { + name: "C0033", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + Exception { + name: "C0034", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + Exception { + name: "C0038", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + Exception { + name: "C0039", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + Exception { + name: "C0040", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + Exception { + name: "C0041", + reason: "self-SKIPs without VIRTIO_CONSOLE_F_MULTIPORT; OpenVMM's console \ + is single-port. TODO(villain): record required_features.", + }, + // --- VIRTIO_F_NOTIFICATION_DATA: self-SKIP without the feature, which OpenVMM + // does not negotiate for any device. TODO(villain): record required_features. + Exception { + name: "F0036", + reason: "self-SKIPs without VIRTIO_F_NOTIFICATION_DATA, which OpenVMM does \ + not negotiate. TODO(villain): record required_features.", + }, + Exception { + name: "T0098", + reason: "self-SKIPs without VIRTIO_F_NOTIFICATION_DATA, which OpenVMM does \ + not negotiate. TODO(villain): record required_features.", + }, + Exception { + name: "T0101", + reason: "self-SKIPs without VIRTIO_F_NOTIFICATION_DATA, which OpenVMM does \ + not negotiate. TODO(villain): record required_features.", + }, + Exception { + name: "T0102", + reason: "self-SKIPs without VIRTIO_F_NOTIFICATION_DATA, which OpenVMM does \ + not negotiate. TODO(villain): record required_features.", + }, + Exception { + name: "T0103", + reason: "self-SKIPs without VIRTIO_F_NOTIFICATION_DATA, which OpenVMM does \ + not negotiate. TODO(villain): record required_features.", + }, + // --- MMIO feature self-skips (previously hand-listed in known_failures as + // M0031/M0032). tests.tsv records reqf = 0 for these MMIO tests, so the + // pre-check can't catch them. TODO(villain): record required_features. + Exception { + name: "M0031", + reason: "MMIO QueueNotify-with-notification-data test self-SKIPs without \ + VIRTIO_F_NOTIFICATION_DATA, which OpenVMM does not negotiate. \ + TODO(villain): record required_features (tsv reqf = 0).", + }, + Exception { + name: "M0032", + reason: "MMIO config-change-interrupt test self-SKIPs when the device \ + raises no config-change interrupt; OpenVMM's kitchen-sink \ + devices don't. TODO(villain): gate on a config-change capability \ + (tsv reqf = 0).", + }, + // --- Net offloads / stats: self-SKIP without the specific offload/stats + // feature, none of which OpenVMM's net device offers. TODO(villain): record + // required_features. + Exception { + name: "N0123", + reason: "self-SKIPs without VIRTIO_NET_F_HOST_ECN; OpenVMM net does not \ + offer it. TODO(villain): record required_features.", + }, + Exception { + name: "N0124", + reason: "self-SKIPs without VIRTIO_NET_F_HOST_UFO (deprecated); OpenVMM \ + net does not offer it. TODO(villain): record required_features.", + }, + Exception { + name: "N0125", + reason: "self-SKIPs without VIRTIO_NET_F_DEVICE_STATS; OpenVMM net does \ + not offer it. TODO(villain): record required_features.", + }, + Exception { + name: "N0126", + reason: "self-SKIPs without VIRTIO_NET_F_GUEST_HDRLEN; OpenVMM net does \ + not offer it. TODO(villain): record required_features.", + }, + Exception { + name: "N0130", + reason: "self-SKIPs when no vIOMMU device is present to translate under \ + VIRTIO_F_ACCESS_PLATFORM; the kitchen-sink VM attaches no \ + vIOMMU. TODO(villain): gate on a vIOMMU capability.", + }, + // --- PCI transport admin virtqueue: these probe the modern PCI common + // config admin queue (common_length >= 0x48 / VIRTIO_F_ADMIN_VQ) and + // self-SKIP when that PCI transport support is absent. OpenVMM does not + // implement the admin virtqueue. TODO(villain): record required_features / + // an admin-vq capability. + Exception { + name: "PCI0076", + reason: "self-SKIPs when the PCI common-config admin-queue registers are \ + absent (common_length < 0x48); OpenVMM has no admin virtqueue. \ + TODO(villain): gate on an admin-vq capability.", + }, + Exception { + name: "PCI0077", + reason: "self-SKIPs when the admin virtqueue is absent (common_length < \ + 0x48 / admin_num == 0); OpenVMM has no admin virtqueue. \ + TODO(villain): gate on an admin-vq capability.", + }, + Exception { + name: "PCI0078", + reason: "self-SKIPs when the admin virtqueue is absent (common_length < \ + 0x48 / admin_num == 0); OpenVMM has no admin virtqueue. \ + TODO(villain): gate on an admin-vq capability.", + }, + Exception { + name: "PCI0079", + reason: "self-SKIPs when the admin virtqueue is absent (common_length < \ + 0x48 / admin_num == 0); OpenVMM has no admin virtqueue. \ + TODO(villain): gate on an admin-vq capability.", + }, + Exception { + name: "PCI0117", + reason: "self-SKIPs without VIRTIO_F_ADMIN_VQ; OpenVMM does not implement \ + the admin virtqueue. TODO(villain): record required_features.", + }, + // --- Legacy PCI I/O BAR: self-SKIP when the legacy I/O-port resource can't be + // opened (fd < 0). OpenVMM's modern virtio-pci exposes no legacy I/O BAR. + // TODO(villain): gate these on a legacy-transport capability. + Exception { + name: "PCI0026", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0042", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0043", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0044", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0045", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0046", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0047", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0048", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + Exception { + name: "PCI0049", + reason: "self-SKIPs when the legacy PCI I/O-port resource can't be opened \ + (fd < 0); OpenVMM's modern virtio-pci has no legacy I/O BAR. \ + TODO(villain): gate on a legacy-transport capability.", + }, + // --- Other PCI-transport preconditions OpenVMM's layout doesn't satisfy. + Exception { + name: "PCI0070", + reason: "per-vector MSI-X mask test self-SKIPs when it can't find the \ + MSI-X table entry / queue-0 vector binding it needs; OpenVMM's \ + virtio-pci MSI-X layout doesn't satisfy the precondition. \ + TODO(villain): investigate the MSI-X table discovery.", + }, + Exception { + name: "PCI0072", + reason: "self-SKIPs when notify_off_multiplier reads 0; OpenVMM reports 0 \ + (a single shared notify register). TODO(villain): allow a zero \ + multiplier layout.", + }, +]; + +/// Look up an [`Exception`] by name in `list`. +fn find_exception(list: &'static [Exception], name: &str) -> Option<&'static Exception> { + list.iter().find(|e| e.name == name) +} + +/// Whether a villain test should be `#[ignore]`d because it can only `[SKIP]` on +/// the kitchen-sink VM, returning the reason if so. Returns `None` for tests we +/// expect to actually run (and must not hide behind `#[ignore]`). +/// +/// The decision order is: +/// 1. `FORCE_RUN` override β€” never ignore a test we know passes. +/// 2. `FORCE_IGNORE` override β€” ignore tests whose runtime precondition +/// `tests.tsv` understates. +/// 3. Device-agnostic PCI tests (`device_id == 0`, non-MMIO): villain's +/// `virtio_pci_find(0)` looks for a literal PCI device `0x0000`, which no +/// virtio device is, so these always `[SKIP]` on every hypervisor. This is a +/// villain harness bug (MMIO tests use `virtio_mmio_find`, which finds any +/// device; PCI has no equivalent "any device" path). TODO(villain): give the +/// PCI runner an any-device path for `device_id 0` transport tests. +/// 4. A device we don't attach β€” the test can only skip. +/// 5. The metadata pre-check: a required feature we don't offer, or more queues +/// than we expose. +pub fn expected_skip( + name: &str, + device_id: u16, + is_mmio: bool, + required_features: u64, + min_queues: u32, +) -> Option<&'static str> { + // 1. Never ignore a test we know runs and passes. + if find_exception(FORCE_RUN, name).is_some() { + return None; + } + + // 2. Ignore tests whose runtime precondition tests.tsv understates. + if let Some(e) = find_exception(FORCE_IGNORE, name) { + return Some(e.reason); + } + + // 3. Non-MMIO transport tests registered with device_id 0 can never resolve a + // device in villain's PCI runner, so they always skip. + if device_id == 0 && !is_mmio { + return Some( + "villain harness bug: a PCI-transport test registered with device_id \ + 0 is looked up via virtio_pci_find(0), which searches for a literal \ + PCI device 0x0000 that no virtio device is, so it always SKIPs on \ + every hypervisor. TODO(villain): add an any-device path to the PCI \ + runner (MMIO already has one via virtio_mmio_find).", + ); + } + + // Device-agnostic tests (device_id 0) that reach here are MMIO transport + // tests, which run against whatever device virtio_mmio_find locates; they are + // not device-gated, so the pre-check below doesn't apply. + if device_id == 0 { + return None; + } + + // 4. A device the kitchen-sink VM doesn't attach can only skip. + let Some(caps) = device_caps(device_id) else { + return Some("device is not attached by the kitchen-sink VM"); + }; + + // 5. Metadata pre-check against what OpenVMM actually offers. + if required_features & !caps.offered_features() != 0 { + return Some( + "requires a virtio feature OpenVMM's device model does not offer \ + (required_features has bits outside DeviceCaps::offered_features)", + ); + } + if min_queues > u32::from(caps.num_queues) { + return Some( + "requires more virtqueues than OpenVMM's device model exposes \ + (min_queues > DeviceCaps::num_queues)", + ); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use test_with_tracing::test; + + #[test] + fn attached_devices_have_caps_and_are_not_skipped() { + for caps in DEVICE_CAPS.iter() { + // A no-requirement test on an attached device must run. + assert!( + expected_skip("X0000", caps.device_id, false, 0, 0).is_none(), + "0x{:04x} is attached", + caps.device_id + ); + } + } + + #[test] + fn common_features_are_always_offered() { + // VERSION_1 (32) and ACCESS_PLATFORM (33) are offered for every device + // even though the models don't list them (the transport adds them). + for caps in DEVICE_CAPS.iter() { + assert_eq!(caps.offered_features() & COMMON_FEATURES, COMMON_FEATURES); + } + } + + #[test] + fn device_caps_anchor_values_match_virtio_spec() { + let net = device_caps( + virtio_spec::pci::VIRTIO_PCI_DEVICE_ID_BASE + virtio_spec::VirtioDeviceType::NET.0, + ) + .unwrap(); + let block = device_caps( + virtio_spec::pci::VIRTIO_PCI_DEVICE_ID_BASE + virtio_spec::VirtioDeviceType::BLK.0, + ) + .unwrap(); + + assert_eq!( + net.device_id, + virtio_spec::pci::VIRTIO_PCI_DEVICE_ID_BASE + virtio_spec::VirtioDeviceType::NET.0 + ); + assert_eq!( + block.device_id, + virtio_spec::pci::VIRTIO_PCI_DEVICE_ID_BASE + virtio_spec::VirtioDeviceType::BLK.0 + ); + assert!(virtio_spec::VirtioDeviceFeatures::from_bits(net.offered_features()).version_1()); + } + + #[test] + fn absent_device_is_expected_to_skip() { + // 0x1063 is present in tests.tsv but not attached by the kitchen sink. + assert!(expected_skip("D0001", 0x1063, false, 0, 0).is_some()); + } + + #[test] + fn device_agnostic_mmio_tests_run() { + assert!(expected_skip("M0001", 0x0000, true, 0, 0).is_none()); + } + + #[test] + fn device_agnostic_pci_tests_are_skipped() { + // device_id 0 + non-MMIO: villain's virtio_pci_find(0) never resolves. + assert!(expected_skip("PCI0050", 0x0000, false, 0, 0).is_some()); + } + + #[test] + fn missing_feature_is_expected_to_skip() { + // Require a bit block doesn't offer (VIRTIO_NET_F_RSS-ish high bit). + let missing_feature = virtio_spec::VirtioDeviceFeatures::new() + .with_device_specific_high(1 << 9) + .into_bits(); + assert!(expected_skip("X", 0x1042, false, missing_feature, 0).is_some()); + } + + #[test] + fn too_many_queues_is_expected_to_skip() { + // Block exposes 1 queue; a test needing 2 must be ignored. + assert!(expected_skip("X", 0x1042, false, 0, 2).is_some()); + } + + #[test] + fn force_run_overrides_missing_feature() { + // N0161 declares RSC_EXT (bit 61), which net doesn't offer, yet it must + // still run (it passes vacuously). + let rsc_ext = virtio_spec::VirtioDeviceFeatures::new() + .with_device_specific_high(1 << 10) + .into_bits(); + assert!(expected_skip("N0161", 0x1041, false, rsc_ext, 0).is_none()); + } + + #[test] + fn force_ignore_overrides_pre_check() { + // B0044 declares min_queues = 0 but self-skips at runtime. + assert!(expected_skip("B0044", 0x1042, false, 0, 0).is_some()); + } + + #[test] + fn exception_names_are_unique_and_disjoint() { + // A name must not appear twice, nor in both lists (contradictory). + let mut all: Vec<&str> = FORCE_RUN + .iter() + .chain(FORCE_IGNORE) + .map(|e| e.name) + .collect(); + let count = all.len(); + all.sort_unstable(); + all.dedup(); + assert_eq!(all.len(), count, "duplicate or overlapping exception names"); + } +} diff --git a/vmm_tests/virtio_villain_tests/src/villain.rs b/vmm_tests/virtio_villain_tests/src/villain.rs new file mode 100644 index 0000000000..f3c37c0847 --- /dev/null +++ b/vmm_tests/virtio_villain_tests/src/villain.rs @@ -0,0 +1,546 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Enumeration of virtio-villain tests and parsing of their serial verdicts. + +use anyhow::Context as _; +use std::path::Path; + +/// A single villain test, as described by one row of `tests.tsv`. +/// +/// `tests.tsv` is emitted by `init --list-tsv` (see villain `bin/init.c`) with +/// one tab-separated row per test and no header: +/// +/// ```text +/// name desc version spec_section device_id flags required_features min_queues +/// ``` +#[derive(Debug, Clone)] +pub struct VillainTest { + /// Test identifier, passed to the guest via `vv.test=` and matched + /// against the serial verdict marker. + pub name: String, + /// Human-readable description. + pub desc: String, + /// virtio spec version the test targets (`tests.tsv` column 3, free text, + /// e.g. `"1.2"`). + pub version: String, + /// virtio spec section the test references (`tests.tsv` column 4, e.g. + /// `"2.6.5"`). + pub spec_section: String, + /// Virtio PCI device ID the test targets (derived from + /// `virtio_spec::pci::VIRTIO_PCI_DEVICE_ID_BASE` and + /// `virtio_spec::VirtioDeviceType`). + pub device_id: u16, + /// Test flags bitfield (`tests.tsv` column 6; see villain `tests/test.h` + /// `TEST_FLAG_*`). Only [`TEST_FLAG_MMIO`] currently affects the harness. + pub flags: u8, + /// virtio feature bits the test requires (`tests.tsv` column 7). Not yet + /// consumed by the harness beyond logging. + pub required_features: u64, + /// Minimum virtqueue count the test needs (`tests.tsv` column 8). Not yet + /// consumed by the harness beyond logging. + pub min_queues: u32, +} + +/// villain `TEST_FLAG_MMIO` (`tests/test.h`): the test drives the virtio-MMIO +/// transport rather than PCI, so the VM must attach its devices on the MMIO bus +/// (otherwise the guest finds no MMIO device and the test self-`[SKIP]`s). +pub const TEST_FLAG_MMIO: u8 = 0x2; + +impl VillainTest { + /// Whether this test targets the virtio-MMIO transport (`TEST_FLAG_MMIO`). + /// + /// Such tests must be run in a VM whose virtio devices are attached over + /// the MMIO bus; see [`crate::run::run_one`]. + pub fn is_mmio(&self) -> bool { + self.flags & TEST_FLAG_MMIO != 0 + } +} + +/// Parse `tests.tsv` into the list of villain tests. +pub fn parse_tsv(path: &Path) -> anyhow::Result> { + let text = fs_err::read_to_string(path)?; + let mut tests = Vec::new(); + for (lineno, line) in text.lines().enumerate() { + let line = line.trim_end_matches('\r'); + if line.is_empty() { + continue; + } + let mut cols = line.split('\t'); + let name = cols + .next() + .filter(|s| !s.is_empty()) + .with_context(|| format!("{}:{}: missing test name", path.display(), lineno + 1))?; + // Columns: name, desc, version, spec_section, device_id, flags, + // required_features, min_queues. Parse strictly so a corrupt or + // format-changed tests.tsv fails loudly rather than silently producing + // bogus values. + let desc = cols.next().with_context(|| { + format!( + "{}:{}: missing description column", + path.display(), + lineno + 1 + ) + })?; + let version = cols.next().filter(|s| !s.is_empty()).with_context(|| { + format!("{}:{}: missing version column", path.display(), lineno + 1) + })?; + let spec_section = cols.next().filter(|s| !s.is_empty()).with_context(|| { + format!( + "{}:{}: missing spec_section column", + path.display(), + lineno + 1 + ) + })?; + let device_id = cols.next().with_context(|| { + format!( + "{}:{}: missing device_id column", + path.display(), + lineno + 1 + ) + })?; + let device_id = device_id + .strip_prefix("0x") + .and_then(|h| u16::from_str_radix(h, 16).ok()) + .with_context(|| { + format!( + "{}:{}: invalid device_id {:?} (expected 0x-prefixed u16 hex)", + path.display(), + lineno + 1, + device_id, + ) + })?; + let flags = cols + .next() + .with_context(|| format!("{}:{}: missing flags column", path.display(), lineno + 1))?; + let flags = flags.parse::().with_context(|| { + format!( + "{}:{}: invalid flags {:?} (expected decimal u8)", + path.display(), + lineno + 1, + flags, + ) + })?; + // required_features (col 7) and min_queues (col 8) are not yet consumed + // by the harness beyond being logged at the start of each test, but + // validate their presence and format so a column added or removed after + // `flags` fails loudly rather than silently shifting the remaining + // fields. + let required_features = cols.next().with_context(|| { + format!( + "{}:{}: missing required_features column", + path.display(), + lineno + 1 + ) + })?; + let required_features = required_features + .strip_prefix("0x") + .and_then(|h| u64::from_str_radix(h, 16).ok()) + .with_context(|| { + format!( + "{}:{}: invalid required_features {:?} (expected 0x-prefixed u64 hex)", + path.display(), + lineno + 1, + required_features, + ) + })?; + let min_queues = cols.next().with_context(|| { + format!( + "{}:{}: missing min_queues column", + path.display(), + lineno + 1 + ) + })?; + let min_queues = min_queues.parse::().with_context(|| { + format!( + "{}:{}: invalid min_queues {:?} (expected decimal integer)", + path.display(), + lineno + 1, + min_queues, + ) + })?; + + // min_queues (col 8) is the last column villain's `list_tests_tsv` + // emits. Reject any trailing column so a schema addition fails loudly + // here rather than being silently dropped. + if let Some(extra) = cols.next() { + anyhow::bail!( + "{}:{}: unexpected trailing column {:?} after min_queues \ + (tests.tsv schema changed?)", + path.display(), + lineno + 1, + extra, + ); + } + + tests.push(VillainTest { + name: name.to_string(), + desc: desc.to_string(), + version: version.to_string(), + spec_section: spec_section.to_string(), + device_id, + flags, + required_features, + min_queues, + }); + } + anyhow::ensure!(!tests.is_empty(), "no tests found in {}", path.display()); + Ok(tests) +} + +/// The verdict the guest emits on the serial console for a test. +/// +/// The guest prints `[] ` (villain `bin/init.c`), where `` is +/// one of these values. The device-health re-probe that distinguishes REJECT +/// from WEDGED is computed by the guest itself; the host only reads the marker. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// Device accepted the (valid) input and behaved correctly. + Pass, + /// Device correctly rejected the malformed input and stayed alive. + Reject, + /// Device absent or precondition unmet; the test did not run. + Skip, + /// Device accepted malformed input / misbehaved. + Fail, + /// Device stopped responding after the malformed input. + Wedged, + /// Known-failing test that failed as expected (guest-side xfail). + Xfail, + /// A test flagged as a known xfail that unexpectedly passed. Villain treats + /// this as a *failure* (`verdict_failed()` in `bin/init.c`): an XPASS means + /// the underlying bug is gone and the stale xfail marker must be removed. + Xpass, +} + +impl Verdict { + fn from_tag(tag: &str) -> Option { + Some(match tag { + "PASS" => Verdict::Pass, + "REJECT" => Verdict::Reject, + "SKIP" => Verdict::Skip, + "FAIL" => Verdict::Fail, + "WEDGED" => Verdict::Wedged, + "XFAIL" => Verdict::Xfail, + "XPASS" => Verdict::Xpass, + _ => return None, + }) + } + + /// Whether this verdict is a "good" outcome (the device model behaved + /// correctly or it is a guest-side xfail β€” neither of which indicates an + /// OpenVMM device-model bug). + /// + /// Note `Verdict::Skip` is deliberately **not** good: on the kitchen-sink VM + /// a skip means the device was absent or a precondition was unmet, so the + /// test exercised nothing. That is always a failure; tests for devices we do + /// not attach are `#[ignore]`d up front (see [`crate::supported_devices`]). + /// + /// `Verdict::Xpass` is likewise **not** good: villain only emits it for an + /// xfail-flagged test that unexpectedly passed, which it counts as a failure + /// so the stale marker gets noticed and removed. Treating it as success here + /// would silently mask that signal. + pub fn is_good(self) -> bool { + matches!(self, Verdict::Pass | Verdict::Reject | Verdict::Xfail) + } +} + +/// The result of scanning a serial log for a single test's verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VerdictScan { + /// The test's own `[TAG] ` marker was found. + Found(Verdict), + /// No villain output appeared at all β€” not even the startup banner β€” so + /// the guest never reached villain (e.g. it failed to boot). + NoMarkers, + /// Villain ran (its startup banner or other tests' markers appeared) but + /// this test's verdict line never printed β€” the device wedged or killed + /// the guest mid-test before emitting a verdict. + MarkerMissing, +} + +/// Incremental scanner for villain serial verdict markers. +#[derive(Default)] +pub struct VerdictScanner { + saw_any_marker: bool, + villain_started: bool, +} + +impl VerdictScanner { + /// Scans one serial log line. + /// + /// Returns `Some` as soon as `name`'s verdict marker is found; otherwise + /// the caller should continue scanning and call [`Self::finish`] at EOF. + pub fn scan_line(&mut self, line: &str, name: &str) -> Option { + let line = line.trim(); + // Markers look like "[PASS] blk.split.bad_desc". Find the closing + // bracket and split into tag + remainder. + let rest = line.strip_prefix('[')?; + let (tag, remainder) = rest.split_once(']')?; + let Some(verdict) = Verdict::from_tag(tag) else { + // Not a verdict tag. Villain's `[vv] ...` startup banner proves it + // booted and began running, so even if it later wedges before this + // test's verdict we can report `MarkerMissing` (started, no verdict) + // rather than `NoMarkers` (never ran). + if tag == "vv" { + self.villain_started = true; + } + return None; + }; + self.saw_any_marker = true; + // The name is the first whitespace-delimited token after the tag. + // Some markers carry a trailing reason, e.g. + // "[SKIP] D0001 (no device 0x1063)", so match only the first token + // rather than the whole remainder. Villain test names never contain + // whitespace (see `tests.tsv`). + (remainder.split_whitespace().next() == Some(name)).then_some(VerdictScan::Found(verdict)) + } + + /// Returns the scan result once EOF is reached without finding `name`. + pub fn finish(self) -> VerdictScan { + if self.saw_any_marker || self.villain_started { + VerdictScan::MarkerMissing + } else { + VerdictScan::NoMarkers + } + } +} + +/// Scan serial console log lines for `name`'s verdict marker. +pub fn scan_verdict_lines<'a>(lines: impl IntoIterator, name: &str) -> VerdictScan { + let mut scanner = VerdictScanner::default(); + for line in lines { + if let Some(scan) = scanner.scan_line(line, name) { + return scan; + } + } + scanner.finish() +} + +/// Scan a captured serial console log for `name`'s verdict marker. +/// +/// Villain prints one `[] ` line per test. A test may share the log +/// with unrelated boot output; we match the exact test name. +pub fn scan_verdict(log: &str, name: &str) -> VerdictScan { + scan_verdict_lines(log.lines(), name) +} + +/// Turn a scan result into a pass/fail outcome for one villain test. +/// +/// Fast-fail rule: a `SKIP` means the device was absent or a precondition was +/// unmet, so the test exercised nothing β€” always a **failure** (an unexercised +/// test must not masquerade as a pass). Tests for devices the kitchen-sink VM +/// does not attach are `#[ignore]`d up front (see [`crate::supported_devices`]) +/// so they never reach here in a normal run; force-running them with +/// `--run-ignored` correctly reports the absent-device skip as a failure. +/// +/// Otherwise [`Verdict::is_good`] decides, and a missing/absent marker (the +/// guest wedged or never booted) is a failure. +pub fn evaluate(scan: VerdictScan) -> anyhow::Result<()> { + match scan { + VerdictScan::Found(Verdict::Skip) => anyhow::bail!( + "SKIP: the device was absent or a precondition was unmet, so this test \ + exercised nothing. On the kitchen-sink VM a skip means a device we meant \ + to attach silently wasn't (a harness/config bug); if the device is \ + deliberately not attached, the test should be ignored via \ + supported_devices::expected_skip" + ), + VerdictScan::Found(v) => { + if v.is_good() { + Ok(()) + } else { + anyhow::bail!("{v:?}") + } + } + VerdictScan::MarkerMissing => { + anyhow::bail!("WEDGED (no verdict marker for this test)") + } + VerdictScan::NoMarkers => { + anyhow::bail!("FAIL (guest emitted no villain markers)") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + // Use the tracing-aware `#[test]` so `tracing` output appears in test logs + // (repo convention; see .github/copilot-instructions.md). + use test_with_tracing::test; + + #[test] + fn evaluate_skip_is_always_a_failure() { + // SKIP means nothing was exercised. Unsupported-device tests are ignored + // up front; anything that actually runs and skips is a failure. + assert!(evaluate(VerdictScan::Found(Verdict::Skip)).is_err()); + } + + #[test] + fn evaluate_good_and_bad_verdicts() { + assert!(evaluate(VerdictScan::Found(Verdict::Pass)).is_ok()); + assert!(evaluate(VerdictScan::Found(Verdict::Reject)).is_ok()); + assert!(evaluate(VerdictScan::Found(Verdict::Xfail)).is_ok()); + // XPASS is a failure by villain's own semantics (stale xfail marker). + assert!(evaluate(VerdictScan::Found(Verdict::Xpass)).is_err()); + assert!(evaluate(VerdictScan::Found(Verdict::Fail)).is_err()); + assert!(evaluate(VerdictScan::Found(Verdict::Wedged)).is_err()); + assert!(evaluate(VerdictScan::MarkerMissing).is_err()); + assert!(evaluate(VerdictScan::NoMarkers).is_err()); + } + + #[test] + fn scan_finds_exact_marker() { + let log = "\ +[vv] virtio-villain +[SKIP] blk.other +[PASS] blk.split.bad_desc +"; + assert_eq!( + scan_verdict(log, "blk.split.bad_desc"), + VerdictScan::Found(Verdict::Pass) + ); + assert_eq!( + scan_verdict(log, "blk.other"), + VerdictScan::Found(Verdict::Skip) + ); + } + + #[test] + fn scan_marker_with_trailing_reason() { + // Villain appends a reason to some SKIP markers; the name is still the + // first token (bin/init.c: "[SKIP] %s (no device 0x%04x)"). + let log = "[vv] virtio-villain\n[SKIP] D0001 (no device 0x1063)\n"; + assert_eq!( + scan_verdict(log, "D0001"), + VerdictScan::Found(Verdict::Skip) + ); + // A name that is a prefix of the marked test must not match. + assert_eq!(scan_verdict(log, "D000"), VerdictScan::MarkerMissing); + } + + #[test] + fn scan_banner_only_is_marker_missing() { + // Villain printed its startup banner but wedged before emitting any + // verdict: the guest *ran* villain, so this is MarkerMissing, not + // NoMarkers (which means villain never started). + let log = "booting...\n[vv] virtio-villain\nkernel panic\n"; + assert_eq!(scan_verdict(log, "M0030"), VerdictScan::MarkerMissing); + } + + #[test] + fn scan_missing_vs_no_markers() { + assert_eq!(scan_verdict("[FAIL] a\n", "b"), VerdictScan::MarkerMissing); + assert_eq!(scan_verdict("booting...\n", "b"), VerdictScan::NoMarkers); + } + + #[test] + fn parse_tsv_row() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + std::fs::write( + &p, + "blk.split.bad_desc\tvalidates descriptor\t1.2\t2.6.5\t0x1042\t0\t0x0000000000000000\t1\n", + ) + .unwrap(); + let tests = parse_tsv(&p).unwrap(); + assert_eq!(tests.len(), 1); + assert_eq!(tests[0].name, "blk.split.bad_desc"); + assert_eq!(tests[0].device_id, 0x1042); + assert_eq!(tests[0].flags, 0); + assert!(!tests[0].is_mmio()); + } + + #[test] + fn parse_tsv_mmio_flag() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // An MMIO-transport test: device-agnostic (0x0000) with flags == 2 + // (TEST_FLAG_MMIO), exactly as villain emits for the `M####` tests. + std::fs::write( + &p, + "M0001\tNon-32-bit access to MMIO control registers\t1.2\t4.2.2.2\t0x0000\t2\t0x0000000000000000\t0\n", + ) + .unwrap(); + let tests = parse_tsv(&p).unwrap(); + assert_eq!(tests[0].flags, TEST_FLAG_MMIO); + assert!(tests[0].is_mmio()); + } + + #[test] + fn parse_tsv_rejects_bad_flags() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // flags column present but not a decimal u8. + std::fs::write(&p, "t\tdesc\t1.2\t2.6.5\t0x0002\tnope\t0\t1\n").unwrap(); + let err = parse_tsv(&p).unwrap_err(); + assert!( + err.to_string().contains("invalid flags"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn parse_tsv_rejects_bad_device_id() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // device_id column present but not valid 0x-prefixed hex. + std::fs::write(&p, "t\tdesc\t1.2\t2.6.5\tnope\t0\t0\t1\n").unwrap(); + let err = parse_tsv(&p).unwrap_err(); + assert!( + err.to_string().contains("invalid device_id"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn parse_tsv_rejects_truncated_row() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // Has name/desc/version/spec_section but the device_id column is missing. + std::fs::write(&p, "t\tdesc\t1.2\t2.6.5\n").unwrap(); + let err = parse_tsv(&p).unwrap_err(); + assert!( + err.to_string().contains("missing device_id"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn parse_tsv_rejects_missing_trailing_columns() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // name..flags present (6 cols) but required_features/min_queues absent. + std::fs::write(&p, "t\tdesc\t1.2\t2.6.5\t0x1042\t0\n").unwrap(); + let err = parse_tsv(&p).unwrap_err(); + assert!( + err.to_string().contains("missing required_features"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn parse_tsv_rejects_bad_min_queues() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // min_queues column present but not a decimal integer. + std::fs::write(&p, "t\tdesc\t1.2\t2.6.5\t0x1042\t0\t0x0\tnope\n").unwrap(); + let err = parse_tsv(&p).unwrap_err(); + assert!( + err.to_string().contains("invalid min_queues"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn parse_tsv_rejects_trailing_column() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("tests.tsv"); + // All 8 columns present plus an unexpected 9th column. + std::fs::write(&p, "t\tdesc\t1.2\t2.6.5\t0x1042\t0\t0x0\t1\textra\n").unwrap(); + let err = parse_tsv(&p).unwrap_err(); + assert!( + err.to_string().contains("unexpected trailing column"), + "unexpected error: {err:#}" + ); + } +} diff --git a/vmm_tests/virtio_villain_tests/tests/villain.rs b/vmm_tests/virtio_villain_tests/tests/villain.rs new file mode 100644 index 0000000000..5bb2359e34 --- /dev/null +++ b/vmm_tests/virtio_villain_tests/tests/villain.rs @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Standalone test runner that drives [virtio-villain] against OpenVMM. +//! +//! virtio-villain is a guest-side virtio protocol fault-injection / conformance +//! suite: a static musl `init` (PID 1) that walks the virtio transports itself +//! and injects out-of-spec virtqueue inputs, printing a verdict marker per test +//! on the serial console before powering off. +//! +//! This binary uses **petri as a library** (like `burette`) plus +//! **libtest-mimic** to expose one test case per villain test. Each case boots +//! a single "kitchen-sink" OpenVMM VM (every supported virtio device attached; +//! a device that is absent makes the guest emit `[SKIP]`, which the harness +//! treats as a failure unless the device is one the kitchen-sink VM +//! deliberately does not attach β€” see [`supported_devices`]) with +//! `vv.test=` on the kernel command line, waits for the VM to halt, and +//! reads the `[TAG] ` verdict from petri's teed serial log. The virtio +//! devices are attached over PCIe, except for villain's virtio-MMIO transport +//! tests (`TEST_FLAG_MMIO` / `M####`), which are booted with the devices on the +//! virtio-MMIO bus so the guest actually exposes an MMIO transport to probe +//! (see [`villain::VillainTest::is_mmio`] and [`run::run_one`]). Villain tests +//! that OpenVMM is known to fail ([`known_failures`]) are marked *ignored*, so +//! CI skips them but they can still be run locally with `--run-ignored`. +//! +//! The villain `initramfs.cpio.gz` and `tests.tsv` are resolved from the +//! `openvmm-deps` release artifact via petri's known-paths resolver (staged +//! into `VMM_TESTS_CONTENT_DIR` by flowey), the same way the guest kernel (the +//! existing OpenVMM linux-direct test `vmlinux`) is. For local development +//! against a custom villain build, the `VILLAIN_INITRAMFS` and `VILLAIN_TSV` +//! environment variables override the resolved artifacts. +//! +//! [virtio-villain]: https://github.com/weltling/virtio-villain + +mod run { + pub use virtio_villain_tests::run::*; +} + +use anyhow::Context as _; +use libtest_mimic::Failed; +use libtest_mimic::Trial; +use petri_artifacts_common::tags::MachineArch; +use std::path::PathBuf; +use virtio_villain_tests::known_failures; +use virtio_villain_tests::supported_devices; +use virtio_villain_tests::villain; + +/// 512 MiB is enough for the linux-direct kernel plus the villain initramfs and +/// keeps the many one-VM-per-test trials lightweight. +const VILLAIN_MEM_BYTES: u64 = 512 * 1024 * 1024; + +/// Two-pass artifact resolution (the petri-tool / burette pattern): resolve the +/// reused OpenVMM linux-direct kernel for the host architecture. +fn register_artifacts(resolver: &petri::ArtifactResolver<'_>) { + let firmware = petri::Firmware::linux_direct(resolver, MachineArch::host()); + petri::PetriVmArtifacts::::new( + resolver, + firmware, + MachineArch::host(), + false, + ); +} + +/// Require the host-architecture villain initramfs artifact. +fn require_villain_initrd(resolver: &petri::ArtifactResolver<'_>) -> petri::ResolvedArtifact { + use petri_artifacts_vmm_test::artifacts::virtio_villain; + match MachineArch::host() { + MachineArch::X86_64 => resolver + .require(virtio_villain::VIRTIO_VILLAIN_INITRD_X64) + .erase(), + MachineArch::Aarch64 => resolver + .require(virtio_villain::VIRTIO_VILLAIN_INITRD_AARCH64) + .erase(), + } +} + +/// Require the host-architecture villain `tests.tsv` artifact. +fn require_villain_tsv(resolver: &petri::ArtifactResolver<'_>) -> petri::ResolvedArtifact { + use petri_artifacts_vmm_test::artifacts::virtio_villain; + match MachineArch::host() { + MachineArch::X86_64 => resolver + .require(virtio_villain::VIRTIO_VILLAIN_TSV_X64) + .erase(), + MachineArch::Aarch64 => resolver + .require(virtio_villain::VIRTIO_VILLAIN_TSV_AARCH64) + .erase(), + } +} + +/// Resolve a single villain artifact path via petri's known-paths resolver +/// (staged under `VMM_TESTS_CONTENT_DIR`). Used for the tsv when it is not +/// overridden by an env var; this does not initialize tracing. +fn resolve_villain_file( + pick: impl Fn(&petri::ArtifactResolver<'_>) -> petri::ResolvedArtifact, +) -> anyhow::Result { + let resolver = + petri_artifact_resolver_openvmm_known_paths::OpenvmmKnownPathsTestArtifactResolver::new(""); + let mut requirements = petri::TestArtifactRequirements::new(); + pick(&petri::ArtifactResolver::collector(&mut requirements)); + let artifacts = requirements + .resolve(&resolver) + .context("failed to resolve villain artifact")?; + Ok(pick(&petri::ArtifactResolver::resolver(&artifacts)) + .get() + .to_path_buf()) +} + +fn resolve_trial_artifacts(name: &str) -> anyhow::Result<(petri::TestArtifacts, PathBuf)> { + let resolver = + petri_artifact_resolver_openvmm_known_paths::OpenvmmKnownPathsTestArtifactResolver::new( + name, + ); + let mut requirements = petri::TestArtifactRequirements::new(); + register_artifacts(&petri::ArtifactResolver::collector(&mut requirements)); + require_villain_initrd(&petri::ArtifactResolver::collector(&mut requirements)); + requirements.require( + petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY, + petri::RemoteAccess::LocalOnly, + false, + ); + let artifacts = requirements + .resolve(&resolver) + .context("failed to resolve test artifacts")?; + let resolver = petri::ArtifactResolver::resolver(&artifacts); + register_artifacts(&resolver); + let initrd = require_villain_initrd(&resolver).get().to_path_buf(); + Ok((artifacts, initrd)) +} + +fn main() -> anyhow::Result<()> { + let mut args = libtest_mimic::Arguments::from_args(); + + let initramfs_override = std::env::var_os("VILLAIN_INITRAMFS").map(PathBuf::from); + let tsv_override = std::env::var_os("VILLAIN_TSV").map(PathBuf::from); + + // The test list comes from the tsv, which is needed for both `--list` and + // running. Prefer the override; otherwise resolve it from the artifact. + let tsv = match tsv_override { + Some(tsv) => tsv, + None => resolve_villain_file(require_villain_tsv) + .context("failed to resolve villain tests.tsv")?, + }; + let tests = villain::parse_tsv(&tsv)?; + let trials: Vec = tests + .into_iter() + .map(|test| { + let initramfs_override = initramfs_override.clone(); + let name = test.name.clone(); + let desc = test.desc.clone(); + let version = test.version.clone(); + let spec_section = test.spec_section.clone(); + let device_id = test.device_id; + let flags = test.flags; + let required_features = test.required_features; + let min_queues = test.min_queues; + let mmio = test.is_mmio(); + // Ignore (don't even boot a VM for) tests that can only self-SKIP on + // the kitchen-sink VM: a device we don't attach, a feature/queue count + // we don't offer, or a villain harness limitation. They report as + // ignored, not as false passes; see `supported_devices::expected_skip` + // for the rule and the reason string it returns. Known product + // failures are ignored too (see `known_failures`). + let expected_skip = supported_devices::expected_skip( + &name, + device_id, + mmio, + required_features, + min_queues, + ); + if let Some(reason) = expected_skip { + tracing::debug!(name, reason, "ignoring villain test (expected skip)"); + } + let ignored = expected_skip.is_some() || known_failures::lookup(&name).is_some(); + Trial::test(test.name.clone(), move || -> Result<(), Failed> { + let (artifacts, initrd) = + resolve_trial_artifacts(&name).map_err(|e| Failed::from(format!("{e:#}")))?; + let output_dir = + artifacts.get(petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY); + let logger = + petri::try_init_tracing(output_dir, tracing::level_filters::LevelFilter::INFO) + .context("failed to initialize tracing") + .map_err(|e| Failed::from(format!("{e:#}")))?; + logger.log_test_start(&name); + tracing::info!( + name, + desc, + version, + spec_section, + device_id = format_args!("{device_id:#06x}"), + flags, + required_features = format_args!("{required_features:#018x}"), + min_queues, + mmio, + "running villain test" + ); + let initramfs = initramfs_override.clone().unwrap_or_else(|| initrd.clone()); + let params = run::VmParams { + initramfs, + arch: MachineArch::host(), + mem_bytes: VILLAIN_MEM_BYTES, + }; + let result = run::run_one(¶ms, &artifacts, &logger, &name, mmio) + .and_then(villain::evaluate); + // Write the petri.passed/petri.failed marker (and log the + // outcome to petri.jsonl) so the logview uploader counts this + // test. villain tests are never "unstable" β€” known failures are + // skipped via the ignored flag instead. + logger.log_test_result(&result, false); + result.map_err(|e| Failed::from(format!("{e:#}"))) + }) + .with_ignored_flag(ignored) + }) + .collect(); + + // These VMs are heavy; run them one at a time in-process. Under nextest, + // each selected test runs in its own process. Running this binary directly + // with multiple non-ignored trials is not supported because petri tracing is + // process-global and can only be initialized once. + if args.test_threads.is_none() { + args.test_threads = Some(1); + } + + libtest_mimic::run(&args, trials).exit(); +}