[WIP] Run virtio-villain tests via petri in CI

.config/nextest.toml

@@ -23,6 +23,24 @@
 # https://nexte.st/docs/configuration/per-test-overrides/#override-precedence
 # https://nexte.st/docs/configuration/?h=hierar#hierarchical-configuration
 
+# virtio-villain tests each boot a dedicated VM that runs one fault-injection
+# test and powers off. A test that never halts indicates a genuinely wedged
+# guest or device model (e.g. a malformed-descriptor host hang that OpenVMM
+# cannot recover from or even tear down) -- a real product bug. Terminate such
+# a test so it is reported as a failure rather than hanging the run. Require
+# two threads per test to bound concurrency and avoid vCPU starvation that
+# would otherwise cause spurious slow boots. (The package name does not contain
+# "vmm_tests", so the package(~vmm_tests) overrides below do not apply here.)
+[[profile.default.overrides]]
+filter = 'package(virtio_villain_tests)'
+slow-timeout = { period = "20s", terminate-after = 3 }
+threads-required = 2
+
+[[profile.ci.overrides]]
+filter = 'package(virtio_villain_tests)'
+slow-timeout = { period = "20s", terminate-after = 3 }
+threads-required = 2
+
 [[profile.ci.overrides]]
 # use fuzzy-matching for the package() to allow out-of-tree tests to use the
 # same profile

Cargo.lock

@@ -9816,6 +9816,29 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "virtio_villain_tests"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "disk_backend_resources",
+ "fs-err",
+ "libtest-mimic",
+ "net_backend_resources",
+ "openvmm_defs",
+ "pal_async",
+ "petri",
+ "petri_artifact_resolver_openvmm_known_paths",
+ "petri_artifacts_common",
+ "serial_core",
+ "tempfile",
+ "tracing",
+ "unix_socket",
+ "virtio_resources",
+ "vm_resource",
+]
+
 [[package]]
 name = "virtio_vsock"
 version = "0.0.0"

Cargo.toml

@@ -24,6 +24,7 @@ members = [
   # VMM tests
   "vmm_tests/prep_steps",
   "vmm_tests/vmm_tests",
+  "vmm_tests/virtio_villain_tests",
   # Performance tests
   "petri/burette",
   # hyper-v tooling

flowey/flowey_hvlite/src/pipelines/checkin_gates.rs

@@ -203,6 +203,14 @@ impl IntoPipeline for CheckinGatesCli {
         let (pub_vmm_tests_archive_linux_aarch64, use_vmm_tests_archive_linux_aarch64) =
             pipeline.new_typed_artifact("aarch64-linux-vmm-tests-archive");
 
+        // virtio-villain nextest archive (built on the linux build machine,
+        // consumed by the KVM villain test job) + the OpenVMM binary it drives,
+        // reused from the linux build job below.
+        let (pub_villain_archive_linux_x86, use_villain_archive_linux_x86) =
+            pipeline.new_typed_artifact("x64-linux-virtio-villain-tests-archive");
+        let mut pub_villain_archive_linux_x86 = Some(pub_villain_archive_linux_x86);
+        let mut use_openvmm_linux_x86 = None;
+
         // wrap each publish handle in an option, so downstream code can
         // `.take()` the handle when emitting the corresponding job
         let mut pub_vmm_tests_archive_linux_x86 = Some(pub_vmm_tests_archive_linux_x86);
@@ -803,6 +811,7 @@ impl IntoPipeline for CheckinGatesCli {
             match arch {
                 CommonArch::X86_64 => {
                     vmm_tests_artifacts_linux_x86.use_openvmm = Some(use_openvmm.clone());
+                    use_openvmm_linux_x86 = Some(use_openvmm.clone());
                     vmm_tests_artifacts_linux_x86.use_openvmm_vhost =
                         Some(use_openvmm_vhost.clone());
                     vmm_tests_artifacts_linux_x86.use_prep_steps = Some(use_prep_steps.clone());
@@ -986,6 +995,23 @@ impl IntoPipeline for CheckinGatesCli {
                             ),
                         }
                     });
+
+                    // Also build the virtio-villain nextest archive on this
+                    // build machine so the KVM villain test job can consume it
+                    // without a Rust toolchain.
+                    let pub_villain_archive_linux_x86 =
+                        pub_villain_archive_linux_x86.take().unwrap();
+                    job = job.publish(pub_villain_archive_linux_x86, |archive| {
+                        flowey_lib_hvlite::build_nextest_virtio_villain_tests::Request {
+                            target: CommonTriple::Common {
+                                arch,
+                                platform: CommonPlatform::LinuxGnu,
+                            }
+                            .as_triple(),
+                            profile: CommonProfile::from_release(release),
+                            archive,
+                        }
+                    });
                 }
                 CommonArch::Aarch64 => {
                     let pub_vmm_tests_archive_linux_aarch64 =
@@ -1760,6 +1786,60 @@ impl IntoPipeline for CheckinGatesCli {
             }
         }
 
+        // virtio-villain conformance suite: a separate, parallel per-PR job that
+        // drives OpenVMM under KVM against the guest-side villain test matrix.
+        //
+        // It consumes the villain nextest archive + the OpenVMM binary built by
+        // the linux build job above, so this test job needs no Rust toolchain
+        // (matching the vmm_tests build/consume split). It runs alongside the
+        // vmm_tests jobs for parallelism. The `-vmm-tests` suffix on the label
+        // makes the published logs artifact
+        // (`x64-linux-kvm-virtio-villain-vmm-tests-logs`) match the
+        // `upload-petri-results` workflow glob, so villain results land on the
+        // logview website alongside the regular VMM test results.
+        //
+        // NOTE: inert until openvmm-deps ships the virtio-villain artifact and
+        // `OPENVMM_DEPS` is bumped; until then this job fails at artifact
+        // resolution. It is wired into PR CI now (per @jstarks) so its results
+        // can seed the known-failure list once the release lands.
+        {
+            let label = "x64-linux-kvm-virtio-villain";
+            let test_label = format!("{label}-vmm-tests");
+
+            let pub_villain_results = if matches!(backend_hint, PipelineBackendHint::Local) {
+                Some(pipeline.new_artifact(&test_label).0)
+            } else {
+                None
+            };
+
+            let use_openvmm = use_openvmm_linux_x86
+                .clone()
+                .expect("linux x86 openvmm artifact built above");
+
+            let villain_job = pipeline
+                .new_job(
+                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
+                    FlowArch::X86_64,
+                    format!("run vmm-tests [{label}]"),
+                )
+                .gh_set_pool(gh_pools::linux_amd_v7_1es())
+                .ado_set_pool(ado_pools::linux_amd_v6_1es())
+                .dep_on(|ctx| {
+                    flowey_lib_hvlite::_jobs::consume_and_test_nextest_virtio_villain_archive::Params {
+                        junit_test_label: test_label,
+                        nextest_villain_archive: ctx
+                            .use_typed_artifact(&use_villain_archive_linux_x86),
+                        openvmm: ctx.use_typed_artifact(&use_openvmm),
+                        arch: CommonArch::X86_64,
+                        run_ignored: false,
+                        artifact_dir: pub_villain_results.map(|x| ctx.publish_artifact(x)),
+                        done: ctx.new_done_handle(),
+                    }
+                })
+                .finish();
+            all_jobs.push(villain_job);
+        }
+
         // test the flowey local backend by running cargo xflowey build-igvm on x64
         {
             if matches!(backend_hint, PipelineBackendHint::Github) {

flowey/flowey_hvlite/src/pipelines/mod.rs

@@ -4,6 +4,7 @@
 use cca_tests::CcaTestsCli;
 use flowey::pipeline::prelude::*;
 use restore_packages::RestorePackagesCli;
+use virtio_villain_run::VirtioVillainRunCli;
 use vmm_tests_run::VmmTestsRunCli;
 
 pub mod build_docs;
@@ -13,6 +14,7 @@ pub mod cca_tests;
 pub mod checkin_gates;
 pub mod custom_vmfirmwareigvm_dll;
 pub mod restore_packages;
+pub mod virtio_villain_run;
 pub mod vmm_tests_run;
 
 #[derive(clap::Subcommand)]
@@ -39,6 +41,9 @@ pub enum OpenvmmPipelines {
     /// Build and run VMM tests with automatic artifact discovery
     VmmTestsRun(VmmTestsRunCli),
 
+    /// Build and run the virtio-villain guest conformance tests against OpenVMM
+    VirtioVillainRun(VirtioVillainRunCli),
+
     /// Build and run CCA tests with installation of emulation environment supported
     CcaTests(CcaTestsCli),
 }
@@ -69,6 +74,7 @@ impl IntoPipeline for OpenvmmPipelines {
             },
             OpenvmmPipelines::RestorePackages(cmd) => cmd.into_pipeline(pipeline_hint),
             OpenvmmPipelines::VmmTestsRun(cmd) => cmd.into_pipeline(pipeline_hint),
+            OpenvmmPipelines::VirtioVillainRun(cmd) => cmd.into_pipeline(pipeline_hint),
             OpenvmmPipelines::CcaTests(cmd) => cmd.into_pipeline(pipeline_hint),
         }
     }

flowey/flowey_hvlite/src/pipelines/virtio_villain_run.rs

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Build OpenVMM and run the virtio-villain guest conformance / fault-injection
+//! test suite against it.
+//!
+//! virtio-villain ships as a versioned artifact from the `openvmm-deps` GitHub
+//! release. This pipeline downloads that artifact, builds OpenVMM, stages the
+//! guest test kernel, and runs the `virtio_villain_tests` nextest suite.
+//!
+//! Note: this is inert until `openvmm-deps` cuts a release that includes the
+//! virtio-villain artifact (and `cfg_versions::OPENVMM_DEPS` is bumped to it).
+
+use flowey::node::prelude::ReadVar;
+use flowey::pipeline::prelude::*;
+use flowey_lib_hvlite::common::CommonArch;
+
+/// Build and run the virtio-villain test suite against OpenVMM.
+#[derive(clap::Args)]
+pub struct VirtioVillainRunCli {
+    /// Also run known-failing (ignored) villain tests. These are skipped by
+    /// default because they exercise OpenVMM bugs (some wedge the VM); enable
+    /// this during fix development.
+    #[clap(long)]
+    pub run_ignored: bool,
+
+    /// Optional nextest filter expression to run only a subset of tests (e.g.
+    /// `test(B0001)` or `test(/^B00/)`). Omit to run the whole suite.
+    #[clap(long)]
+    pub filter: Option<String>,
+
+    /// Verbose pipeline output.
+    #[clap(long)]
+    pub verbose: bool,
+}
+
+impl IntoPipeline for VirtioVillainRunCli {
+    fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
+        let Self {
+            run_ignored,
+            filter,
+            verbose,
+        } = self;
+
+        // Phase 1 is Linux-only (villain drives OpenVMM under KVM).
+        let arch = match (
+            FlowArch::host(backend_hint),
+            FlowPlatform::host(backend_hint),
+        ) {
+            (FlowArch::X86_64, FlowPlatform::Linux(_)) => CommonArch::X86_64,
+            (FlowArch::Aarch64, FlowPlatform::Linux(_)) => CommonArch::Aarch64,
+            _ => anyhow::bail!("virtio-villain tests currently require a Linux host"),
+        };
+
+        let openvmm_repo = flowey_lib_common::git_checkout::RepoSource::ExistingClone(
+            ReadVar::from_static(crate::repo_root()),
+        );
+
+        let mut pipeline = Pipeline::new();
+
+        pipeline
+            .new_job(
+                FlowPlatform::host(backend_hint),
+                FlowArch::host(backend_hint),
+                "virtio-villain: run tests",
+            )
+            .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init)
+            .dep_on(
+                |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params {
+                    hvlite_repo_source: openvmm_repo.clone(),
+                },
+            )
+            .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params {
+                local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams {
+                    interactive: true,
+                    auto_install: true,
+                    ignore_rust_version: true,
+                }),
+                verbose: ReadVar::from_static(verbose),
+                locked: false,
+                deny_warnings: false,
+                no_incremental: false,
+            })
+            .dep_on(
+                |ctx| flowey_lib_hvlite::_jobs::run_virtio_villain_tests::Params {
+                    arch,
+                    run_ignored,
+                    nextest_filter_expr: filter,
+                    done: ctx.new_done_handle(),
+                },
+            )
+            .finish();
+
+        Ok(pipeline)
+    }
+}

flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs

@@ -30,7 +30,7 @@ pub const NODEJS: &str = "24.x";
 // None disables hcl-dev builds and tests; Some(version) enables them.
 pub const OPENHCL_KERNEL_DEV_VERSION: Option<&str> = None;
 pub const OPENHCL_KERNEL_STABLE_VERSION: &str = "6.18.37.2";
-pub const OPENVMM_DEPS: &str = "0.3.0-110";
+pub const OPENVMM_DEPS: &str = "0.3.0-112";
 pub const PROTOC: &str = "27.1";
 
 flowey_request! {
@@ -64,6 +64,7 @@ impl FlowNode for Node {
         ctx.import::<crate::resolve_openvmm_test_initrd::Node>();
         ctx.import::<crate::resolve_openvmm_test_linux_kernel::Node>();
         ctx.import::<crate::resolve_openvmm_test_virtio_win::Node>();
+        ctx.import::<crate::resolve_virtio_villain::Node>();
         ctx.import::<crate::download_uefi_mu_msvm::Node>();
         ctx.import::<crate::cfg_rustup_version::Node>();
         ctx.import::<flowey_lib_common::download_azcopy::Node>();
@@ -223,6 +224,12 @@ impl FlowNode for Node {
             version: Some(OPENVMM_DEPS.into()),
             ..Default::default()
         });
+        // Inert until openvmm-deps cuts a release that includes the
+        // virtio-villain artifact; the resolve node only downloads on demand.
+        ctx.config(crate::resolve_virtio_villain::Config {
+            version: Some(OPENVMM_DEPS.into()),
+            ..Default::default()
+        });
         ctx.config(crate::resolve_openvmm_qemu::Config {
             version: Some(OPENVMM_DEPS.into()),
             ..Default::default()