From f64005926175f7497364084292f04792842c2361 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 02:13:12 +0900 Subject: [PATCH 1/8] rootless: fix containerd startup when the host has run a rootful containerd v2 Rootful containerd creates /run/nri on the host (NRI is enabled by default since containerd v2.0). RootlessKit's copy-up of /run turns that directory into a symlink pointing back at the root-owned original, inside which the rootless containerd cannot bind its own NRI socket: failed to create socket "/var/run/nri/nri.sock": listen unix /var/run/nri/nri.sock: bind: permission denied Remove the symlink in the child namespace, like the existing /run/containerd one, so that a fresh, writable directory gets created on the copy-up tmpfs instead. Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- extras/rootless/containerd-rootless.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/rootless/containerd-rootless.sh b/extras/rootless/containerd-rootless.sh index b992b5744be..bfa7acc4578 100755 --- a/extras/rootless/containerd-rootless.sh +++ b/extras/rootless/containerd-rootless.sh @@ -161,7 +161,7 @@ else # Remove the *symlinks* for the existing files in the parent namespace if any, # so that we can create our own files in our mount namespace. # The actual files in the parent namespace are *not removed* by this rm command. - rm -f /run/containerd /run/xtables.lock \ + rm -f /run/containerd /run/nri /run/xtables.lock \ /var/lib/containerd /var/lib/cni /etc/containerd # Bind-mount /etc/ssl. From 18e6645b27059ce0a27058e87a701dc151802cce Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 06:30:50 +0900 Subject: [PATCH 2/8] test-integration.sh: do not print the output of the non-failing tests gotestsum was invoked with the "testname" format, which it silently upgrades to "github-actions" when GITHUB_ACTIONS=true (with no option to disable the upgrade), printing the output of every test: the logs were too large to be rendered by the GitHub Actions web UI. Use the "pkgname-and-test-fails" format by default: it prints a single line per package, plus the output of the failing tests (which is also repeated in the "=== Failed" summary at the end of the run), and is not subjected to the "github-actions" upgrade. Set GOTESTSUM_FORMAT to override the format. Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- hack/test-integration.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hack/test-integration.sh b/hack/test-integration.sh index cdbeb61957f..470ac567ef5 100755 --- a/hack/test-integration.sh +++ b/hack/test-integration.sh @@ -30,8 +30,16 @@ readonly timeout="30m" readonly retries="2" readonly needsudo="${WITH_SUDO:-}" +# Do not print the output of the non-failing tests: the full logs are too large to be +# rendered by the GitHub Actions web UI. The "pkgname-and-test-fails" format prints a +# single line per package, plus the output of the failing tests (which is repeated in +# the "=== Failed" summary at the end of the run). +# Note that the "testname" format must be avoided in the CI: gotestsum silently upgrades +# it to "github-actions" when GITHUB_ACTIONS=true, printing the output of every test. +# Set GOTESTSUM_FORMAT to override the format. +# # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization -args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...) +args=(--format="${GOTESTSUM_FORMAT:-pkgname-and-test-fails}" --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...) # FIXME: not working on windows. Need to change approach: move away from --post-run-command and # just process the log file. This might also allow multi-steps/multi-target results aggregation. [ "$(uname -s)" != "Linux" ] || args+=(--post-run-command "$root"/github/gotestsum-reporter.sh) From 7655a251ae8cf013b6761568f2507728a8f9bb52 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 05:27:19 +0900 Subject: [PATCH 3/8] test: fix the SELinux tests TestRunSelinux, TestRunSelinuxWithSecurityOpt, and TestRunSelinuxWithVolumeLabel had never actually run in the CI, as the containerized test environment did not enable SELinux. Now that the integration tests run directly on Enterprise Linux Lima guests, they run for the first time, revealing that: - the image argument was missing from the `nerdctl run` command lines ("sleep" was interpreted as the image name, failing to pull) - the volume directory was created at "/", which is not writable in rootless mode: create it under the test temporary directory instead - a volume relabeled with :Z gets container_file_t, which does not contain the expected "container_t" substring; `ls -Z` also needs -d to print the label of the (empty) directory itself Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- .../container/container_run_security_linux_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/nerdctl/container/container_run_security_linux_test.go b/cmd/nerdctl/container/container_run_security_linux_test.go index 3f0a6e0c891..55a1492e0ae 100644 --- a/cmd/nerdctl/container/container_run_security_linux_test.go +++ b/cmd/nerdctl/container/container_run_security_linux_test.go @@ -250,7 +250,7 @@ func TestRunSelinuxWithSecurityOpt(t *testing.T) { { Description: "test run with selinux-enabled", Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { - return helpers.Command("--selinux-enabled", "run", "-d", "--security-opt", "label=type:container_t", "--name", testContainer, "sleep", "infinity") + return helpers.Command("--selinux-enabled", "run", "-d", "--security-opt", "label=type:container_t", "--name", testContainer, testutil.CommonImage, "sleep", "infinity") }, Cleanup: func(data test.Data, helpers test.Helpers) { helpers.Anyhow("rm", "-f", testContainer) @@ -283,7 +283,7 @@ func TestRunSelinux(t *testing.T) { { Description: "test run with selinux-enabled", Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { - return helpers.Command("--selinux-enabled", "run", "-d", "--name", testContainer, "sleep", "infinity") + return helpers.Command("--selinux-enabled", "run", "-d", "--name", testContainer, testutil.CommonImage, "sleep", "infinity") }, Cleanup: func(data test.Data, helpers test.Helpers) { helpers.Anyhow("rm", "-f", testContainer) @@ -317,21 +317,23 @@ func TestRunSelinuxWithVolumeLabel(t *testing.T) { { Description: "test run with selinux-enabled", Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { - return helpers.Command("--selinux-enabled", "run", "-d", "-v", fmt.Sprintf("/%s:/%s:Z", testContainer, testContainer), "--name", testContainer, "sleep", "infinity") + // The volume directory must live somewhere writable by the (possibly + // rootless) user running the tests: nerdctl creates it on `run`. + hostDir := data.Temp().Path("volume") + return helpers.Command("--selinux-enabled", "run", "-d", "-v", fmt.Sprintf("%s:/mnt:Z", hostDir), "--name", testContainer, testutil.CommonImage, "sleep", "infinity") }, Cleanup: func(data test.Data, helpers test.Helpers) { helpers.Anyhow("rm", "-f", testContainer) - os.RemoveAll(fmt.Sprintf("/%s", testContainer)) }, Expected: func(data test.Data, helpers test.Helpers) *test.Expected { return &test.Expected{ ExitCode: expect.ExitCodeSuccess, Output: expect.All( func(stdout string, t tig.T) { - cmd := exec.Command("ls", "-Z", fmt.Sprintf("/%s", testContainer)) + cmd := exec.Command("ls", "-dZ", data.Temp().Path("volume")) lsStdout, err := cmd.CombinedOutput() assert.NilError(t, err) - assert.Equal(t, strings.Contains(string(lsStdout), "container_t"), true) + assert.Equal(t, strings.Contains(string(lsStdout), "container_file_t"), true) }, ), } From d8b7940dfad3ff3a87f493f6cef25fb349791f84 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 17:05:52 +0900 Subject: [PATCH 4/8] test: TestBuildFromStdin: do not assume direct export to the image store The test asserted that the image tag appears on stderr, which only happens when BuildKit exports the build result directly to the containerd image store ("naming to docker.io/library/..."). When nerdctl determines that the image is not sharable between the BuildKit worker and the client (eg: on EL 8 rootless, where the BuildKit worker uses the fuse-overlayfs snapshotter while the client uses the default overlayfs), it falls back to loading a tarball, and the tag is printed on stdout ("Loaded image: ...") instead. Assert the actual outcome instead: the image built from the Dockerfile fed on stdin exists and runs its CMD. Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- cmd/nerdctl/builder/builder_build_test.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/nerdctl/builder/builder_build_test.go b/cmd/nerdctl/builder/builder_build_test.go index 06676463e7b..84a0d17f860 100644 --- a/cmd/nerdctl/builder/builder_build_test.go +++ b/cmd/nerdctl/builder/builder_build_test.go @@ -17,7 +17,6 @@ package builder import ( - "errors" "fmt" "os" "path/filepath" @@ -247,19 +246,23 @@ CMD ["echo", "nerdctl-build-test-stdin"]`, testutil.CommonImage) testCase := &test.Case{ Require: nerdtest.Build, + Setup: func(data test.Data, helpers test.Helpers) { + cmd := helpers.Command("build", "-t", data.Identifier(), "-f", "-", ".") + cmd.Feed(strings.NewReader(dockerfile)) + cmd.Run(&test.Expected{ExitCode: expect.ExitCodeSuccess}) + }, Cleanup: func(data test.Data, helpers test.Helpers) { helpers.Anyhow("rmi", "-f", data.Identifier()) }, + // Run the image to prove that the build consumed the Dockerfile fed on stdin. + // Note: do not assert on the tag appearing in the build output: it is only + // printed on stderr ("naming to ...") when BuildKit exports directly to the + // containerd image store, not when nerdctl falls back to loading a tarball + // (eg: when the BuildKit worker snapshotter does not match the client one). Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { - cmd := helpers.Command("build", "-t", data.Identifier(), "-f", "-", ".") - cmd.Feed(strings.NewReader(dockerfile)) - return cmd - }, - Expected: func(data test.Data, helpers test.Helpers) *test.Expected { - return &test.Expected{ - Errors: []error{errors.New(data.Identifier())}, - } + return helpers.Command("run", "--rm", data.Identifier()) }, + Expected: test.Expects(expect.ExitCodeSuccess, nil, expect.Equals("nerdctl-build-test-stdin\n")), } testCase.Run(t) From e59ffa650de48f6676432c29b401251879f2f97a Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 01:15:56 +0900 Subject: [PATCH 5/8] test: adapt to running directly on hosts TestHostNetworkDnsPreserved: GitHub Actions hosts run systemd-resolved, so /etc/resolv.conf only contains the 127.0.0.53 stub. For host-network containers, nerdctl deliberately provides the resolv.conf that systemd-resolved generates with the actual upstream nameservers (see pkg/resolvconf.Path()), while Docker keeps the stub. Capture the nameservers accordingly in the test setup. This could not happen while the tests ran inside a Docker container with a Docker-generated resolv.conf. Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- .../container_run_network_linux_test.go | 29 ++++++++++++++----- cmd/nerdctl/network/network_inspect_test.go | 4 ++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cmd/nerdctl/container/container_run_network_linux_test.go b/cmd/nerdctl/container/container_run_network_linux_test.go index 13d94a2cab0..25c62db3733 100644 --- a/cmd/nerdctl/container/container_run_network_linux_test.go +++ b/cmd/nerdctl/container/container_run_network_linux_test.go @@ -964,14 +964,27 @@ func TestHostNetworkDnsPreserved(t *testing.T) { Setup: func(data test.Data, helpers test.Helpers) { // In some rootless CI job, slirp provides 10.0.2.3 as DNS server. // We cannot simply parse host /etc/resolv.conf here. - helpers.Command("run", "--rm", - "-v", "/etc/resolv.conf:/mnt/resolv.conf:ro", - testutil.AlpineImage, - "grep", "-E", "^nameserver\\s+", "/mnt/resolv.conf").Run(&test.Expected{ - Output: func(stdout string, t tig.T) { - data.Labels().Set("nameservers", stdout) - }, - }) + captureNameservers := func(resolvConfPath string) string { + var nameservers string + helpers.Command("run", "--rm", + "-v", resolvConfPath+":/mnt/resolv.conf:ro", + testutil.AlpineImage, + "grep", "-E", "^nameserver\\s+", "/mnt/resolv.conf").Run(&test.Expected{ + Output: func(stdout string, t tig.T) { + nameservers = stdout + }, + }) + return nameservers + } + nameservers := captureNameservers("/etc/resolv.conf") + // Mirror pkg/resolvconf.Path(): when 127.0.0.53 is the only nameserver, the host + // runs systemd-resolved, and nerdctl uses the resolv.conf that systemd-resolved + // generates with the actual upstream nameservers. + // Docker, on the other hand, keeps the stub for host-network containers. + if !nerdtest.IsDocker() && strings.TrimSpace(nameservers) == "nameserver 127.0.0.53" { + nameservers = captureNameservers("/run/systemd/resolve/resolv.conf") + } + data.Labels().Set("nameservers", nameservers) }, Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { return helpers.Command("run", "--rm", diff --git a/cmd/nerdctl/network/network_inspect_test.go b/cmd/nerdctl/network/network_inspect_test.go index 84e5e6f9dcb..8811bec39bf 100644 --- a/cmd/nerdctl/network/network_inspect_test.go +++ b/cmd/nerdctl/network/network_inspect_test.go @@ -500,7 +500,9 @@ func TestNetworkInspectDualStack(t *testing.T) { Setup: func(data test.Data, helpers test.Helpers) { helpers.Ensure("network", "create", "--ipv6", - "--subnet", "10.1.0.0/24", + // Not 10.1.0.0/24: the eth0 of the GitHub Actions runners lives in + // 10.1.0.0/20, which the subnet overlap check rejects + "--subnet", "10.24.0.0/24", "--subnet", "fd00::/64", data.Identifier("test-dual-stack")) From f21b32bb49bc013ce34117caddf35a09b3f0d748 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 2 Jul 2026 22:22:17 +0900 Subject: [PATCH 6/8] CI: run test-integration[-rootless] directly on hosts and Lima guests The integration tests are no longer wrapped inside a Docker container. Docker is still used to build the test dependencies (the new `out-test-integration-artifacts` Dockerfile stage), which are then installed on the host with hack/provisioning/linux/test-integration-env.sh, and the tests now run with `go test` via hack/test-integration.sh: - job-test-in-container.yml is removed; its matrix is migrated to job-test-in-host.yml - job-test-in-lima.yml no longer nests Docker inside the guest VM (issue 3858); the artifacts are built on the host and installed in the guest, which is started in plain mode - the test-integration* Dockerfile stages are removed, along with the unused build-minimal stage (the demo stage is retained) - hack/test-integration-rootless.sh (moved from Dockerfile.d) runs the tests inside the systemd user session of the unprivileged user, which is available without a login, as the provisioning script enables lingering for that user Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- .github/workflows/job-test-in-container.yml | 210 -------------- .github/workflows/job-test-in-host.yml | 236 +++++++++------ .github/workflows/job-test-in-lima.yml | 101 ++++--- .github/workflows/workflow-flaky.yml | 1 + .github/workflows/workflow-test.yml | 23 +- Dockerfile | 125 ++------ ...containerd.service.d_port-slirp4netns.conf | 3 - ...-integration-buildkit-nerdctl-test.service | 2 +- .../test-integration-ipfs-offline.service | 2 +- Dockerfile.d/test-integration-rootless.sh | 76 ----- .../test-integration-soci-snapshotter.service | 2 +- docs/testing/README.md | 20 +- hack/build-integration-canary.sh | 2 +- hack/provisioning/README.md | 3 +- .../linux/test-integration-env.sh | 271 ++++++++++++++++++ hack/test-integration-rootless.sh | 135 +++++++++ 16 files changed, 668 insertions(+), 544 deletions(-) delete mode 100644 .github/workflows/job-test-in-container.yml delete mode 100644 Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf delete mode 100755 Dockerfile.d/test-integration-rootless.sh create mode 100755 hack/provisioning/linux/test-integration-env.sh create mode 100755 hack/test-integration-rootless.sh diff --git a/.github/workflows/job-test-in-container.yml b/.github/workflows/job-test-in-container.yml deleted file mode 100644 index 76789ef09c7..00000000000 --- a/.github/workflows/job-test-in-container.yml +++ /dev/null @@ -1,210 +0,0 @@ -# This job runs integration tests inside a container, for all supported variants (ipv6, canary, etc) -# Note that it is linux and nerdctl (+/- gomodjail) only. -name: job-test-in-container - -on: - workflow_call: - inputs: - timeout: - required: true - type: number - runner: - required: true - type: string - canary: - required: false - default: false - type: boolean - target: - required: false - default: '' - type: string - binary: - required: false - default: nerdctl - type: string - containerd-version: - required: false - default: '' - type: string - rootlesskit-version: - required: false - default: '' - type: string - ipv6: - required: false - default: false - type: boolean - skip-flaky: - required: false - default: false - type: boolean - -env: - GOTOOLCHAIN: local - -jobs: - test: - name: | - ${{ inputs.binary != 'nerdctl' && format('{0} < ', inputs.binary) || '' }} - ${{ inputs.target }} - ${{ contains(inputs.runner, 'arm') && '(arm)' || '' }} - ${{ contains(inputs.runner, '22.04') && '(old ubuntu)' || '' }} - ${{ inputs.ipv6 && ' (ipv6)' || '' }} - ${{ inputs.canary && ' (canary)' || '' }} - ${{ inputs.containerd-version && format(' (ctd: {0})', inputs.containerd-version) || '' }} - ${{ inputs.rootlesskit-version && format(' (rlk: {0})', inputs.rootlesskit-version) || '' }} - timeout-minutes: ${{ inputs.timeout }} - runs-on: ${{ inputs.runner }} - defaults: - run: - shell: bash - - env: - # https://github.com/containerd/nerdctl/issues/622 - # The only case when rootlesskit-version is force-specified is when we downgrade explicitly to v1 - WORKAROUND_ISSUE_622: ${{ inputs.rootlesskit-version }} - - steps: - - name: "Init: checkout" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 1 - persist-credentials: false - - - name: "Init: expose GitHub Runtime variables for gha" - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 - - - name: "Init: install br-netfilter" - run: | - # This ensures that bridged traffic goes through netfilter - sudo modprobe br-netfilter - - name: "Init: register QEMU (tonistiigi/binfmt)" - run: | - # `--install all` will only install emulation for architectures that cannot be natively executed - # Since some arm64 platforms do provide native fallback execution for 32 bits, - # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`. - # To avoid that, we explicitly list the architectures we do want emulation for. - docker run --privileged --rm tonistiigi/binfmt --install linux/amd64 - docker run --privileged --rm tonistiigi/binfmt --install linux/arm64 - docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7 - - if: ${{ inputs.canary }} - name: "Init (canary): prepare updated test image" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - . ./hack/build-integration-canary.sh - canary::build::integration - - if: ${{ ! inputs.canary }} - name: "Init: prepare test image" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - INPUTS_CONTAINERD_VERSION: ${{ inputs.containerd-version }} - INPUTS_TARGET: ${{ inputs.target }} - INPUTS_ROOTLESSKIT_VERSION: ${{ inputs.rootlesskit-version }} - run: | - buildargs=() - # If the runner is old, use old ubuntu inside the container as well - [ "${{ contains(inputs.runner, '22.04') }}" != "true" ] || buildargs=(--build-arg UBUNTU_VERSION=22.04) - # Honor if we want old containerd - [ "${INPUTS_CONTAINERD_VERSION}" == "" ] || buildargs+=(--build-arg CONTAINERD_VERSION=${INPUTS_CONTAINERD_VERSION}) - # Honor custom targets and if we want old rootlesskit - target=test-integration - if [ "${INPUTS_TARGET}" != "rootful" ]; then - target+=-${INPUTS_TARGET} - if [ "${INPUTS_ROOTLESSKIT_VERSION}" != "" ]; then - buildargs+=(--build-arg ROOTLESSKIT_VERSION=${INPUTS_ROOTLESSKIT_VERSION}) - fi - fi - # Cache is sharded per-architecture - arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }} - docker buildx create --name with-gha --use - docker buildx build \ - --secret id=github_token,env=GITHUB_TOKEN \ - --output=type=docker \ - --cache-from type=gha,scope=test-integration-dependencies-"$arch" \ - -t "$target" --target "$target" \ - "${buildargs[@]}" \ - . - # Rootful needs to disable snap - - if: ${{ inputs.target == 'rootful' }} - name: "Init: remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)" - run: | - sudo systemctl disable --now snapd.service snapd.socket - sudo apt-get purge -qq snapd - sudo losetup -Dv - sudo losetup -lv - # Rootless on modern ubuntu wants apparmor - - if: ${{ inputs.target != 'rootful' && ! contains(inputs.runner, '22.04') }} - name: "Init: prepare apparmor for rootless + ubuntu 24+" - run: | - cat <, - include - /usr/local/bin/rootlesskit flags=(unconfined) { - userns, - # Site-specific additions and overrides. See local/README for details. - include if exists - } - EOT - sudo systemctl restart apparmor.service - # ipv6 wants... ipv6 - - if: ${{ inputs.ipv6 }} - name: "Init: ipv6" - run: | - # Enable ipv4 and ipv6 forwarding - sudo sysctl -w net.ipv6.conf.all.forwarding=1 - sudo sysctl -w net.ipv4.ip_forward=1 - # Enable IPv6 for Docker, and configure docker to use containerd for gha - sudo mkdir -p /etc/docker - echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64", "ip6tables": true}' | sudo tee /etc/docker/daemon.json - - name: "Init: enable Docker experimental features" - run: | - sudo mkdir -p /etc/docker - if [ -f /etc/docker/daemon.json ]; then - tmpfile="$(sudo mktemp)" - sudo jq '.experimental = true' /etc/docker/daemon.json | sudo tee "$tmpfile" >/dev/null - sudo mv "$tmpfile" /etc/docker/daemon.json - else - echo '{"experimental": true}' | sudo tee /etc/docker/daemon.json >/dev/null - fi - sudo systemctl restart docker - - name: "Run: integration tests" - run: | - . ./hack/github/action-helpers.sh - github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY" - - # IPV6 note: nested IPv6 network inside docker and qemu is complex and needs a bunch of sysctl config. - # Therefore, it's hard to debug why the IPv6 tests fail in such an isolation layer. - # On the other side, using the host network is easier at configuration. - # Besides, each job is running on a different instance, which means using host network here - # is safe and has no side effects on others. - [ "${INPUTS_TARGET}" == "rootful" ] \ - && args=(test-integration ./hack/test-integration.sh -test.allow-modify-users=true) \ - || args=(test-integration-${INPUTS_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh) - if [ "${{ inputs.ipv6 }}" == true ]; then - docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${INPUTS_BINARY} - else - docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${INPUTS_BINARY} - fi - env: - INPUTS_TARGET: ${{ inputs.target }} - INPUTS_BINARY: ${{ inputs.binary }} - # FIXME: this NEEDS to go away - - name: "Run: integration tests (flaky)" - if: ${{ !fromJSON(inputs.skip-flaky) }} - run: | - . ./hack/github/action-helpers.sh - github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY" - - [ "${INPUTS_TARGET}" == "rootful" ] \ - && args=(test-integration ./hack/test-integration.sh) \ - || args=(test-integration-${INPUTS_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh) - if [ "${{ inputs.ipv6 }}" == true ]; then - docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${INPUTS_BINARY} - else - docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${INPUTS_BINARY} - fi - env: - INPUTS_TARGET: ${{ inputs.target }} - INPUTS_BINARY: ${{ inputs.binary }} diff --git a/.github/workflows/job-test-in-host.yml b/.github/workflows/job-test-in-host.yml index 4d12106dab0..bd4e29e352b 100644 --- a/.github/workflows/job-test-in-host.yml +++ b/.github/workflows/job-test-in-host.yml @@ -1,5 +1,10 @@ -# This currently test docker and nerdctl on windows (w/o canary) -# Structure is in to allow testing nerdctl on linux as well, though more work is required to make it functional. +# This job runs integration tests directly on the host, with `go test`. +# It covers: +# - windows (w/o canary) +# - docker on linux +# - nerdctl on linux, for all supported variants (rootful, rootless, ipv6, canary, etc.) +# On linux, Docker is only used to build the test dependencies (see the +# `out-test-integration-artifacts` Dockerfile stage), and is disabled before the tests start. name: job-test-in-host on: @@ -19,39 +24,50 @@ on: required: false default: false type: boolean + # Leave empty for windows and docker. Set to rootful, rootless, or + # rootless-port-slirp4netns to test nerdctl on a linux host. + target: + required: false + default: '' + type: string binary: required: false default: nerdctl type: string + containerd-version: + required: false + default: '' + type: string + rootlesskit-version: + required: false + default: '' + type: string + ipv6: + required: false + default: false + type: boolean + skip-flaky: + required: false + default: false + type: boolean go-version: required: true type: string docker-version: - required: true + required: false + default: '' type: string windows-containerd-version: - required: true + required: false + default: '' type: string windows-containerd-sha: - required: true - type: string - linux-containerd-version: - required: true - type: string - linux-containerd-sha: - required: true - type: string - linux-containerd-service-sha: - required: true + required: false + default: '' type: string windows-cni-version: - required: true - type: string - linux-cni-version: - required: true - type: string - linux-cni-sha: - required: true + required: false + default: '' type: string env: @@ -61,10 +77,14 @@ jobs: test: name: | ${{ inputs.binary != 'nerdctl' && format('{0} < ', inputs.binary) || '' }} + ${{ inputs.target }} ${{ contains(inputs.runner, 'ubuntu') && ' linux' || ' windows' }} ${{ contains(inputs.runner, 'arm') && '(arm)' || '' }} ${{ contains(inputs.runner, '22.04') && '(old ubuntu)' || '' }} + ${{ inputs.ipv6 && ' (ipv6)' || '' }} ${{ inputs.canary && ' (canary)' || '' }} + ${{ inputs.containerd-version && format(' (ctd: {0})', inputs.containerd-version) || '' }} + ${{ inputs.rootlesskit-version && format(' (rlk: {0})', inputs.rootlesskit-version) || '' }} timeout-minutes: ${{ inputs.timeout }} runs-on: "${{ inputs.runner }}" defaults: @@ -74,12 +94,13 @@ jobs: env: SHOULD_RUN: "yes" GO_VERSION: ${{ inputs.go-version }} - # Both Docker and nerdctl on linux need rootful right now - WITH_SUDO: ${{ contains(inputs.runner, 'ubuntu') }} + # Docker on linux needs rootful. So does nerdctl, unless the target is rootless. + WITH_SUDO: ${{ contains(inputs.runner, 'ubuntu') && !startsWith(inputs.target, 'rootless') }} WINDOWS_CONTAINERD_VERSION: ${{ inputs.windows-containerd-version }} WINDOWS_CONTAINERD_SHA: ${{ inputs.windows-containerd-sha }} - LINUX_CONTAINERD_VERSION: ${{ inputs.linux-containerd-version }} - LINUX_CONTAINERD_SHA: ${{ inputs.linux-containerd-sha }} + # https://github.com/containerd/nerdctl/issues/622 + # The only case when rootlesskit-version is force-specified is when we downgrade explicitly to v1 + WORKAROUND_ISSUE_622: ${{ inputs.rootlesskit-version }} steps: - name: "Init: checkout" @@ -107,18 +128,22 @@ jobs: if [[ "${INPUTS_RUNNER}" == *windows* ]]; then containerd_version="$WINDOWS_CONTAINERD_VERSION" - else - containerd_version="$LINUX_CONTAINERD_VERSION" - fi - [ "${latest_containerd:1}" == "$containerd_version" ] || { - if [[ "${INPUTS_RUNNER}" == *windows* ]]; then + [ "${latest_containerd:1}" == "$containerd_version" ] || { printf "WINDOWS_CONTAINERD_VERSION=%s\n" "${latest_containerd:1}" >> "$GITHUB_ENV" printf "WINDOWS_CONTAINERD_SHA=canary is volatile and I accept the risk\n" >> "$GITHUB_ENV" - else - printf "LINUX_CONTAINERD_VERSION=%s\n" "${latest_containerd:1}" >> "$GITHUB_ENV" - printf "LINUX_CONTAINERD_SHA=canary is volatile and I accept the risk\n" >> "$GITHUB_ENV" - fi - } + } + else + # On linux, containerd is built from source, as part of the test artifacts + containerd_version="$(grep -m1 '^ARG CONTAINERD_VERSION=' Dockerfile | cut -d= -f2)" + containerd_version="${containerd_version%%@*}" + containerd_version="${containerd_version:1}" + [ "${latest_containerd:1}" == "$containerd_version" ] || \ + printf "CANARY_CONTAINERD_VERSION=%s\n" "$latest_containerd" >> "$GITHUB_ENV" + # The golang docker image tag lags behind the golang releases + latest_go_hub="$(. ./hack/build-integration-canary.sh; canary::golang::hublatest)" + [ "$latest_go_hub" == "" ] || \ + printf "CANARY_GO_VERSION=%s\n" "$latest_go_hub" >> "$GITHUB_ENV" + fi if [ "$latest_go" == "" ] && [ "${latest_containerd:1}" == "$containerd_version" ]; then echo "::warning title=No canary::There is currently no canary versions to test. Steps will not run."; printf "SHOULD_RUN=no\n" >> "$GITHUB_ENV" @@ -134,52 +159,35 @@ jobs: # XXX RUNNER_OS and generally env is too unreliable # - if: ${{ env.RUNNER_OS == 'Linux' }} - if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }} - name: "Init (linux): prepare host" + name: "Init (linux): register QEMU (tonistiigi/binfmt)" run: | - if [ "${{ contains(inputs.binary, 'docker') }}" == true ]; then - echo "::group:: configure cdi and experimental for docker" - sudo mkdir -p /etc/docker - sudo jq -n '.features.cdi = true | .experimental = true' | sudo tee /etc/docker/daemon.json - echo "::endgroup::" - echo "::group:: downgrade docker to the specific version we want to test (${INPUTS_DOCKER_VERSION})" - sudo apt-get update -qq - sudo apt-get install -qq ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo cp ./hack/provisioning/gpg/docker /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" \ - | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update -qq - sudo apt-get install -qq --allow-downgrades docker-ce=${INPUTS_DOCKER_VERSION} docker-ce-cli=${INPUTS_DOCKER_VERSION} - sudo systemctl restart docker - echo "::endgroup::" - else - # FIXME: this is missing runc (see top level workflow note about the state of this) - echo "::group:: install dependencies" - sudo ./hack/provisioning/linux/containerd.sh uninstall - ./hack/provisioning/linux/containerd.sh rootful "$LINUX_CONTAINERD_VERSION" "amd64" "$LINUX_CONTAINERD_SHA" "${INPUTS_LINUX_CONTAINERD_SERVICE_SHA}" - sudo ./hack/provisioning/linux/cni.sh uninstall - ./hack/provisioning/linux/cni.sh install "${INPUTS_LINUX_CNI_VERSION}" "amd64" "${INPUTS_LINUX_CNI_SHA}" - echo "::endgroup::" - - echo "::group:: build nerctl" - go install ./cmd/nerdctl - echo "$HOME/go/bin" >> "$GITHUB_PATH" - # Since tests are going to run root, we need nerdctl to be in a PATH that will survive `sudo` - sudo cp "$(which nerdctl)" /usr/local/bin - echo "::endgroup::" - fi - - # Register QEMU (tonistiigi/binfmt) # `--install all` will only install emulation for architectures that cannot be natively executed # Since some arm64 platforms do provide native fallback execution for 32 bits, # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`. # To avoid that, we explicitly list the architectures we do want emulation for. - echo "::group:: install binfmt" docker run --quiet --privileged --rm tonistiigi/binfmt --install linux/amd64 docker run --quiet --privileged --rm tonistiigi/binfmt --install linux/arm64 docker run --quiet --privileged --rm tonistiigi/binfmt --install linux/arm/v7 + + - if: ${{ contains(inputs.runner, 'ubuntu') && inputs.target == '' && env.SHOULD_RUN == 'yes' }} + name: "Init (linux docker): prepare host" + run: | + echo "::group:: configure cdi and experimental for docker" + sudo mkdir -p /etc/docker + sudo jq -n '.features.cdi = true | .experimental = true' | sudo tee /etc/docker/daemon.json + echo "::endgroup::" + echo "::group:: downgrade docker to the specific version we want to test (${INPUTS_DOCKER_VERSION})" + sudo apt-get update -qq + sudo apt-get install -qq ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo cp ./hack/provisioning/gpg/docker /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update -qq + sudo apt-get install -qq --allow-downgrades docker-ce=${INPUTS_DOCKER_VERSION} docker-ce-cli=${INPUTS_DOCKER_VERSION} + sudo systemctl restart docker echo "::endgroup::" # FIXME: remove expect when we are done removing unbuffer from tests @@ -192,9 +200,6 @@ jobs: sudo modprobe br-netfilter env: INPUTS_DOCKER_VERSION: ${{ inputs.docker-version }} - INPUTS_LINUX_CONTAINERD_SERVICE_SHA: ${{ inputs.linux-containerd-service-sha }} - INPUTS_LINUX_CNI_VERSION: ${{ inputs.linux-cni-version }} - INPUTS_LINUX_CNI_SHA: ${{ inputs.linux-cni-sha }} - if: ${{ contains(inputs.runner, 'windows') && env.SHOULD_RUN == 'yes' }} name: "Init (windows): prepare host" @@ -220,15 +225,65 @@ jobs: choco install jq + # Rootful needs to disable snap + - if: ${{ inputs.target == 'rootful' && env.SHOULD_RUN == 'yes' }} + name: "Init (rootful): remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)" + run: | + sudo systemctl disable --now snapd.service snapd.socket + sudo apt-get purge -qq snapd + sudo losetup -Dv + sudo losetup -lv + + # ipv6 wants... ipv6 + - if: ${{ inputs.ipv6 && env.SHOULD_RUN == 'yes' }} + name: "Init (ipv6): enable ipv4 and ipv6 forwarding" + run: | + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + sudo sysctl -w net.ipv4.ip_forward=1 + + - if: ${{ inputs.target != '' && env.SHOULD_RUN == 'yes' }} + name: "Init (linux): Set up Docker Buildx" + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Unlike a raw `docker buildx build --cache-from type=gha`, the action provides + # the GitHub runtime token and cache url to BuildKit by itself. + # The cache is sharded per-architecture; empty build-args lines are ignored, and + # the canary containerd version (if any) takes precedence over the input. + - if: ${{ inputs.target != '' && env.SHOULD_RUN == 'yes' }} + name: "Init (linux): build test artifacts" + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + # push is false by default; stated explicitly for zizmor's cache-poisoning audit + push: false + target: out-test-integration-artifacts + outputs: type=local,dest=/tmp/nerdctl-test-artifacts + cache-from: type=gha,scope=test-integration-dependencies-${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }} + secrets: | + github_token=${{ secrets.GITHUB_TOKEN }} + build-args: | + ${{ (env.CANARY_CONTAINERD_VERSION && format('CONTAINERD_VERSION={0}', env.CANARY_CONTAINERD_VERSION)) || (inputs.containerd-version && format('CONTAINERD_VERSION={0}', inputs.containerd-version)) || '' }} + ${{ inputs.rootlesskit-version && format('ROOTLESSKIT_VERSION={0}', inputs.rootlesskit-version) || '' }} + ${{ env.CANARY_GO_VERSION && format('GO_VERSION={0}', env.CANARY_GO_VERSION) || '' }} + + # Note that Docker cannot be used anymore past this point. + - if: ${{ inputs.target != '' && env.SHOULD_RUN == 'yes' }} + name: "Init (linux): provision the host" + env: + INPUTS_TARGET: ${{ inputs.target }} + run: | + sudo env GITHUB_ACTIONS="$GITHUB_ACTIONS" ./hack/provisioning/linux/test-integration-env.sh install /tmp/nerdctl-test-artifacts "${INPUTS_TARGET}" + - if: ${{ env.SHOULD_RUN == 'yes' }} name: "Init: install dev tools" run: | echo "::group:: make install-dev-tools" make install-dev-tools + [ "$(uname -s)" != "Linux" ] || echo "$HOME/go/bin" >> "$GITHUB_PATH" echo "::endgroup::" - # ipv6 is tested only on linux - - if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }} + # ipv6 is tested only on linux. For nerdctl, this is done through the ipv6 input instead. + - if: ${{ contains(inputs.runner, 'ubuntu') && inputs.target == '' && env.SHOULD_RUN == 'yes' }} name: "Run (linux): integration tests (IPv6)" run: | . ./hack/github/action-helpers.sh @@ -244,17 +299,36 @@ jobs: . ./hack/github/action-helpers.sh github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY" - ./hack/test-integration.sh -test.target=${INPUTS_BINARY} -test.only-flaky=false + args=(-test.target=${INPUTS_BINARY} -test.only-flaky=false) + [ "${INPUTS_IPV6}" != "true" ] || args+=(-test.only-ipv6) + [ "${INPUTS_TARGET}" != "rootful" ] || args+=(-test.allow-modify-users=true) + if [[ "${INPUTS_TARGET}" == rootless* ]]; then + ./hack/test-integration-rootless.sh ./hack/test-integration.sh "${args[@]}" + else + ./hack/test-integration.sh "${args[@]}" + fi env: INPUTS_BINARY: ${{ inputs.binary }} + INPUTS_TARGET: ${{ inputs.target }} + INPUTS_IPV6: ${{ inputs.ipv6 }} + CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER: ${{ inputs.target == 'rootless-port-slirp4netns' && 'slirp4netns' || '' }} # FIXME: this must go - - if: ${{ env.SHOULD_RUN == 'yes' }} + - if: ${{ env.SHOULD_RUN == 'yes' && !fromJSON(inputs.skip-flaky) }} name: "Run: integration tests (flaky)" run: | . ./hack/github/action-helpers.sh github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY" - ./hack/test-integration.sh -test.target=${INPUTS_BINARY} -test.only-flaky=true + args=(-test.target=${INPUTS_BINARY} -test.only-flaky=true) + [ "${INPUTS_IPV6}" != "true" ] || args+=(-test.only-ipv6) + if [[ "${INPUTS_TARGET}" == rootless* ]]; then + ./hack/test-integration-rootless.sh ./hack/test-integration.sh "${args[@]}" + else + ./hack/test-integration.sh "${args[@]}" + fi env: INPUTS_BINARY: ${{ inputs.binary }} + INPUTS_TARGET: ${{ inputs.target }} + INPUTS_IPV6: ${{ inputs.ipv6 }} + CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER: ${{ inputs.target == 'rootless-port-slirp4netns' && 'slirp4netns' || '' }} diff --git a/.github/workflows/job-test-in-lima.yml b/.github/workflows/job-test-in-lima.yml index 8e8ce68adc8..50ad41903fe 100644 --- a/.github/workflows/job-test-in-lima.yml +++ b/.github/workflows/job-test-in-lima.yml @@ -1,4 +1,6 @@ # Currently, Lima job test only for EL, though in the future it could be used to also test FreeBSD or other linux-es +# The test artifacts are built on the host with Docker, then installed inside the guest VM, +# where the integration tests run directly with `go test` (no Docker inside the VM). name: job-test-in-lima on: @@ -16,6 +18,9 @@ on: guest: required: true type: string + go-version: + required: true + type: string skip-flaky: required: false default: false @@ -29,6 +34,7 @@ jobs: env: TARGET: ${{ inputs.target }} GUEST: ${{ inputs.guest }} + GO_VERSION: ${{ inputs.go-version }} steps: - name: "Init: checkout" uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -49,84 +55,75 @@ jobs: - name: "Init: start the guest VM" run: | set -eux - # containerd=none is set because the built-in containerd support conflicts with Docker + # --plain disables the mounts, the port forwards, containerd, and the Lima boot + # scripts: just a plain VM with ssh. The provisioning and test scripts recreate + # the environment that the boot scripts would have set up (kernel modules, + # sysctls, CONTAINERD_SNAPSHOTTER, ...). limactl start \ --name=default \ + --plain \ --cpus=4 \ --memory=12 \ - --containerd=none \ - --set '.mounts=null | .portForwards=[{"guestSocket":"/var/run/docker.sock","hostSocket":"{{.Dir}}/sock/docker.sock"}]' \ template://${GUEST} - # FIXME: the tests should be directly executed in the VM without nesting Docker inside it - # https://github.com/containerd/nerdctl/issues/3858 - - name: "Init: install dockerd in the guest VM" + - name: "Init: Set up Docker Buildx" + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Unlike a raw `docker buildx build --cache-from type=gha`, the action provides the + # GitHub runtime token and cache url to BuildKit by itself. + - name: "Init: build test artifacts (on the host, with Docker)" + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + # push is false by default; stated explicitly for zizmor's cache-poisoning audit + push: false + target: out-test-integration-artifacts + outputs: type=local,dest=/tmp/nerdctl-test-artifacts + cache-from: type=gha,scope=test-integration-dependencies-amd64 + secrets: | + github_token=${{ secrets.GITHUB_TOKEN }} + + - name: "Init: copy source and artifacts into the guest VM" run: | set -eux - lima sudo mkdir -p /etc/systemd/system/docker.socket.d - cat <<-EOF | lima sudo tee /etc/systemd/system/docker.socket.d/override.conf - [Socket] - SocketUser=$(whoami) - EOF - lima sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - lima sudo dnf -q -y install docker-ce --nobest - lima sudo systemctl enable --now docker + limactl copy -r "$PWD" default:nerdctl + limactl copy -r /tmp/nerdctl-test-artifacts default:nerdctl-test-artifacts + # Run all the subsequent lima commands from the copied source + echo "LIMA_WORKDIR=$(lima pwd)/nerdctl" >>"$GITHUB_ENV" - - name: "Init: configure the host to use dockerd in the guest VM" + - name: "Init: install go in the guest VM" run: | set -eux - sudo systemctl disable --now docker.service docker.socket - export DOCKER_HOST="unix://$(limactl ls --format '{{.Dir}}/sock/docker.sock' default)" - echo "DOCKER_HOST=${DOCKER_HOST}" >>$GITHUB_ENV - docker info - docker version + lima bash -c 'command -v tar' || lima sudo dnf install -q -y tar + gover="$(curl -fsSL --proto '=https' --tlsv1.2 'https://go.dev/dl/?mode=json&include=all' | jq -r --arg prefix "go${GO_VERSION}." '[.[].version | select(startswith($prefix))] | first')" + [ "$gover" != "null" ] + curl -fsSL --proto '=https' --tlsv1.2 "https://go.dev/dl/${gover}.linux-amd64.tar.gz" | lima sudo tar -xzf- -C /usr/local - - name: "Init: install br-netfilter in the guest VM" + - name: "Init: provision the guest VM" run: | - lima sudo modprobe br-netfilter - - - name: "Init: expose GitHub Runtime variables for gha" - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 + set -eux + lima sudo env GITHUB_ACTIONS=true ./hack/provisioning/linux/test-integration-env.sh install ../nerdctl-test-artifacts "$TARGET" - - name: "Init: prepare integration tests" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Init: install dev tools in the guest VM" run: | set -eux + lima bash -c 'PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" make install-dev-tools' - sudo losetup -Dv - sudo losetup -lv - - [ "$TARGET" = "rootless" ] && TARGET=test-integration-rootless || TARGET=test-integration - docker buildx create --name with-gha --use - docker buildx build \ - --secret id=github_token,env=GITHUB_TOKEN \ - --output=type=docker \ - --cache-from type=gha,scope=test-integration-dependencies-amd64 \ - -t test-integration --target "${TARGET}" \ - . - - - name: "Run integration tests" - # Presumably, something is broken with the way docker exposes /dev to the container, as it appears to only - # randomly work. Mounting /dev does workaround the issue. - # This might be due to the old kernel shipped with Alma (4.18), or something else between centos/docker. + - name: "Run: integration tests" run: | set -eux if [ "$TARGET" = "rootless" ]; then - echo "rootless" - docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false + lima bash -c 'export GITHUB_ACTIONS=true PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" && ./hack/test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false' else - echo "rootful" - docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=false + lima bash -c 'export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" WITH_SUDO=true && ./hack/test-integration.sh -test.only-flaky=false' fi + - name: "Run: integration tests (flaky)" if: ${{ !fromJSON(inputs.skip-flaky) }} run: | set -eux if [ "$TARGET" = "rootless" ]; then - echo "rootless" - docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true + lima bash -c 'export GITHUB_ACTIONS=true PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" && ./hack/test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true' else - echo "rootful" - docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true + lima bash -c 'export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" WITH_SUDO=true && ./hack/test-integration.sh -test.only-flaky=true' fi diff --git a/.github/workflows/workflow-flaky.yml b/.github/workflows/workflow-flaky.yml index 5d507c9d281..2bbf64fe5b1 100644 --- a/.github/workflows/workflow-flaky.yml +++ b/.github/workflows/workflow-flaky.yml @@ -32,6 +32,7 @@ jobs: runner: ubuntu-24.04 guest: ${{ matrix.guest }} target: ${{ matrix.target }} + go-version: 1.26 skip-flaky: true # skip the most flaky ones for now test-integration-freebsd: diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index 9fda60e7c26..22f45575dc8 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -58,9 +58,9 @@ jobs: containerd-version: ${{ matrix.containerd-version }} timeout: 20 - test-integration-container: - name: "in-container${{ inputs.hack }}" - uses: ./.github/workflows/job-test-in-container.yml + test-integration-host-linux: + name: "in-host${{ inputs.hack }}" + uses: ./.github/workflows/job-test-in-host.yml needs: build-dependencies strategy: fail-fast: false @@ -120,6 +120,7 @@ jobs: ipv6: ${{ matrix.ipv6 && true || false }} canary: ${{ matrix.canary && true || false }} skip-flaky: ${{ matrix.skip-flaky && true || false }} + go-version: 1.26 test-integration-host: name: "in-host${{ inputs.hack }}" @@ -136,14 +137,6 @@ jobs: # Test docker on linux - runner: ubuntu-24.04 binary: docker - - # FIXME: running nerdctl on the host is work in progress - # (we miss runc to be installed on the host - and obviously other deps) - # Plan is to pause this for now and first consolidate dependencies management (wrt Dockerfile vs. host-testing CI) - # before we can really start testing linux nerdctl on the host. - # - runner: ubuntu-24.04 - # - runner: ubuntu-24.04 - # canary: true with: timeout: 45 runner: ${{ matrix.runner }} @@ -161,11 +154,3 @@ jobs: # https://github.com/containerd/containerd/issues/13254 windows-containerd-version: 2.2.5 windows-containerd-sha: 8724c3a873b4984f5ee092c8f15c1a98ebbb0f968106cf8f5849ea100f3a0236 - linux-containerd-version: 2.3.2 - # FIXME: containerd SHAs are not verified for authenticity (only affects tests) - # https://github.com/containerd/nerdctl/issues/4666 - # Note: these are for amd64 - linux-containerd-sha: 75625e6f6595bb95f3fb9c8123a60534af4a8d9b52d7617065967bcefe71a17a - linux-containerd-service-sha: 1941362cbaa89dd591b99c32b050d82c583d3cd2e5fa63085d7017457ec5fca8 - linux-cni-version: v1.9.1 - linux-cni-sha: b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303 diff --git a/Dockerfile b/Dockerfile index 0b68758e902..acbc86dfc28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,6 @@ ARG GOMODJAIL_VERSION=v0.3.2@c145bb1e36fe0939c5fa0467f2477878dea8e3d9 ARG GO_VERSION=1.26 ARG UBUNTU_VERSION=24.04 ARG CONTAINERIZED_SYSTEMD_VERSION=v0.1.1 -ARG GOTESTSUM_VERSION=v1.13.0 ARG NYDUS_VERSION=v2.4.3 ARG SOCI_SNAPSHOTTER_VERSION=0.14.1 ARG KUBO_VERSION=v0.42.0 @@ -133,10 +132,6 @@ RUN xx-go --wrap && \ make build && \ xx-verify --static cmd/ipfs/ipfs && cp -a cmd/ipfs/ipfs /out/${TARGETARCH} -FROM build-base AS build-minimal -RUN BINDIR=/out/bin make binaries install -# We do not set CMD to `go test` here, because it requires systemd - FROM build-base AS build-dependencies ARG TARGETARCH ENV GOARCH=${TARGETARCH} @@ -270,6 +265,36 @@ RUN (cd /out && find ! -type d | sort | xargs sha256sum > /tmp/SHA256SUMS ) && \ FROM scratch AS out-full COPY --from=build-full /out / +# build-test-integration-artifacts assembles, on top of the full distribution, the additional +# binaries that are only needed to run the integration test suite (cosign, soci, ipfs, nydus). +# It is meant to be exported with `--output=type=local` and installed under /usr/local on a +# (CI) host or VM, so that the integration tests can run directly on the host with `go test`. +FROM build-full AS build-test-integration-artifacts +ARG TARGETARCH +# copy cosign binary for integration test +COPY --from=ghcr.io/sigstore/cosign/cosign:v3.0.5@sha256:be924970ba7438c22e18067dec5637946d6566eac711f5bedd1584e7137008fb /ko-app/cosign /out/bin/cosign +# installing soci for integration test +# (the static build, so that it also runs on EL hosts, whose glibc is older than what +# the default build requires) +ARG SOCI_SNAPSHOTTER_VERSION +RUN fname="soci-snapshotter-${SOCI_SNAPSHOTTER_VERSION}-${TARGETOS:-linux}-${TARGETARCH:-amd64}-static.tar.gz" && \ + curl -o "${fname}" -fsSL --retry 5 --retry-delay 5 --retry-max-time 120 --connect-timeout 20 --proto '=https' --tlsv1.2 "https://github.com/awslabs/soci-snapshotter/releases/download/v${SOCI_SNAPSHOTTER_VERSION}/${fname}" && \ + tar -C /out/bin -xvf "${fname}" soci soci-snapshotter-grpc && \ + rm -f "${fname}" +# enable offline ipfs for integration test +COPY --from=build-kubo /out/${TARGETARCH:-amd64}/* /out/bin/ +# install nydus components +ARG NYDUS_VERSION +RUN curl -o nydus-static.tgz -fsSL --retry 5 --retry-delay 5 --retry-max-time 120 --connect-timeout 20 --proto '=https' --tlsv1.2 "https://github.com/dragonflyoss/image-service/releases/download/${NYDUS_VERSION}/nydus-static-${NYDUS_VERSION}-linux-${TARGETARCH}.tgz" && \ + tar xzf nydus-static.tgz && \ + mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /out/bin/ && \ + rm -rf nydus-static.tgz nydus-static +# tests need a tini-custom binary +RUN cp /out/bin/tini /out/bin/tini-custom + +FROM scratch AS out-test-integration-artifacts +COPY --from=build-test-integration-artifacts /out / + FROM ubuntu:${UBUNTU_VERSION} AS base # fuse3 is required by stargz snapshotter RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \ @@ -295,94 +320,4 @@ VOLUME /var/lib/nerdctl ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["bash", "--login", "-i"] -FROM base AS test-integration -ARG DEBIAN_FRONTEND=noninteractive -# `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing -# `jq` is required to generate test summaries -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - software-properties-common \ - gnupg \ - gpg-agent \ - ca-certificates && \ - add-apt-repository ppa:criu/ppa && \ - apt-get update -qq && apt-get install -qq --no-install-recommends \ - expect \ - jq \ - git \ - make \ - criu -# We wouldn't need this if Docker Hub could have "golang:${GO_VERSION}-ubuntu" -COPY --from=build-base /usr/local/go /usr/local/go -ARG TARGETARCH -ENV PATH=/usr/local/go/bin:$PATH -ARG GOTESTSUM_VERSION -RUN GOBIN=/usr/local/bin go install gotest.tools/gotestsum@${GOTESTSUM_VERSION} -COPY . /go/src/github.com/containerd/nerdctl -WORKDIR /go/src/github.com/containerd/nerdctl -VOLUME /tmp -ENV CGO_ENABLED=0 -# copy cosign binary for integration test -COPY --from=ghcr.io/sigstore/cosign/cosign:v3.0.5@sha256:be924970ba7438c22e18067dec5637946d6566eac711f5bedd1584e7137008fb /ko-app/cosign /usr/local/bin/cosign -# installing soci for integration test -ARG SOCI_SNAPSHOTTER_VERSION -RUN fname="soci-snapshotter-${SOCI_SNAPSHOTTER_VERSION}-${TARGETOS:-linux}-${TARGETARCH:-amd64}.tar.gz" && \ - curl -o "${fname}" -fsSL --retry 5 --retry-delay 5 --retry-max-time 120 --connect-timeout 20 --proto '=https' --tlsv1.2 "https://github.com/awslabs/soci-snapshotter/releases/download/v${SOCI_SNAPSHOTTER_VERSION}/${fname}" && \ - tar -C /usr/local/bin -xvf "${fname}" soci soci-snapshotter-grpc && \ - mkdir -p /etc/soci-snapshotter-grpc && \ - touch /etc/soci-snapshotter-grpc/config.toml && \ - echo "\n[pull_modes]\n [pull_modes.soci_v1]\n enable = true" >> /etc/soci-snapshotter-grpc/config.toml -# enable offline ipfs for integration test -COPY --from=build-kubo /out/${TARGETARCH:-amd64}/* /usr/local/bin/ -COPY ./Dockerfile.d/test-integration-etc_containerd-stargz-grpc_config.toml /etc/containerd-stargz-grpc/config.toml -COPY ./Dockerfile.d/test-integration-ipfs-offline.service /usr/local/lib/systemd/system/ -COPY ./Dockerfile.d/test-integration-buildkit-nerdctl-test.service /usr/local/lib/systemd/system/ -COPY ./Dockerfile.d/test-integration-soci-snapshotter.service /usr/local/lib/systemd/system/ -RUN cp /usr/local/bin/tini /usr/local/bin/tini-custom -# using test integration containerd config -COPY ./Dockerfile.d/test-integration-etc_containerd_config.toml /etc/containerd/config.toml -# install ipfs service. avoid using 5001(api)/8080(gateway) which are reserved by tests. -RUN systemctl enable test-integration-ipfs-offline test-integration-buildkit-nerdctl-test test-integration-soci-snapshotter && \ - ipfs init && \ - ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888" && \ - ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889" -# install nydus components -ARG NYDUS_VERSION -RUN curl -o nydus-static.tgz -fsSL --retry 5 --retry-delay 5 --retry-max-time 120 --connect-timeout 20 --proto '=https' --tlsv1.2 "https://github.com/dragonflyoss/image-service/releases/download/${NYDUS_VERSION}/nydus-static-${NYDUS_VERSION}-linux-${TARGETARCH}.tgz" && \ - tar xzf nydus-static.tgz && \ - mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \ - rm nydus-static.tgz -CMD ["./hack/test-integration.sh"] - -FROM test-integration AS test-integration-rootless -# Install SSH for creating systemd user session. -# (`sudo` does not work for this purpose, -# OTOH `machinectl shell` can create the session but does not propagate exit code) -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - uidmap \ - openssh-server \ - openssh-client -# Install slirp4netns only if rootlesskit is prior to v3.0 -RUN if ! rootlesskit --help | grep -q gvisor-tap-vsock; then apt-get install -qq --no-install-recommends slirp4netns; fi -# TODO: update containerized-systemd to enable sshd by default, or allow `systemctl wants ssh` here -RUN ssh-keygen -q -t rsa -f /root/.ssh/id_rsa -N '' && \ - useradd -m -s /bin/bash rootless && \ - mkdir -p -m 0700 /home/rootless/.ssh && \ - cp -a /root/.ssh/id_rsa.pub /home/rootless/.ssh/authorized_keys && \ - mkdir -p /home/rootless/.local/share && \ - chown -R rootless:rootless /home/rootless -COPY ./Dockerfile.d/etc_systemd_system_user@.service.d_delegate.conf /etc/systemd/system/user@.service.d/delegate.conf -# ipfs daemon for rootless containerd will be enabled in /test-integration-rootless.sh -RUN systemctl disable test-integration-ipfs-offline -VOLUME /home/rootless/.local/share -COPY ./Dockerfile.d/test-integration-rootless.sh / -RUN chmod a+rx /test-integration-rootless.sh -CMD ["/test-integration-rootless.sh", "./hack/test-integration.sh"] - -# test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns -FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - slirp4netns -COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf -RUN chown -R rootless:rootless /home/rootless/.config - FROM base AS demo diff --git a/Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf b/Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf deleted file mode 100644 index e4c40b7eb24..00000000000 --- a/Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf +++ /dev/null @@ -1,3 +0,0 @@ -[Service] -# Change the port driver from "builtin" to "slirp4netns". Only used in CI. -Environment="CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns" diff --git a/Dockerfile.d/test-integration-buildkit-nerdctl-test.service b/Dockerfile.d/test-integration-buildkit-nerdctl-test.service index 23d0ffc81c5..a9d6ec4dbe8 100644 --- a/Dockerfile.d/test-integration-buildkit-nerdctl-test.service +++ b/Dockerfile.d/test-integration-buildkit-nerdctl-test.service @@ -38,4 +38,4 @@ TasksMax=infinity OOMScoreAdjust=-999 [Install] -WantedBy=docker-entrypoint.target +WantedBy=multi-user.target diff --git a/Dockerfile.d/test-integration-ipfs-offline.service b/Dockerfile.d/test-integration-ipfs-offline.service index af0662250c5..b6b27d19f26 100644 --- a/Dockerfile.d/test-integration-ipfs-offline.service +++ b/Dockerfile.d/test-integration-ipfs-offline.service @@ -6,4 +6,4 @@ ExecStart=ipfs daemon --init --offline Environment=IPFS_PATH="%h/.ipfs" [Install] -WantedBy=docker-entrypoint.target +WantedBy=multi-user.target diff --git a/Dockerfile.d/test-integration-rootless.sh b/Dockerfile.d/test-integration-rootless.sh deleted file mode 100755 index 63f383462cc..00000000000 --- a/Dockerfile.d/test-integration-rootless.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux -o pipefail -if [[ "$(id -u)" = "0" ]]; then - # Ensure securityfs is mounted for apparmor to work - if ! mountpoint -q /sys/kernel/security; then - mount -tsecurityfs securityfs /sys/kernel/security - fi - if [ -e /sys/kernel/security/apparmor/profiles ]; then - # Load the "nerdctl-default" profile for TestRunApparmor - nerdctl apparmor load - fi - - : "${WORKAROUND_ISSUE_622:=}" - if [[ "$WORKAROUND_ISSUE_622" != "" ]]; then - touch /workaround-issue-622 - fi - - # Switch to the rootless user via SSH - systemctl start ssh - exec ssh -o StrictHostKeyChecking=no rootless@localhost "$0" "$@" -else - containerd-rootless-setuptool.sh install - if grep -q "options use-vc" /etc/resolv.conf; then - containerd-rootless-setuptool.sh nsenter -- sh -euc 'echo "options use-vc" >>/etc/resolv.conf' - fi - - if [[ -e /workaround-issue-622 ]]; then - echo "WORKAROUND_ISSUE_622: Not enabling BuildKit (https://github.com/containerd/nerdctl/issues/622)" >&2 - else - CONTAINERD_NAMESPACE="nerdctl-test" containerd-rootless-setuptool.sh install-buildkit-containerd - fi - containerd-rootless-setuptool.sh install-stargz - if [ ! -f "/home/rootless/.config/containerd/config.toml" ] ; then - echo "version = 2" > /home/rootless/.config/containerd/config.toml - fi - cat <>/home/rootless/.config/containerd/config.toml -[proxy_plugins] - [proxy_plugins."stargz"] - type = "snapshot" - address = "/run/user/$(id -u)/containerd-stargz-grpc/containerd-stargz-grpc.sock" - [proxy_plugins.stargz.exports] - root = "/home/rootless/.local/share/containerd-stargz-grpc/" - enable_remote_snapshot_annotations = "true" -[[plugins."io.containerd.transfer.v1.local".unpack_config]] - platform = "linux" - snapshotter = "overlayfs" -[[plugins."io.containerd.transfer.v1.local".unpack_config]] - platform = "linux" - snapshotter = "stargz" -EOF - systemctl --user restart containerd.service - containerd-rootless-setuptool.sh -- install-ipfs --init --offline # offline ipfs daemon for testing - echo "ipfs = true" >>/home/rootless/.config/containerd-stargz-grpc/config.toml - systemctl --user restart stargz-snapshotter.service - export IPFS_PATH="/home/rootless/.local/share/ipfs" - containerd-rootless-setuptool.sh install-bypass4netnsd - # Once ssh-ed, we lost the Dockerfile working dir, so, get back in the nerdctl checkout - cd /go/src/github.com/containerd/nerdctl - # We also lose the PATH (and SendEnv=PATH would require sshd config changes) - exec env PATH="/usr/local/go/bin:$PATH" "$@" -fi diff --git a/Dockerfile.d/test-integration-soci-snapshotter.service b/Dockerfile.d/test-integration-soci-snapshotter.service index 5964702ac6a..d4465e3d2f0 100644 --- a/Dockerfile.d/test-integration-soci-snapshotter.service +++ b/Dockerfile.d/test-integration-soci-snapshotter.service @@ -12,4 +12,4 @@ Restart=always RestartSec=5 [Install] -WantedBy=docker-entrypoint.target +WantedBy=multi-user.target diff --git a/docs/testing/README.md b/docs/testing/README.md index 0ab8fcd7647..82a2b35e86f 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -76,11 +76,25 @@ Note that this is different from the `--parallel` flag, which controls the amoun parallelization that a single go test binary will use when faced with tests that do explicitly allow it (with a call to `t.Parallel()`). -### Or test in a container +### Or provision a test environment with Docker-built artifacts + +Docker can be used to build all the dependencies needed to run the integration tests +(containerd, runc, CNI plugins, BuildKit, snapshotters, etc.), which can then be installed +on the host (this is what the CI does). These scripts substantially and irreversibly modify +the host, so they refuse to run unless `GITHUB_ACTIONS=true` is set - only do this on a +disposable machine: + +```bash +docker buildx build --target out-test-integration-artifacts --output type=local,dest=/tmp/nerdctl-test-artifacts . +sudo env GITHUB_ACTIONS=true ./hack/provisioning/linux/test-integration-env.sh install /tmp/nerdctl-test-artifacts rootful +./hack/test-integration.sh -test.target=nerdctl -test.only-flaky=false +``` + +For rootless (`rootless`, or `rootless-port-slirp4netns`): ```bash -docker build -t test-integration --target test-integration . -docker run -t --rm --privileged test-integration +sudo env GITHUB_ACTIONS=true ./hack/provisioning/linux/test-integration-env.sh install /tmp/nerdctl-test-artifacts rootless +GITHUB_ACTIONS=true ./hack/test-integration-rootless.sh ./hack/test-integration.sh -test.target=nerdctl -test.only-flaky=false ``` ### Principles diff --git a/hack/build-integration-canary.sh b/hack/build-integration-canary.sh index 7f55cc6e868..052140fafb7 100755 --- a/hack/build-integration-canary.sh +++ b/hack/build-integration-canary.sh @@ -214,7 +214,7 @@ assets::get(){ ###################### canary::build::integration(){ - docker_args=(docker build -t test-integration --target test-integration) + docker_args=(docker build -t test-integration-artifacts --target build-test-integration-artifacts) for dep in "${dependencies[@]}"; do local bl="" diff --git a/hack/provisioning/README.md b/hack/provisioning/README.md index 314ffc9fed4..03640804bc2 100644 --- a/hack/provisioning/README.md +++ b/hack/provisioning/README.md @@ -10,6 +10,7 @@ Use provided installation scripts instead (see user documentation). ## Contents - `/version` allows retrieving latest (or experimental) versions of certain products (golang, containerd, etc) -- `/linux` allows updating in-place containerd, cni (future: buildkit) +- `/linux` allows updating in-place containerd, cni (future: buildkit), and provisioning a full +integration testing environment (`test-integration-env.sh`) from Docker-built artifacts - `/windows` allows install WinCNI, containerd - `/kube` allows spinning-up a Kind cluster \ No newline at end of file diff --git a/hack/provisioning/linux/test-integration-env.sh b/hack/provisioning/linux/test-integration-env.sh new file mode 100755 index 00000000000..530ffd5e827 --- /dev/null +++ b/hack/provisioning/linux/test-integration-env.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# test-integration-env.sh provisions a linux host (a GitHub Actions runner, or a Lima guest) +# so that the integration test suite can run directly on it with `go test` +# (through hack/test-integration.sh), without being wrapped inside a Docker container. +# +# It expects a directory containing the artifacts exported from the +# `out-test-integration-artifacts` Dockerfile stage - Docker is only used to *build* them: +# docker buildx build --target out-test-integration-artifacts --output type=local,dest=DIR . +# +# Usage (as root, through sudo from the unprivileged user meant to run the tests): +# sudo ./hack/provisioning/linux/test-integration-env.sh install DIR [rootful|rootless|rootless-port-slirp4netns] +# +# Supported distributions: Ubuntu (GitHub Actions runners), and Enterprise Linux (Lima guests). + +set -o errexit -o errtrace -o functrace -o nounset -o pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)" +readonly root +# lib.sh requires shasum at source time, which EL does not ship by default +if ! command -v shasum >/dev/null && command -v dnf >/dev/null; then + dnf install -q -y perl-Digest-SHA +fi +# shellcheck source=/dev/null +. "$root/../../scripts/lib.sh" + +readonly repo_root="$root/../../.." + +host::packages(){ + if command -v apt-get >/dev/null; then + # `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing + # `jq` is required to generate test summaries + apt-get update -qq >/dev/null + add-apt-repository -y ppa:criu/ppa >/dev/null + apt-get install -qq --no-install-recommends \ + apparmor \ + criu \ + dbus-user-session \ + expect \ + fuse3 \ + git \ + jq \ + make \ + openssh-server \ + uidmap >/dev/null + elif command -v dnf >/dev/null; then + # `container-selinux` provides the contexts (container_t, ...) applied by the SELinux + # tests - the labels are applied even in permissive mode + dnf install -q -y \ + container-selinux \ + criu \ + expect \ + fuse3 \ + git \ + iptables \ + jq \ + make \ + openssh-server \ + shadow-utils \ + tar + else + log::error "Unsupported distribution (neither apt-get nor dnf found)" + return 1 + fi +} + +host::slirp4netns(){ + if command -v apt-get >/dev/null; then + apt-get install -qq --no-install-recommends slirp4netns >/dev/null + else + dnf install -q -y slirp4netns + fi +} + +host::artifacts(){ + local artifacts="$1" + + # The distribution-shipped containerd and Docker (if any) conflict with the containerd + # under test. Note that Docker cannot be used anymore past this point. + systemctl disable --now docker.socket 2>/dev/null || true + systemctl disable --now docker.service 2>/dev/null || true + systemctl disable --now containerd.service 2>/dev/null || true + + # /usr/local/lib/systemd/system is part of the default systemd unit search path, + # so, the containerd, buildkit and stargz-snapshotter units are usable right away. + # --no-overwrite-dir keeps the metadata of pre-existing directories (notably the + # permissions of /usr/local itself - the buildx local exporter creates the artifacts + # directory with mode 0700). + # --no-same-owner makes the files owned by root, rather than by the user that ran buildx. + (cd "$artifacts" && tar -cf- .) | tar -C /usr/local -xf- --no-same-owner --no-overwrite-dir +} + +host::configuration(){ + # Test-specific containerd, buildkit, stargz and soci configurations + mkdir -p /etc/containerd /etc/buildkit /etc/containerd-stargz-grpc /etc/soci-snapshotter-grpc + cp "$repo_root/Dockerfile.d/test-integration-etc_containerd_config.toml" /etc/containerd/config.toml + cp "$repo_root/Dockerfile.d/etc_buildkit_buildkitd.toml" /etc/buildkit/buildkitd.toml + cp "$repo_root/Dockerfile.d/test-integration-etc_containerd-stargz-grpc_config.toml" /etc/containerd-stargz-grpc/config.toml + printf '\n[pull_modes]\n [pull_modes.soci_v1]\n enable = true\n' > /etc/soci-snapshotter-grpc/config.toml + mkdir -p /etc/cni && chmod 0755 /etc/cni + + # Test-specific systemd units (started explicitly by host::services) + cp "$repo_root"/Dockerfile.d/test-integration-*.service /etc/systemd/system/ + + # On EL, be permissive: the test environment is not currently designed to run enforcing + # (the containerized test environment used to run privileged) + if command -v setenforce >/dev/null; then + setenforce 0 || true + [ ! -e /etc/selinux/config ] || sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + fi + + # Tests are run by an unprivileged user, wrapping privileged invocations with + # `sudo`: the binaries under test must be resolvable then. + printf 'Defaults secure_path="/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"\n' \ + > /etc/sudoers.d/99-test-integration + chmod 0440 /etc/sudoers.d/99-test-integration + # Note: only validate our own drop-in - other, pre-existing files might not be valid + # (eg: /etc/sudoers.d/runner on GitHub Actions runners has "bad" permissions) + visudo -c -f /etc/sudoers.d/99-test-integration >/dev/null + + # Preload the kernel modules needed by the tests: br-netfilter ensures that bridged + # traffic goes through netfilter, and rootless containerd cannot trigger module + # auto-loading from inside a user namespace. The list mirrors Lima's 00-modprobe.sh + # boot script, which does not run for guests started in plain mode. + local module + for module in \ + br_netfilter \ + fuse \ + tun tap \ + bridge veth \ + ip_tables ip6_tables iptable_nat ip6table_nat iptable_filter ip6table_filter \ + nf_tables \ + x_tables xt_MASQUERADE xt_addrtype xt_comment xt_conntrack xt_mark xt_multiport xt_nat xt_tcpudp \ + overlay; do + modprobe "$module" || log::warning "Failed to load the $module module (negligible if it is built-in the kernel)" + done + + # Ensure securityfs is mounted, so that AppArmor detection works + mountpoint -q /sys/kernel/security || mount -t securityfs securityfs /sys/kernel/security + # Load the "nerdctl-default" AppArmor profile for TestRunApparmor: in rootless mode, + # the tests cannot load it themselves + if [ -e /sys/kernel/security/apparmor/profiles ]; then + /usr/local/bin/nerdctl apparmor load + fi +} + +host::services(){ + local target="$1" + local unit + local units=( + test-integration-soci-snapshotter.service + containerd.service + buildkit.service + stargz-snapshotter.service + test-integration-buildkit-nerdctl-test.service + ) + + if [ "$target" == "rootful" ]; then + # Offline ipfs daemon for testing. Avoid using 5001(api)/8080(gateway) which are + # reserved by tests. In rootless mode, this is handled by containerd-rootless-setuptool.sh. + if [ ! -e /root/.ipfs/config ]; then + env IPFS_PATH=/root/.ipfs /usr/local/bin/ipfs init >/dev/null + fi + env IPFS_PATH=/root/.ipfs /usr/local/bin/ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888" + env IPFS_PATH=/root/.ipfs /usr/local/bin/ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889" + units+=(test-integration-ipfs-offline.service) + fi + + systemctl daemon-reload + for unit in "${units[@]}"; do + systemctl restart "$unit" + done +} + +host::rootless(){ + local target="$1" + # The unprivileged user that is going to run the tests + local user="${SUDO_USER:-}" + + if [ "$user" == "" ] || [ "$user" == "root" ]; then + log::error "The $target target requires this script to be run with sudo, from the unprivileged user meant to run the tests" + return 1 + fi + + # Rootless containerd needs subordinate uids/gids for the testing user + grep -q "^$user:" /etc/subuid 2>/dev/null || echo "$user:100000:65536" >> /etc/subuid + grep -q "^$user:" /etc/subgid 2>/dev/null || echo "$user:100000:65536" >> /etc/subgid + + # Since Ubuntu 23.10+, apparmor restricts unprivileged user namespaces creation + if [ -e /etc/apparmor.d/abi/4.0 ]; then + cat < "/etc/apparmor.d/usr.local.bin.rootlesskit" +abi , +include +/usr/local/bin/rootlesskit flags=(unconfined) { + userns, + # Site-specific additions and overrides. See local/README for details. + include if exists +} +EOT + systemctl restart apparmor.service + fi + + # cgroup v2 delegation, for resource limits to work in rootless mode + mkdir -p /etc/systemd/system/user@.service.d + cp "$repo_root/Dockerfile.d/etc_systemd_system_user@.service.d_delegate.conf" /etc/systemd/system/user@.service.d/delegate.conf + systemctl daemon-reload + + # Keep the systemd user session (and thus the rootless daemons installed by + # containerd-rootless-setuptool.sh) alive in-between ssh sessions + loginctl enable-linger "$user" + + # Some tests publish ports 80 and 443, which the rootlesskit port driver has to bind + # as the unprivileged user. The containerized test environment used to get this for + # free: Docker sets this sysctl to 0 inside containers. + sysctl -w net.ipv4.ip_unprivileged_port_start=0 >/dev/null + + # Allow unprivileged ICMP Echo sockets (EL disables them by default; this mirrors + # Lima's 20-rootless-base.sh boot script, which does not run for guests started in + # plain mode) + sysctl -w "net.ipv4.ping_group_range=0 2147483647" >/dev/null + + # Without slirp4netns installed, rootlesskit v3.0+ falls back to its experimental + # gvisor-tap-vsock network driver: always provide the stable slirp4netns driver. + # slirp4netns is also required by the slirp4netns port driver, and by rootlesskit prior to v3.0. + host::slirp4netns +} + +provision::test-integration-env::install(){ + local artifacts="${1:-}" + local target="${2:-rootful}" + + [ "$(id -u)" == 0 ] || { + log::error "You need to be root" + return 1 + } + + # This script substantially and irreversibly modifies the host it runs on: + # it is only safe to run on a disposable CI machine + [ "${GITHUB_ACTIONS:-}" == "true" ] || { + log::error "Refusing to run outside of GitHub Actions (export GITHUB_ACTIONS=true to force)" + return 1 + } + + if [ "$artifacts" == "" ] || [ ! -d "$artifacts" ]; then + log::error "You need to point at a directory containing the test artifacts (built with: docker buildx build --target out-test-integration-artifacts --output type=local,dest=DIR .)" + return 1 + fi + + host::packages + host::artifacts "$artifacts" + host::configuration + [ "$target" == "rootful" ] || host::rootless "$target" + host::services "$target" +} + +com="$1" +shift +provision::test-integration-env::"$com" "$@" diff --git a/hack/test-integration-rootless.sh b/hack/test-integration-rootless.sh new file mode 100755 index 00000000000..3bf6bd4d981 --- /dev/null +++ b/hack/test-integration-rootless.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# test-integration-rootless.sh runs COMMAND in rootless mode, inside the systemd user +# session of the current, unprivileged user. +# +# It must be started as an unprivileged user with passwordless sudo, on a host +# provisioned with hack/provisioning/linux/test-integration-env.sh (which, among other +# things, enables lingering for the user, so that the systemd user session is available +# even when this script does not run from a logind session, eg: on a GitHub Actions +# runner). +# +# Usage: test-integration-rootless.sh COMMAND [ARGS...] +set -eux -o pipefail + +[ "$(id -u)" != "0" ] || { + echo "This script must be started as an unprivileged user with passwordless sudo" >&2 + exit 1 +} + +# This script substantially and irreversibly modifies the host (and the current user +# account): it is only safe to run on a disposable CI machine +[ "${GITHUB_ACTIONS:-}" == "true" ] || { + echo "Refusing to run outside of GitHub Actions (export GITHUB_ACTIONS=true to force)" >&2 + exit 1 +} + +# systemctl --user (and the dbus clients) locate the systemd user session through +# XDG_RUNTIME_DIR and DBUS_SESSION_BUS_ADDRESS. They are inherited when the script runs +# from a logind session (eg: an ssh session into a Lima guest), but not necessarily +# otherwise (eg: a GitHub Actions runner job): if unset, default them to the standard +# locations of the user session, which exists in any case, as the provisioning script +# enabled lingering. +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=${XDG_RUNTIME_DIR}/bus}" + +# The environment below mirrors Lima's 20-rootless-base.sh boot script, which does not +# run in the CI guest VMs (they are started in plain mode). +# Make sure iptables and mount.fuse3 are resolvable (on EL, non-login shells do not get +# the sbin directories in their PATH). +export PATH="$PATH:/usr/sbin:/sbin" +# fuse-overlayfs is the most stable snapshotter for rootless, on kernel < 5.13 (eg: EL 8) +# https://rootlesscontaine.rs/how-it-works/overlayfs/ +if [ -z "${CONTAINERD_SNAPSHOTTER:-}" ]; then + kernel="$(uname -r)" + kernel="${kernel%%-*}" + if [ "$(printf '%s\n' "$kernel" "5.13" | sort -V | head -n1)" != "5.13" ]; then + export CONTAINERD_SNAPSHOTTER="fuse-overlayfs" + fi +fi + +export IPFS_PATH="$HOME/.local/share/ipfs" + +# If anything fails below, the systemd user journal usually knows why +trap 'sudo journalctl --no-pager --lines=200 _UID="$(id -u)" >&2 || true' ERR + +# This script gets invoked repeatedly (eg: non-flaky, then flaky test runs). +# The installation below is not idempotent (specifically, the containerd configuration +# must not be appended twice), so, only perform it once. +if [ ! -e "$HOME/.config/nerdctl-test-setup-done" ]; then + # The rootlesskit port driver is configured through the environment of the (generated) + # containerd systemd user unit, so, it has to be baked into a unit drop-in. + if [ "${CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER:-builtin}" != "builtin" ]; then + mkdir -p "$HOME/.config/systemd/user/containerd.service.d" + cat <<-EOF >"$HOME/.config/systemd/user/containerd.service.d/port-driver.conf" + [Service] + Environment="CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=${CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER}" + EOF + systemctl --user daemon-reload + fi + + containerd-rootless-setuptool.sh install + if grep -q "options use-vc" /etc/resolv.conf; then + containerd-rootless-setuptool.sh nsenter -- sh -euc 'echo "options use-vc" >>/etc/resolv.conf' + fi + + if [ "${WORKAROUND_ISSUE_622:-}" != "" ]; then + echo "WORKAROUND_ISSUE_622: Not enabling BuildKit (https://github.com/containerd/nerdctl/issues/622)" >&2 + else + CONTAINERD_NAMESPACE="nerdctl-test" containerd-rootless-setuptool.sh install-buildkit-containerd + fi + containerd-rootless-setuptool.sh install-stargz + # The fuse-overlayfs snapshotter is required on hosts that cannot mount overlayfs + # in a user namespace (eg: EL 8) + containerd-rootless-setuptool.sh install-fuse-overlayfs + if [ ! -f "$HOME/.config/containerd/config.toml" ]; then + mkdir -p "$HOME/.config/containerd" + echo "version = 2" >"$HOME/.config/containerd/config.toml" + fi + cat <>"$HOME/.config/containerd/config.toml" +[proxy_plugins] + [proxy_plugins."stargz"] + type = "snapshot" + address = "/run/user/$(id -u)/containerd-stargz-grpc/containerd-stargz-grpc.sock" + [proxy_plugins.stargz.exports] + root = "$HOME/.local/share/containerd-stargz-grpc/" + enable_remote_snapshot_annotations = "true" + [proxy_plugins."fuse-overlayfs"] + type = "snapshot" + address = "/run/user/$(id -u)/containerd-fuse-overlayfs.sock" + [proxy_plugins."fuse-overlayfs".exports] + root = "$HOME/.local/share/containerd-fuse-overlayfs/" +[[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux" + snapshotter = "overlayfs" +[[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux" + snapshotter = "stargz" +[[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux" + snapshotter = "fuse-overlayfs" +EOF + systemctl --user restart containerd.service + containerd-rootless-setuptool.sh -- install-ipfs --init --offline # offline ipfs daemon for testing + echo "ipfs = true" >>"$HOME/.config/containerd-stargz-grpc/config.toml" + systemctl --user restart stargz-snapshotter.service + containerd-rootless-setuptool.sh install-bypass4netnsd + + touch "$HOME/.config/nerdctl-test-setup-done" +fi + +exec "$@" From 0d818115a0c5ee4a4b334b4bc685648cc4406065 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 23:52:56 +0900 Subject: [PATCH 7/8] Dockerfile: run the entrypoint command only after the daemons are ready The command passed to `docker run` is executed by the generated docker-entrypoint.service, which had no ordering dependency on the containerd, buildkit, and stargz-snapshotter units, so `docker run -t --rm --privileged ghcr.io/containerd/nerdctl nerdctl run ...` was racy: it failed with "cannot access containerd socket" whenever the command won the race against containerd. Add the ordering through a unit drop-in. Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index acbc86dfc28..9e9c49c0f03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -308,6 +308,12 @@ COPY --from=build-full /docker-entrypoint.sh /docker-entrypoint.sh COPY --from=out-full / /usr/local/ RUN perl -pi -e 's/multi-user.target/docker-entrypoint.target/g' /usr/local/lib/systemd/system/*.service && \ systemctl enable containerd buildkit stargz-snapshotter && \ + mkdir -p /etc/systemd/system/docker-entrypoint.service.d && \ + { echo "# docker-entrypoint.service runs the command passed to \`docker run\`: delay it"; \ + echo "# until the daemons are ready, so that \`docker run ... nerdctl run ...\` works"; \ + echo "[Unit]"; \ + echo "After=containerd.service buildkit.service stargz-snapshotter.service"; \ + } >/etc/systemd/system/docker-entrypoint.service.d/10-after-daemons.conf && \ mkdir -p /etc/bash_completion.d && \ nerdctl completion bash >/etc/bash_completion.d/nerdctl && \ mkdir -p -m 0755 /etc/cni From e08b9b81cf6d1603023b4ed1464afb76c13a3e19 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 4 Jul 2026 22:47:08 +0900 Subject: [PATCH 8/8] CI: smoke test the image built by the image workflow Build the image for the host platform, load it into Docker, and check that `nerdctl run` works inside it, before the multi-platform image is built and (except on pull requests) published to ghcr.io. Assisted-by: Claude Fable 5 Signed-off-by: Akihiro Suda --- .github/workflows/ghcr-image-build-and-publish.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ghcr-image-build-and-publish.yml b/.github/workflows/ghcr-image-build-and-publish.yml index 914dfd660db..383dd33cade 100644 --- a/.github/workflows/ghcr-image-build-and-publish.yml +++ b/.github/workflows/ghcr-image-build-and-publish.yml @@ -60,6 +60,20 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # Build the image for the host platform and load it into Docker, to smoke test + # it before the multi-platform image is built and published + - name: Build Docker image for the smoke test + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + load: true + tags: nerdctl-smoke-test + secrets: | + github_token=${{ secrets.GITHUB_TOKEN }} + + - name: Smoke test + run: docker run -t --rm --privileged nerdctl-smoke-test nerdctl run --rm hello-world + # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action - name: Build and push Docker image