test: Add nightly builds and tests

.github/workflows/nightly-build.yaml

@@ -0,0 +1,135 @@
+name: Nightly Build (reusable)
+
+# Reusable build for nightly WESTest testing: derives one version string, builds
+# and pushes the operator image, bakes that tag into the chart, then packages and
+# pushes the chart to the same OCI repo the release path uses. See
+# docs/nightly-westest-testing.md.
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: "Git ref to build (blank = the caller's checkout ref)."
+        type: string
+        required: false
+        default: ""
+    outputs:
+      image_tag:
+        description: "Nightly operator image tag (== chart_version)."
+        value: ${{ jobs.build.outputs.image_tag }}
+      chart_version:
+        description: "Nightly operator chart version (== image_tag)."
+        value: ${{ jobs.build.outputs.chart_version }}
+
+env:
+  IMAGE_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/operator
+  CHART_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/charts
+
+jobs:
+  build:
+    name: Build & publish nightly image + chart
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    outputs:
+      image_tag: ${{ steps.version.outputs.image_tag }}
+      chart_version: ${{ steps.version.outputs.chart_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          ref: ${{ inputs.ref }}
+
+      - name: Derive nightly version
+        id: version
+        shell: bash
+        run: |
+          set -euo pipefail
+          ts="$(date -u +%Y%m%d-%H%M%S)"
+          sha="$(git rev-parse --short=7 HEAD)"
+          # Lead with the current chart's release line (strip any prerelease suffix).
+          # TODO: once 2.0.0 is GA (Chart.yaml has no prerelease), lead with the NEXT
+          # semver (bump patch/minor) so nightlies sort ABOVE the last release rather
+          # than below it as a 2.0.0 prerelease.
+          base="$(awk '$1 == "version:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '"' | cut -d- -f1)"
+          # One version string for BOTH the image tag and the chart version.
+          version="${base}-nightly-${ts}"
+          echo "image_tag=${version}" >> "${GITHUB_OUTPUT}"
+          echo "chart_version=${version}" >> "${GITHUB_OUTPUT}"
+          {
+            echo "### Nightly build"
+            echo "- version (image + chart): \`${version}\`"
+            echo "- image: \`${IMAGE_REPOSITORY}:${version}\`"
+            echo "- chart: \`${CHART_REPOSITORY}/operator:${version}\`"
+            echo "- commit: \`${sha}\`"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Install Helm
+        uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4
+        with:
+          version: v3.19.0
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
+
+      - id: auth
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
+        with:
+          create_credentials_file: "true"
+          token_format: access_token
+          project_id: wandb-production
+          workload_identity_provider: ${{ secrets.CI_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.CI_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}
+
+      - name: Authorize Docker for Artifact Registry
+        run: gcloud auth configure-docker us-docker.pkg.dev --quiet
+
+      - name: Bake nightly image tag into chart values
+        env:
+          IMAGE_TAG: ${{ steps.version.outputs.image_tag }}
+        run: |
+          set -euo pipefail
+          # yq v4 is preinstalled on ubuntu-latest.
+          yq -i '.["wandb-operator"].image.tag = strenv(IMAGE_TAG)' deploy/operator/values.yaml
+
+      - name: Build and push nightly image
+        env:
+          IMAGE_TAG: ${{ steps.version.outputs.image_tag }}
+        run: make docker-build docker-push IMG="${IMAGE_REPOSITORY}:${IMAGE_TAG}"
+
+      - name: Resolve chart dependencies
+        run: |
+          set -euo pipefail
+          helm repo add ci-wandb https://charts.wandb.ai/
+          helm repo add ci-moco https://cybozu-go.github.io/moco/
+          helm repo add ci-ot-container-kit https://ot-container-kit.github.io/helm-charts
+          helm repo add ci-seaweedfs https://seaweedfs.github.io/seaweedfs-operator/
+          helm repo add ci-prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo add ci-altinity https://helm.altinity.com
+          helm repo add ci-victoria-metrics https://victoriametrics.github.io/helm-charts/
+          helm repo add ci-grafana https://grafana.github.io/helm-charts
+          helm dependency build deploy/operator
+
+      - name: Package, verify, and push nightly chart
+        env:
+          IMAGE_TAG: ${{ steps.version.outputs.image_tag }}
+          CHART_VERSION: ${{ steps.version.outputs.chart_version }}
+        run: |
+          set -euo pipefail
+          mkdir -p dist
+          helm package deploy/operator \
+            --version "${CHART_VERSION}" \
+            --app-version "${CHART_VERSION}" \
+            --destination dist
+          # Guard against the "green but stale" failure mode: the packaged chart
+          # must reference the nightly operator image, not the committed default.
+          if ! helm template "dist/operator-${CHART_VERSION}.tgz" \
+              | grep -qF "operator:${IMAGE_TAG}"; then
+            echo "Packaged chart does not reference ${IMAGE_REPOSITORY}:${IMAGE_TAG}" >&2
+            exit 1
+          fi
+          helm push "dist/operator-${CHART_VERSION}.tgz" "oci://${CHART_REPOSITORY}"

.github/workflows/nightly.yaml

@@ -0,0 +1,128 @@
+name: Nightly WESTest
+
+# Nightly end-to-end testing of the operator via wandb/westest. Builds a nightly
+# operator image+chart, then runs WESTest scenarios against it on large runners.
+# See docs/nightly-westest-testing.md for the full plan, prerequisites, and phases.
+#
+# Currently workflow_dispatch-only. Do NOT enable the schedule trigger until the
+# prerequisites in docs/nightly-westest-testing.md §7 are met (notably: the
+# WESTEST_APP_ID / WESTEST_APP_PRIVATE_KEY secrets for the wandb/westest App token,
+# and ubuntu-latest-8-cores availability).
+
+on:
+  workflow_dispatch:
+    inputs:
+      scenario:
+        description: "Single scenario to run (blank = full Phase 1 smoke set)."
+        type: string
+        required: false
+        default: ""
+  # schedule:
+  #   - cron: "0 7 * * *" # 07:00 UTC daily
+
+concurrency:
+  group: nightly-westest
+  cancel-in-progress: false
+
+env:
+  # Pin the WESTest release; the action defaults to `latest`, which resolves by
+  # creation date and would make nightlies non-reproducible.
+  WESTEST_VERSION: v0.2.0
+
+jobs:
+  build:
+    uses: ./.github/workflows/nightly-build.yaml
+    permissions:
+      contents: read
+      id-token: write
+    secrets: inherit
+
+  setup:
+    name: Resolve scenario matrix
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Build scenario matrix
+        id: matrix
+        env:
+          SCENARIO: ${{ inputs.scenario }}
+        run: |
+          set -euo pipefail
+          if [ -n "${SCENARIO}" ]; then
+            # Phase 0 bootstrap: a single scenario chosen at dispatch time.
+            matrix="$(printf '["%s"]' "${SCENARIO}")"
+          else
+            # Phase 1 smoke set (self-contained, no license, no cloud).
+            matrix='["local-kind-ingress","local-kind-gateway","local-kind-oidc","local-kind-proxy"]'
+          fi
+          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
+
+  test:
+    name: ${{ matrix.scenario }}
+    needs: [build, setup]
+    # The full local-kind stack needs >=8 vCPU; a standard runner leaves MySQL and
+    # the ClickHouse keeper Pending on Insufficient cpu.
+    runs-on: ubuntu-latest-8-cores
+    permissions:
+      contents: read
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        scenario: ${{ fromJSON(needs.setup.outputs.matrix) }}
+    env:
+      # Point every remote-chart scenario at the nightly chart. WESTest reads this
+      # env var (config.go); upgrade scenarios additionally override per-phase via
+      # --param (see docs/nightly-westest-testing.md §12 for the Phase 2/3 matrix).
+      WESTEST_OPERATOR_CHART_VERSION: ${{ needs.build.outputs.chart_version }}
+    steps:
+      - name: Mint a token to read wandb/westest releases
+        id: westest-token
+        # GitHub App installation token scoped to wandb/westest so the action can
+        # download the release binary (the operator repo's GITHUB_TOKEN can't).
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.WESTEST_APP_ID }}
+          private-key: ${{ secrets.WESTEST_APP_PRIVATE_KEY }}
+          owner: wandb
+          repositories: westest
+      - name: Run scenario
+        # Pinned to the westest main SHA that adds the github-token input (post-v0.2.0,
+        # no release tag yet); bump to a release tag once westest cuts one.
+        uses: wandb/westest/actions/run@662b59b6199d99a84435d889e7d8cf6e94ebf74c
+        with:
+          scenario: ${{ matrix.scenario }}
+          github-token: ${{ steps.westest-token.outputs.token }}
+          westest-version: ${{ env.WESTEST_VERSION }}
+          upload-artifacts: "true"
+
+  report:
+    needs: [build, setup, test]
+    if: always()
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Summarize
+        run: |
+          echo "build=${{ needs.build.result }} chart=${{ needs.build.outputs.chart_version }}"
+          echo "test=${{ needs.test.result }}"
+      - name: Notify on failure
+        if: needs.build.result != 'success' || needs.test.result != 'success'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.WESTEST_SLACK_WEBHOOK }}
+        run: |
+          set -euo pipefail
+          if [ -z "${SLACK_WEBHOOK_URL:-}" ]; then
+            echo "no WESTEST_SLACK_WEBHOOK configured; skipping notification"
+            exit 0
+          fi
+          run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl -fsSL -X POST -H 'Content-type: application/json' \
+            --data "{\"text\":\"Nightly WESTest failed — chart ${{ needs.build.outputs.chart_version }} — ${run_url}\"}" \
+            "${SLACK_WEBHOOK_URL}"

Makefile

@@ -348,4 +348,13 @@ $(GINKGO): $(LOCALBIN)
 	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@latest
 
 
+## westest e2e binary (github.com/wandb/westest)
+WESTEST ?= $(LOCALBIN)/westest
+WESTEST_VERSION ?= latest
+WESTEST_REPO ?= wandb/westest
+
+.PHONY: westest
+westest: $(LOCALBIN) ## Download the westest e2e binary from wandb/westest releases (set WESTEST_VERSION to pin).
+	WESTEST_VERSION="$(WESTEST_VERSION)" WESTEST_REPO="$(WESTEST_REPO)" ./hack/scripts/download-westest.sh "$(WESTEST)"
+
 include dep-management.mk olm.mk