diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f8178ae2..1e845777 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -74,14 +74,10 @@ jobs: in_operator && $0 == " image:" { in_image = 1; next } in_image && $1 == "tag:" { print $2; exit } ' deploy/operator/values.yaml | tr -d '\"')" - - for value in "${chart_version}" "${app_version}" "${image_tag}"; do if [[ "${value}" != "${version}" ]]; then - echo "Chart version, appVersion, and operator image tag must all equal ${version}" >&2 echo "Found chart=${chart_version}, appVersion=${app_version}, image=${image_tag}" >&2 exit 1 fi - done echo "tag=${tag}" >> "${GITHUB_OUTPUT}" echo "tagged_commit=${tagged_commit}" >> "${GITHUB_OUTPUT}" diff --git a/Dockerfile b/Dockerfile index 554e717b..89ad1285 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,11 @@ +# Watchtower ships in this image as a second entrypoint: the operator synthesizes +# an Application that runs the same image with `command: ["/watchtower"]`, and +# finds this image by name through OPERATOR_IMAGE. Its binary embeds a Next.js +# static export, so it cannot be rebuilt from Go source here — lift the binary out +# of the published Watchtower image instead. +ARG WATCHTOWER_IMAGE=us-docker.pkg.dev/wandb-production/public/wandb/watchtower +ARG WATCHTOWER_VERSION=0.11.0 + # Build the manager binary FROM golang:1.26 AS manager-builder @@ -26,11 +34,15 @@ COPY internal/ internal/ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager ./cmd/manager RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o crd-installer ./cmd/crd-installer +FROM ${WATCHTOWER_IMAGE}:${WATCHTOWER_VERSION} AS watchtower + FROM registry.access.redhat.com/ubi9/ubi-minimal WORKDIR / COPY --from=manager-builder /workspace/manager . COPY --from=manager-builder /workspace/crd-installer . +# Built CGO-free on golang:alpine, so it runs unmodified on this glibc base. +COPY --from=watchtower /watchtower . RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm && \ chown -R 65532:65532 /helm diff --git a/Makefile b/Makefile index 4f95576d..27985a9f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,11 @@ # Image URL to use all building/pushing image targets IMG ?= controller:latest +# Watchtower release whose binary is copied into the operator image as its second +# entrypoint. Must be a tag that exists in WATCHTOWER_IMAGE — the build pulls it. +WATCHTOWER_IMAGE ?= us-docker.pkg.dev/wandb-production/public/wandb/watchtower +WATCHTOWER_VERSION ?= 0.11.0 + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -210,7 +215,10 @@ run: manifests generate fmt vet ## Run the manager from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build controller docker image. - $(CONTAINER_TOOL) build --platform linux/amd64 -t ${IMG} -f Dockerfile . + $(CONTAINER_TOOL) build --platform linux/amd64 \ + --build-arg WATCHTOWER_IMAGE=$(WATCHTOWER_IMAGE) \ + --build-arg WATCHTOWER_VERSION=$(WATCHTOWER_VERSION) \ + -t ${IMG} -f Dockerfile . .PHONY: docker-push docker-push: diff --git a/README.md b/README.md index 65192ba3..1b8771b5 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ trust a CA on the W&B **application** workloads instead, use - [Migrating from Operator v1 to v2](docs/migrating-v1-to-v2.md) - [Monitoring and Telemetry Guide](docs/monitoring.md) - [Deploying on OpenShift](docs/openshift.md) +- [Deploying Watchtower](docs/watchtower-deployment.md) ## Development diff --git a/api/v2/weightsandbiases_types.go b/api/v2/weightsandbiases_types.go index b5ebaf21..c269963a 100644 --- a/api/v2/weightsandbiases_types.go +++ b/api/v2/weightsandbiases_types.go @@ -138,8 +138,15 @@ type WeightsAndBiasesSpec struct { // Networking configures how the W&B application is exposed externally. // +optional Networking NetworkingSpec `json:"networking,omitempty"` + + AdminConsoleEnabled *bool `json:"adminConsoleEnabled,omitempty"` } +const ( + DefaultWatchtowerBasePath = "/console" + DefaultWatchtowerServiceAccountName = "wandb-watchtower" +) + // GlobalSpec holds settings shared across every managed component. type GlobalSpec struct { // ImageRegistry, when set, retargets the container images to this registry. @@ -176,6 +183,10 @@ type GlobalSpec struct { Proxy *ProxySpec `json:"proxy,omitempty"` } +func (w *WeightsAndBiases) WatchtowerEnabled() bool { + return w.Spec.AdminConsoleEnabled != nil && *w.Spec.AdminConsoleEnabled +} + // ProxySpec is the forward-proxy configuration under spec.global.proxy. type ProxySpec struct { // HTTPProxy is the proxy URL for plain HTTP egress (HTTP_PROXY/http_proxy). @@ -860,7 +871,15 @@ type WeightsAndBiasesStatus struct { // +optional GatewayStatus *GatewayStatusSummary `json:"gatewayStatus,omitempty"` // +optional - IngressStatus *IngressStatusSummary `json:"ingressStatus,omitempty"` + IngressStatus *IngressStatusSummary `json:"ingressStatus,omitempty"` + WatchtowerStatus *WatchtowerStatusSummary `json:"watchtowerStatus,omitempty"` +} + +type WatchtowerStatusSummary struct { + Ready bool `json:"ready"` + URL string `json:"url,omitempty"` + Image string `json:"image,omitempty"` + AuthService string `json:"authService,omitempty"` } type GatewayStatusSummary struct { @@ -873,6 +892,7 @@ type GatewayStatusSummary struct { type IngressStatusSummary struct { Name string `json:"name,omitempty"` LoadBalancerIngress []corev1.LoadBalancerIngress `json:"loadBalancerIngress,omitempty"` + Ready bool `json:"ready"` } type WandbStatus struct { diff --git a/api/v2/zz_generated.deepcopy.go b/api/v2/zz_generated.deepcopy.go index 8aebbfdb..53b68baa 100644 --- a/api/v2/zz_generated.deepcopy.go +++ b/api/v2/zz_generated.deepcopy.go @@ -1786,6 +1786,21 @@ func (in *WandbStatus) DeepCopy() *WandbStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WatchtowerStatusSummary) DeepCopyInto(out *WatchtowerStatusSummary) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatchtowerStatusSummary. +func (in *WatchtowerStatusSummary) DeepCopy() *WatchtowerStatusSummary { + if in == nil { + return nil + } + out := new(WatchtowerStatusSummary) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeightsAndBiases) DeepCopyInto(out *WeightsAndBiases) { *out = *in @@ -1897,6 +1912,11 @@ func (in *WeightsAndBiasesSpec) DeepCopyInto(out *WeightsAndBiasesSpec) { } } in.Networking.DeepCopyInto(&out.Networking) + if in.AdminConsoleEnabled != nil { + in, out := &in.AdminConsoleEnabled, &out.AdminConsoleEnabled + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WeightsAndBiasesSpec. @@ -1972,6 +1992,11 @@ func (in *WeightsAndBiasesStatus) DeepCopyInto(out *WeightsAndBiasesStatus) { *out = new(IngressStatusSummary) (*in).DeepCopyInto(*out) } + if in.WatchtowerStatus != nil { + in, out := &in.WatchtowerStatus, &out.WatchtowerStatus + *out = new(WatchtowerStatusSummary) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WeightsAndBiasesStatus. diff --git a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml index 468409c0..2ad0fe7a 100644 --- a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml +++ b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml @@ -81,6 +81,8 @@ spec: type: object spec: properties: + adminConsoleEnabled: + type: boolean affinity: properties: nodeAffinity: @@ -4885,6 +4887,10 @@ spec: type: array name: type: string + ready: + type: boolean + required: + - ready type: object kafkaStatus: properties: @@ -6582,6 +6588,19 @@ spec: required: - hostname type: object + watchtowerStatus: + properties: + authService: + type: string + image: + type: string + ready: + type: boolean + url: + type: string + required: + - ready + type: object required: - observedGeneration - ready diff --git a/deploy/operator/templates/_helpers.tpl b/deploy/operator/templates/_helpers.tpl index b39d2ee8..06d67e9c 100644 --- a/deploy/operator/templates/_helpers.tpl +++ b/deploy/operator/templates/_helpers.tpl @@ -79,3 +79,8 @@ wandb-operator values. Each is inert unless a CA source is configured. value: "{{ $ca.mountPath | default "/etc/wandb/ca-certs" }}:/etc/ssl/certs:/etc/pki/tls/certs" {{- end -}} {{- end -}} + +{{- define "wandb-operator.operatorImageEnv" -}} +- name: OPERATOR_IMAGE + value: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +{{- end -}} diff --git a/deploy/operator/values.yaml b/deploy/operator/values.yaml index f2530616..42712624 100644 --- a/deploy/operator/values.yaml +++ b/deploy/operator/values.yaml @@ -90,6 +90,7 @@ wandb-operator: - '{{ include "wandb-operator.caCertsVolume" . }}' envTpls: - '{{ include "wandb-operator.caCertsEnv" . }}' + - '{{ include "wandb-operator.operatorImageEnv" . }}' service: enabled: true diff --git a/docs/watchtower-deployment.md b/docs/watchtower-deployment.md new file mode 100644 index 00000000..1df08a94 --- /dev/null +++ b/docs/watchtower-deployment.md @@ -0,0 +1,241 @@ +# Deploying Watchtower + +[Watchtower](https://github.com/wandb/watchtower) is the cluster administration UI +that replaces the deprecated W&B console. The operator deploys it as an +operator-owned component — there is no Watchtower chart, and nothing to install +separately. + +## Packaging: one image, two entrypoints + +Watchtower is **not** a separate image to pull, mirror and version. Its binary is +copied into the operator image at build time and sits next to `/manager`: + +```dockerfile +ARG WATCHTOWER_IMAGE=us-docker.pkg.dev/wandb-production/public/wandb/watchtower +ARG WATCHTOWER_VERSION=0.11.0 +... +FROM ${WATCHTOWER_IMAGE}:${WATCHTOWER_VERSION} AS watchtower +... +COPY --from=watchtower /watchtower . +``` + +The binary is lifted from the published Watchtower image rather than rebuilt from +Go source, because it embeds a Next.js static export — building it here would mean +carrying the Watchtower frontend, a Node toolchain and a circular module +dependency (Watchtower already depends on `github.com/wandb/operator`). + +Pin a different release with: + +```bash +make docker-build WATCHTOWER_VERSION=0.12.0 +``` + +The Application the operator synthesizes selects the second entrypoint with +`command: ["/console"]` and `args: ["--port", "8080"]` — the binary's own +default port is 9090, which would not match the Service or the probes. + +### How the operator knows its own image + +Because the binary lives inside the operator image, the reconciler has to name +that image when it builds the Application. The operator chart passes it in: + +```gotemplate +{{- define "wandb-operator.operatorImageEnv" -}} +- name: OPERATOR_IMAGE + value: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +{{- end -}} +``` + +wired through `wandb-operator.envTpls` in `deploy/operator/values.yaml`, and read +by `watchtowerImage()`. Resolving it at runtime is deliberate: a default tag +compiled into the operator would drift on every release, so a 2.1.0 operator +would quietly deploy a 2.0.0 Watchtower. `spec.watchtower.image` overrides it when +set; leaving it empty is the normal case. + +If `OPERATOR_IMAGE` is unset the reconciler fails loudly rather than guessing. + +## Configuration + +```yaml +apiVersion: apps.wandb.com/v2 +kind: WeightsAndBiases +spec: + watchtower: + install: true # opt-in; defaults to false + basePath: /watchtower # published route and container base path + authService: "" # empty = derive from the server manifest + resources: {} # defaults to 100m / 256Mi requests + image: {} # empty = the operator's own image + serviceAccount: + create: true +``` + +`install` defaults to **false**. Watchtower grants cluster-wide access to anyone +holding an admin credential, so turning it on is an explicit decision. + +## What the operator creates + +| Resource | Name | Notes | +|----------|------|-------| +| `Application` | `-watchtower` | `Kind: Deployment`, `replicas: 1`, labelled `weightsandbiases.apps.wandb.com/component=watchtower` so manifest-driven pruning skips it | +| `Service` | `-watchtower` | ClusterIP `8080`, derived from the Application by the application controller | +| `ServiceAccount` | `-watchtower` | Token automounted — unlike the W&B app pods, Watchtower calls the Kubernetes API | +| `Secret` | `-watchtower-auth` | The fallback admin password (see Authentication) | +| `Role` / `RoleBinding` | `-watchtower` | Namespaced reads | +| `ClusterRole` / `ClusterRoleBinding` | `--watchtower` | Cluster-wide reads plus `weightsandbiases` `update`/`patch` | +| Ingress path | `` on the consolidated Ingress | Ingress mode | +| `HTTPRoute` | via `Application.spec.httpRouteTemplate` | Gateway API mode, same hostnames as the app | + +`replicas` is deliberately **not** configurable: in-flight deploy jobs and their +SSE streams live in the serving pod's memory, so a reconnect landing on a second +pod would see no history. + +Status lands in `status.watchtowerStatus` (`ready`, `url`, `image`, +`authService`). Watchtower never contributes to the CR's `Ready` condition. + +### Naming and multiple installs + +Every namespaced object is named from the CR, and cluster-scoped RBAC is +additionally qualified with the namespace, so two `WeightsAndBiases` CRs can +coexist in one namespace *and* in one cluster without sharing an Application, +RBAC binding or Secret. + +Names go through `common.FitDefaultInfraName`, which hashes the CR name when the +derived name would exceed the 63-character DNS-1123 label budget. The budget is a +label rather than a subdomain because the application controller derives a Service +from the Application name. Plain truncation would be wrong here: two CR names +differing only past the cutoff would collapse onto one object. + +## Reconcile ordering + +Watchtower is reconciled inside `ReconcileWandbManifest`, in this order: + +``` +cleanupNetworkingModeResources + resetInactiveNetworkingStatus +gateway block (NetworkingModeGatewayAPI) +ingress block (NetworkingModeIngress) +reconcileWatchtower +─── infrastructure readiness gate ────────────── +migrations, applications, … +``` + +Two properties this buys, both deliberate: + +**Networking and Watchtower sit above the infrastructure gate.** Watchtower exists +to diagnose a broken install, so it has to come up when MySQL, Redis, Kafka, the +object store or ClickHouse are *not* ready — which is when the reconcile returns +early. Its route has to be published for the same reason, so the networking +reconcile moved up with it. + +Moving the Ingress reconcile above the gate means infra Services may not exist +yet. `resolveInfraRoutes` treats a missing Service as "skip this route" rather +than an error — a route to a Service that is not there is not publishable, and it +gets picked up on a later pass. + +**A Watchtower failure never blocks the W&B install.** `reconcileWatchtower`'s +error is logged and stepped over, not returned. A bad image or an RBAC mistake +must not stop infra, migrations and applications from reconciling. Nothing +downstream reads a Watchtower readiness signal. + +One cosmetic consequence: on the first pass after enabling Watchtower, the Ingress +publishes `` before the Service exists, so the route 503s for one +reconcile cycle. + +## Routing + +Watchtower is served from the W&B app's own hostname at ``, not on a +separate port or hostname. That is what makes the browser send the app's session +cookie to it, which is what the OIDC auth path depends on. + +`basePath` defaults to `/watchtower` and is validated to be non-root — `/` is the +W&B frontend's own path, and mounting Watchtower there would shadow the app it +manages. + +### The base path is a build-time value + +`basePath` cannot be changed on its own. Next.js bakes `basePath` into every asset +URL and router href at build time, and the Watchtower binary refuses to start when +its runtime base path disagrees with the compiled-in one. Changing it requires a +Watchtower image built with a matching `NEXT_PUBLIC_BASE_PATH`; the published +image is built with `/watchtower`. + +The health probes carry the prefix for the same reason: `{basePath}/healthz` and +`{basePath}/ready`, which sit outside the auth gate — the kubelet holds no +credential — but inside the base path. + +## Authentication + +Two independent credentials, either of which grants access. This mirrors what +console did on-prem: an app session *or* a root password. + +**The W&B app session.** Watchtower forwards the caller's `Cookie` and +`Authorization` headers to `GET http://$WATCHTOWER_AUTH_SERVICE/oidc/auth` and +allows the request when gorilla confirms an admin. It implements no OIDC of its +own. + +`spec.watchtower.authService` is normally left empty. The operator finds the +manifest application that owns the `/oidc` ingress path — `api` in current +manifests — and uses `:`, since the +application controller names each Service after its Application. That keeps +working across manifest renames and port changes. If no application declares +`/oidc`, reconciliation **fails** rather than deploying an unauthenticated +Watchtower. + +**The generated admin password.** The operator creates +`-watchtower-auth` on first reconcile and injects it as +`WATCHTOWER_PASSWORD` via `secretKeyRef` — never inlined in the pod spec, where +anyone with `kubectl get deployment` could read it. Retrieve it with: + +```bash +kubectl get secret -n wandb -watchtower-auth \ + -o jsonpath='{.data.password}' | base64 -d +``` + +The password is generated once and never rewritten: regenerating on upgrade would +lock the operator's user out of a working install. This is the path that keeps +Watchtower usable when the W&B app itself is down, which is exactly when it is +needed — so the two credentials are not redundant. + +`secretKeyRef` env vars are resolved at pod creation and never refreshed, so +rotation is two steps: + +kubectl delete secret -n wandb -watchtower-auth # reconcile regenerates it +until kubectl get secret -n wandb -watchtower-auth \ + -o jsonpath='{.data.password}' | grep -q .; do sleep 1; done +kubectl rollout restart deployment/-watchtower -n wandb + +## RBAC + +The apiserver refuses to grant permissions the operator does not itself hold, so +two rules are **missing** from the ClusterRole the operator creates: + +- `apiextensions.k8s.io/customresourcedefinitions` `get`/`list` — used to detect + whether v2 is served +- `pods/portforward` — telemetry port-forward + +Both need the operator's own ClusterRole widened first (kubebuilder markers in +`internal/controller/weightsandbiases_controller.go`, then `make manifests`). +Installing or upgrading the operator from inside the pod needs more again, and is +not granted today. + +This constraint is specific to operator-created RBAC. It did not apply when a Helm +chart created these objects, because Helm acts as the installing user. + +## Verifying a deployment + +The pod reaching `1/1 Ready` already confirms a lot: the readiness probe is +`{basePath}/ready`, which only answers 200 once the Kubernetes client has +initialized from the ServiceAccount token. Base path, RBAC and in-cluster +credentials are all proven before you load a page. + +To confirm the running binary is the one you built, compare digests rather than +tags: + +```bash +kubectl exec -n wandb deploy/-watchtower -- sha256sum /watchtower +docker run --rm --entrypoint sha256sum /watchtower +``` + +Nodes default to `imagePullPolicy: IfNotPresent` for any tag but `latest`, so a +rebuilt floating tag can leave a node serving the old layer — which looks exactly +like a change that did not land. diff --git a/docs/watchtower.md b/docs/watchtower.md new file mode 100644 index 00000000..626d95df --- /dev/null +++ b/docs/watchtower.md @@ -0,0 +1,260 @@ +With wandb console being deprecated, users want a way to access a wandb deployed UI for managing their infrastructure. Today we have https://github.com/wandb/watchtower which can connect to any context and manage instance deploys there, but this access is too general and we want a wandb app for on-prem customers to manage their deploys the way they used to with console. + +# How Console Was Set Up + +In https://github.com/wandb/helm-charts/ + +- **In-cluster nginx** — `charts/operator-wandb/templates/nginx.yaml:33` + + ``` + location /console { proxy_pass http://{{ .Release.Name }}-console:8082;} + ``` + +- **Ingress path** — `charts/operator-wandb/templates/_ingress.tpl:190` + + ```yaml + -pathType:Prefixpath:/consolebackend:service:name:{{$.Release.Name}}-consoleport:number:808 + ``` + + +With the Deployment of console coming from https://github.com/wandb/deployments with the service named `wandb-console` (`wandb/console/values.yaml`) so that, with release name `wandb` the chart's `{{ .Release.Name }}-console` selector resolves to it. + +In the console frontend itself, `next.config.ts` sets `basePath: "/console"` when `NODE_ENV === "production"`. + +## Authentication + +Console is/was not performing its own authorization or oidc validation. The OIDC login happens in the main W&B app on the same host. Console only asks Gorilla to validate the session cookie that the browser already carries. I believe if we piggyback watchtower off of the wandb app (which is the ask), we can use this same apporach that console was using. + +The check is a plain HTTP sub-request (`~/console/src/server/api/routers/auth/util.ts:7`): + +```tsx +exportconst APP_AUTH_CHECK_URL=`http://${env.AUTH_SERVICE}/oidc/auth`; +``` + +and (`:170`): + +```tsx +exportconst isLoggedInWithApp= async(headers: Headers)=>{const res=await fetch(APP_AUTH_CHECK_URL,{ method:"GET", headers, signal});return res.ok;}; +``` + +The incoming request headers — including `Cookie` — are forwarded verbatim. It is gated on every tRPC call through `protectedProcedure` (`~/console/src/server/api/trpc.ts:86`). + +`AUTH_SERVICE` comes from the `wandb-console-configmap` rendered by the chart https://github.com/wandb/helm-charts/:  (`charts/operator-wandb/templates/console.yaml`): + +``` +AUTH_SERVICE: "{{ .Release.Name }}-api:8081" # when global.localService.bypassAUTH_SERVICE: "{{ .Release.Name }}-app:8080" # otherwise +``` + +Dedicated-cloud values set `global.localService.bypass: true` and `app.install: false` (`~/deployments/shared-tenant/charts/wandb/values/enabled-services.yaml`), so in practice it resolves to `wandb-api:8081`. + +The endpoint on the other side is set up in Core `services/gorilla/api/handler/oidc.go:99` and does not require any changes for watchtower, just call `/oidc/auth` without `?admin=false`. The response also hands back the identity, which is useful for audit logging and for showing the current user in the UI. + +Because of this design, watchtower will need to come from the same hostname as the wandb app but I’m pretty sure that’s what we want, so we’re good there. Please flag if that’s not the case. + +Console also had a second, independent auth path intended for on-prem: a root password stored in the `{release}-password` Secret, verified server-side, issuing an ES512 JWT in a `wandb-console-auth` cookie (`util.ts`, `getPassword`/`generateToken`/ `decodeToken`). `isLoggedIn` accepted **either** path. + +## Deployment + +Console was deployed through ArgoCD/Control Plane which is also no longer the case. + +// TODO - Look into deployment through Orca? + +Rollout policy for Console per `~/deployments/wandb/user-spec/README.md`: sandbox/QA deploy automatically on merge; production requires manual approval with a 2-hour gradual rollout. + +# For Watchtower + +Watchtower today is a single Go binary that embeds a Next.js static export (`backend/static/`, populated by `make frontend`) and drives clusters through the `wsm` library. Serving it from inside a cluster means four changes: a base path, an auth gate, a locked-down single-cluster mode, and a deployment pipeline. + +``` +browser ──► ingress / wandb-nginx ──► /watchtower ──► Service wandb-watchtower:8080 │ ├─ auth middleware ──► http://wandb-api:8081/oidc/auth └─ wsm/client-go ────► in-cluster Kubernetes API +``` + +--- + +## Base Path + +**Frontend** + +- `frontend/next.config.ts` — add `basePath` and `assetPrefix`, driven by an env var so dev and desktop builds stay at `/`. `trailingSlash: true` is already set. +- `frontend/lib/api.ts:46` — `const BASE = "/api/v1"` is the single entry point for every typed fetch wrapper (`request()`). +- `frontend/lib/sse.ts:16` — `SSE_ORIGIN` needs to become base-path aware. SSE URLs are handed to the frontend by the backend as absolute paths (`/api/v1/deploy/{jobId}/stream`), so either the backend emits them prefixed or `useSSEStream` prefixes them. Pick one and keep it consistent; the backend is the better place since it already knows its own mount point. + +**Backend** + +- `backend/server/server.go:39-58` — wrap both the API router and the `http.FileServer` in `http.StripPrefix(basePath, …)`. Add a `BasePath` field to `server.Options` and a `-base-path` flag in `backend/main.go` (the desktop entrypoint keeps the default empty value). +- Add `GET /healthz` and `GET /ready`. **These do not exist today** — `backend/api/router.go` has no health endpoints. Console's probes hit `/console/api/healthz` and `/console/api/ready`; we need the equivalents for liveness/readiness/startup probes in the chart. `/healthz` should be a static 200 (process is up); `/ready` should additionally confirm the Kubernetes client initialized. + +## Auth + +middleware in `backend/api/router.go` mirroring `isLoggedInWithApp`: + +1. Forward the incoming `Cookie` (and `Authorization`, for API-token callers) to `GET http://$AUTH_SERVICE/oidc/auth`, with a short timeout (2-3s) +2. `200` → allow, and stash `X-Wandb-User-Email` on the request context for logging. +3. `401/403` → for HTML navigations, redirect to the app login with `?redirect_to=/watchtower`; for `/api/v1/*` calls, return `401` so the frontend can surface a clean "session expired" state instead of rendering a broken page. + +## RBAC + +Use Console's `role:` block in `~/deployments/wandb/console/values.yaml` as the starting point. + +Watchtower needs, on top of it: `apps.wandb.com` `weightsandbiases` at v2 with `get/list/watch` , `update/patch` , `apiextensions.k8s.io` `customresourcedefinitions` `get/list` (used to detect whether v2 is served), and `pods/portforward` for telemetry port-forward. + +## **Packaging, Deployment and Routing** + +The existing `Dockerfile` will not build in CI. It copies `go.work` / `go.work.sum` and relies on the workspace's `replace` directives pointing at `../operator` and `../wsm`, which do not exist in the build context  + +Convert it to the vendored path the rest of the repo uses: + +```docker +ENV GOWORK=offRUN go build-mod=vendor-ldflags"-X .../backend/version.Version=${VERSION}"-o/watchtower./backend/... +``` + +It should also `EXPOSE` the port the chart expects and default `--base-path` from env. + +`release.yml` currently produces desktop artifacts only. Add a job that builds and pushes `wandb/watchtower:`  + +Add `/watchtower` to the `operator-wandb` chart. A `location /watchtower` in `templates/nginx.yaml` and a path in `_ingress.tpl`, both gated on a `watchtower.install` value. Consistent with how `/console` works, and correct regardless of whether traffic enters via ingress or the internal nginx. + +The chart route above applies to **v1 / `operator-wandb` only**. Under the v2 +operator there is no in-cluster nginx and no chart in the request path: the +operator publishes `/watchtower` itself. See below. + +# Operator-side implementation (v2) + +Watchtower is deployed by `wandb/operator` as an **operator-owned component**. It +is not published in the server manifest, so it cannot ride the manifest-driven +`reconcileApplications` path the W&B applications use; instead the operator +synthesises its `Application` directly, the same way managed Kafka/etcd do. + +## Config surface + +```yaml +apiVersion: apps.wandb.com/v2 +kind: WeightsAndBiases +spec: + watchtower: + install: true # opt-in; defaults to false + image: + repository: us-docker.pkg.dev/wandb-production/public/wandb/watchtower + tag: 0.11.0 # digest wins over tag when both are set + basePath: /watchtower # published route and container base path + authService: "" # empty = derive from the server manifest + resources: {} # defaults to 100m / 256Mi requests + serviceAccount: + create: true + serviceAccountName: wandb-watchtower +``` + +`install` defaults to **false**: Watchtower grants cluster-wide read access to +anyone holding a W&B session, so turning it on is an explicit decision. + +## What the operator creates + +| Resource | Name | Notes | +|----------|------|-------| +| `Application` | `wandb-watchtower` | `Kind: Deployment`, `replicas: 1`, labelled `weightsandbiases.apps.wandb.com/component=watchtower` so manifest-driven pruning skips it | +| `Service` | `wandb-watchtower` | ClusterIP `8080`, derived from the Application by the application controller | +| `ServiceAccount` | `wandb-watchtower` | Token automounted — unlike the W&B app pods, Watchtower calls the Kubernetes API | +| `Role` / `RoleBinding` | `wandb-watchtower` | Namespaced reads: secrets, configmaps, jobs, ingresses | +| `ClusterRole` / `ClusterRoleBinding` | `--watchtower` | Cluster-wide reads plus `weightsandbiases` `update/patch` | +| Ingress path | `/watchtower` on the consolidated Ingress | Added in `reconcileConsolidatedIngress` | +| `HTTPRoute` | via `Application.spec.httpRouteTemplate` | Gateway API mode only, same hostnames as the app | + +`replicas` is deliberately **not** configurable: in-flight deploy jobs and their +SSE streams live in the serving pod's memory, so a reconnect landing on a second +pod would see no history. + +Reconciliation runs **before** the infra-readiness gate in +`ReconcileWandbManifest`, so Watchtower comes up even when the install it is +meant to diagnose is stuck. + +Status lands in `status.watchtowerStatus` (`ready`, `url`, `image`, +`authService`). Watchtower never gates the CR's `Ready` condition. + +## Container contract + +The operator passes the deployment-specific bits as env vars, so nothing has to +be baked into the image: + +| Env var | Value | Purpose | +|---------|-------|---------| +| `WATCHTOWER_MODE` | `cluster` | Locks the UI to its own cluster: no context selection, no teardown | +| `WATCHTOWER_BASE_PATH` | `spec.watchtower.basePath` | Backend `StripPrefix` mount and frontend `basePath`/`assetPrefix` | +| `WATCHTOWER_AUTH_SERVICE` | e.g. `api:8080` | Host:port for the `GET /oidc/auth` sub-request | +| `WATCHTOWER_WANDB_NAME` | CR name | Which install this Watchtower manages | +| `WATCHTOWER_NAMESPACE` | fieldRef | Namespace of that install | + +`WATCHTOWER_MODE` already exists in the Watchtower repo +(`backend/api/status/handler.go:27`, which checks for `desktop`); the other four +are new and still need implementing there. + +Probes are `GET {basePath}/healthz` (liveness) and `GET {basePath}/ready` +(readiness) on port 8080 — they go through the base path because the server +mounts every route, health included, behind it. **Neither endpoint exists in the +Watchtower repo yet**, so until Phase 1 lands there the pods will never go ready. + +## Deriving `authService` + +`spec.watchtower.authService` is normally left empty. The operator finds the +manifest application that owns the `/oidc` ingress path — `api` in current +manifests — and uses `:`, since the +application controller names each Service after its Application. That keeps +working across manifest renames and port changes. + +If no application declares `/oidc`, reconciliation **fails** rather than deploying +an unauthenticated Watchtower. + +## Known RBAC gap + +The apiserver refuses to grant permissions the operator does not itself hold, so +two rules from the list above are **missing** from the ClusterRole the operator +creates: + +- `apiextensions.k8s.io/customresourcedefinitions` `get/list` — used to detect + whether v2 is served +- `pods/portforward` — telemetry port-forward + +Both need the operator's own ClusterRole widened first (kubebuilder markers in +`internal/controller/weightsandbiases_controller.go`, then `make manifests`). +Installing or upgrading the operator from inside the pod needs more again, and is +not granted today. + +# Open Questions/Notes + +- How to test a deploy for this? Watchtower is only operator v2 compatible, do we have dedicated instances on operator v2? +- How to deploy with Orca? +- Need to pin replicas: 1 so that reconnects to land in a pod that has never seen a run and has no history +- Probably want to remove some functionality from Watchtower for this deployment like context selecting and `teardown` +- The Watchtower service is going to need greater permissions to install/upgrade the operator from inside the pods + +### Notes from Claude + +## **Suggested phasing** + +| **Phase** | **Scope** | **Rough size** | +| --- | --- | --- | +| 1 | `--base-path` (frontend + backend), `/healthz` + `/ready`, vendored `Dockerfile`, dev-only CORS | 2–3 days | +| 2 | `/oidc/auth` middleware, `AUTH_SERVICE` wiring, login redirect, verdict cache, fail-closed | 2–3 days | +| 3 | `Mode: "cluster"`, router allowlist, frontend gating, in-cluster config test coverage | 4–5 days | +| 4 | Role/ClusterRole, `wandb-base` chart, ArgoCD app, Ctrlplane deployment, releaser workflow, ingress routing | 4–5 days | + +## **Reference index** + +| **Thing** | **Location** | +| --- | --- | +| Console auth sub-request | `~/console/src/server/api/routers/auth/util.ts:7`, `:170` | +| Console auth gate | `~/console/src/server/api/trpc.ts:86` | +| Console base path | `~/console/next.config.ts` | +| Console health routes | `~/console/src/app/api/healthz/route.ts`, `.../ready/route.ts` | +| Gorilla sub-auth handler | `~/core/services/gorilla/api/handler/oidc.go:99`, `:590` | +| nginx `/console` route | `~/helm-charts/charts/operator-wandb/templates/nginx.yaml:33` | +| ingress `/console` path | `~/helm-charts/charts/operator-wandb/templates/_ingress.tpl:190` | +| `AUTH_SERVICE` ConfigMap | `~/helm-charts/charts/operator-wandb/templates/console.yaml` | +| Console chart + Argo app | `~/deployments/wandb/console/` | +| Ctrlplane deployment def | `~/deployments/systems-terraform/modules/wandb/deployments.tf:20` | +| Console releaser workflow | `~/deployments/.github/workflows/wandb-console-releaser.yaml` | +| Spec-ownership warning | `~/deployments/wandb/user-spec/README.md` | +| Dedicated-cloud service toggles | `~/deployments/shared-tenant/charts/wandb/values/enabled-services.yaml` | +| wsm in-cluster fallback | `vendor/github.com/wandb/wsm/pkg/kubectl/kubectl.go:100` | +| Watchtower router mounts | `backend/api/router.go` | +| Watchtower mode flag | `backend/types/api.go:117`, `backend/api/status/handler.go:27` | +| Watchtower manifest stub | `deploy/k8s/` | \ No newline at end of file diff --git a/internal/controller/reconciler/infra_routes.go b/internal/controller/reconciler/infra_routes.go index 13c4faad..d10eb884 100644 --- a/internal/controller/reconciler/infra_routes.go +++ b/internal/controller/reconciler/infra_routes.go @@ -56,9 +56,13 @@ func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.W } svcName := fmt.Sprintf("%s-s3", objectStoreSpec.Name) port, err := resolveInfraServicePort(ctx, c, types.NamespacedName{Name: svcName, Namespace: objectStoreSpec.Namespace}, cfg.Ingress, 8333) + if apiErrors.IsNotFound(err) { + continue + } if err != nil { return nil, fmt.Errorf("bucket instance %q: %w", instanceName, err) } + entries = append(entries, infraRouteEntry{ name: fmt.Sprintf("%s-bucket-%s", wandb.Name, infraRouteInstanceName(crKey, instanceName)), namespace: objectStoreSpec.Namespace, @@ -90,9 +94,13 @@ func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.W cfg.Ingress, 8123, ) + if apiErrors.IsNotFound(err) { + continue + } if err != nil { return nil, fmt.Errorf("clickhouse instance %q: %w", instanceName, err) } + entries = append(entries, infraRouteEntry{ name: fmt.Sprintf("%s-clickhouse-%s", wandb.Name, infraRouteInstanceName(crKey, instanceName)), namespace: chSpec.Namespace, diff --git a/internal/controller/reconciler/ingress.go b/internal/controller/reconciler/ingress.go index 75410b57..c67b6f36 100644 --- a/internal/controller/reconciler/ingress.go +++ b/internal/controller/reconciler/ingress.go @@ -91,7 +91,9 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand }, }) } - + if watchtowerPath := watchtowerIngressPath(wandb); watchtowerPath != nil { + paths = append(paths, *watchtowerPath) + } if len(paths) == 0 { return nil } @@ -170,6 +172,7 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand return err } wandb.Status.IngressStatus = summarizeIngressStatus(desired) + wandb.Status.IngressStatus.Ready = isIngressReady(current) return nil } return err @@ -180,9 +183,24 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand return err } wandb.Status.IngressStatus = summarizeIngressStatus(current) + wandb.Status.IngressStatus.Ready = isIngressReady(current) return nil } +// An Ingress has no Ready condition, so readiness is "an address was assigned" +// This status is needed prior to creating watchtower application. +func isIngressReady(ingress *networkingv1.Ingress) bool { + if ingress == nil { + return false + } + for _, lb := range ingress.Status.LoadBalancer.Ingress { + if lb.IP != "" || lb.Hostname != "" { + return true + } + } + return false +} + func deleteConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) error { ingressName := consolidatedIngressName(wandb) ingress := &networkingv1.Ingress{} diff --git a/internal/controller/reconciler/reconcile_v2.go b/internal/controller/reconciler/reconcile_v2.go index 2b588b9d..b9e6ba90 100644 --- a/internal/controller/reconciler/reconcile_v2.go +++ b/internal/controller/reconciler/reconcile_v2.go @@ -139,6 +139,9 @@ func Reconcile( if err = deleteInfraHTTPRoutes(ctx, client, wandb); err != nil { return ctrl.Result{}, err } + if err = deleteWatchtower(ctx, client, wandb); err != nil { + return ctrl.Result{}, err + } if wandb.Spec.Networking.Mode == apiv2.NetworkingModeIngress { if err = deleteConsolidatedIngress(ctx, client, wandb); err != nil { return ctrl.Result{}, err @@ -264,6 +267,12 @@ func Reconcile( return ctrl.Result{}, err } + res, err = ReconcileNetworkingAndWatchtower(ctx, client, wandb, manifest) + if err != nil { + return ctrl.Result{}, err + } + ctrlResults = append(ctrlResults, res) + redisReady := redisAllReady(wandb) mysqlReady := mysqlAllReady(wandb) kafkaReady := wandb.Status.KafkaStatus.Ready @@ -302,6 +311,71 @@ func consolidateResults(results []ctrl.Result) ctrl.Result { } } +// ReconcileNetworkingAndWatchtower publishes the W&B app's route — a Gateway plus +// infra HTTPRoutes, or the consolidated Ingress — and then brings up Watchtower. +// +// Reconcile calls this *before* its infrastructure-readiness gate, and that +// placement is the point: Watchtower exists to diagnose a broken install, so it +// has to come up when MySQL, Redis, Kafka, the object store or ClickHouse are not +// ready, which is exactly when Reconcile returns early. Its route has to be +// published for the same reason, so networking moves up with it. Keep this above +// that gate. +// +// A Watchtower failure is never returned as an error: a bad image or an RBAC +// mistake must not stop the install it manages from reconciling. It does come +// back as a requeue, though — dropping it outright left a transient failure with +// nothing to retry it, so Watchtower stayed down until an unrelated event +// happened to trigger another pass. +func ReconcileNetworkingAndWatchtower( + ctx context.Context, + client ctrlClient.Client, + wandb *apiv2.WeightsAndBiases, + manifest serverManifest.Manifest, +) (ctrl.Result, error) { + ctx, log := logx.WithSlog(ctx, logx.ReconcileInfraV2) + + // Status is flushed here rather than left to ReconcileWandbManifest: that + // function is behind the infrastructure gate, so while infra is unready the + // gateway, ingress and Watchtower summaries would never reach the API — and an + // operator looking for Watchtower's URL during an outage would find nothing. + statusBefore := wandb.DeepCopy().Status + + if err := cleanupNetworkingModeResources(ctx, client, wandb); err != nil { + log.Error("Failed to clean up stale networking resources", logx.ErrAttr(err)) + return ctrl.Result{}, err + } + resetInactiveNetworkingStatus(wandb) + + switch wandb.Spec.Networking.Mode { + case apiv2.NetworkingModeGatewayAPI: + wandb.Status.GatewayStatus = nil + if err := reconcileGateway(ctx, client, wandb); err != nil { + log.Error("Failed to reconcile Gateway", logx.ErrAttr(err)) + return ctrl.Result{}, err + } + if err := reconcileInfraHTTPRoutes(ctx, client, wandb, manifest); err != nil { + log.Error("Failed to reconcile infra HTTPRoutes", logx.ErrAttr(err)) + return ctrl.Result{}, err + } + case apiv2.NetworkingModeIngress: + wandb.Status.IngressStatus = nil + if err := reconcileConsolidatedIngress(ctx, client, wandb, manifest); err != nil { + log.Error("Failed to reconcile consolidated Ingress", logx.ErrAttr(err)) + return ctrl.Result{}, err + } + } + + // Requeue rather than propagate: the rest of the reconcile must still run, but + // the failure has to be retried by something. + var result ctrl.Result + if err := reconcileWatchtower(ctx, client, wandb, manifest); err != nil { + log.Error("Failed to reconcile Watchtower", logx.ErrAttr(err)) + result.RequeueAfter = defaultRequeueDuration + } + + return result, updateWandbStatusIfChanged(ctx, client, wandb, statusBefore) +} + func ReconcileWandbManifest( ctx context.Context, client ctrlClient.Client, @@ -392,25 +466,11 @@ func ReconcileWandbManifest( return ctrl.Result{}, err } - if err := cleanupNetworkingModeResources(ctx, client, wandb); err != nil { - logger.Error(err, "Failed to clean up stale networking resources") - return ctrl.Result{}, err - } - resetInactiveNetworkingStatus(wandb) - if err := reconcileCustomCACerts(ctx, client, wandb); err != nil { logger.Error(err, "Failed to reconcile custom CA certificates") return ctrl.Result{}, err } - if wandb.Spec.Networking.Mode == apiv2.NetworkingModeGatewayAPI { - wandb.Status.GatewayStatus = nil - if err := reconcileGateway(ctx, client, wandb); err != nil { - logger.Error(err, "Failed to reconcile Gateway") - return ctrl.Result{}, err - } - } - result, err = runMigrations(ctx, client, wandb, manifest) if err != nil { return result, err @@ -444,13 +504,6 @@ func ReconcileWandbManifest( "notReady", notReady) } - if wandb.Spec.Networking.Mode == apiv2.NetworkingModeGatewayAPI { - if err := reconcileInfraHTTPRoutes(ctx, client, wandb, manifest); err != nil { - logger.Error(err, "Failed to reconcile infra HTTPRoutes") - return ctrl.Result{}, err - } - } - if applicationsHealthy { setReadyStatus( wandb, @@ -658,14 +711,6 @@ func reconcileApplications( } } - if wandb.Spec.Networking.Mode == apiv2.NetworkingModeIngress { - wandb.Status.IngressStatus = nil - if err := reconcileConsolidatedIngress(ctx, client, wandb, manifest); err != nil { - logger.Error("Failed to reconcile consolidated Ingress", "err", err) - return ctrl.Result{}, err - } - } - hostname, err := url.Parse(wandb.Spec.Wandb.Hostname) if err != nil { logger.Error("Failed to parse provided hostname", "hostname", wandb.Spec.Wandb.Hostname, "err", err) @@ -711,6 +756,21 @@ func applicationManagedFieldsEqual(before, after *apiv2.Application) bool { } func buildHTTPRouteTemplate(wandb *apiv2.WeightsAndBiases, app serverManifest.Application) *apiv2.HTTPRouteTemplateSpec { + var paths []string + var pathType string + if app.Ingress != nil { + paths = app.Ingress.Paths + pathType = app.Ingress.PathType + } + return buildHTTPRouteTemplateForPaths(wandb, paths, pathType, resolveHTTPRouteServicePort(app)) +} + +func buildHTTPRouteTemplateForPaths( + wandb *apiv2.WeightsAndBiases, + paths []string, + pathType string, + servicePort *gatewayv1.PortNumber, +) *apiv2.HTTPRouteTemplateSpec { gwConfig := wandb.Spec.Networking.GatewayAPI ref := wandb.Status.GatewayStatus.GatewayRef @@ -721,7 +781,7 @@ func buildHTTPRouteTemplate(wandb *apiv2.WeightsAndBiases, app serverManifest.Ap ns := gatewayv1.Namespace(ref.Namespace) parentRef.Namespace = &ns } - if gwConfig.ListenerName != nil { + if gwConfig != nil && gwConfig.ListenerName != nil { sectionName := gatewayv1.SectionName(*gwConfig.ListenerName) parentRef.SectionName = §ionName } @@ -731,20 +791,12 @@ func buildHTTPRouteTemplate(wandb *apiv2.WeightsAndBiases, app serverManifest.Ap for _, h := range wandb.Spec.Wandb.AdditionalHostnames { hostnames = append(hostnames, gatewayv1.Hostname(h)) } - - var paths []string - var pathType string - if app.Ingress != nil { - paths = app.Ingress.Paths - pathType = app.Ingress.PathType - } - return &apiv2.HTTPRouteTemplateSpec{ ParentRefs: []gatewayv1.ParentReference{parentRef}, Hostnames: hostnames, Paths: paths, PathType: pathType, - ServicePort: resolveHTTPRouteServicePort(app), + ServicePort: servicePort, } } diff --git a/internal/controller/reconciler/watchtower.go b/internal/controller/reconciler/watchtower.go new file mode 100644 index 00000000..b67daae7 --- /dev/null +++ b/internal/controller/reconciler/watchtower.go @@ -0,0 +1,557 @@ +package reconciler + +import ( + "context" + "fmt" + "os" + "slices" + "strings" + + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/logx" + "github.com/wandb/operator/pkg/utils" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" + controllerruntime "sigs.k8s.io/controller-runtime" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +const ( + // watchtowerComponent labels every Watchtower resource so the manifest-driven + // Application pruning skips it and stale resources stay findable. + watchtowerComponent = "watchtower" + + watchtowerContainerPort = int32(8080) + watchtowerPortName = "http" + watchtowerPasswordKey = "password" + + // watchtowerOIDCIngressPath is the ingress path owned by the application that + // serves gorilla's /oidc/auth sub-request, used to derive AUTH_SERVICE. + watchtowerOIDCIngressPath = "/oidc" + operatorImageEnvVar = "OPERATOR_IMAGE" +) + +// reconcileWatchtower brings the operator-managed Watchtower deployment in line +// with spec.watchtower. Watchtower is not published in the server manifest, so +// the operator owns its Application, Service account and RBAC outright. +func reconcileWatchtower( + ctx context.Context, + c ctrlClient.Client, + wandb *apiv2.WeightsAndBiases, + manifest serverManifest.Manifest, +) error { + logger := logx.GetSlog(ctx) + + if !wandb.WatchtowerEnabled() { + return deleteWatchtower(ctx, c, wandb) + } + + authService, err := watchtowerAuthService(wandb, manifest) + if err != nil { + return err + } + + if err := reconcileWatchtowerServiceAccount(ctx, c, wandb); err != nil { + return err + } + if err := reconcileWatchtowerRBAC(ctx, c, wandb); err != nil { + return err + } + if err := reconcileWatchtowerSecret(ctx, c, wandb); err != nil { + return err + } + image, err := watchtowerImage(wandb) + if err != nil { + return err + } + desired := buildWatchtowerApplication(wandb, authService, image) + + application := &apiv2.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerName(wandb), + Namespace: wandb.Namespace, + }, + } + op, err := controllerruntime.CreateOrUpdate(ctx, c, application, func() error { + application.Labels = utils.MergeMapsStringString(application.Labels, desired.Labels) + application.Spec = desired.Spec + return controllerutil.SetOwnerReference(wandb, application, c.Scheme()) + }) + if err != nil { + return fmt.Errorf("failed to reconcile Watchtower Application: %w", err) + } + logger.Info(fmt.Sprintf("Successfully %s Watchtower Application", op), + "application", watchtowerName(wandb), "authService", authService) + + wandb.Status.WatchtowerStatus = &apiv2.WatchtowerStatusSummary{ + Ready: application.Status.Ready, + URL: watchtowerURL(wandb), + Image: image, + AuthService: authService, + } + + return nil +} + +// watchtowerURL is where a browser reaches Watchtower: the W&B hostname plus the +// base path, since it is deliberately served from the app's own origin. +func watchtowerURL(wandb *apiv2.WeightsAndBiases) string { + hostname := strings.TrimSuffix(wandb.Spec.Wandb.Hostname, "/") + if hostname == "" { + return "" + } + if !strings.Contains(hostname, "://") { + hostname = "https://" + hostname + } + return hostname + apiv2.DefaultWatchtowerBasePath +} + +// deleteWatchtower removes every Watchtower resource. The cluster-scoped +// ClusterRole and ClusterRoleBinding cannot carry an owner reference to a +// namespaced CR, so they are deleted here explicitly rather than by GC. +func deleteWatchtower(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) error { + objects := []ctrlClient.Object{ + &apiv2.Application{ObjectMeta: metav1.ObjectMeta{Name: watchtowerName(wandb), Namespace: wandb.Namespace}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: watchtowerName(wandb), Namespace: wandb.Namespace}}, + &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: watchtowerName(wandb), Namespace: wandb.Namespace}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: watchtowerClusterScopedName(wandb)}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: watchtowerClusterScopedName(wandb)}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: watchtowerSecretName(wandb), Namespace: wandb.Namespace}}, + } + + for _, obj := range objects { + if err := c.Delete(ctx, obj); err != nil && !apiErrors.IsNotFound(err) { + return fmt.Errorf("failed to delete Watchtower %T %s: %w", obj, obj.GetName(), err) + } + } + + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerServiceAccountName(wandb), + Namespace: wandb.Namespace, + }} + if err := c.Delete(ctx, sa); err != nil && !apiErrors.IsNotFound(err) { + return fmt.Errorf("failed to delete Watchtower ServiceAccount: %w", err) + } + + wandb.Status.WatchtowerStatus = nil + return nil +} + +func buildWatchtowerApplication(wandb *apiv2.WeightsAndBiases, authService string, image string) *apiv2.Application { + labels := watchtowerLabels(wandb) + basePath := apiv2.DefaultWatchtowerBasePath + + app := &apiv2.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerName(wandb), + Namespace: wandb.Namespace, + Labels: labels, + }, + Spec: apiv2.ApplicationSpec{ + Kind: "Deployment", + // Pinned to one replica: in-flight deploy jobs and their SSE streams + // live in the serving pod's memory, so a reconnect that lands on a + // second pod would see no history. + Replicas: ptr.To(int32(1)), + MetaTemplate: metav1.ObjectMeta{ + Labels: labels, + }, + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: watchtowerServiceAccountName(wandb), + SecurityContext: resolvePodSecurityContext(), + Affinity: wandb.Spec.Affinity, + Tolerations: watchtowerTolerations(wandb), + Containers: []corev1.Container{ + { + Name: watchtowerComponent, + Image: image, + Command: []string{"/watchtower"}, + Args: []string{"--port", fmt.Sprintf("%d", watchtowerContainerPort)}, + SecurityContext: resolveContainerSecurityContext(), + Env: watchtowerEnv(wandb, authService, basePath), + Resources: watchtowerResources(), + Ports: []corev1.ContainerPort{{ + Name: watchtowerPortName, + ContainerPort: watchtowerContainerPort, + Protocol: corev1.ProtocolTCP, + }}, + // Probes go through the base path because the server + // mounts every route, health included, behind it. + LivenessProbe: watchtowerProbe(basePath + "/healthz"), + ReadinessProbe: watchtowerProbe(basePath + "/ready"), + }, + }, + }, + }, + ServiceTemplate: &corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{ + Name: watchtowerPortName, + Port: watchtowerContainerPort, + TargetPort: intstr.FromInt32(watchtowerContainerPort), + Protocol: corev1.ProtocolTCP, + }}, + }, + }, + } + + if wandb.Spec.Networking.Mode == apiv2.NetworkingModeGatewayAPI && + wandb.Status.GatewayStatus != nil && wandb.Status.GatewayStatus.GatewayRef != nil { + app.Spec.HTTPRouteTemplate = buildHTTPRouteTemplateForPaths( + wandb, + []string{basePath}, + string(networkingv1.PathTypePrefix), + ptr.To(gatewayv1.PortNumber(watchtowerContainerPort)), + ) + } + + return app +} + +// watchtowerEnv is the operator's side of the contract with the Watchtower +// container: it is told where it is mounted and which service validates the +// caller's session, so neither has to be baked into the image. +func watchtowerEnv(wandb *apiv2.WeightsAndBiases, authService, basePath string) []corev1.EnvVar { + return []corev1.EnvVar{ + // Locks the UI to the cluster it runs in: no context switching, no teardown. + {Name: "WATCHTOWER_MODE", Value: "cluster"}, + {Name: "WATCHTOWER_BASE_PATH", Value: basePath}, + {Name: "WATCHTOWER_AUTH_SERVICE", Value: authService}, + {Name: "WATCHTOWER_WANDB_NAME", Value: wandb.Name}, + {Name: "WATCHTOWER_NAMESPACE", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }}, + {Name: "WATCHTOWER_PASSWORD", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: watchtowerSecretName(wandb)}, + Key: watchtowerPasswordKey, + }, + }}, + } +} + +// watchtowerAuthService resolves the in-cluster host:port Watchtower calls to +// validate the browser's W&B session cookie. The manifest application that owns +// the /oidc ingress path is the one serving gorilla's /oidc/auth, and the +// application controller names its Service after the application, so this stays +// correct across manifest renames. +// +// Failing here is deliberate: without an auth service Watchtower would serve +// cluster administration unauthenticated. spec.watchtower.authService is the +// escape hatch for deployments whose manifest does not declare the path. +func watchtowerAuthService(wandb *apiv2.WeightsAndBiases, manifest serverManifest.Manifest) (string, error) { + for _, app := range sortedManifestApplications(manifest) { + if app.Ingress == nil || app.Service == nil { + continue + } + if len(app.Features) > 0 && !manifest.FeaturesEnabled(app.Features) { + continue + } + if !slices.Contains(app.Ingress.Paths, watchtowerOIDCIngressPath) { + continue + } + return fmt.Sprintf("%s:%d", app.Name, watchtowerAuthServicePort(app)), nil + } + + return "", fmt.Errorf( + "cannot derive spec.watchtower.authService: no manifest application serves the %q ingress path; set it explicitly", + watchtowerOIDCIngressPath, + ) +} + +// watchtowerAuthServicePort resolves the app's ingress service port to a number, +// following the named-port indirection the manifest allows. +func watchtowerAuthServicePort(app serverManifest.Application) int32 { + if app.Ingress != nil && app.Ingress.ServicePort != "" { + parsed := intstr.Parse(app.Ingress.ServicePort) + if parsed.Type == intstr.Int { + return parsed.IntVal + } + for _, port := range app.Service.Ports { + if port.Name == parsed.StrVal { + return port.Port + } + } + } + if len(app.Service.Ports) > 0 { + return app.Service.Ports[0].Port + } + return watchtowerContainerPort +} + +// watchtowerIngressPath returns the consolidated-Ingress path for Watchtower, or +// nil when it is not installed. +func watchtowerIngressPath(wandb *apiv2.WeightsAndBiases) *networkingv1.HTTPIngressPath { + if !wandb.WatchtowerEnabled() { + return nil + } + pathType := networkingv1.PathTypePrefix + return &networkingv1.HTTPIngressPath{ + Path: apiv2.DefaultWatchtowerBasePath, + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: watchtowerName(wandb), + Port: networkingv1.ServiceBackendPort{Number: watchtowerContainerPort}, + }, + }, + } +} + +func watchtowerProbe(path string) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: path, + Port: intstr.FromInt32(watchtowerContainerPort), + }, + }, + TimeoutSeconds: 3, + PeriodSeconds: 10, + FailureThreshold: 3, + } +} + +// watchtowerResources keeps the UI modest by default; it is an admin console +// whose heavy work happens in the cluster, not in this pod. +func watchtowerResources() corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } +} + +func watchtowerImage(wandb *apiv2.WeightsAndBiases) (string, error) { + image := os.Getenv(operatorImageEnvVar) + if image == "" { + return "", fmt.Errorf( + "%s is unset: cannot determine which image carries the Watchtower binary", + operatorImageEnvVar, + ) + } + return image, nil +} + +func watchtowerTolerations(wandb *apiv2.WeightsAndBiases) []corev1.Toleration { + if wandb.Spec.Tolerations == nil { + return nil + } + return *wandb.Spec.Tolerations +} + +func watchtowerServiceAccountName(wandb *apiv2.WeightsAndBiases) string { + return watchtowerName(wandb) +} + +func watchtowerName(wandb *apiv2.WeightsAndBiases) string { + return common.FitDefaultInfraName(wandb.Name, "-watchtower", validation.DNS1123LabelMaxLength) +} + +func watchtowerSecretName(wandb *apiv2.WeightsAndBiases) string { + return common.FitDefaultInfraName(wandb.Name, "-watchtower-auth", validation.DNS1123LabelMaxLength) +} + +// watchtowerClusterScopedName qualifies cluster-scoped RBAC with the CR's +// namespace so two W&B installs in one cluster do not fight over one object. +// +// The separator is "." because a namespace is a DNS-1123 label and so cannot +// contain one, which makes the first "." the unambiguous end of the namespace. +// Joining with "-" would not: namespace "a-b" with CR "c" and namespace "a" with +// CR "b-c" both render "a-b-c", reintroducing the collision this exists to stop. +// FitDefaultInfraName then bounds the result to the 253 characters the apiserver +// allows, hashing the joined key rather than truncating it. +func watchtowerClusterScopedName(wandb *apiv2.WeightsAndBiases) string { + return common.FitDefaultInfraName( + wandb.Namespace+"."+wandb.Name, + "-watchtower", + validation.DNS1123SubdomainMaxLength, + ) +} + +func watchtowerLabels(wandb *apiv2.WeightsAndBiases) map[string]string { + labels := common.BuildWandbLabels(wandb, watchtowerComponent) + labels["app.kubernetes.io/managed-by"] = "wandb-operator" + labels["app.kubernetes.io/instance"] = wandb.Name + labels["app.kubernetes.io/part-of"] = "wandb" + return labels +} + +func reconcileWatchtowerSecret(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerSecretName(wandb), + Namespace: wandb.Namespace, + }, + } + _, err := controllerruntime.CreateOrUpdate(ctx, c, secret, func() error { + secret.Labels = utils.MergeMapsStringString(secret.Labels, watchtowerLabels(wandb)) + secret.Type = corev1.SecretTypeOpaque + if len(secret.Data[watchtowerPasswordKey]) == 0 { + password, err := utils.GenerateRandomPassword(32) + if err != nil { + return err + } + if secret.StringData == nil { + secret.StringData = map[string]string{} + } + secret.StringData[watchtowerPasswordKey] = password + } + return controllerutil.SetControllerReference(wandb, secret, c.Scheme()) + }) + if err != nil { + return fmt.Errorf("failed to reconcile Watchtower Secret: %w", err) + } + return nil +} + +func reconcileWatchtowerServiceAccount(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) error { + serviceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerServiceAccountName(wandb), + Namespace: wandb.Namespace, + }, + } + _, err := controllerruntime.CreateOrUpdate(ctx, c, serviceAccount, func() error { + serviceAccount.Labels = utils.MergeMapsStringString(serviceAccount.Labels, watchtowerLabels(wandb)) + // Watchtower talks to the Kubernetes API with this token, so unlike the + // W&B application pods it must have one mounted. + serviceAccount.AutomountServiceAccountToken = ptr.To(true) + return controllerutil.SetControllerReference(wandb, serviceAccount, c.Scheme()) + }) + if err != nil { + return fmt.Errorf("failed to reconcile Watchtower ServiceAccount: %w", err) + } + return nil +} + +// reconcileWatchtowerRBAC grants Watchtower what the operator itself holds and +// can therefore delegate: the apiserver rejects a binding that would escalate +// beyond the operator's own permissions. +// +// Deliberately absent, because the operator does not hold them today: +// apiextensions.k8s.io/customresourcedefinitions (v2 served-version detection) +// and pods/portforward (telemetry port-forward). Both need the operator's own +// ClusterRole widened first. +func reconcileWatchtowerRBAC(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) error { + labels := watchtowerLabels(wandb) + serviceAccountName := watchtowerServiceAccountName(wandb) + clusterScopedName := watchtowerClusterScopedName(wandb) + + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: watchtowerName(wandb), Namespace: wandb.Namespace}} + if _, err := controllerruntime.CreateOrUpdate(ctx, c, role, func() error { + role.Labels = utils.MergeMapsStringString(role.Labels, labels) + // Secrets and ConfigMaps stay namespace-scoped: Watchtower reads the + // install's license and connection material, not the whole cluster's. + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"batch"}, + Resources: []string{"jobs", "cronjobs"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + return controllerutil.SetOwnerReference(wandb, role, c.Scheme()) + }); err != nil { + return fmt.Errorf("failed to reconcile Watchtower Role: %w", err) + } + + roleBinding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: watchtowerName(wandb), Namespace: wandb.Namespace}} + if _, err := controllerruntime.CreateOrUpdate(ctx, c, roleBinding, func() error { + roleBinding.Labels = utils.MergeMapsStringString(roleBinding.Labels, labels) + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: watchtowerName(wandb), + } + roleBinding.Subjects = []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: wandb.Namespace, + }} + return controllerutil.SetOwnerReference(wandb, roleBinding, c.Scheme()) + }); err != nil { + return fmt.Errorf("failed to reconcile Watchtower RoleBinding: %w", err) + } + + clusterRole := &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterScopedName}} + if _, err := controllerruntime.CreateOrUpdate(ctx, c, clusterRole, func() error { + clusterRole.Labels = utils.MergeMapsStringString(clusterRole.Labels, labels) + clusterRole.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{"apps.wandb.com"}, + Resources: []string{"weightsandbiases"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"apps.wandb.com"}, + Resources: []string{"applications"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Only "get": the operator itself holds get/update/patch on these + // subresources, and it cannot grant verbs it does not have. + APIGroups: []string{"apps.wandb.com"}, + Resources: []string{"weightsandbiases/status", "applications/status"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "pods", "pods/log", "services", "events"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "statefulsets", "replicasets", "daemonsets"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + // Cluster-scoped objects cannot own-reference a namespaced CR; cleanup + // runs through deleteWatchtower instead. + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile Watchtower ClusterRole: %w", err) + } + + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: clusterScopedName}} + if _, err := controllerruntime.CreateOrUpdate(ctx, c, clusterRoleBinding, func() error { + clusterRoleBinding.Labels = utils.MergeMapsStringString(clusterRoleBinding.Labels, labels) + clusterRoleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: clusterScopedName, + } + clusterRoleBinding.Subjects = []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: wandb.Namespace, + }} + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile Watchtower ClusterRoleBinding: %w", err) + } + + return nil +} diff --git a/internal/controller/reconciler/watchtower_test.go b/internal/controller/reconciler/watchtower_test.go new file mode 100644 index 00000000..a3ff196c --- /dev/null +++ b/internal/controller/reconciler/watchtower_test.go @@ -0,0 +1,548 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package reconciler + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + apiv2 "github.com/wandb/operator/api/v2" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func watchtowerTestClient(t *testing.T, objects ...ctrlClient.Object) ctrlClient.Client { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, rbacv1.AddToScheme(scheme)) + require.NoError(t, apiv2.AddToScheme(scheme)) + + builder := fake.NewClientBuilder().WithScheme(scheme) + if len(objects) > 0 { + builder = builder.WithObjects(objects...) + } + return builder.Build() +} + +func watchtowerTestCR(name, namespace string) *apiv2.WeightsAndBiases { + return &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: apiv2.WeightsAndBiasesSpec{ + AdminConsoleEnabled: ptr.To(true), + Wandb: apiv2.WandbAppSpec{Hostname: "wandb.example.com"}, + }, + } +} + +// manifestWithOIDC returns a manifest whose "api" application owns /oidc, which +// is what watchtowerAuthService derives AUTH_SERVICE from. +func manifestWithOIDC() serverManifest.Manifest { + return serverManifest.Manifest{ + Applications: map[string]serverManifest.Application{ + "api": { + Name: "api", + Ingress: &serverManifest.AppIngressSpec{ + Paths: []string{"/oidc", "/graphql"}, + ServicePort: "http", + }, + Service: &serverManifest.ServiceSpec{ + Ports: []corev1.ServicePort{{Name: "http", Port: 8081}}, + }, + }, + }, + } +} + +// --- naming: same-namespace multi-install ----------------------------------- + +// Two CRs in one namespace must not share an Application, RBAC or Secret. Every +// namespaced Watchtower name is derived from the CR name for this reason. +func TestWatchtowerNamesAreDistinctPerCRInOneNamespace(t *testing.T) { + first := watchtowerTestCR("alpha", "wandb") + second := watchtowerTestCR("beta", "wandb") + + require.NotEqual(t, watchtowerName(first), watchtowerName(second)) + require.NotEqual(t, watchtowerSecretName(first), watchtowerSecretName(second)) + require.NotEqual(t, watchtowerServiceAccountName(first), watchtowerServiceAccountName(second)) + require.Equal(t, "alpha-watchtower", watchtowerName(first)) + require.Equal(t, "alpha-watchtower-auth", watchtowerSecretName(first)) +} + +// The application controller derives a Service from the Application name, and +// Service names are DNS-1123 labels, so long CR names must be shortened — but +// shortening must not collapse two distinct CRs onto one name. +func TestWatchtowerNamesStayWithinLabelBudgetAndDistinct(t *testing.T) { + longA := watchtowerTestCR("production-cluster-that-is-really-quite-long-east-region-one", "wandb") + longB := watchtowerTestCR("production-cluster-that-is-really-quite-long-west-region-two", "wandb") + + for _, name := range []string{ + watchtowerName(longA), watchtowerName(longB), + watchtowerSecretName(longA), watchtowerSecretName(longB), + } { + require.LessOrEqual(t, len(name), validation.DNS1123LabelMaxLength) + require.Empty(t, validation.IsDNS1123Label(name), "name %q is not a valid label", name) + } + + require.NotEqual(t, watchtowerName(longA), watchtowerName(longB)) + require.NotEqual(t, watchtowerSecretName(longA), watchtowerSecretName(longB)) +} + +// The Secret name is derived from the CR name with its own suffix rather than by +// appending to watchtowerName: composing on an already-hashed name would push +// the suffix past the budget and truncate the hash back off. +func TestWatchtowerSecretNameIsNotAPrefixCollisionOfAppName(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + require.NotEqual(t, watchtowerName(wandb), watchtowerSecretName(wandb)) +} + +// Cluster-scoped RBAC is global, so two installs in *different* namespaces must +// not collide either. +func TestWatchtowerClusterScopedNameIncludesNamespace(t *testing.T) { + prod := watchtowerTestCR("wandb", "wandb") + staging := watchtowerTestCR("wandb", "wandb-staging") + + require.Equal(t, "wandb-wandb-b8716-watchtower", watchtowerClusterScopedName(prod)) + require.Equal(t, "wandb-staging-wandb-2b09f-watchtower", watchtowerClusterScopedName(staging)) +} + +// The namespace and CR name are joined with "." rather than "-" because a +// namespace cannot contain one. Joined with "-" these two pairs would both +// render "a-b-c-watchtower" and share a single ClusterRole. +func TestWatchtowerClusterScopedNameSeparatorIsUnambiguous(t *testing.T) { + nsHasHyphen := watchtowerTestCR("c", "a-b") + nameHasHyphen := watchtowerTestCR("b-c", "a") + + require.NotEqual(t, + watchtowerClusterScopedName(nsHasHyphen), + watchtowerClusterScopedName(nameHasHyphen), + ) + require.Equal(t, "a-b-c-fb187-watchtower", watchtowerClusterScopedName(nsHasHyphen)) + require.Equal(t, "a-b-c-ab8bb-watchtower", watchtowerClusterScopedName(nameHasHyphen)) +} + +// ClusterRole names are DNS-1123 subdomains, so a long CR name must be hashed +// down rather than pushed past what the apiserver accepts. +func TestWatchtowerClusterScopedNameStaysWithinSubdomainBudget(t *testing.T) { + longNS := strings.Repeat("n", validation.DNS1123LabelMaxLength) + longName := strings.Repeat("c", validation.DNS1123SubdomainMaxLength) + + first := watchtowerTestCR(longName, longNS) + second := watchtowerTestCR(longName+"x", longNS) + + for _, name := range []string{ + watchtowerClusterScopedName(first), + watchtowerClusterScopedName(second), + } { + require.LessOrEqual(t, len(name), validation.DNS1123SubdomainMaxLength) + require.Empty(t, validation.IsDNS1123Subdomain(name), "name %q is not a valid subdomain", name) + } + + // Truncation alone would collapse these onto one name; the hash keeps them apart. + require.NotEqual(t, + watchtowerClusterScopedName(first), + watchtowerClusterScopedName(second), + ) +} + +// --- the generated admin password ------------------------------------------- + +func TestReconcileWatchtowerSecretGeneratesAPassword(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + c := watchtowerTestClient(t, wandb) + + require.NoError(t, reconcileWatchtowerSecret(context.Background(), c, wandb)) + + secret := &corev1.Secret{} + require.NoError(t, c.Get(context.Background(), types.NamespacedName{ + Name: watchtowerSecretName(wandb), Namespace: wandb.Namespace, + }, secret)) + + password := secretPassword(secret) + require.NotEmpty(t, password, "expected a generated password") + require.Len(t, password, 32) + require.Equal(t, corev1.SecretTypeOpaque, secret.Type) +} + +// The whole point of create-if-not-found: an upgrade must not rotate the password +// out from under whoever is holding it. +func TestReconcileWatchtowerSecretPreservesAnExistingPassword(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + existing := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerSecretName(wandb), + Namespace: wandb.Namespace, + }, + Type: corev1.SecretTypeOpaque, + // Seeded via Data, which is how the apiserver returns a Secret written + // with StringData. + Data: map[string][]byte{watchtowerPasswordKey: []byte("do-not-rotate-me")}, + } + c := watchtowerTestClient(t, wandb, existing) + + require.NoError(t, reconcileWatchtowerSecret(context.Background(), c, wandb)) + + secret := &corev1.Secret{} + require.NoError(t, c.Get(context.Background(), types.NamespacedName{ + Name: watchtowerSecretName(wandb), Namespace: wandb.Namespace, + }, secret)) + require.Equal(t, "do-not-rotate-me", secretPassword(secret)) +} + +func TestReconcileWatchtowerSecretIsOwnedByTheCR(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + c := watchtowerTestClient(t, wandb) + + require.NoError(t, reconcileWatchtowerSecret(context.Background(), c, wandb)) + + secret := &corev1.Secret{} + require.NoError(t, c.Get(context.Background(), types.NamespacedName{ + Name: watchtowerSecretName(wandb), Namespace: wandb.Namespace, + }, secret)) + require.NotEmpty(t, secret.OwnerReferences) + require.Equal(t, wandb.Name, secret.OwnerReferences[0].Name) +} + +// secretPassword reads the password from whichever field it landed in. The fake +// client does not perform the apiserver's StringData -> Data conversion, so a +// freshly written Secret carries StringData while a seeded one carries Data. +func secretPassword(secret *corev1.Secret) string { + if value, ok := secret.Data[watchtowerPasswordKey]; ok && len(value) > 0 { + return string(value) + } + return secret.StringData[watchtowerPasswordKey] +} + +// --- container environment --------------------------------------------------- + +func TestWatchtowerEnvReferencesThePasswordSecret(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + env := watchtowerEnv(wandb, "api:8081", "/console") + + byName := map[string]corev1.EnvVar{} + for _, e := range env { + byName[e.Name] = e + } + + password, ok := byName["WATCHTOWER_PASSWORD"] + require.True(t, ok, "expected WATCHTOWER_PASSWORD to be set") + require.Nil(t, password.ValueFrom.SecretKeyRef.Optional) + require.Empty(t, password.Value, "the password must never be inlined in the pod spec") + require.Equal(t, watchtowerSecretName(wandb), password.ValueFrom.SecretKeyRef.Name) + require.Equal(t, watchtowerPasswordKey, password.ValueFrom.SecretKeyRef.Key) + + require.Equal(t, "cluster", byName["WATCHTOWER_MODE"].Value) + require.Equal(t, "/console", byName["WATCHTOWER_BASE_PATH"].Value) + require.Equal(t, "api:8081", byName["WATCHTOWER_AUTH_SERVICE"].Value) + require.Equal(t, "wandb", byName["WATCHTOWER_WANDB_NAME"].Value) + require.Equal(t, + "metadata.namespace", + byName["WATCHTOWER_NAMESPACE"].ValueFrom.FieldRef.FieldPath, + ) +} + +// --- auth service derivation ------------------------------------------------- + +func TestWatchtowerAuthServiceDerivesFromTheOIDCApplication(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + authService, err := watchtowerAuthService(wandb, manifestWithOIDC()) + + require.NoError(t, err) + require.Equal(t, "api:8081", authService) +} +func TestWatchtowerAuthServiceFailsWhenNoApplicationServesOIDC(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + manifest := serverManifest.Manifest{ + Applications: map[string]serverManifest.Application{ + "frontend": { + Name: "frontend", + Ingress: &serverManifest.AppIngressSpec{Paths: []string{"/"}}, + Service: &serverManifest.ServiceSpec{ + Ports: []corev1.ServicePort{{Name: "http", Port: 80}}, + }, + }, + }, + } + + _, err := watchtowerAuthService(wandb, manifest) + + require.Error(t, err) + require.Contains(t, err.Error(), "/oidc") +} + +// --- routing ----------------------------------------------------------------- + +// The Ingress backend has to name the Service the Application produces. If these +// drift the route silently points at nothing, or at another install's Watchtower. +func TestWatchtowerIngressPathTargetsTheApplicationService(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + path := watchtowerIngressPath(wandb) + + require.NotNil(t, path) + require.Equal(t, "/console", path.Path) + require.Equal(t, networkingv1.PathTypePrefix, *path.PathType) + require.Equal(t, watchtowerName(wandb), path.Backend.Service.Name) + require.Equal(t, watchtowerContainerPort, path.Backend.Service.Port.Number) +} + +func TestWatchtowerIngressPathIsNilWhenDisabled(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + wandb.Spec.AdminConsoleEnabled = ptr.To(false) + + require.Nil(t, watchtowerIngressPath(wandb)) +} + +func TestWatchtowerURL(t *testing.T) { + for _, tc := range []struct { + name string + hostname string + basePath string + want string + }{ + {"adds a scheme", "wandb.example.com", "", "https://wandb.example.com/console"}, + {"keeps an explicit scheme", "http://wandb.example.com", "", "http://wandb.example.com/console"}, + {"strips a trailing slash", "https://wandb.example.com/", "", "https://wandb.example.com/console"}, + {"empty hostname yields no URL", "", "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + wandb.Spec.Wandb.Hostname = tc.hostname + + require.Equal(t, tc.want, watchtowerURL(wandb)) + }) + } +} + +// --- image resolution ------------------------------------------------------- + +const testOperatorImage = "us-docker.pkg.dev/wandb-production/public/wandb/operator:2.0.0-beta.3" + +// The Watchtower binary ships inside the operator image, so an unconfigured CR +// resolves to whatever image this operator is itself running. +func TestWatchtowerImageFallsBackToTheOperatorImage(t *testing.T) { + t.Setenv(operatorImageEnvVar, testOperatorImage) + + image, err := watchtowerImage(watchtowerTestCR("wandb", "wandb")) + + require.NoError(t, err) + require.Equal(t, testOperatorImage, image) +} + +// Failing beats guessing: an empty image would be rejected by the apiserver with +// a message that never mentions the missing environment variable. +func TestWatchtowerImageFailsWhenOperatorImageIsUnset(t *testing.T) { + t.Setenv(operatorImageEnvVar, "") + + _, err := watchtowerImage(watchtowerTestCR("wandb", "wandb")) + + require.ErrorContains(t, err, operatorImageEnvVar) +} + +// --- the Application -------------------------------------------------------- + +// Replicas is deliberately not configurable: in-flight deploy jobs and their SSE +// streams live in the serving pod's memory. +func TestBuildWatchtowerApplicationPinsOneReplica(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + app := buildWatchtowerApplication(wandb, "api:8081", testOperatorImage) + + require.Equal(t, int32(1), *app.Spec.Replicas) + require.Equal(t, "Deployment", app.Spec.Kind) + require.Equal(t, watchtowerName(wandb), app.Name) +} + +// The component label is what makes manifest-driven Application pruning skip +// this Application instead of deleting it every reconcile. +func TestBuildWatchtowerApplicationCarriesTheComponentLabel(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + app := buildWatchtowerApplication(wandb, "api:8081", testOperatorImage) + + require.Equal(t, watchtowerComponent, app.Labels["weightsandbiases.apps.wandb.com/component"]) +} + +// The operator image's entrypoint is /manager, so without an explicit command the +// pod would come up healthy running a second operator instead of Watchtower. +func TestBuildWatchtowerApplicationSelectsTheWatchtowerEntrypoint(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + container := buildWatchtowerApplication(wandb, "api:8081", testOperatorImage). + Spec.PodTemplate.Spec.Containers[0] + + require.Equal(t, testOperatorImage, container.Image) + // The binary inside the image, not the URL prefix — those are independent. + require.Equal(t, []string{"/watchtower"}, container.Command) + // The binary defaults to 9090; the Service, container port and probes are 8080. + require.Equal(t, []string{"--port", "8080"}, container.Args) + require.Equal(t, watchtowerContainerPort, container.Ports[0].ContainerPort) +} + +func TestBuildWatchtowerApplicationProbesGoThroughTheBasePath(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + + container := buildWatchtowerApplication(wandb, "api:8081", testOperatorImage). + Spec.PodTemplate.Spec.Containers[0] + + require.Equal(t, "/console/healthz", container.LivenessProbe.HTTPGet.Path) + require.Equal(t, "/console/ready", container.ReadinessProbe.HTTPGet.Path) +} + +// --- teardown --------------------------------------------------------------- + +// deleteWatchtower runs when Watchtower is *disabled*, not just when the CR is +// deleted, so owner-reference GC does not cover it — every object has to be +// listed explicitly. +func TestDeleteWatchtowerRemovesEveryOwnedObject(t *testing.T) { + wandb := watchtowerTestCR("wandb", "wandb") + name := watchtowerName(wandb) + clusterName := watchtowerClusterScopedName(wandb) + + c := watchtowerTestClient(t, wandb, + &apiv2.Application{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: wandb.Namespace}}, + &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: wandb.Namespace}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: wandb.Namespace}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterName}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: clusterName}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerSecretName(wandb), Namespace: wandb.Namespace, + }}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: watchtowerServiceAccountName(wandb), Namespace: wandb.Namespace, + }}, + ) + + require.NoError(t, deleteWatchtower(context.Background(), c, wandb)) + + ctx := context.Background() + for _, tc := range []struct { + what string + obj ctrlClient.Object + key types.NamespacedName + }{ + {"Application", &apiv2.Application{}, types.NamespacedName{Name: name, Namespace: wandb.Namespace}}, + {"Role", &rbacv1.Role{}, types.NamespacedName{Name: name, Namespace: wandb.Namespace}}, + {"RoleBinding", &rbacv1.RoleBinding{}, types.NamespacedName{Name: name, Namespace: wandb.Namespace}}, + {"ClusterRole", &rbacv1.ClusterRole{}, types.NamespacedName{Name: clusterName}}, + {"ClusterRoleBinding", &rbacv1.ClusterRoleBinding{}, types.NamespacedName{Name: clusterName}}, + {"Secret", &corev1.Secret{}, types.NamespacedName{Name: watchtowerSecretName(wandb), Namespace: wandb.Namespace}}, + {"ServiceAccount", &corev1.ServiceAccount{}, types.NamespacedName{Name: watchtowerServiceAccountName(wandb), Namespace: wandb.Namespace}}, + } { + err := c.Get(ctx, tc.key, tc.obj) + require.Error(t, err, "%s should have been deleted", tc.what) + } + + require.Nil(t, wandb.Status.WatchtowerStatus) +} +func TestIsIngressReady(t *testing.T) { + for _, tc := range []struct { + name string + ingress *networkingv1.Ingress + want bool + }{ + {"nil ingress", nil, false}, + { + "no load balancer entries", + &networkingv1.Ingress{}, + false, + }, + { + // Some controllers append an entry before filling in the address, so a + // non-empty slice is not on its own proof of readiness. + "entry present but no address", + &networkingv1.Ingress{Status: networkingv1.IngressStatus{ + LoadBalancer: networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{{}}, + }, + }}, + false, + }, + { + "IP assigned", + &networkingv1.Ingress{Status: networkingv1.IngressStatus{ + LoadBalancer: networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{{IP: "203.0.113.10"}}, + }, + }}, + true, + }, + { + "hostname assigned", + &networkingv1.Ingress{Status: networkingv1.IngressStatus{ + LoadBalancer: networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{{Hostname: "lb.example.com"}}, + }, + }}, + true, + }, + { + "second entry carries the address", + &networkingv1.Ingress{Status: networkingv1.IngressStatus{ + LoadBalancer: networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{{}, {IP: "203.0.113.11"}}, + }, + }}, + true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, isIngressReady(tc.ingress)) + }) + } +} + +// --- independence from infrastructure readiness ------------------------------- + +// Watchtower exists to diagnose a broken install, so nothing in its reconcile +// path may depend on the infrastructure being healthy. This is the property that +// the two infra gates in Reconcile/ReconcileWandbManifest kept swallowing: the +// code was correct, but unreachable. +func TestReconcileWatchtowerIgnoresInfraReadiness(t *testing.T) { + t.Setenv(operatorImageEnvVar, testOperatorImage) + + wandb := watchtowerTestCR("wandb", "wandb") + // Declare an object store instance with no ready status behind it. Without a + // declared instance allInstancesReady is vacuously true, and the precondition + // below would pass for the wrong reason. + wandb.Spec.ObjectStore = map[string]apiv2.ObjectStoreSpec{"default": {}} + + require.False(t, wandb.Status.KafkaStatus.Ready, "precondition: kafka unready") + require.False(t, objectStoreAllReady(wandb), "precondition: object store unready") + + c := watchtowerTestClient(t, wandb) + require.NoError(t, reconcileWatchtower(context.Background(), c, wandb, manifestWithOIDC())) + + ctx := context.Background() + ns := wandb.Namespace + + require.NoError(t, c.Get(ctx, types.NamespacedName{Name: watchtowerSecretName(wandb), Namespace: ns}, + &corev1.Secret{}), "the admin password must exist even with infra down") + require.NoError(t, c.Get(ctx, types.NamespacedName{Name: watchtowerName(wandb), Namespace: ns}, + &apiv2.Application{}), "the Application must exist even with infra down") + require.NoError(t, c.Get(ctx, types.NamespacedName{Name: watchtowerServiceAccountName(wandb), Namespace: ns}, + &corev1.ServiceAccount{}), "the ServiceAccount must exist even with infra down") + require.NoError(t, c.Get(ctx, types.NamespacedName{Name: watchtowerName(wandb), Namespace: ns}, + &rbacv1.Role{}), "the Role must exist even with infra down") +} diff --git a/internal/controller/weightsandbiases_controller_networking_test.go b/internal/controller/weightsandbiases_controller_networking_test.go index ab18c5ed..fcc38d41 100644 --- a/internal/controller/weightsandbiases_controller_networking_test.go +++ b/internal/controller/weightsandbiases_controller_networking_test.go @@ -15,6 +15,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -219,6 +220,47 @@ var _ = Describe("WeightsAndBiases Networking", func() { Expect(wandb.Status.IngressStatus.LoadBalancerIngress).To(HaveLen(1)) Expect(wandb.Status.IngressStatus.LoadBalancerIngress[0].IP).To(Equal("34.118.10.1")) }) + + // Watchtower must not be able to take down the install it manages, but a + // failure still has to be retried by something — dropping it outright left a + // transient error with nothing to pick it back up. + It("requeues without failing the reconcile when Watchtower cannot be reconciled", func() { + ctx := context.Background() + wandbName := "network-watchtower-failure" + ingressClassName := "nginx" + + wandb, service := newNetworkingWandb(wandbName, "") + wandb.Spec.AdminConsoleEnabled = ptr.To(true) + wandb.Spec.Networking = apiv2.NetworkingSpec{ + Mode: apiv2.NetworkingModeIngress, + Ingress: &apiv2.IngressConfig{IngressClassName: &ingressClassName}, + } + Expect(k8sClient.Create(ctx, wandb)).To(Succeed()) + Expect(k8sClient.Create(ctx, service)).To(Succeed()) + DeferCleanup(deleteIfPresent, ctx, wandb) + + // OPERATOR_IMAGE is what tells the reconciler which image carries the + // Watchtower binary; unset, watchtowerImage() fails. + GinkgoT().Setenv("OPERATOR_IMAGE", "") + + wandb = markWandbReadyForNetworking(ctx, wandbName, wandbNamespace) + wandbManifest, err := manifest.GetServerManifest(ctx, wandb.Spec.Wandb.ManifestRepository, wandb.Spec.Wandb.Version, nil) + Expect(err).NotTo(HaveOccurred()) + + result, err := v2.ReconcileNetworkingAndWatchtower(ctx, k8sClient, wandb, wandbManifest) + + // The error is swallowed so the rest of the reconcile still runs... + Expect(err).NotTo(HaveOccurred()) + // ...but it comes back as a requeue so the failure is retried. + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + + // And networking was still published — the failure did not abort early. + ingress := &networkingv1.Ingress{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: wandbName, + Namespace: wandbNamespace, + }, ingress)).To(Succeed()) + }) }) func newNetworkingWandb(name string, infraNamespace string) (*apiv2.WeightsAndBiases, *corev1.Service) { @@ -341,6 +383,11 @@ func reconcileNetworkingManifest(ctx context.Context, wandb *apiv2.WeightsAndBia wandbManifest, err := manifest.GetServerManifest(ctx, wandb.Spec.Wandb.ManifestRepository, wandb.Spec.Wandb.Version, nil) Expect(err).NotTo(HaveOccurred()) + // Networking lives above Reconcile's infrastructure gate, in its own function, + // so it has to be driven separately from the manifest reconcile. + _, err = v2.ReconcileNetworkingAndWatchtower(ctx, k8sClient, wandb, wandbManifest) + Expect(err).NotTo(HaveOccurred()) + _, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, telemetry.DefaultTelemetryRuntimeConfig()) Expect(err).NotTo(HaveOccurred()) } diff --git a/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml b/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml index cb6fa8e7..2ad0fe7a 100644 --- a/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml +++ b/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml @@ -81,6 +81,8 @@ spec: type: object spec: properties: + adminConsoleEnabled: + type: boolean affinity: properties: nodeAffinity: @@ -3977,6 +3979,26 @@ spec: items: type: string type: array + applications: + additionalProperties: + properties: + autoscaling: + properties: + maxReplicas: + format: int32 + minimum: 1 + type: integer + minReplicas: + format: int32 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: minReplicas must be <= maxReplicas + rule: '!has(self.minReplicas) || !has(self.maxReplicas) + || self.minReplicas <= self.maxReplicas' + type: object + type: object bucketProxy: type: boolean features: @@ -4865,6 +4887,10 @@ spec: type: array name: type: string + ready: + type: boolean + required: + - ready type: object kafkaStatus: properties: @@ -6562,6 +6588,19 @@ spec: required: - hostname type: object + watchtowerStatus: + properties: + authService: + type: string + image: + type: string + ready: + type: boolean + url: + type: string + required: + - ready + type: object required: - observedGeneration - ready