feat: Watchtower charts

.github/workflows/release.yaml

@@ -74,14 +74,10 @@ jobs:
             in_operator && $0 == "  image:" { in_image = 1; next }
             in_image && $1 == "tag:" { print $2; exit }
           ' deploy/operator/values.yaml | tr -d '\"')"
-
-          for value in "${chart_version}" "${app_version}" "${image_tag}"; do
             if [[ "${value}" != "${version}" ]]; then
-              echo "Chart version, appVersion, and operator image tag must all equal ${version}" >&2
               echo "Found chart=${chart_version}, appVersion=${app_version}, image=${image_tag}" >&2
               exit 1
             fi
-          done
 
           echo "tag=${tag}" >> "${GITHUB_OUTPUT}"
           echo "tagged_commit=${tagged_commit}" >> "${GITHUB_OUTPUT}"

Dockerfile

@@ -1,3 +1,11 @@
+# Watchtower ships in this image as a second entrypoint: the operator synthesizes
+# an Application that runs the same image with `command: ["/watchtower"]`, and
+# finds this image by name through OPERATOR_IMAGE. Its binary embeds a Next.js
+# static export, so it cannot be rebuilt from Go source here — lift the binary out
+# of the published Watchtower image instead.
+ARG WATCHTOWER_IMAGE=us-docker.pkg.dev/wandb-production/public/wandb/watchtower
+ARG WATCHTOWER_VERSION=0.11.0
+
 # Build the manager binary
 FROM golang:1.26 AS manager-builder
 
@@ -26,11 +34,15 @@ COPY internal/ internal/
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager ./cmd/manager
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o crd-installer ./cmd/crd-installer
 
+FROM ${WATCHTOWER_IMAGE}:${WATCHTOWER_VERSION} AS watchtower
+
 FROM registry.access.redhat.com/ubi9/ubi-minimal
 
 WORKDIR /
 COPY --from=manager-builder /workspace/manager .
 COPY --from=manager-builder /workspace/crd-installer .
+# Built CGO-free on golang:alpine, so it runs unmodified on this glibc base.
+COPY --from=watchtower /watchtower .
 
 RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm && \
     chown -R 65532:65532 /helm

Makefile

@@ -1,6 +1,11 @@
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 
+# Watchtower release whose binary is copied into the operator image as its second
+# entrypoint. Must be a tag that exists in WATCHTOWER_IMAGE — the build pulls it.
+WATCHTOWER_IMAGE ?= us-docker.pkg.dev/wandb-production/public/wandb/watchtower
+WATCHTOWER_VERSION ?= 0.11.0
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -210,7 +215,10 @@ run: manifests generate fmt vet ## Run the manager from your host.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build controller docker image.
-	$(CONTAINER_TOOL) build --platform linux/amd64 -t ${IMG} -f Dockerfile .
+	$(CONTAINER_TOOL) build --platform linux/amd64 \
+		--build-arg WATCHTOWER_IMAGE=$(WATCHTOWER_IMAGE) \
+		--build-arg WATCHTOWER_VERSION=$(WATCHTOWER_VERSION) \
+		-t ${IMG} -f Dockerfile .
 
 .PHONY: docker-push
 docker-push:

README.md

@@ -129,6 +129,7 @@ trust a CA on the W&B **application** workloads instead, use
 - [Migrating from Operator v1 to v2](docs/migrating-v1-to-v2.md)
 - [Monitoring and Telemetry Guide](docs/monitoring.md)
 - [Deploying on OpenShift](docs/openshift.md)
+- [Deploying Watchtower](docs/watchtower-deployment.md)
 
 ## Development
 

api/v2/weightsandbiases_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v2
 
 import (
+	"strings"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -138,8 +140,63 @@ type WeightsAndBiasesSpec struct {
 	// Networking configures how the W&B application is exposed externally.
 	// +optional
 	Networking NetworkingSpec `json:"networking,omitempty"`
+
+	Watchtower WatchtowerSpec `json:"watchtower,omitempty"`
+}
+
+type WatchtowerSpec struct {
+	Install        *bool                       `json:"install,omitempty"`
+	Image          WatchtowerImageSpec         `json:"image,omitempty"`
+	BasePath       string                      `json:"basePath,omitempty"`
+	AuthService    string                      `json:"authService,omitempty"`
+	Resources      corev1.ResourceRequirements `json:"resources,omitempty"`
+	ServiceAccount ManagedServiceAccountSpec   `json:"serviceAccount,omitempty"`
+}
+
+type WatchtowerImageSpec struct {
+	// +optional
+	Repository string `json:"repository,omitempty"`
+	// +optional
+	Tag string `json:"tag,omitempty"`
+	// +optional
+	Digest string `json:"digest,omitempty"`
+}
+
+func (s WatchtowerSpec) ResolvedBasePath() string {
+	basePath := s.BasePath
+	if basePath == "" {
+		basePath = DefaultWatchtowerBasePath
+	}
+	if !strings.HasPrefix(basePath, "/") {
+		basePath = "/" + basePath
+	}
+	return strings.TrimSuffix(basePath, "/")
 }
 
+// GetImage returns an explicitly configured Watchtower image, or "" when none is
+// set. Empty is the normal case. Binary shisp inside the operator's own image
+func (s WatchtowerSpec) GetImage(globalImageRegistry string) string {
+	if s.Image.Repository == "" {
+		return ""
+	}
+	repository := s.Image.Repository
+	if globalImageRegistry != "" {
+		repository = globalImageRegistry + "/" + repository
+	}
+	if s.Image.Digest != "" {
+		return repository + "@" + s.Image.Digest
+	}
+	if s.Image.Tag != "" {
+		return repository + ":" + s.Image.Tag
+	}
+	return repository
+}
+
+const (
+	DefaultWatchtowerBasePath           = "/watchtower"
+	DefaultWatchtowerServiceAccountName = "wandb-watchtower"
+)
+
 // GlobalSpec holds settings shared across every managed component.
 type GlobalSpec struct {
 	// ImageRegistry, when set, retargets the container images to this registry.
@@ -176,6 +233,10 @@ type GlobalSpec struct {
 	Proxy *ProxySpec `json:"proxy,omitempty"`
 }
 
+func (w *WeightsAndBiases) WatchtowerEnabled() bool {
+	return w.Spec.Watchtower.Install != nil && *w.Spec.Watchtower.Install
+}
+
 // ProxySpec is the forward-proxy configuration under spec.global.proxy.
 type ProxySpec struct {
 	// HTTPProxy is the proxy URL for plain HTTP egress (HTTP_PROXY/http_proxy).
@@ -854,7 +915,15 @@ type WeightsAndBiasesStatus struct {
 	// +optional
 	GatewayStatus *GatewayStatusSummary `json:"gatewayStatus,omitempty"`
 	// +optional
-	IngressStatus *IngressStatusSummary `json:"ingressStatus,omitempty"`
+	IngressStatus    *IngressStatusSummary    `json:"ingressStatus,omitempty"`
+	WatchtowerStatus *WatchtowerStatusSummary `json:"watchtowerStatus,omitempty"`
+}
+
+type WatchtowerStatusSummary struct {
+	Ready       bool   `json:"ready"`
+	URL         string `json:"url,omitempty"`
+	Image       string `json:"image,omitempty"`
+	AuthService string `json:"authService,omitempty"`
 }
 
 type GatewayStatusSummary struct {
@@ -867,6 +936,7 @@ type GatewayStatusSummary struct {
 type IngressStatusSummary struct {
 	Name                string                       `json:"name,omitempty"`
 	LoadBalancerIngress []corev1.LoadBalancerIngress `json:"loadBalancerIngress,omitempty"`
+	Ready               bool                         `json:"ready"`
 }
 
 type WandbStatus struct {

config/crd/bases/apps.wandb.com_weightsandbiases.yaml

@@ -4566,6 +4566,68 @@ spec:
                 - hostname
                 - version
                 type: object
+              watchtower:
+                properties:
+                  authService:
+                    type: string
+                  basePath:
+                    type: string
+                  image:
+                    properties:
+                      digest:
+                        type: string
+                      repository:
+                        type: string
+                      tag:
+                        type: string
+                    type: object
+                  install:
+                    type: boolean
+                  resources:
+                    properties:
+                      claims:
+                        items:
+                          properties:
+                            name:
+                              type: string
+                            request:
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        type: object
+                    type: object
+                  serviceAccount:
+                    properties:
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      create:
+                        type: boolean
+                      serviceAccountName:
+                        type: string
+                    type: object
+                type: object
             required:
             - retentionPolicy
             type: object
@@ -4833,6 +4895,10 @@ spec:
                     type: array
                   name:
                     type: string
+                  ready:
+                    type: boolean
+                required:
+                - ready
                 type: object
               kafkaStatus:
                 properties:
@@ -6530,6 +6596,19 @@ spec:
                 required:
                 - hostname
                 type: object
+              watchtowerStatus:
+                properties:
+                  authService:
+                    type: string
+                  image:
+                    type: string
+                  ready:
+                    type: boolean
+                  url:
+                    type: string
+                required:
+                - ready
+                type: object
             required:
             - observedGeneration
             - ready

deploy/operator/templates/_helpers.tpl

@@ -79,3 +79,8 @@ wandb-operator values. Each is inert unless a CA source is configured.
   value: "{{ $ca.mountPath | default "/etc/wandb/ca-certs" }}:/etc/ssl/certs:/etc/pki/tls/certs"
 {{- end -}}
 {{- end -}}
+
+{{- define "wandb-operator.operatorImageEnv" -}}
+- name: OPERATOR_IMAGE
+  value: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+{{- end -}}

deploy/operator/values.yaml

@@ -90,6 +90,7 @@ wandb-operator:
     - '{{ include "wandb-operator.caCertsVolume" . }}'
   envTpls:
     - '{{ include "wandb-operator.caCertsEnv" . }}'
+    - '{{ include "wandb-operator.operatorImageEnv" . }}'
 
   service:
     enabled: true