diff --git a/api/v2/weightsandbiases_types.go b/api/v2/weightsandbiases_types.go index c269963a..bc4c624f 100644 --- a/api/v2/weightsandbiases_types.go +++ b/api/v2/weightsandbiases_types.go @@ -400,6 +400,10 @@ type WandbAppSpec struct { // +optional Security SecuritySpec `json:"security,omitempty"` + // Retention Spec for the WandB application + // +optional + Retention *RetentionSpec `json:"retention,omitempty"` + // LegacyOverrides holds env/resource overrides extracted from v1 // spec.values, keyed by manifest application name plus the reserved // "global" key (env only, applied to every application). Unknown keys are @@ -563,6 +567,11 @@ type SlackSpec struct { ClientSecret corev1.SecretKeySelector `json:"clientSecret,omitempty"` } +type RetentionSpec struct { + ArtifactGarbageCollection bool `json:"artifactGarbageCollection,omitempty"` + DataRetentionPeriod *metav1.Duration `json:"dataRetentionPeriod,omitempty"` +} + type ManagedInfraSpec struct { RetentionPolicy *RetentionPolicy `json:"retentionPolicy,omitempty"` diff --git a/api/v2/zz_generated.deepcopy.go b/api/v2/zz_generated.deepcopy.go index 53b68baa..cdc9c6f6 100644 --- a/api/v2/zz_generated.deepcopy.go +++ b/api/v2/zz_generated.deepcopy.go @@ -1455,6 +1455,26 @@ func (in *RetentionPolicy) DeepCopy() *RetentionPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RetentionSpec) DeepCopyInto(out *RetentionSpec) { + *out = *in + if in.DataRetentionPeriod != nil { + in, out := &in.DataRetentionPeriod, &out.DataRetentionPeriod + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RetentionSpec. +func (in *RetentionSpec) DeepCopy() *RetentionSpec { + if in == nil { + return nil + } + out := new(RetentionSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SeaweedObjectStoreSpec) DeepCopyInto(out *SeaweedObjectStoreSpec) { *out = *in @@ -1658,6 +1678,11 @@ func (in *WandbAppSpec) DeepCopyInto(out *WandbAppSpec) { (*in).DeepCopyInto(*out) } out.Security = in.Security + if in.Retention != nil { + in, out := &in.Retention, &out.Retention + *out = new(RetentionSpec) + (*in).DeepCopyInto(*out) + } if in.LegacyOverrides != nil { in, out := &in.LegacyOverrides, &out.LegacyOverrides *out = make(map[string]LegacyOverrides, len(*in)) diff --git a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml index 2ad0fe7a..d2ea6161 100644 --- a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml +++ b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml @@ -4550,6 +4550,13 @@ spec: type: integer type: object type: object + retention: + properties: + artifactGarbageCollection: + type: boolean + dataRetentionPeriod: + type: string + type: object security: properties: allowAnonymousPublicProjects: diff --git a/hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.0/manifest.yaml b/hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.1/manifest.yaml similarity index 95% rename from hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.0/manifest.yaml rename to hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.1/manifest.yaml index b2d3e7f1..4632b21b 100644 --- a/hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.0/manifest.yaml +++ b/hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.1/manifest.yaml @@ -292,6 +292,16 @@ commonEnvvars: - name: security type: custom-resource field: spec.wandb.security.hideUpgradeBanner + - name: GORILLA_ARTIFACT_GC_ENABLED + sources: + - name: retention + type: custom-resource + field: spec.wandb.retention.artifactGarbageCollection + - name: GORILLA_DATA_RETENTION_PERIOD + sources: + - name: retention + type: custom-resource + field: spec.wandb.retention.dataRetentionPeriod - name: GORILLA_BUCKET_ATTRIBUTION_DISABLED sources: - name: object-store @@ -542,6 +552,8 @@ commonEnvvars: value: "/tmp/wandb-home" - name: XDG_CACHE_HOME value: "/tmp/wandb-cache" + - name: WANDB_RUNTIME_CA_BUNDLE + value: "/tmp/wandb-ca-bundle.pem" pythonRuntimePaths: - name: PYTHONDONTWRITEBYTECODE value: "1" @@ -580,7 +592,7 @@ applications: - runtimeTmp image: repository: us-docker.pkg.dev/wandb-production/public/wandb/anaconda2 - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: anaconda2 ports: @@ -632,7 +644,7 @@ applications: type: mysql image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: api args: @@ -678,7 +690,7 @@ applications: - runtimeTmp image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: executor args: @@ -698,7 +710,7 @@ applications: - runtimeTmp image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 filestream: name: filestream features: @@ -716,7 +728,7 @@ applications: - runtimeTmp image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: filestream flat-run-fields-updater: @@ -739,14 +751,14 @@ applications: value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=flat-run-fields-updater" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: flat-run-fields-updater frontend: name: frontend image: repository: us-docker.pkg.dev/wandb-production/public/wandb/frontend-nginx - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 commonEnvs: - frontend env: @@ -839,7 +851,7 @@ applications: value: "memory://" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: glue metric-observer: @@ -862,7 +874,7 @@ applications: value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=metric-observer" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: metric-observer parquet: @@ -882,7 +894,7 @@ applications: - runtimeTmp image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 containers: - name: parquet ports: @@ -904,7 +916,7 @@ applications: name: weave image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-python - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 commonEnvs: - runtimePaths - pythonRuntimePaths @@ -936,6 +948,7 @@ applications: path: /__weave/hello - name: weave-cache-clear command: + - /usr/local/bin/wandb-entrypoint - python - weave-public/weave_query/scripts/clear_cache.py resources: @@ -960,7 +973,7 @@ applications: name: weave-trace image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 commonEnvs: - clickhouse - kafka @@ -1009,7 +1022,7 @@ applications: name: weave-trace-worker image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 env: - name: DD_TRACE_ENABLED value: "false" @@ -1032,7 +1045,7 @@ applications: legacyKey: weave-evaluate-model-worker image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 env: - name: DD_TRACE_ENABLED value: "false" @@ -1055,7 +1068,7 @@ migrations: gorilla: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 args: - "migrate" - "--db=$(GORILLA_METADATA_STORE)" @@ -1072,7 +1085,7 @@ migrations: internal-signer: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 args: - "secret-generation-job" env: @@ -1089,7 +1102,7 @@ migrations: weave-trace: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.84.0-notifications-security-flags.0 + tag: 0.84.0-notifications-security-flags.1 args: - python - migrator.py diff --git a/hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.0/sizing.yaml b/hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.1/sizing.yaml similarity index 100% rename from hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.0/sizing.yaml rename to hack/testing-manifests/server-manifest/0.84.0-notifications-security-flags.1/sizing.yaml diff --git a/hack/testing-manifests/wandb/wandb-dev-v2.yaml b/hack/testing-manifests/wandb/wandb-dev-v2.yaml index def5c60f..f2253f28 100644 --- a/hack/testing-manifests/wandb/wandb-dev-v2.yaml +++ b/hack/testing-manifests/wandb/wandb-dev-v2.yaml @@ -5,12 +5,15 @@ metadata: spec: wandb: hostname: http://localhost + retention: + artifactGarbageCollection: true + dataRetentionPeriod: 168h internalServiceAuth: enabled: true features: proxy: true manifestRepository: "file:///server-manifest" - version: "0.79.0" + version: "0.84.0-notifications-security-flags.1" size: dev retentionPolicy: onDelete: detach @@ -34,3 +37,4 @@ spec: managedClickhouse: telemetry: enabled: true + diff --git a/internal/controller/reconciler/retention_env_test.go b/internal/controller/reconciler/retention_env_test.go new file mode 100644 index 00000000..de8496d2 --- /dev/null +++ b/internal/controller/reconciler/retention_env_test.go @@ -0,0 +1,72 @@ +package reconciler + +import ( + "context" + "testing" + "time" + + apiv2 "github.com/wandb/operator/api/v2" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestResolveEnvvarsCustomResourceRetention(t *testing.T) { + retentionPeriod := metav1.Duration{Duration: 168 * time.Hour} + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default"}, + } + wandb.Spec.Wandb.Retention = &apiv2.RetentionSpec{ + ArtifactGarbageCollection: true, + DataRetentionPeriod: &retentionPeriod, + } + + tests := []struct { + name string + field string + expected string + }{ + { + name: "GORILLA_ARTIFACT_GC_ENABLED", + field: "spec.wandb.retention.artifactGarbageCollection", + expected: "true", + }, + { + name: "GORILLA_DATA_RETENTION_PERIOD", + field: "spec.wandb.retention.dataRetentionPeriod", + expected: "168h0m0s", + }, + } + + envs := make([]serverManifest.EnvVar, 0, len(tests)) + for _, tc := range tests { + envs = append(envs, serverManifest.EnvVar{ + Name: tc.name, + Sources: []serverManifest.EnvSource{{ + Type: "custom-resource", Field: tc.field, + }}, + }) + } + + resolved, err := resolveEnvvars( + context.Background(), + fake.NewClientBuilder().Build(), + wandb, + serverManifest.Manifest{}, + nil, + envs, + ) + if err != nil { + t.Fatal(err) + } + + for _, tc := range tests { + env := mustFindEnvVar(t, resolved, tc.name) + if env.Value != tc.expected { + t.Errorf("expected %s=%q, got %q", tc.name, tc.expected, env.Value) + } + if env.ValueFrom != nil { + t.Errorf("expected %s to be a plain value, got valueFrom %+v", tc.name, env.ValueFrom) + } + } +} diff --git a/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml b/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml index 2ad0fe7a..d2ea6161 100644 --- a/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml +++ b/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml @@ -4550,6 +4550,13 @@ spec: type: integer type: object type: object + retention: + properties: + artifactGarbageCollection: + type: boolean + dataRetentionPeriod: + type: string + type: object security: properties: allowAnonymousPublicProjects: