diff --git a/README.md b/README.md index 1f925ae..06a877d 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,55 @@ Supports `ReplicatedMergeTree` and all replicated table engines. | `24.8` | LTS | | | `25.5` | Latest | | +## Backups + +Backup and restore is implemented via [clickhouse-backup](https://github.com/Altinity/clickhouse-backup), +run as a sidecar container on the ClickHouse pod and driven through its REST +API (`ProviderManaged` execution mode). + +The chart does not yet ship a `BackupClass` CR (tracked as a follow-up) — apply +one by hand first: + +```yaml +apiVersion: backup.openeverest.io/v1alpha1 +kind: BackupClass +metadata: + name: altinity-clickhouse-backups +spec: + displayName: "ClickHouse Backups" + executionMode: ProviderManaged + supportedProviders: + - altinity-clickhouse + providerManaged: + supportsPITR: false + limits: + maxStorages: 1 +``` + +Then enable it by setting `spec.backup` on the Instance, referencing a +namespaced `BackupStorage` (S3-compatible): + +```yaml +spec: + backup: + enabled: true + classRef: + name: altinity-clickhouse-backups + storages: + - storageRef: + name: my-s3-storage +``` + +v1 limitations: + +- Exactly one storage — clickhouse-backup's REST API only supports one + static storage destination, fixed via env vars when the sidecar starts. +- No point-in-time recovery and no scheduled backups yet (on-demand `Backup`/ + `Restore` CRs only). +- Backups must be configured at Instance creation. The sidecar is wired into + the CHI pod template once, at creation time; enabling backups on an + already-running Instance is rejected — recreate the instance instead. + ## Quick Start ```bash diff --git a/internal/provider/backup.go b/internal/provider/backup.go new file mode 100644 index 0000000..81bef9c --- /dev/null +++ b/internal/provider/backup.go @@ -0,0 +1,278 @@ +// Copyright (C) 2026 The OpenEverest Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + backupv1alpha1 "github.com/openeverest/openeverest/v2/api/backup/v1alpha1" + "github.com/openeverest/openeverest/v2/provider-runtime/controller" + + "github.com/scaledb-io/provider-altinity-clickhouse/internal/common" +) + +// SyncBackup implements controller.BackupProvider. It drives the +// clickhouse-backup sidecar's REST API on this Instance's replica-0 pod (see +// buildBackupContainer for how the sidecar is wired in at Instance creation). +// SyncBackup is idempotent: it only triggers create_remote once per Backup +// name, then polls clickhouse-backup's own status for completion. +func (p *Provider) SyncBackup(c *controller.Context, backup *backupv1alpha1.Backup) (controller.BackupExecutionStatus, error) { + client := newBackupClient(chiReplicaZeroHost(c)) + + entries, err := client.status(c.Context()) + if err != nil { + return controller.BackupExecutionStatus{}, fmt.Errorf("query clickhouse-backup status: %w", err) + } + if entry := findCommand(entries, "create_remote", backup.Name); entry != nil { + return backupExecutionStatus(*entry), nil + } + + if err := client.createRemote(c.Context(), backup.Name); err != nil { + return controller.BackupExecutionStatus{}, fmt.Errorf("trigger backup: %w", err) + } + now := metav1.NewTime(time.Now()) + return controller.BackupExecutionStatus{State: backupv1alpha1.BackupStatePending, StartedAt: &now}, nil +} + +// SyncRestore implements controller.BackupProvider. Restores always target +// this Instance's replica-0 pod; the resolved backup name is the referenced +// Backup CR's name, which SyncBackup used as the clickhouse-backup backup +// name too. +func (p *Provider) SyncRestore(c *controller.Context, restore *backupv1alpha1.Restore) (controller.RestoreExecutionStatus, error) { + if restore.Spec.DataSource.Type != backupv1alpha1.DataSourceTypeBackup || restore.Spec.DataSource.Backup == nil { + return controller.RestoreExecutionStatus{}, fmt.Errorf("unsupported restore data source type %q", restore.Spec.DataSource.Type) + } + backupName := restore.Spec.DataSource.Backup.BackupRef.Name + + client := newBackupClient(chiReplicaZeroHost(c)) + + entries, err := client.status(c.Context()) + if err != nil { + return controller.RestoreExecutionStatus{}, fmt.Errorf("query clickhouse-backup status: %w", err) + } + if entry := findCommand(entries, "restore_remote", backupName); entry != nil { + return restoreExecutionStatus(*entry), nil + } + + if err := client.restoreRemote(c.Context(), backupName); err != nil { + return controller.RestoreExecutionStatus{}, fmt.Errorf("trigger restore: %w", err) + } + now := metav1.NewTime(time.Now()) + return controller.RestoreExecutionStatus{State: backupv1alpha1.RestoreStatePending, StartedAt: &now}, nil +} + +// CleanupBackup implements controller.BackupProvider. Backup data lives in +// remote (S3) storage, not as an in-cluster resource, so cleanup is a single +// best-effort delete call rather than a polled operation. +func (p *Provider) CleanupBackup(c *controller.Context, backup *backupv1alpha1.Backup) (bool, error) { + if c.ShouldRetainBackupData(backup) { + return true, nil + } + client := newBackupClient(chiReplicaZeroHost(c)) + if err := client.deleteRemote(c.Context(), backup.Name); err != nil { + return false, fmt.Errorf("delete remote backup: %w", err) + } + return true, nil +} + +// CleanupRestore implements controller.BackupProvider. Restores are +// run-to-completion; there is nothing to clean up. +func (p *Provider) CleanupRestore(_ *controller.Context, _ *backupv1alpha1.Restore) (bool, error) { + return true, nil +} + +// chiReplicaZeroHost returns the stable per-pod DNS name of shard 0 / replica +// 0 for this Instance's CHI, following the same per-pod headless service +// naming convention documented on keeperZookeeperNodes. +func chiReplicaZeroHost(c *controller.Context) string { + return fmt.Sprintf("chi-%s-%s-0-0.%s.svc", c.Name(), common.CHIClusterName, c.Namespace()) +} + +// backupExecutionStatus maps a clickhouse-backup status entry onto the +// runtime's BackupExecutionStatus. +func backupExecutionStatus(entry backupStatusEntry) controller.BackupExecutionStatus { + status := controller.BackupExecutionStatus{ + State: mapBackupState(entry.Status), + Message: entry.Error, + } + if t, ok := parseBackupTime(entry.Finish); ok { + status.CompletedAt = &t + } + return status +} + +// restoreExecutionStatus is the Restore counterpart of backupExecutionStatus. +func restoreExecutionStatus(entry backupStatusEntry) controller.RestoreExecutionStatus { + status := controller.RestoreExecutionStatus{ + State: mapRestoreState(entry.Status), + Message: entry.Error, + } + if t, ok := parseBackupTime(entry.Finish); ok { + status.CompletedAt = &t + } + return status +} + +func mapBackupState(status string) backupv1alpha1.BackupState { + switch status { + case "success": + return backupv1alpha1.BackupStateSucceeded + case "error": + return backupv1alpha1.BackupStateFailed + default: + return backupv1alpha1.BackupStateRunning + } +} + +func mapRestoreState(status string) backupv1alpha1.RestoreState { + switch status { + case "success": + return backupv1alpha1.RestoreStateSucceeded + case "error": + return backupv1alpha1.RestoreStateFailed + default: + return backupv1alpha1.RestoreStateRunning + } +} + +func parseBackupTime(s string) (metav1.Time, bool) { + if s == "" { + return metav1.Time{}, false + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return metav1.Time{}, false + } + return metav1.NewTime(t), true +} + +// ============================================================================= +// clickhouse-backup REST client +// ============================================================================= + +// backupClient talks to a clickhouse-backup sidecar's REST API (`clickhouse-backup +// server`, see https://github.com/Altinity/clickhouse-backup/blob/master/Examples.md). +// The API exposes a single, static storage destination fixed via the +// sidecar's env vars at startup, so every call targets one specific pod. +type backupClient struct { + host string + port int + httpClient *http.Client +} + +func newBackupClient(host string) *backupClient { + return newBackupClientWithPort(host, backupRESTPort) +} + +// newBackupClientWithPort allows tests to point at an httptest.Server's +// random port instead of the fixed backupRESTPort. +func newBackupClientWithPort(host string, port int) *backupClient { + return &backupClient{ + host: host, + port: port, + httpClient: &http.Client{Timeout: 30 * time.Second}, + } +} + +func (b *backupClient) baseURL() string { + return fmt.Sprintf("http://%s:%d", b.host, b.port) +} + +// backupStatusEntry is one entry of clickhouse-backup's GET /backup/status response. +type backupStatusEntry struct { + Command string `json:"command"` + Status string `json:"status"` // "in progress", "success", "error" + Start string `json:"start,omitempty"` + Finish string `json:"finish,omitempty"` + Error string `json:"error,omitempty"` +} + +// createRemote triggers a combined create+upload for the named backup. +func (b *backupClient) createRemote(ctx context.Context, name string) error { + return b.post(ctx, "/backup/create_remote?name="+url.QueryEscape(name)) +} + +// restoreRemote triggers a combined download+restore for the named backup. +func (b *backupClient) restoreRemote(ctx context.Context, name string) error { + return b.post(ctx, "/backup/restore_remote/"+url.PathEscape(name)) +} + +// deleteRemote removes the named backup from remote storage. +func (b *backupClient) deleteRemote(ctx context.Context, name string) error { + return b.post(ctx, "/backup/delete/remote/"+url.PathEscape(name)) +} + +// status returns clickhouse-backup's currently tracked operations. +func (b *backupClient) status(ctx context.Context) ([]backupStatusEntry, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.baseURL()+"/backup/status", nil) + if err != nil { + return nil, err + } + resp, err := b.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("clickhouse-backup status: %s: %s", resp.Status, string(body)) + } + var entries []backupStatusEntry + if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil { + return nil, fmt.Errorf("decode clickhouse-backup status: %w", err) + } + return entries, nil +} + +func (b *backupClient) post(ctx context.Context, path string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, b.baseURL()+path, nil) + if err != nil { + return err + } + resp, err := b.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("clickhouse-backup %s: %s: %s", path, resp.Status, string(body)) + } + return nil +} + +// findCommand returns the status entry for a given action (e.g. +// "create_remote") and backup name, if present. clickhouse-backup reports the +// full invocation (action + arguments) in the "command" field, so this +// matches on substrings rather than assuming an exact format. +func findCommand(entries []backupStatusEntry, action, name string) *backupStatusEntry { + for i := range entries { + if strings.Contains(entries[i].Command, action) && strings.Contains(entries[i].Command, name) { + return &entries[i] + } + } + return nil +} diff --git a/internal/provider/backup_test.go b/internal/provider/backup_test.go new file mode 100644 index 0000000..2c7e688 --- /dev/null +++ b/internal/provider/backup_test.go @@ -0,0 +1,373 @@ +// Copyright (C) 2026 The OpenEverest Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + chiv1 "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + backupv1alpha1 "github.com/openeverest/openeverest/v2/api/backup/v1alpha1" + apicommon "github.com/openeverest/openeverest/v2/api/common/v1alpha1" + corev1alpha1 "github.com/openeverest/openeverest/v2/api/core/v1alpha1" + "github.com/openeverest/openeverest/v2/provider-runtime/controller" +) + +func newTestContext(t *testing.T, in *corev1alpha1.Instance, objs ...client.Object) *controller.Context { + t.Helper() + scheme := runtime.NewScheme() + if err := corev1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("register corev1alpha1 scheme: %v", err) + } + if err := backupv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("register backupv1alpha1 scheme: %v", err) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + return controller.NewContext(context.Background(), c, in, "altinity-clickhouse") +} + +func TestBuildBackupContainer_Disabled(t *testing.T) { + tests := []struct { + name string + backup *corev1alpha1.InstanceBackupSpec + }{ + {name: "nil backup spec", backup: nil}, + {name: "not enabled", backup: &corev1alpha1.InstanceBackupSpec{Enabled: false}}, + {name: "enabled but no storages", backup: &corev1alpha1.InstanceBackupSpec{Enabled: true}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := &corev1alpha1.Instance{ + ObjectMeta: metav1.ObjectMeta{Name: "itest", Namespace: "default"}, + Spec: corev1alpha1.InstanceSpec{Backup: tt.backup}, + } + c := newTestContext(t, in) + container, err := buildBackupContainer(c) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if container != nil { + t.Fatalf("expected nil container, got %+v", container) + } + }) + } +} + +func TestBuildBackupContainer_HappyPath(t *testing.T) { + storage := &backupv1alpha1.BackupStorage{ + ObjectMeta: metav1.ObjectMeta{Name: "my-s3-storage", Namespace: "default"}, + Spec: backupv1alpha1.BackupStorageSpec{ + Type: backupv1alpha1.BackupStorageTypeS3, + S3: &backupv1alpha1.BackupStorageS3Spec{ + Bucket: "my-bucket", + Region: "us-east-1", + EndpointURL: "https://s3.amazonaws.com", + CredentialsSecretRef: apicommon.SecretRef{Name: "my-s3-creds"}, + }, + }, + } + in := &corev1alpha1.Instance{ + ObjectMeta: metav1.ObjectMeta{Name: "itest", Namespace: "default"}, + Spec: corev1alpha1.InstanceSpec{ + Backup: &corev1alpha1.InstanceBackupSpec{ + Enabled: true, + Storages: []corev1alpha1.InstanceBackupStorage{ + {StorageRef: apicommon.ObjectRef{Name: "my-s3-storage"}}, + }, + }, + }, + } + c := newTestContext(t, in, storage) + + container, err := buildBackupContainer(c) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if container == nil { + t.Fatal("expected a non-nil container") + } + if container.Name != backupContainerName { + t.Errorf("Name = %q, want %q", container.Name, backupContainerName) + } + wantEnv := map[string]string{ + "REMOTE_STORAGE": "s3", + "S3_ENDPOINT": "https://s3.amazonaws.com", + "S3_BUCKET": "my-bucket", + "S3_REGION": "us-east-1", + } + for _, e := range container.Env { + if want, ok := wantEnv[e.Name]; ok && e.Value != want { + t.Errorf("env %s = %q, want %q", e.Name, e.Value, want) + } + } + if len(container.Ports) != 1 || container.Ports[0].ContainerPort != backupRESTPort { + t.Errorf("unexpected ports: %+v", container.Ports) + } +} + +func TestBuildBackupContainer_NonS3Storage(t *testing.T) { + storage := &backupv1alpha1.BackupStorage{ + ObjectMeta: metav1.ObjectMeta{Name: "not-s3", Namespace: "default"}, + Spec: backupv1alpha1.BackupStorageSpec{Type: backupv1alpha1.BackupStorageTypeS3}, + } + in := &corev1alpha1.Instance{ + ObjectMeta: metav1.ObjectMeta{Name: "itest", Namespace: "default"}, + Spec: corev1alpha1.InstanceSpec{ + Backup: &corev1alpha1.InstanceBackupSpec{ + Enabled: true, + Storages: []corev1alpha1.InstanceBackupStorage{ + {StorageRef: apicommon.ObjectRef{Name: "not-s3"}}, + }, + }, + }, + } + c := newTestContext(t, in, storage) + + if _, err := buildBackupContainer(c); err == nil { + t.Error("expected an error when BackupStorage.Spec.S3 is nil") + } +} + +func TestChiHasBackupContainer(t *testing.T) { + withSidecar := &chiv1.ClickHouseInstallation{ + Spec: chiv1.ChiSpec{ + Templates: &chiv1.Templates{ + PodTemplates: []chiv1.PodTemplate{ + {Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "clickhouse"}, + {Name: backupContainerName}, + }}}, + }, + }, + }, + } + withoutSidecar := &chiv1.ClickHouseInstallation{ + Spec: chiv1.ChiSpec{ + Templates: &chiv1.Templates{ + PodTemplates: []chiv1.PodTemplate{ + {Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "clickhouse"}}}}, + }, + }, + }, + } + noTemplates := &chiv1.ClickHouseInstallation{} + + if !chiHasBackupContainer(withSidecar) { + t.Error("expected true when the sidecar container is present") + } + if chiHasBackupContainer(withoutSidecar) { + t.Error("expected false when the sidecar container is absent") + } + if chiHasBackupContainer(noTemplates) { + t.Error("expected false when Templates is nil") + } +} + +func TestChiReplicaZeroHost(t *testing.T) { + in := &corev1alpha1.Instance{ObjectMeta: metav1.ObjectMeta{Name: "itest-ch", Namespace: "ns1"}} + c := newTestContext(t, in) + got := chiReplicaZeroHost(c) + want := "chi-itest-ch-clickhouse-0-0.ns1.svc" + if got != want { + t.Errorf("chiReplicaZeroHost() = %q, want %q", got, want) + } +} + +func TestMapBackupState(t *testing.T) { + tests := []struct { + status string + want backupv1alpha1.BackupState + }{ + {"success", backupv1alpha1.BackupStateSucceeded}, + {"error", backupv1alpha1.BackupStateFailed}, + {"in progress", backupv1alpha1.BackupStateRunning}, + {"", backupv1alpha1.BackupStateRunning}, + } + for _, tt := range tests { + if got := mapBackupState(tt.status); got != tt.want { + t.Errorf("mapBackupState(%q) = %q, want %q", tt.status, got, tt.want) + } + } +} + +func TestMapRestoreState(t *testing.T) { + tests := []struct { + status string + want backupv1alpha1.RestoreState + }{ + {"success", backupv1alpha1.RestoreStateSucceeded}, + {"error", backupv1alpha1.RestoreStateFailed}, + {"in progress", backupv1alpha1.RestoreStateRunning}, + } + for _, tt := range tests { + if got := mapRestoreState(tt.status); got != tt.want { + t.Errorf("mapRestoreState(%q) = %q, want %q", tt.status, got, tt.want) + } + } +} + +func TestParseBackupTime(t *testing.T) { + if _, ok := parseBackupTime(""); ok { + t.Error("expected ok=false for empty string") + } + if _, ok := parseBackupTime("not-a-time"); ok { + t.Error("expected ok=false for invalid time") + } + want := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC) + got, ok := parseBackupTime(want.Format(time.RFC3339)) + if !ok { + t.Fatal("expected ok=true for a valid RFC3339 time") + } + if !got.Time.Equal(want) { + t.Errorf("parseBackupTime() = %v, want %v", got.Time, want) + } +} + +func TestBackupExecutionStatus(t *testing.T) { + entry := backupStatusEntry{Status: "success", Finish: "2026-07-24T12:00:00Z"} + got := backupExecutionStatus(entry) + if got.State != backupv1alpha1.BackupStateSucceeded { + t.Errorf("State = %q, want %q", got.State, backupv1alpha1.BackupStateSucceeded) + } + if got.CompletedAt == nil { + t.Error("expected CompletedAt to be set") + } +} + +func TestFindCommand(t *testing.T) { + entries := []backupStatusEntry{ + {Command: "create_remote my-backup", Status: "success"}, + {Command: "restore_remote other-backup", Status: "in progress"}, + } + if e := findCommand(entries, "create_remote", "my-backup"); e == nil { + t.Error("expected to find the create_remote entry") + } + if e := findCommand(entries, "restore_remote", "my-backup"); e != nil { + t.Error("expected no match for a mismatched backup name") + } + if e := findCommand(entries, "restore_remote", "other-backup"); e == nil { + t.Error("expected to find the restore_remote entry") + } +} + +// ============================================================================= +// backupClient (REST client) tests +// ============================================================================= + +func testClientAndServer(t *testing.T, handler http.HandlerFunc) *backupClient { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + host, portStr, err := net.SplitHostPort(strings.TrimPrefix(srv.URL, "http://")) + if err != nil { + t.Fatalf("split host/port: %v", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("parse port: %v", err) + } + return newBackupClientWithPort(host, port) +} + +func TestBackupClient_CreateRemote(t *testing.T) { + var gotPath string + c := testClientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.RequestURI() + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + w.WriteHeader(http.StatusOK) + }) + + if err := c.createRemote(context.Background(), "my backup"); err != nil { + t.Fatalf("createRemote: %v", err) + } + if want := "/backup/create_remote?name=my+backup"; gotPath != want { + t.Errorf("path = %q, want %q", gotPath, want) + } +} + +func TestBackupClient_RestoreRemote(t *testing.T) { + var gotPath string + c := testClientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.RequestURI() + w.WriteHeader(http.StatusOK) + }) + + if err := c.restoreRemote(context.Background(), "my-backup"); err != nil { + t.Fatalf("restoreRemote: %v", err) + } + if want := "/backup/restore_remote/my-backup"; gotPath != want { + t.Errorf("path = %q, want %q", gotPath, want) + } +} + +func TestBackupClient_DeleteRemote(t *testing.T) { + var gotPath string + c := testClientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.RequestURI() + w.WriteHeader(http.StatusOK) + }) + + if err := c.deleteRemote(context.Background(), "my-backup"); err != nil { + t.Fatalf("deleteRemote: %v", err) + } + if want := "/backup/delete/remote/my-backup"; gotPath != want { + t.Errorf("path = %q, want %q", gotPath, want) + } +} + +func TestBackupClient_ErrorResponse(t *testing.T) { + c := testClientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("boom")) + }) + + err := c.createRemote(context.Background(), "x") + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("expected an error containing the response body, got %v", err) + } +} + +func TestBackupClient_Status(t *testing.T) { + c := testClientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/backup/status" { + t.Errorf("path = %q, want /backup/status", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[{"command":"create_remote x","status":"success"}]`)) + }) + + entries, err := c.status(context.Background()) + if err != nil { + t.Fatalf("status: %v", err) + } + if len(entries) != 1 || entries[0].Status != "success" { + t.Errorf("unexpected entries: %+v", entries) + } +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index e64eccf..041fa52 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -18,21 +18,25 @@ import ( "fmt" "time" - chiv1 "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" chkv1 "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" + chiv1 "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/log" + backupv1alpha1 "github.com/openeverest/openeverest/v2/api/backup/v1alpha1" corev1alpha1 "github.com/openeverest/openeverest/v2/api/core/v1alpha1" "github.com/openeverest/openeverest/v2/provider-runtime/controller" "github.com/scaledb-io/provider-altinity-clickhouse/internal/common" ) -// Compile-time check. -var _ controller.ProviderInterface = (*Provider)(nil) +// Compile-time checks. +var ( + _ controller.ProviderInterface = (*Provider)(nil) + _ controller.BackupProvider = (*Provider)(nil) +) // Provider implements controller.ProviderInterface for ClickHouse via the Altinity operator. type Provider struct { @@ -47,6 +51,7 @@ func New() *Provider { SchemeFuncs: []func(*runtime.Scheme) error{ chiv1.AddToScheme, chkv1.AddToScheme, + backupv1alpha1.AddToScheme, }, // NOTE: We intentionally do NOT watch CHI/CHK here. // Watching them causes a tight feedback loop: operator updates @@ -89,6 +94,50 @@ func (p *Provider) Validate(c *controller.Context) error { } } + if err := validateBackup(c); err != nil { + return err + } + + return nil +} + +// validateBackup enforces the v1 constraints on Instance.Spec.Backup: +// exactly one storage, no schedules (ProviderManaged classes get no free +// CronJob scheduling from the runtime — the provider would need its own +// scheduler, deferred), and no enabling backups on an instance that wasn't +// created with them (the clickhouse-backup sidecar's storage config is baked +// into the CHI pod template at creation and can't be redirected in place). +func validateBackup(c *controller.Context) error { + backup := c.Instance().Spec.Backup + if backup == nil || !backup.Enabled { + return nil + } + + for _, s := range backup.Storages { + if len(s.Schedules) > 0 { + return fmt.Errorf("scheduled backups are not yet supported by this provider") + } + } + + bc, err := c.BackupClassForInstance() + if err != nil { + return fmt.Errorf("resolve BackupClass: %w", err) + } + if err := controller.ValidateInstanceBackupAgainstClass(c.Instance(), bc); err != nil { + return err + } + + existing := &chiv1.ClickHouseInstallation{} + switch err := c.Get(existing, c.Name()); { + case err == nil: + if !chiHasBackupContainer(existing) { + return fmt.Errorf("enabling backups on an existing instance is not supported: " + + "recreate the instance with spec.backup set") + } + case !controller.IsNotFound(err): + return fmt.Errorf("checking existing ClickHouseInstallation during backup validation: %w", err) + } + return nil } @@ -288,6 +337,15 @@ func buildCHI(c *controller.Context, replicasCount int) (*chiv1.ClickHouseInstal Requests: corev1.ResourceList{corev1.ResourceCPU: cpu, corev1.ResourceMemory: memory}, }, } + containers := []corev1.Container{container} + + backupContainer, err := buildBackupContainer(c) + if err != nil { + return nil, fmt.Errorf("build backup sidecar: %w", err) + } + if backupContainer != nil { + containers = append(containers, *backupContainer) + } pvcSpec := corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, @@ -314,7 +372,7 @@ func buildCHI(c *controller.Context, replicasCount int) (*chiv1.ClickHouseInstal Clusters: []*chiv1.Cluster{cluster}, }, Templates: &chiv1.Templates{ - PodTemplates: []chiv1.PodTemplate{{Name: common.PodTemplateName, Spec: corev1.PodSpec{Containers: []corev1.Container{container}}}}, + PodTemplates: []chiv1.PodTemplate{{Name: common.PodTemplateName, Spec: corev1.PodSpec{Containers: containers}}}, VolumeClaimTemplates: []chiv1.VolumeClaimTemplate{{Name: common.DataVolumeClaimTemplateName, Spec: pvcSpec}}, }, } @@ -395,6 +453,100 @@ func buildCHK(c *controller.Context) *chkv1.ClickHouseKeeperInstallation { } } +const ( + backupContainerName = "clickhouse-backup" + // Backup image is fixed, not user-configurable — same rationale as the + // Keeper image above. + backupImage = "altinity/clickhouse-backup:2.6.5" + backupRESTPort = 7171 + backupRESTPortName = "backup-rest" +) + +// buildBackupContainer returns the clickhouse-backup sidecar container for +// Instance.Spec.Backup, or nil if backups are not enabled. +// +// clickhouse-backup's REST API exposes a single, static storage destination +// fixed via env vars at sidecar startup — it cannot be redirected per backup +// request. Validate() only allows exactly one storage entry and rejects +// enabling backups on an already-provisioned CHI, so this only ever runs at +// Instance creation. +func buildBackupContainer(c *controller.Context) (*corev1.Container, error) { + backup := c.Instance().Spec.Backup + if backup == nil || !backup.Enabled || len(backup.Storages) == 0 { + return nil, nil + } + + bs, err := c.BackupStorage(backup.Storages[0].StorageRef.Name) + if err != nil { + return nil, fmt.Errorf("resolve BackupStorage: %w", err) + } + if bs.Spec.S3 == nil { + return nil, fmt.Errorf("BackupStorage %q: only s3 storage is supported", bs.Name) + } + s3 := bs.Spec.S3 + + forcePathStyle := "false" + if s3.ForcePathStyle != nil && *s3.ForcePathStyle { + forcePathStyle = "true" + } + disableCertVerification := "false" + if s3.VerifyTLS != nil && !*s3.VerifyTLS { + disableCertVerification = "true" + } + + return &corev1.Container{ + Name: backupContainerName, + Image: backupImage, + Args: []string{"server"}, + Env: []corev1.EnvVar{ + {Name: "API_LISTEN", Value: fmt.Sprintf("0.0.0.0:%d", backupRESTPort)}, + {Name: "REMOTE_STORAGE", Value: "s3"}, + {Name: "S3_ENDPOINT", Value: s3.EndpointURL}, + {Name: "S3_BUCKET", Value: s3.Bucket}, + {Name: "S3_REGION", Value: s3.Region}, + {Name: "S3_FORCE_PATH_STYLE", Value: forcePathStyle}, + {Name: "S3_DISABLE_CERT_VERIFICATION", Value: disableCertVerification}, + { + Name: "S3_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretRef.Name}, + Key: "AWS_ACCESS_KEY_ID", + }, + }, + }, + { + Name: "S3_SECRET_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretRef.Name}, + Key: "AWS_SECRET_ACCESS_KEY", + }, + }, + }, + }, + Ports: []corev1.ContainerPort{ + {Name: backupRESTPortName, ContainerPort: backupRESTPort}, + }, + }, nil +} + +// chiHasBackupContainer reports whether an existing CHI's pod template +// already carries the clickhouse-backup sidecar. +func chiHasBackupContainer(chi *chiv1.ClickHouseInstallation) bool { + if chi.Spec.Templates == nil { + return false + } + for _, pt := range chi.Spec.Templates.PodTemplates { + for _, ctr := range pt.Spec.Containers { + if ctr.Name == backupContainerName { + return true + } + } + } + return false +} + // ============================================================================= // Helpers // ============================================================================= diff --git a/internal/provider/rbac.go b/internal/provider/rbac.go index 95e83d1..ffa6413 100644 --- a/internal/provider/rbac.go +++ b/internal/provider/rbac.go @@ -44,3 +44,15 @@ package provider // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch + +// Backup/restore (ProviderManaged): the runtime dispatches Backup/Restore CRs +// to SyncBackup/SyncRestore and reads BackupClass/BackupStorage to resolve +// the storage config wired into the clickhouse-backup sidecar. +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=backups,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=backups/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=backups/finalizers,verbs=update +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=restores,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=restores/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=restores/finalizers,verbs=update +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=backupclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups=backup.openeverest.io,resources=backupstorages,verbs=get;list;watch