diff --git a/api/v1/tenant_types.go b/api/v1/tenant_types.go index 89f2297df6..2ea355ef09 100644 --- a/api/v1/tenant_types.go +++ b/api/v1/tenant_types.go @@ -67,6 +67,23 @@ var DataTypes = map[DataType]string{ DataTypePolicyActivity: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", } +var CloudStandardIndices = map[DataType]string{ + DataTypeAlerts: "calico_alerts_standard", + DataTypeAuditLogs: "calico_auditlogs_standard", + DataTypeBGPLogs: "calico_bgplogs_standard", + DataTypeComplianceBenchmarks: "calico_compliance_benchmarks_results_standard", + DataTypeComplianceReports: "calico_compliance_reports_standard", + DataTypeComplianceSnapshots: "calico_compliance_snapshots_standard", + DataTypeDNSLogs: "calico_dnslogs_standard", + DataTypeFlowLogs: "calico_flowlogs_standard", + DataTypeL7Logs: "calico_l7logs_standard", + DataTypeRuntimeReports: "calico_runtime_reports_standard", + DataTypeThreatFeedsDomainSet: "calico_threatfeeds_domainnameset_standard", + DataTypeThreatFeedsIPSet: "calico_threatfeeds_ipset_standard", + DataTypeWAFLogs: "calico_waflogs_standard", + DataTypePolicyActivity: "calico_policy_activity_standard", +} + type TenantSpec struct { // ID is the unique identifier for this tenant. // +required diff --git a/cmd/main.go b/cmd/main.go index 95f2d1860a..7003b76b1f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -550,10 +550,12 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe } elasticIsMigrating := false + indexIsMigrating := false useExternalElastic := discovery.UseExternalElastic(bootConfig) if isCloudBuild() { elasticIsMigrating = discovery.ElasticIsMigrating(bootConfig) + indexIsMigrating = discovery.IndexIsMigrating(bootConfig) if !elasticIsMigrating { if err := verifyElasticSearch(ctx, cs, useExternalElastic); err != nil { setupLog.Error(err, "Elasticsearch configuration verification failed") @@ -580,6 +582,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: useExternalElastic, Cloud: isCloudBuild(), ESMigration: elasticIsMigrating, + IndexMigration: indexIsMigrating, UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, } diff --git a/pkg/common/discovery/discovery.go b/pkg/common/discovery/discovery.go index 74846f228e..a65530aa42 100644 --- a/pkg/common/discovery/discovery.go +++ b/pkg/common/discovery/discovery.go @@ -308,3 +308,19 @@ func ElasticIsMigrating(config *corev1.ConfigMap) bool { } return false } + +// IndexIsMigrating returns true if this cluster is in the last phase of a migration to single-index +// storage, during which the operator must reconfigure Linseed to use the single-index names. +func IndexIsMigrating(config *corev1.ConfigMap) bool { + if config == nil { + return false + } + + // Load the operator bootstrap configuration from its configmap. + if val, ok := config.Data["INDEX_MIGRATION"]; ok && val != "" { + if strings.ToLower(val) == "true" { + return true + } + } + return false +} diff --git a/pkg/controller/compliance/compliance_controller.go b/pkg/controller/compliance/compliance_controller.go index d42ff76d62..bfbf271f19 100644 --- a/pkg/controller/compliance/compliance_controller.go +++ b/pkg/controller/compliance/compliance_controller.go @@ -463,7 +463,7 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read External Elasticsearch config map", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } reqLogger.V(3).Info("rendering components") diff --git a/pkg/controller/intrusiondetection/intrusiondetection_controller.go b/pkg/controller/intrusiondetection/intrusiondetection_controller.go index d1334da2b1..52a3be452e 100644 --- a/pkg/controller/intrusiondetection/intrusiondetection_controller.go +++ b/pkg/controller/intrusiondetection/intrusiondetection_controller.go @@ -437,7 +437,7 @@ func (r *ReconcileIntrusionDetection) Reconcile(ctx context.Context, request rec r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Create a component handler to manage the rendered component. diff --git a/pkg/controller/logstorage/dashboards/dashboards_controller.go b/pkg/controller/logstorage/dashboards/dashboards_controller.go index 703f00c851..09c07856bf 100644 --- a/pkg/controller/logstorage/dashboards/dashboards_controller.go +++ b/pkg/controller/logstorage/dashboards/dashboards_controller.go @@ -288,7 +288,7 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil d.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Determine the host and port from the URL. @@ -321,7 +321,8 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil // Query the username and password this Dashboards Installer instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. key = types.NamespacedName{Name: dashboards.ElasticCredentialsSecret, Namespace: helper.InstallNamespace()} credentials := corev1.Secret{} if err = d.client.Get(ctx, key, &credentials); err != nil && !errors.IsNotFound(err) { diff --git a/pkg/controller/logstorage/initializer/conditions_controller.go b/pkg/controller/logstorage/initializer/conditions_controller.go index a546f22eec..913ed27bba 100644 --- a/pkg/controller/logstorage/initializer/conditions_controller.go +++ b/pkg/controller/logstorage/initializer/conditions_controller.go @@ -17,8 +17,10 @@ package initializer import ( "context" "fmt" + "sort" "time" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -41,9 +43,10 @@ func AddConditionsController(mgr manager.Manager, opts options.ControllerOptions // Create the reconciler r := &LogStorageConditions{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - multiTenant: opts.MultiTenant, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + multiTenant: opts.MultiTenant, + indexMigration: opts.IndexMigration, } return ctrl.NewControllerManagedBy(mgr). @@ -62,6 +65,10 @@ type LogStorageConditions struct { client client.Client scheme *runtime.Scheme multiTenant bool + + // indexMigration indicates that this single-tenant cluster is migrating to single-index storage, + // in which case the log-storage users controller runs and reports status. + indexMigration bool } func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -88,9 +95,22 @@ func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile. } // Compare and update the current StatusCondition if there are any new changes - ls.Status.Conditions = updateConditions(currentConditions, desiredConditions) + conditions := updateConditions(currentConditions, desiredConditions) + + // Skip the write if nothing changed. This controller watches LogStorage, so a no-op write would + // re-trigger it and spin: each reconcile would bump the resourceVersion and enqueue another one. + if equality.Semantic.DeepEqual(ls.Status.Conditions, conditions) { + return reconcile.Result{}, nil + } + ls.Status.Conditions = conditions if err := r.client.Status().Update(ctx, ls); err != nil { + if errors.IsConflict(err) { + // The LogStorage was modified after we read it - our cached copy is stale. Requeue and + // recompute the conditions from the updated object instead of reporting an error. + reqLogger.V(1).Info("Conflict updating LogStorage status conditions, retrying") + return reconcile.Result{Requeue: true}, nil + } log.WithValues("reason", err).Info("Failed to update LogStorage status conditions") return reconcile.Result{}, err } @@ -112,6 +132,10 @@ func (r *LogStorageConditions) getDesiredConditions(ctx context.Context) (map[st expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) } else { expectedInstances = append(expectedInstances, TigeraStatusLogStorageESMetrics, TigeraStatusLogStorageKubeController, TigeraStatusLogStorageDashboards) + if r.indexMigration { + // While migrating to single-index storage, the users controller runs in single-tenant mode too. + expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) + } } // Keep track of which instances are in which state. @@ -197,5 +221,12 @@ func updateConditions(currentConditions, desiredConditions map[string]metav1.Con statusConditions = append(statusConditions, desired) } + + // desiredConditions is a map, so iteration order is random. Sort by type to keep the stored + // conditions stable across reconciles - otherwise every write reorders the list, which counts + // as a change and triggers another reconcile. + sort.Slice(statusConditions, func(i, j int) bool { + return statusConditions[i].Type < statusConditions[j].Type + }) return statusConditions } diff --git a/pkg/controller/logstorage/kubecontrollers/cloud.go b/pkg/controller/logstorage/kubecontrollers/cloud.go index 93a4c4110d..19a183f95b 100644 --- a/pkg/controller/logstorage/kubecontrollers/cloud.go +++ b/pkg/controller/logstorage/kubecontrollers/cloud.go @@ -70,6 +70,11 @@ func (r *ESKubeControllersController) esGatewayAddCloudModificationsToConfig(c * // esKubeControllersAddCloudModificationsToConfig modifies the provided *kubecontrollers.KubeControllersConfiguration to include Calico Cloud specific configuration. func (r *ESKubeControllersController) esKubeControllersAddCloudModificationsToConfig(c *kubecontrollers.KubeControllersConfiguration, reqLogger logr.Logger, ctx context.Context) (reconcile.Result, bool, error) { + // While migrating to single-index storage, es-kube-controllers must not run the elasticsearch + // configuration controller. The operator's log-storage users controller provisions the + // Elasticsearch users instead, so that Linseed gets the RBAC needed for the new indices. + c.IndexMigration = r.indexMigration + if r.cloud && r.elasticExternal && !r.multiTenant { cloudConfig, err := utils.GetCloudConfig(ctx, r.client) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index 2c2ae2c0a1..9855772292 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -63,6 +63,7 @@ type ESKubeControllersController struct { elasticExternal bool multiTenant bool cloud bool + indexMigration bool tierWatchReady *utils.ReadyFlag } @@ -88,6 +89,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { elasticExternal: opts.ElasticExternal, multiTenant: opts.MultiTenant, cloud: opts.Cloud, + indexMigration: opts.IndexMigration, tierWatchReady: &utils.ReadyFlag{}, } r.status.Run(opts.ShutdownContext) diff --git a/pkg/controller/logstorage/linseed/linseed_controller.go b/pkg/controller/logstorage/linseed/linseed_controller.go index 6135175110..d4dac5f506 100644 --- a/pkg/controller/logstorage/linseed/linseed_controller.go +++ b/pkg/controller/logstorage/linseed/linseed_controller.go @@ -69,6 +69,7 @@ type LinseedSubController struct { multiTenant bool elasticExternal bool cloud bool + useSingleIndex bool } func Add(mgr manager.Manager, opts options.ControllerOptions) error { @@ -87,6 +88,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { status: status.New(mgr.GetClient(), "log-storage-access", opts.KubernetesVersion), elasticExternal: opts.ElasticExternal, cloud: opts.Cloud, + useSingleIndex: opts.IndexMigration, } r.status.Run(opts.ShutdownContext) @@ -334,7 +336,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(r.useSingleIndex) } // Determine the host and port from the URL. @@ -361,7 +363,8 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. // Query the username and password this Linseed instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. key = types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.InstallNamespace()} credentials := corev1.Secret{} if err = r.client.Get(ctx, key, &credentials); err != nil && !errors.IsNotFound(err) { @@ -418,7 +421,8 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. // Query the username and password this Linseed instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. // Delay installing Linseed until available. // TODO: Switch single-tenant to using operator-provisioned users. key = types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.InstallNamespace()} @@ -468,6 +472,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. ElasticClientCredentialsSecret: &credentials, LogStorage: logStorage, Cloud: r.cloud, + UseSingleIndex: r.useSingleIndex, } linseedComponent := linseed.Linseed(cfg) diff --git a/pkg/controller/logstorage/users/users_controller.go b/pkg/controller/logstorage/users/users_controller.go index f7a173837d..e831bcd818 100644 --- a/pkg/controller/logstorage/users/users_controller.go +++ b/pkg/controller/logstorage/users/users_controller.go @@ -28,11 +28,13 @@ import ( "github.com/tigera/operator/pkg/render/logstorage/dashboards" corev1 "k8s.io/api/core/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/secret" "k8s.io/apimachinery/pkg/api/errors" @@ -59,6 +61,11 @@ type UserController struct { esClientFn utils.ElasticsearchClientCreator multiTenant bool elasticExternal bool + + // indexMigration indicates that this single-tenant cluster is migrating to single-index storage. + // While migrating, this controller provisions the Elasticsearch users instead of es-kube-controllers, + // so that Linseed gets the RBAC needed for the new indices. + indexMigration bool } type UsersCleanupController struct { @@ -72,9 +79,10 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if !opts.EnterpriseCRDExists { return nil } - if !opts.MultiTenant { - // For now, the operator only creates users in multi-tenant mode. In single-tenant mode, - // user creation is handled by es-kube-controllers instead. + if !opts.MultiTenant && !opts.IndexMigration { + // The operator only creates users in multi-tenant mode, or in single-tenant mode while + // migrating to single-index storage. Otherwise, user creation is handled by + // es-kube-controllers instead. return nil } @@ -83,6 +91,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { client: mgr.GetClient(), scheme: mgr.GetScheme(), multiTenant: opts.MultiTenant, + indexMigration: opts.IndexMigration, status: status.New(mgr.GetClient(), initializer.TigeraStatusLogStorageUsers, opts.KubernetesVersion), esClientFn: utils.NewElasticClient, elasticExternal: opts.ElasticExternal, @@ -119,6 +128,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err = c.WatchObject(&operatorv1.Tenant{}, &handler.EnqueueRequestForObject{}); err != nil { return fmt.Errorf("log-storage-user-controller failed to watch Tenant resource: %w", err) } + } else { + // In single-tenant mode, the tenant configuration - including the Elasticsearch endpoint - + // comes from the cloud config ConfigMap. + if err = utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("log-storage-user-controller failed to watch the ConfigMap resource: %w", err) + } } // Watch for Elasticsearch. @@ -133,6 +148,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("log-storage-user-controller failed to create periodic reconcile watch: %w", err) } + if !opts.MultiTenant { + // The cleanup controller reconciles Tenant resources, which only exist in multi-tenant mode. + return nil + } + // Now that the users controller is set up, we can also set up the controller that cleans up stale users usersCleanupReconciler := &UsersCleanupController{ client: mgr.GetClient(), @@ -179,6 +199,18 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, err } + if !r.multiTenant { + // Single-tenant clusters have no Tenant resource. Build the equivalent tenant configuration + // from the cloud config, so that we provision the users against the right Elasticsearch. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant(r.indexMigration) + tenantID = tenant.Spec.ID + } + // Get LogStorage resource. logStorage := &operatorv1.LogStorage{} err = r.client.Get(ctx, utils.DefaultEnterpriseInstanceKey, logStorage) @@ -215,33 +247,25 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques } } - clusterIDConfigMap := corev1.ConfigMap{} - clusterIDConfigMapKey := client.ObjectKey{Name: "cluster-info", Namespace: "tigera-operator"} - err = r.client.Get(ctx, clusterIDConfigMapKey, &clusterIDConfigMap) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Waiting for ConfigMap %s/%s to be available", clusterIDConfigMapKey.Namespace, clusterIDConfigMapKey.Name), - nil, reqLogger) - return reconcile.Result{}, err - } - - clusterID, ok := clusterIDConfigMap.Data["cluster-id"] - if !ok { - err = fmt.Errorf("%s/%s ConfigMap does not contain expected 'cluster-id' key", - clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) - return reconcile.Result{}, err - } - - if clusterID == "" { - err = fmt.Errorf("%s/%s ConfigMap value for key 'cluster-id' must be non-empty", - clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) - return reconcile.Result{}, err + // Determine the names of the users to provision. In multi-tenant clusters the cluster ID forms part + // of the user names, and is read from the cluster-info ConfigMap written at install time. + // Single-tenant clusters have no such ConfigMap - their user names are derived from the tenant ID + // alone, matching the names es-kube-controllers used before the operator took over provisioning. + var linseedUser, dashboardUser *utils.User + if r.multiTenant { + clusterID, err := r.clusterID(ctx, reqLogger) + if err != nil { + return reconcile.Result{}, err + } + linseedUser = utils.LinseedUser(clusterID, tenantID) + dashboardUser = utils.DashboardUser(clusterID, tenantID) + } else { + linseedUser = utils.LinseedUserSingleTenant(tenantID) + dashboardUser = utils.DashboardUserSingleTenant(tenantID) } // Query any existing username and password for this Linseed instance. If one already exists, we'll simply // use that. Otherwise, generate a new one. - linseedUser := utils.LinseedUser(clusterID, tenantID) linseedUserSecret := corev1.Secret{} var credentialSecrets []client.Object key := types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.TruthNamespace()} @@ -256,14 +280,19 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Make sure we install the generated credentials into the truth namespace. credentialSecrets = append(credentialSecrets, &linseedUserSecret) + } else if string(linseedUserSecret.Data["username"]) != linseedUser.Username { + // The credentials exist, but reference a different Elasticsearch user than the one we provision - + // e.g. because they were created by es-kube-controllers before this cluster started migrating to + // single-index storage. Point them at our user, keeping the existing password. + linseedUserSecret.StringData = map[string]string{"username": linseedUser.Username} + credentialSecrets = append(credentialSecrets, &linseedUserSecret) } // Query any existing username and password for this Dashboards instance. If one already exists, we'll simply // use that. Otherwise, generate a new one. keyDashboardCred := types.NamespacedName{Name: dashboards.ElasticCredentialsSecret, Namespace: helper.TruthNamespace()} - dashboardUser := utils.DashboardUser(clusterID, tenantID) dashboardUserSecret := corev1.Secret{} - if err = r.client.Get(ctx, key, &dashboardUserSecret); err != nil && !errors.IsNotFound(err) { + if err = r.client.Get(ctx, keyDashboardCred, &dashboardUserSecret); err != nil && !errors.IsNotFound(err) { r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error getting Secret %s", keyDashboardCred), err, reqLogger) return reconcile.Result{}, err } else if errors.IsNotFound(err) { @@ -274,6 +303,10 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Make sure we install the generated credentials into the truth namespace. credentialSecrets = append(credentialSecrets, &dashboardUserSecret) + } else if string(dashboardUserSecret.Data["username"]) != dashboardUser.Username { + // As above - point the existing credentials at the user we provision. + dashboardUserSecret.StringData = map[string]string{"username": dashboardUser.Username} + credentialSecrets = append(credentialSecrets, &dashboardUserSecret) } if helper.TruthNamespace() != helper.InstallNamespace() { @@ -297,7 +330,7 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Add a finalizer to the Tenant instance if it exists so that we can clean up the Linseed user when the Tenant // is deleted. The finalizer will be removed by the user cleanup controller when the user is deleted from ES. - if tenant != nil && tenant.GetDeletionTimestamp().IsZero() && !stringsutil.StringInSlice(userCleanupFinalizer, tenant.GetFinalizers()) { + if r.multiTenant && tenant != nil && tenant.GetDeletionTimestamp().IsZero() && !stringsutil.StringInSlice(userCleanupFinalizer, tenant.GetFinalizers()) { tenant.SetFinalizers(append(tenant.GetFinalizers(), userCleanupFinalizer)) if err = r.client.Update(ctx, tenant); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error adding finalizer to Tenant", err, reqLogger) @@ -325,6 +358,35 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, nil } +// clusterID reads the cluster ID from the cluster-info ConfigMap. This ConfigMap is written at install +// time and only exists in multi-tenant clusters. +func (r *UserController) clusterID(ctx context.Context, reqLogger logr.Logger) (string, error) { + clusterIDConfigMap := corev1.ConfigMap{} + clusterIDConfigMapKey := client.ObjectKey{Name: "cluster-info", Namespace: "tigera-operator"} + if err := r.client.Get(ctx, clusterIDConfigMapKey, &clusterIDConfigMap); err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Waiting for ConfigMap %s/%s to be available", clusterIDConfigMapKey.Namespace, clusterIDConfigMapKey.Name), + nil, reqLogger) + return "", err + } + + clusterID, ok := clusterIDConfigMap.Data["cluster-id"] + if !ok { + err := fmt.Errorf("%s/%s ConfigMap does not contain expected 'cluster-id' key", + clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) + return "", err + } + + if clusterID == "" { + err := fmt.Errorf("%s/%s ConfigMap value for key 'cluster-id' must be non-empty", + clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) + return "", err + } + + return clusterID, nil +} + func (r *UserController) createUserLogin(ctx context.Context, elasticEndpoint string, secret *corev1.Secret, user *utils.User, reqLogger logr.Logger) error { esClient, err := r.esClientFn(r.client, ctx, elasticEndpoint, r.elasticExternal) if err != nil { diff --git a/pkg/controller/logstorage/users/users_controller_test.go b/pkg/controller/logstorage/users/users_controller_test.go index 1a8620c2f7..70605f8228 100644 --- a/pkg/controller/logstorage/users/users_controller_test.go +++ b/pkg/controller/logstorage/users/users_controller_test.go @@ -23,15 +23,23 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" apiv1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" tigeraelastic "github.com/tigera/operator/pkg/controller/logstorage/elastic" + "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" + "github.com/tigera/operator/pkg/render/logstorage/dashboards" ) var _ = Describe("LogStorage cleanup controller", func() { @@ -110,3 +118,124 @@ var _ = Describe("LogStorage cleanup controller", func() { Expect(testESClient.AssertExpectations(t)) }) }) + +// fakeESClient records the users provisioned against Elasticsearch. +type fakeESClient struct { + created []*utils.User +} + +func (f *fakeESClient) SetILMPolicies(context.Context, *operatorv1.LogStorage, bool) error { + return nil +} + +func (f *fakeESClient) CreateUser(_ context.Context, user *utils.User) error { + f.created = append(f.created, user) + return nil +} + +func (f *fakeESClient) DeleteUser(context.Context, *utils.User) error { return nil } + +func (f *fakeESClient) GetUsers(context.Context) ([]utils.User, error) { return nil, nil } + +var _ = Describe("LogStorage users controller", func() { + const ( + tenantID = "tenant-a" + ) + + var ( + cli client.Client + ctx context.Context + scheme *runtime.Scheme + esClient *fakeESClient + r *UserController + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(operatorv1.AddToScheme(scheme)).NotTo(HaveOccurred()) + Expect(corev1.AddToScheme(scheme)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + ctx = context.Background() + + ls := &operatorv1.LogStorage{} + ls.Name = "tigera-secure" + ls.Status.State = operatorv1.TigeraStatusReady + Expect(cli.Create(ctx, ls)).NotTo(HaveOccurred()) + + // Note that no cluster-info ConfigMap is created here: single-tenant clusters don't have one, + // and the controller must not require it. + Expect(cli.Create(ctx, cloudconfig.NewCloudConfig(tenantID, "tenant-a-name", "es.example.com", "kb.example.com", false).ConfigMap())).NotTo(HaveOccurred()) + + mockStatus := &status.MockStatus{} + mockStatus.On("OnCRFound").Return() + mockStatus.On("ReadyToMonitor") + mockStatus.On("ClearDegraded") + mockStatus.On("ClearWarning", mock.Anything).Return() + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + + esClient = &fakeESClient{} + r = &UserController{ + client: cli, + scheme: scheme, + status: mockStatus, + esClientFn: func(_ client.Client, _ context.Context, _ string, _ bool) (utils.ElasticClient, error) { + return esClient, nil + }, + multiTenant: false, + elasticExternal: true, + indexMigration: true, + } + }) + + // secretValue reads a value from a Secret. The fake client doesn't convert StringData into Data + // the way the API server does, so we may find the value in either field. + secretValue := func(name, namespace, key string) string { + s := &corev1.Secret{} + ExpectWithOffset(1, cli.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, s)).NotTo(HaveOccurred()) + if v, ok := s.StringData[key]; ok { + return v + } + return string(s.Data[key]) + } + + It("should provision users for a single-tenant cluster migrating to single-index storage", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + // The Linseed user should be created in ES with the name es-kube-controllers used, and with + // privileges on the new single-index calico_* indices. + expected := utils.LinseedUserSingleTenant(tenantID) + Expect(expected.Username).To(Equal("tigera-ee-linseed-tenant-a-secure")) + Expect(esClient.created).To(HaveLen(2)) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles).To(Equal(expected.Roles)) + Expect(esClient.created[0].Roles[0].Name).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(ContainElement("calico_*")) + Expect(esClient.created[1].Username).To(Equal("tigera-ee-dashboards-installer-tenant-a-secure")) + + // The credentials should be written to the Elasticsearch namespace for Linseed to consume. + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "password")).NotTo(BeEmpty()) + Expect(secretValue(dashboards.ElasticCredentialsSecret, render.ElasticsearchNamespace, "username")).To(Equal(utils.DashboardUserSingleTenant(tenantID).Username)) + }) + + It("should re-point existing credentials at the operator provisioned user", func() { + // es-kube-controllers uses a different naming convention for the ES user. + Expect(cli.Create(ctx, &corev1.Secret{ + ObjectMeta: apiv1.ObjectMeta{Name: render.ElasticsearchLinseedUserSecret, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"username": []byte("tigera-ee-linseed"), "password": []byte("existing-password")}, + })).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + expected := utils.LinseedUserSingleTenant(tenantID) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "username")).To(Equal(expected.Username)) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) + + // The existing password is preserved, and used for the user we provision. + Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "password")).To(Equal("existing-password")) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Password).To(Equal("existing-password")) + }) +}) diff --git a/pkg/controller/manager/manager_controller_cloud.go b/pkg/controller/manager/manager_controller_cloud.go index 5f0c56591b..e9384b96b6 100644 --- a/pkg/controller/manager/manager_controller_cloud.go +++ b/pkg/controller/manager/manager_controller_cloud.go @@ -122,7 +122,7 @@ func (r *ReconcileManager) handleCloudReconcile( r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return nil, render.ManagerCloudResources{}, nil, nil, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } else { var err error tenant, err = utils.GetTenantFromCloudAuthConfig(ctx, r.client) diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index b1f1c50b57..ee06a0039d 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -57,6 +57,10 @@ type ControllerOptions struct { // LSS configuration and internal elasticsearch running. Only meaningful when Cloud is set. ESMigration bool + // This field is enabled in the last phase of an index migration for a single tenant format, + // we need to reconfigure + IndexMigration bool + // Whether or not to use crd.projectcalico.org/v1 or projectcalico.org/v3 for Calico CRDs. UseV3CRDs bool diff --git a/pkg/controller/policyrecommendation/policyrecommendation_controller.go b/pkg/controller/policyrecommendation/policyrecommendation_controller.go index 1fca4018cc..23479b5f01 100644 --- a/pkg/controller/policyrecommendation/policyrecommendation_controller.go +++ b/pkg/controller/policyrecommendation/policyrecommendation_controller.go @@ -349,7 +349,7 @@ func (r *ReconcilePolicyRecommendation) Reconcile(ctx context.Context, request r r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, logc) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Create a component handler to manage the rendered component. diff --git a/pkg/controller/utils/elasticsearch.go b/pkg/controller/utils/elasticsearch.go index 0a977f291b..bbe4cde489 100644 --- a/pkg/controller/utils/elasticsearch.go +++ b/pkg/controller/utils/elasticsearch.go @@ -220,8 +220,30 @@ var ( ElasticsearchUserNameDashboardInstaller = "tigera-ee-dashboards-installer" ) +// ElasticsearchSecureUserSuffix is appended to the user names provisioned for single-tenant clusters. +// It maintains the 1:1 mapping between the public user propagated to components and the private user +// swapped in at ES gateway, which strips this suffix. +const ElasticsearchSecureUserSuffix = "secure" + +// formatNameSingleTenant builds the ES user name for a single-tenant cluster: --secure. +// This matches the name previously provisioned by es-kube-controllers, so that existing credentials +// and role mappings keep resolving once the operator takes over user provisioning. +func formatNameSingleTenant(name, tenantID string) string { + return fmt.Sprintf("%s-%s-%s", name, tenantID, ElasticsearchSecureUserSuffix) +} + func LinseedUser(clusterID, tenant string) *User { - username := formatName(ElasticsearchUserNameLinseed, clusterID, tenant) + return linseedUser(formatName(ElasticsearchUserNameLinseed, clusterID, tenant), tenant) +} + +// LinseedUserSingleTenant returns the Linseed user for a single-tenant cluster. It carries the same +// privileges as the multi-tenant user - including access to the single-index calico_* indices - but is +// named the way es-kube-controllers named it, and needs no cluster ID. +func LinseedUserSingleTenant(tenant string) *User { + return linseedUser(formatNameSingleTenant(ElasticsearchUserNameLinseed, tenant), tenant) +} + +func linseedUser(username, tenant string) *User { return &User{ Username: username, Roles: []Role{ @@ -243,7 +265,16 @@ func LinseedUser(clusterID, tenant string) *User { } func DashboardUser(clusterID, tenant string) *User { - username := formatName(ElasticsearchUserNameDashboardInstaller, clusterID, tenant) + return dashboardUser(formatName(ElasticsearchUserNameDashboardInstaller, clusterID, tenant)) +} + +// DashboardUserSingleTenant returns the Dashboards installer user for a single-tenant cluster, named +// the way es-kube-controllers named it. See LinseedUserSingleTenant. +func DashboardUserSingleTenant(tenant string) *User { + return dashboardUser(formatNameSingleTenant(ElasticsearchUserNameDashboardInstaller, tenant)) +} + +func dashboardUser(username string) *User { return &User{ Username: username, Roles: []Role{ diff --git a/pkg/render/common/cloudconfig/cloudconfig.go b/pkg/render/common/cloudconfig/cloudconfig.go index f93dd3209e..7a5a530529 100644 --- a/pkg/render/common/cloudconfig/cloudconfig.go +++ b/pkg/render/common/cloudconfig/cloudconfig.go @@ -16,6 +16,7 @@ package cloudconfig import ( "fmt" + "sort" "strconv" v1 "github.com/tigera/operator/api/v1" @@ -82,8 +83,12 @@ type CloudConfig struct { // ToTenant converts the given CloudConfig structure to a Tenant object. // This allows controllers that have been converted to support multi-tenancy to still leverage // the single-tenant CloudConfig structure using the same code path as in multi-tenancy. -func (c CloudConfig) ToTenant() *v1.Tenant { - return &v1.Tenant{ +// +// useSingleIndex declares the standard single-index base names on the returned Tenant. Only clusters +// migrating to single-index storage should set it: index base names are otherwise not carried on the +// artificial single-tenant Tenant, and components fall back to their default index names. +func (c CloudConfig) ToTenant(useSingleIndex bool) *v1.Tenant { + tenant := &v1.Tenant{ // We don't specify a Namespace for this tenant because it represents a singular tenant installed // in this management cluster. The signals to the render code that this is a single-tenant cluster and not // a cluster capable of multi-tenancy. @@ -98,6 +103,22 @@ func (c CloudConfig) ToTenant() *v1.Tenant { }, }, } + + if !useSingleIndex { + return tenant + } + + for dataType := range v1.DataTypes { + tenant.Spec.Indices = append(tenant.Spec.Indices, v1.Index{DataType: dataType, BaseIndexName: v1.CloudStandardIndices[dataType]}) + } + + // DataTypes is a map, so iteration order is random. Sort by data type to keep the generated + // index list - and therefore the env vars rendered from it - stable across reconciles. + sort.Slice(tenant.Spec.Indices, func(i, j int) bool { + return tenant.Spec.Indices[i].DataType < tenant.Spec.Indices[j].DataType + }) + + return tenant } func (c CloudConfig) TenantId() string { diff --git a/pkg/render/common/cloudconfig/cloudconfig_test.go b/pkg/render/common/cloudconfig/cloudconfig_test.go index da125c5d73..f8d8d5f23d 100644 --- a/pkg/render/common/cloudconfig/cloudconfig_test.go +++ b/pkg/render/common/cloudconfig/cloudconfig_test.go @@ -17,6 +17,7 @@ package cloudconfig import ( "strconv" + v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" corev1 "k8s.io/api/core/v1" @@ -91,6 +92,52 @@ var _ = Describe("CloudConfig ConfigMap tests", func() { }) }) + Context("ToTenant", func() { + var cloudConfig *CloudConfig + + BeforeEach(func() { + cloudConfig = &CloudConfig{ + tenantId: "abc123", + tenantName: "tenant1", + externalESDomain: "externalES.com", + externalKibanaDomain: "externalKibana.com", + enableMTLS: true, + } + }) + + It("should return a single-tenant Tenant with the Elastic configuration from the CloudConfig", func() { + tenant := cloudConfig.ToTenant(false) + Expect(tenant.Name).To(Equal("default")) + Expect(tenant.Namespace).To(BeEmpty()) + Expect(tenant.MultiTenant()).To(BeFalse()) + Expect(tenant.Spec.ID).To(Equal("abc123")) + Expect(tenant.Spec.Name).To(Equal("tenant1")) + Expect(tenant.Spec.Elastic.URL).To(Equal("https://externalES.com:443")) + Expect(tenant.Spec.Elastic.KibanaURL).To(Equal("https://externalKibana.com:443")) + Expect(tenant.Spec.Elastic.MutualTLS).To(BeTrue()) + }) + + It("should not declare any indices when not using single-index storage", func() { + Expect(cloudConfig.ToTenant(false).Spec.Indices).To(BeEmpty()) + }) + + It("should declare the standard index for every data type when using single-index storage", func() { + indices := cloudConfig.ToTenant(true).Spec.Indices + Expect(indices).To(HaveLen(len(v1.DataTypes))) + for _, index := range indices { + Expect(index.BaseIndexName).To(Equal(v1.CloudStandardIndices[index.DataType])) + Expect(index.BaseIndexName).ToNot(BeEmpty()) + } + }) + + It("should declare indices in a stable order", func() { + expected := cloudConfig.ToTenant(true).Spec.Indices + for i := 0; i < 10; i++ { + Expect(cloudConfig.ToTenant(true).Spec.Indices).To(Equal(expected)) + } + }) + }) + Context("ConfigMap from CloudConfig", func() { var cloudConfig *CloudConfig diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index bb0fecd785..8ef970ff84 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -124,6 +124,11 @@ type KubeControllersConfiguration struct { // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. Cloud bool + // IndexMigration indicates that this cluster is migrating to single-index storage. While migrating, + // Elasticsearch configuration - including provisioning of the Linseed user - is handled by the + // operator's log-storage users controller rather than by es-kube-controllers. + IndexMigration bool + MetricsServerTLS certificatemanagement.KeyPairInterface // Namespace to be installed into. @@ -282,7 +287,12 @@ func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeCon var enabledControllers []string if !cfg.Tenant.MultiTenant() { // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") + enabledControllers = append(enabledControllers, "authorization") + if !cfg.IndexMigration { + // While migrating to single-index storage, the operator's log-storage users controller + // provisions the Elasticsearch users so that they get the RBAC needed for the new indices. + enabledControllers = append(enabledControllers, "elasticsearchconfiguration") + } if cfg.ManagementCluster != nil && cfg.Tenant == nil { // Enterprise will require the managedcluster controller to push licenses enabledControllers = append(enabledControllers, "managedcluster") diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index e2c484492d..ee17bba0ad 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -560,6 +560,24 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) + It("should not enable the elasticsearchconfiguration controller when migrating to single-index storage", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.LogStorageExists = true + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + cfg.IndexMigration = true + + component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "authorization", + })) + }) + It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { expectedResources := []struct { name string diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 66e8338046..a3fc1e8041 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -117,6 +117,7 @@ type Config struct { // Tenant configuration, if running for a particular tenant. Tenant *operatorv1.Tenant ExternalElastic bool + UseSingleIndex bool // Secret containing client certificate and key for connecting to the Elastic cluster. If configured, // mTLS is used between Linseed and the external Elastic cluster. @@ -426,6 +427,13 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { if l.cfg.Tenant.Spec.ControlPlaneReplicas != nil { replicas = l.cfg.Tenant.Spec.ControlPlaneReplicas } + } else if l.cfg.UseSingleIndex { + // For single-tenant clusters migrating to multi-tenant style indices, + // use the elastic-single-index backend and configure index base names. + envVars = append(envVars, corev1.EnvVar{Name: "BACKEND", Value: "elastic-single-index"}) + for _, index := range l.cfg.Tenant.Spec.Indices { + envVars = append(envVars, index.EnvVar()) + } } } sc := securitycontext.NewNonRootContext()