diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index f9aef63d84..d65140117e 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -473,6 +473,19 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } } + // Detect the allow-tigera -> calico-system apiserver migration window. Gate on the Tier + // watch being ready, matching the includeV3NetworkPolicy read above: before the watch is + // ready the aggregated v3 API may be unavailable, and we must not degrade/return on that. + // During a real migration the old apiserver is serving, so the watch is ready. + migrationInProgress := false + if !r.opts.UseV3CRDs && r.tierWatchReady.IsReady() { + migrationInProgress, err = utils.APIServerMigrationInProgress(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error checking apiserver migration state", err, reqLogger) + return reconcile.Result{}, err + } + } + err = utils.PopulateK8sServiceEndPoint(r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading services endpoint configmap", err, reqLogger) @@ -577,6 +590,12 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } + // The bridge must precede the workload component so the apply loop's fail-fast + // behavior requeues before any cutover if a bridge write does not land. + if migrationInProgress { + components = append(components, render.APIServerMigrationBridge(&apiServerCfg)) + } + components = append(components, component, rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -594,7 +613,10 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re // dependency on the API server deployment. // // We do this last to avoid transient errors with policy preventing progression of the controller. - if r.opts.UseV3CRDs || includeV3NetworkPolicy { + // While migrating, the bridge owns calico-system.apiserver-access and the transitional + // allow-tigera.apiserver-access must stay; skipping this component avoids the double-write + // and suppresses its deletion of allow-tigera.apiserver-access until the apiserver is stable. + if (r.opts.UseV3CRDs || includeV3NetworkPolicy) && !migrationInProgress { components = append(components, render.APIServerPolicy(&apiServerCfg)) } diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index ebcd062b5b..438d69670b 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -32,7 +32,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" @@ -1030,4 +1032,106 @@ var _ = Describe("apiserver controller tests", func() { Expect(err.Error()).To(ContainSubstring("CalicoWebhooksDeployment")) }) }) + + Context("apiserver migration bridge", func() { + // Reconciler wired as in the other tests, with both watches ready. + newReconciler := func(cli client.Client) ReconcileAPIServer { + return ReconcileAPIServer{ + client: cli, + scheme: scheme, + status: mockStatus, + tierWatchReady: ready, + migrationWatchReady: ready, + opts: options.ControllerOptions{ + EnterpriseCRDExists: true, + DetectedProvider: operatorv1.ProviderNone, + ClusterDomain: dns.DefaultClusterDomain, + }, + } + } + + seedMigrationState := func(cli client.Client) { + // Deprecated tier still present; apiserver not yet Ready in calico-system. + Expect(cli.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}})).NotTo(HaveOccurred()) + Expect(cli.Create(ctx, &v3.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera.default-deny", Namespace: "calico-system"}, + Spec: v3.NetworkPolicySpec{Tier: "allow-tigera", Selector: "all()"}, + })).NotTo(HaveOccurred()) + Expect(cli.Create(ctx, installation)).To(BeNil()) + } + + It("writes the bridge and does not tear down allow-tigera.apiserver-access while migrating", func() { + seedMigrationState(cli) + r := newReconciler(cli) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // default-deny rewritten to exclude the apiserver. + dd := &v3.NetworkPolicy{} + Expect(cli.Get(ctx, client.ObjectKey{Name: "allow-tigera.default-deny", Namespace: "calico-system"}, dd)).NotTo(HaveOccurred()) + Expect(dd.Spec.Selector).To(Equal("k8s-app != 'calico-apiserver'")) + + // order-1 transitional allow created in the deprecated tier and NOT deleted. + at := &v3.NetworkPolicy{} + Expect(cli.Get(ctx, client.ObjectKey{Name: "allow-tigera.apiserver-access", Namespace: "calico-system"}, at)).NotTo(HaveOccurred()) + Expect(at.Spec.Tier).To(Equal("allow-tigera")) + + // steady-state allow created by the bridge. + cs := &v3.NetworkPolicy{} + Expect(cli.Get(ctx, client.ObjectKey{Name: "calico-system.apiserver-access", Namespace: "calico-system"}, cs)).NotTo(HaveOccurred()) + }) + + It("requeues without moving when a bridge write fails", func() { + // A bridge write failure surfaces through SetDegraded(ResourceUpdateError, ...), + // which the shared mock (set up for other error codes only) does not expect. + mockStatus.On("SetDegraded", operatorv1.ResourceUpdateError, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + + failing := ctrlrfake.DefaultFakeClientBuilder(scheme).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if np, ok := obj.(*v3.NetworkPolicy); ok && np.GetName() == "allow-tigera.apiserver-access" { + return kerror.NewServiceUnavailable("the server is currently unable to handle the request") + } + return c.Create(ctx, obj, opts...) + }, + }).Build() + + // Re-seed prerequisites on the failing client (mirrors BeforeEach essentials): the + // operator CA (so certificatemanager.Create, which is not called with + // AllowCACreation from within Reconcile, does not fail first), and the APIServer CR + // (without it Reconcile short-circuits on "not found" before ever reaching the + // bridge). Authentication/dex/oidc are not required: GetAuthentication's NotFound is + // tolerated and the keyValidatorConfig lookup is skipped. + certificateManager, err := certificatemanager.Create(failing, nil, "cluster.local", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(failing.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + Expect(failing.Create(ctx, &operatorv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}})).NotTo(HaveOccurred()) + Expect(failing.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + seedMigrationState(failing) + + r := newReconciler(failing) + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).To(HaveOccurred()) + + // The workload cutover must NOT have happened. The bridge is ordered before the + // workload component and the apply loop is fail-fast, so render.APIServer never runs + // and the v3.projectcalico.org APIService is never created this reconcile. + apiService := &apiregv1.APIService{} + getErr := failing.Get(ctx, client.ObjectKey{Name: "v3.projectcalico.org"}, apiService) + Expect(kerror.IsNotFound(getErr)).To(BeTrue(), "workload cutover must not run when a bridge write fails") + }) + + It("lets the tail create calico-system.apiserver-access when no bridge is needed", func() { + // No allow-tigera tier => not migrating; calico-system tier already seeded by BeforeEach. + Expect(cli.Create(ctx, installation)).To(BeNil()) + r := newReconciler(cli) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + cs := &v3.NetworkPolicy{} + Expect(cli.Get(ctx, client.ObjectKey{Name: "calico-system.apiserver-access", Namespace: "calico-system"}, cs)).NotTo(HaveOccurred()) + Expect(cs.Spec.Tier).To(Equal("calico-system")) + }) + }) }) diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 386dde4350..70de12e88e 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1629,8 +1629,13 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile if nonclusterhost != nil { components = append(components, render.NewTyphaNonClusterHostPolicy(&typhaCfg)) } + migrationInProgress, err := utils.APIServerMigrationInProgress(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error checking apiserver migration state", err, reqLogger) + return reconcile.Result{}, err + } components = append(components, - kubecontrollers.NewCalicoKubeControllersPolicy(&kubeControllersCfg, calicoSystemDefaultDenyForCalicoSystem()), + kubecontrollers.NewCalicoKubeControllersPolicy(&kubeControllersCfg, calicoSystemDefaultDenyForCalicoSystem(), migrationInProgress), ) } diff --git a/pkg/controller/tiers/tiers_controller.go b/pkg/controller/tiers/tiers_controller.go index e7e7bcbb43..41629d1e15 100644 --- a/pkg/controller/tiers/tiers_controller.go +++ b/pkg/controller/tiers/tiers_controller.go @@ -96,6 +96,14 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("tiers-controller failed to watch node-local-dns daemonset: %v", err) } + // Perform periodic reconciliation. This acts as a backstop to catch reconcile issues, and also + // makes sure we spot when things change that might not trigger a reconciliation - notably, deleting + // the deprecated allow-tigera tier once the transitional apiserver-migration policies within it have + // been cleaned up (that cleanup is driven by other controllers and does not enqueue this one). + if err := utils.AddPeriodicReconcile(c, utils.PeriodicReconcileTime, &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("tiers-controller failed to create periodic reconcile watch: %w", err) + } + return nil } @@ -136,13 +144,21 @@ func (r *ReconcileTiers) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, err } - // Try to delete allow-tigera deprecated tier - err = componentHandler.CreateOrUpdateOrDelete(ctx, render.NewDeletionPassthrough(&v3.Tier{ - TypeMeta: metav1.TypeMeta{Kind: "Tier", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}, - }), nil) - if err != nil { - log.V(1).Info("Unable to delete deprecated allow-tigera tier at this time", "error", err) + // Delete the deprecated allow-tigera tier, but not while the apiserver is still migrating + // out of it: the migration bridge (apiserver controller) writes transitional policies into + // this tier, so removing it mid-migration would fight the bridge. Resume once the apiserver + // is stable in calico-system. + migrationInProgress, migErr := utils.APIServerMigrationInProgress(ctx, r.client) + if migErr != nil { + log.V(1).Info("Unable to determine apiserver migration state; deferring allow-tigera tier deletion", "error", migErr) + } else if !migrationInProgress { + err = componentHandler.CreateOrUpdateOrDelete(ctx, render.NewDeletionPassthrough(&v3.Tier{ + TypeMeta: metav1.TypeMeta{Kind: "Tier", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}, + }), nil) + if err != nil { + log.V(1).Info("Unable to delete deprecated allow-tigera tier at this time", "error", err) + } } r.status.ReadyToMonitor() diff --git a/pkg/controller/tiers/tiers_controller_test.go b/pkg/controller/tiers/tiers_controller_test.go index 449ced2b21..053054d3dc 100644 --- a/pkg/controller/tiers/tiers_controller_test.go +++ b/pkg/controller/tiers/tiers_controller_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/mock" appsv1 "k8s.io/api/apps/v1" + kerror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -204,4 +205,43 @@ var _ = Describe("tier controller tests", func() { Expect(err).ShouldNot(HaveOccurred()) mockStatus.AssertExpectations(GinkgoT()) }) + + It("does not delete the allow-tigera tier while the apiserver migration is in progress", func() { + mockStatus.On("ReadyToMonitor") + mockStatus.On("ClearDegraded") + + // Migration in progress: deprecated tier present, calico-apiserver not Ready. + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}})).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, + Status: appsv1.DeploymentStatus{ReadyReplicas: 0}, + })).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + tier := &v3.Tier{} + Expect(c.Get(ctx, client.ObjectKey{Name: "allow-tigera"}, tier)).NotTo(HaveOccurred()) + mockStatus.AssertExpectations(GinkgoT()) + }) + + It("deletes the allow-tigera tier once the apiserver is stable in calico-system", func() { + mockStatus.On("ReadyToMonitor") + mockStatus.On("ClearDegraded") + + // Migration complete: deprecated tier present, calico-apiserver Ready. + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}})).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, + Status: appsv1.DeploymentStatus{ReadyReplicas: 1}, + })).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + tier := &v3.Tier{} + err = c.Get(ctx, client.ObjectKey{Name: "allow-tigera"}, tier) + Expect(kerror.IsNotFound(err)).To(BeTrue()) + mockStatus.AssertExpectations(GinkgoT()) + }) }) diff --git a/pkg/controller/utils/apiserver_migration.go b/pkg/controller/utils/apiserver_migration.go new file mode 100644 index 0000000000..8020c7213f --- /dev/null +++ b/pkg/controller/utils/apiserver_migration.go @@ -0,0 +1,62 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/render" +) + +// APIServerMigrationInProgress reports whether the cluster is mid-way through the +// allow-tigera -> calico-system apiserver migration performed on a direct +// Calico Enterprise 3.21.x -> 3.23.x upgrade: the deprecated "allow-tigera" Tier +// still exists AND the calico-apiserver Deployment is not yet Ready in +// calico-system. Controllers use it to sequence the migration bridge and to +// suppress teardown of the transitional policies until the apiserver is stable. +// +// It reads the "allow-tigera" Tier and the calico-apiserver Deployment through the +// caller's cached client; every controller that calls it already watches those +// types. +func APIServerMigrationInProgress(ctx context.Context, c client.Client) (bool, error) { + tier := &v3.Tier{} + if err := c.Get(ctx, client.ObjectKey{Name: render.DeprecatedAPIServerTierName}, tier); err != nil { + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + // No deprecated tier: fresh install or already-migrated cluster. + return false, nil + } + return false, err + } + + // The deprecated tier exists. Migration is in progress until calico-apiserver + // is Ready in calico-system. + d := &appsv1.Deployment{} + err := c.Get(ctx, client.ObjectKey{Name: render.APIServerName, Namespace: render.APIServerNamespace}, d) + if err != nil { + if apierrors.IsNotFound(err) { + // Not created (or moved) yet. + return true, nil + } + return false, err + } + return d.Status.ReadyReplicas == 0, nil +} diff --git a/pkg/controller/utils/apiserver_migration_test.go b/pkg/controller/utils/apiserver_migration_test.go new file mode 100644 index 0000000000..3125a1ecfd --- /dev/null +++ b/pkg/controller/utils/apiserver_migration_test.go @@ -0,0 +1,87 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/apis" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" +) + +var _ = Describe("APIServerMigrationInProgress", func() { + var ( + ctx context.Context + c client.Client + scheme *runtime.Scheme + ) + + allowTigeraTier := func() *v3.Tier { + return &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}} + } + apiserverDeployment := func(ready int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, + Status: appsv1.DeploymentStatus{ReadyReplicas: ready}, + } + } + + BeforeEach(func() { + ctx = context.Background() + scheme = runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + Expect(appsv1.AddToScheme(scheme)).NotTo(HaveOccurred()) + c = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + }) + + It("returns false when the allow-tigera tier does not exist", func() { + inProgress, err := APIServerMigrationInProgress(ctx, c) + Expect(err).NotTo(HaveOccurred()) + Expect(inProgress).To(BeFalse()) + }) + + It("returns true when the allow-tigera tier exists but calico-apiserver is absent", func() { + Expect(c.Create(ctx, allowTigeraTier())).NotTo(HaveOccurred()) + inProgress, err := APIServerMigrationInProgress(ctx, c) + Expect(err).NotTo(HaveOccurred()) + Expect(inProgress).To(BeTrue()) + }) + + It("returns true when the allow-tigera tier exists and calico-apiserver has zero ready replicas", func() { + Expect(c.Create(ctx, allowTigeraTier())).NotTo(HaveOccurred()) + Expect(c.Create(ctx, apiserverDeployment(0))).NotTo(HaveOccurred()) + inProgress, err := APIServerMigrationInProgress(ctx, c) + Expect(err).NotTo(HaveOccurred()) + Expect(inProgress).To(BeTrue()) + }) + + It("returns false when the allow-tigera tier exists and calico-apiserver is ready", func() { + Expect(c.Create(ctx, allowTigeraTier())).NotTo(HaveOccurred()) + Expect(c.Create(ctx, apiserverDeployment(1))).NotTo(HaveOccurred()) + inProgress, err := APIServerMigrationInProgress(ctx, c) + Expect(err).NotTo(HaveOccurred()) + Expect(inProgress).To(BeFalse()) + }) +}) diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index 6ee5473730..4253abdb08 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -70,7 +70,11 @@ const ( QueryserverServiceName = "calico-api" // Use the same API server container name for both OSS and Enterprise. - APIServerName = "calico-apiserver" + APIServerName = "calico-apiserver" + + // DeprecatedAPIServerTierName is the pre-3.22 policy tier the migration bridge writes into. + DeprecatedAPIServerTierName = "allow-tigera" + APIServerContainerName ContainerName = "calico-apiserver" TigeraAPIServerQueryServerContainerName ContainerName = "tigera-queryserver" @@ -125,6 +129,66 @@ func APIServerPolicy(cfg *APIServerConfiguration) Component { ) } +// APIServerMigrationBridge renders the transitional policies that keep calico-apiserver +// reachable during a direct allow-tigera -> calico-system upgrade. They are written +// through the still-serving old apiserver *before* the workload cutover, reproducing the +// validated 3.22 steady state: +// - allow-tigera.default-deny rewritten to exclude calico-apiserver, +// - order-1 allow-tigera.apiserver-access in the deprecated tier, +// - calico-system.apiserver-access in the calico-system tier. +// +// See docs/superpowers/specs/2026-07-20-ev6821-apiserver-upgrade-allow-gap-design.md. +func APIServerMigrationBridge(cfg *APIServerConfiguration) Component { + return NewPassthrough( + []client.Object{ + allowTigeraDefaultDenyBridgePolicy(), + allowTigeraAPIServerBridgePolicy(cfg), + calicoSystemAPIServerPolicy(cfg), + }, + nil, + ) +} + +// allowTigeraDefaultDenyBridgePolicy rewrites the leftover 3.21-era allow-tigera.default-deny +// in calico-system so its selector excludes calico-apiserver (replacing the all() selector), +// stopping the deprecated tier's default-deny from trapping the moved apiserver pod. +func allowTigeraDefaultDenyBridgePolicy() *v3.NetworkPolicy { + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: DeprecatedAPIServerTierName + ".default-deny", + Namespace: APIServerNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Tier: DeprecatedAPIServerTierName, + Selector: "k8s-app != 'calico-apiserver'", + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + }, + } +} + +// allowTigeraAPIServerBridgePolicy is the transitional order-1 allow for calico-apiserver in +// the deprecated allow-tigera tier. It shares calicoSystemAPIServerPolicy's rules so the two +// allows cannot drift. +func allowTigeraAPIServerBridgePolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy { + ingress, egress := apiServerPolicyIngressEgress(cfg) + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: DeprecatedAPIServerTierName + ".apiserver-access", + Namespace: APIServerNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: DeprecatedAPIServerTierName, + Selector: networkpolicy.KubernetesAppSelector(APIServerName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingress, + Egress: egress, + }, + } +} + // APIServerConfiguration contains all the config information needed to render the component. type APIServerConfiguration struct { K8SServiceEndpoint k8sapi.ServiceEndpoint @@ -549,10 +613,14 @@ func (c *apiServerComponent) linseedAccessClusterRoleBinding() *rbacv1.ClusterRo return rcomp.ClusterRoleBinding(APIServerLinseedAccessClusterRoleName, APIServerLinseedAccessClusterRoleName, APIServerServiceAccountName, c.cfg.BindingNamespaces) } -func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy { - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) - egressRules = append(egressRules, []v3.Rule{ +// apiServerPolicyIngressEgress builds the ingress and egress rules shared by the +// steady-state calico-system.apiserver-access policy and the transitional +// allow-tigera.apiserver-access policy written by the migration bridge. Keeping a +// single builder prevents the two allows from drifting. +func apiServerPolicyIngressEgress(cfg *APIServerConfiguration) (ingress, egress []v3.Rule) { + egress = []v3.Rule{} + egress = networkpolicy.AppendDNSEgressRules(egress, cfg.OpenShift) + egress = append(egress, []v3.Rule{ { Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, @@ -578,35 +646,54 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy if cfg.KeyValidatorConfig != nil { if parsedURL, err := url.Parse(cfg.KeyValidatorConfig.Issuer()); err == nil { - oidcEgressRule := networkpolicy.GetOIDCEgressRule(parsedURL) - egressRules = append(egressRules, oidcEgressRule) + egress = append(egress, networkpolicy.GetOIDCEgressRule(parsedURL)) } } if r, err := cfg.K8SServiceEndpoint.DestinationEntityRule(); r != nil && err == nil { - egressRules = append(egressRules, v3.Rule{ + egress = append(egress, v3.Rule{ Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Destination: *r, }) } - // add pass after all egress rules - egressRules = append(egressRules, v3.Rule{ - // Pass to subsequent tiers for further enforcement - Action: v3.Pass, - }) + // Pass to subsequent tiers for further enforcement. + egress = append(egress, v3.Rule{Action: v3.Pass}) apiServerContainerPort := getContainerPort(cfg, APIServerContainerName).ContainerPort queryServerContainerPort := getContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort l7AdmCtrlContainerPort := getContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort - // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. + // The ports the API Server and Query Server listen on. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) if cfg.IsSidecarInjectionEnabled() { ingressPorts = append(ingressPorts, numorstring.Port{MinPort: uint16(l7AdmCtrlContainerPort), MaxPort: uint16(l7AdmCtrlContainerPort)}) } + ingress = []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{Nets: []string{"0.0.0.0/0"}}, + Destination: v3.EntityRule{ + Ports: ingressPorts, + }, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{Nets: []string{"::/0"}}, + Destination: v3.EntityRule{ + Ports: ingressPorts, + }, + }, + } + return ingress, egress +} + +func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy { + ingress, egress := apiServerPolicyIngressEgress(cfg) return &v3.NetworkPolicy{ TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -618,30 +705,8 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy Tier: networkpolicy.CalicoTierName, Selector: networkpolicy.KubernetesAppSelector(APIServerName), Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - // This policy allows Calico Enterprise API Server access from anywhere. - Source: v3.EntityRule{ - Nets: []string{"0.0.0.0/0"}, - }, - Destination: v3.EntityRule{ - Ports: ingressPorts, - }, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: v3.EntityRule{ - Nets: []string{"::/0"}, - }, - Destination: v3.EntityRule{ - Ports: ingressPorts, - }, - }, - }, - Egress: egressRules, + Ingress: ingress, + Egress: egress, }, } } diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index fb68455876..aa20ed08d6 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -852,6 +852,42 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { })) }) + It("renders the migration bridge as three policies", func() { + component := render.APIServerMigrationBridge(cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + toCreate, toDelete := component.Objects() + Expect(toDelete).To(BeEmpty()) + Expect(toCreate).To(HaveLen(3)) + + byName := map[string]*calicov3.NetworkPolicy{} + for _, o := range toCreate { + np, ok := o.(*calicov3.NetworkPolicy) + Expect(ok).To(BeTrue()) + byName[np.Name] = np + } + + defaultDeny := byName["allow-tigera.default-deny"] + Expect(defaultDeny).ToNot(BeNil()) + Expect(defaultDeny.Namespace).To(Equal("calico-system")) + Expect(defaultDeny.Spec.Tier).To(Equal("allow-tigera")) + Expect(defaultDeny.Spec.Selector).To(Equal("k8s-app != 'calico-apiserver'")) + + allowTigera := byName["allow-tigera.apiserver-access"] + Expect(allowTigera).ToNot(BeNil()) + Expect(allowTigera.Namespace).To(Equal("calico-system")) + Expect(allowTigera.Spec.Tier).To(Equal("allow-tigera")) + Expect(allowTigera.Spec.Selector).To(Equal("k8s-app == 'calico-apiserver'")) + Expect(*allowTigera.Spec.Order).To(Equal(1.0)) + + calicoSystem := byName["calico-system.apiserver-access"] + Expect(calicoSystem).ToNot(BeNil()) + Expect(calicoSystem.Spec.Tier).To(Equal("calico-system")) + + // The transitional allow shares the steady-state allow's rules (shared builder). + Expect(allowTigera.Spec.Ingress).To(Equal(calicoSystem.Spec.Ingress)) + Expect(allowTigera.Spec.Egress).To(Equal(calicoSystem.Spec.Egress)) + }) + It("should not set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with proxy.local", func() { cfg.K8SServiceEndpoint.Host = "proxy.local" cfg.K8SServiceEndpoint.Port = "1234" diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index b08d9ec692..9a19e73852 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -105,21 +105,25 @@ type KubeControllersConfiguration struct { Tenant *operatorv1.Tenant } -func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { +func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy, migrationInProgress bool) render.Component { toCreate := []client.Object{kubeControllersCalicoSystemPolicy(cfg)} if defaultDeny != nil { toCreate = append(toCreate, defaultDeny) } - return render.NewPassthrough( - toCreate, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), - }, - ) + toDelete := []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), + } + if !migrationInProgress { + // While the apiserver migration bridge owns allow-tigera.default-deny (it rewrites the + // selector to exclude calico-apiserver), suppress this deletion to avoid create<->delete + // oscillation. Cleanup resumes once the apiserver is stable in calico-system. + toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace)) + } + + return render.NewPassthrough(toCreate, toDelete) } func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 8206245ed8..e65a6fec4f 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -1055,7 +1055,7 @@ var _ = Describe("kube-controllers rendering tests", func() { }, } - component := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, defaultDenyPolicy) + component := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, defaultDenyPolicy, false) resources, _ := component.Objects() Expect(resources).To(HaveLen(2)) Expect(resources).Should(ContainElement(defaultDenyPolicy)) @@ -1081,19 +1081,38 @@ var _ = Describe("kube-controllers rendering tests", func() { It("policy should omit prometheus ingress rule when metrics port is 0", func() { // Baseline cfg.MetricsPort = 9094 - component := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil) + component := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil, false) resources, _ := component.Objects() Expect(resources).To(HaveLen(1)) baselinePolicy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) // Zeroed policy cfg.MetricsPort = 0 - component = kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil) + component = kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil, false) resources, _ = component.Objects() zeroedPolicy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(len(zeroedPolicy.Spec.Ingress)).To(Equal(len(baselinePolicy.Spec.Ingress) - 1)) }) + + It("omits the deprecated allow-tigera.default-deny deletion while migrating", func() { + names := func(objs []client.Object) []string { + var out []string + for _, o := range objs { + out = append(out, o.GetName()) + } + return out + } + + _, notMigrating := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil, false).Objects() + Expect(names(notMigrating)).To(ContainElement("allow-tigera.default-deny")) + Expect(names(notMigrating)).To(ContainElement("allow-tigera.kube-controller-access")) + + _, migrating := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil, true).Objects() + Expect(names(migrating)).ToNot(ContainElement("allow-tigera.default-deny")) + // The kube-controller-access deletion is unconditional either way. + Expect(names(migrating)).To(ContainElement("allow-tigera.kube-controller-access")) + }) }) Context("es-kube-controllers calico-system rendering", func() { @@ -1163,7 +1182,7 @@ var _ = Describe("kube-controllers rendering tests", func() { It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" - objects, _ := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil).Objects() + objects, _ := kubecontrollers.NewCalicoKubeControllersPolicy(&cfg, nil, false).Objects() Expect(objects).To(HaveLen(1)) policy, ok := objects[0].(*v3.NetworkPolicy) Expect(ok).To(BeTrue())