diff --git a/api/v1/installation_types.go b/api/v1/installation_types.go index 4708fdb735..9c11e80320 100644 --- a/api/v1/installation_types.go +++ b/api/v1/installation_types.go @@ -253,6 +253,26 @@ type InstallationSpec struct { // it installs to protect the Calico components it manages. // +optional NetworkPolicy *NetworkPolicySpec `json:"networkPolicy,omitempty"` + + // NetworkReadyTaint, when Enabled, taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule + // until Calico networking is ready on the node, so that workloads don't schedule onto a node before Calico + // can service them. Calico removes the taint once networking is ready. Defaults to Disabled. + // +kubebuilder:validation:Enum=Enabled;Disabled + // +optional + NetworkReadyTaint *NetworkReadyTaintType `json:"networkReadyTaint,omitempty"` +} + +// NetworkReadyTaintType specifies whether Calico taints nodes until networking is ready. +type NetworkReadyTaintType string + +const ( + NetworkReadyTaintEnabled NetworkReadyTaintType = "Enabled" + NetworkReadyTaintDisabled NetworkReadyTaintType = "Disabled" +) + +// NetworkReadyTaintIsEnabled reports whether the network-ready taint feature is enabled. +func (s InstallationSpec) NetworkReadyTaintIsEnabled() bool { + return s.NetworkReadyTaint != nil && *s.NetworkReadyTaint == NetworkReadyTaintEnabled } // BPFNetworkBootstrapType defines how the initial networking configuration is executed. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index dadaa9d46a..587ff9c628 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -6113,6 +6113,11 @@ func (in *InstallationSpec) DeepCopyInto(out *InstallationSpec) { *out = new(NetworkPolicySpec) (*in).DeepCopyInto(*out) } + if in.NetworkReadyTaint != nil { + in, out := &in.NetworkReadyTaint, &out.NetworkReadyTaint + *out = new(NetworkReadyTaintType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstallationSpec. diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a766f061f4..dfd7d2d6a6 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -2401,6 +2401,19 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont } desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion) + if !install.Spec.NetworkReadyTaintIsEnabled() { + // The node-taint policy ships in the embedded set but must only be installed when the feature + // is enabled - otherwise nodes get tainted with nothing to remove the taint. Dropping it from + // the desired set here also uninstalls it if the feature is later turned off. + filtered := desired[:0] + for _, obj := range desired { + if name := obj.GetName(); name == admission.NetworkReadyTaintPolicyName || name == admission.NetworkReadyTaintBindingName { + continue + } + filtered = append(filtered, obj) + } + desired = filtered + } existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log) diff --git a/pkg/imports/admission/admission.go b/pkg/imports/admission/admission.go index 9f7c9474b1..0dd45d304d 100644 --- a/pkg/imports/admission/admission.go +++ b/pkg/imports/admission/admission.go @@ -66,6 +66,12 @@ const ( KindValidatingPolicy = "ValidatingAdmissionPolicy" // KindValidatingBinding is the ValidatingAdmissionPolicyBinding kind. KindValidatingBinding = "ValidatingAdmissionPolicyBinding" + + // NetworkReadyTaintPolicyName and NetworkReadyTaintBindingName identify the node-taint policy, + // which ships in the embedded set but is only installed when the network-ready taint feature is + // enabled in the Installation (see updateMutatingAdmissionPolicies). + NetworkReadyTaintPolicyName = "networkreadytaint.projectcalico.org" + NetworkReadyTaintBindingName = "networkreadytaint-binding" ) // PolicyGroupKind is the GroupKind for MutatingAdmissionPolicy. Exposed so the API discovery diff --git a/pkg/imports/admission/admission_test.go b/pkg/imports/admission/admission_test.go index ed28fd6901..08fbce8e5f 100644 --- a/pkg/imports/admission/admission_test.go +++ b/pkg/imports/admission/admission_test.go @@ -29,7 +29,7 @@ var _ = Describe("MutatingAdmissionPolicies", func() { Describe("GetMutatingAdmissionPolicies", func() { It("returns Calico v1beta1 MAPs when v3=true", func() { objs := GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1Beta1) - Expect(objs).To(HaveLen(4)) + Expect(objs).To(HaveLen(6)) var mapCount, mapbCount int for _, obj := range objs { @@ -41,13 +41,13 @@ var _ = Describe("MutatingAdmissionPolicies", func() { } Expect(obj.GetLabels()).To(HaveKeyWithValue(ManagedMAPLabel, ManagedMAPLabelValue)) } - Expect(mapCount).To(Equal(2)) - Expect(mapbCount).To(Equal(2)) + Expect(mapCount).To(Equal(3)) + Expect(mapbCount).To(Equal(3)) }) It("returns Calico v1 MAPs when discovered version is v1", func() { objs := GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1) - Expect(objs).To(HaveLen(4)) + Expect(objs).To(HaveLen(6)) var mapCount, mapbCount int for _, obj := range objs { @@ -61,13 +61,13 @@ var _ = Describe("MutatingAdmissionPolicies", func() { } Expect(obj.GetLabels()).To(HaveKeyWithValue(ManagedMAPLabel, ManagedMAPLabelValue)) } - Expect(mapCount).To(Equal(2)) - Expect(mapbCount).To(Equal(2)) + Expect(mapCount).To(Equal(3)) + Expect(mapbCount).To(Equal(3)) }) It("returns Calico v1alpha1 MAPs when discovered version is v1alpha1", func() { objs := GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1Alpha1) - Expect(objs).To(HaveLen(4)) + Expect(objs).To(HaveLen(6)) var mapCount, mapbCount int for _, obj := range objs { @@ -81,13 +81,13 @@ var _ = Describe("MutatingAdmissionPolicies", func() { } Expect(obj.GetLabels()).To(HaveKeyWithValue(ManagedMAPLabel, ManagedMAPLabelValue)) } - Expect(mapCount).To(Equal(2)) - Expect(mapbCount).To(Equal(2)) + Expect(mapCount).To(Equal(3)) + Expect(mapbCount).To(Equal(3)) }) It("returns Enterprise MAPs at the chosen version", func() { objs := GetMutatingAdmissionPolicies(opv1.CalicoEnterprise, true, VersionV1) - Expect(objs).To(HaveLen(4)) + Expect(objs).To(HaveLen(6)) var mapCount, mapbCount int for _, obj := range objs { @@ -98,8 +98,8 @@ var _ = Describe("MutatingAdmissionPolicies", func() { mapbCount++ } } - Expect(mapCount).To(Equal(2)) - Expect(mapbCount).To(Equal(2)) + Expect(mapCount).To(Equal(3)) + Expect(mapbCount).To(Equal(3)) }) It("returns empty when v3=false", func() { @@ -117,6 +117,14 @@ var _ = Describe("MutatingAdmissionPolicies", func() { Expect(obj.GetName()).ToNot(BeEmpty()) } }) + + It("includes the node-taint policy (gating is applied by the controller, not here)", func() { + var names []string + for _, obj := range GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1) { + names = append(names, obj.GetName()) + } + Expect(names).To(ContainElements(NetworkReadyTaintPolicyName, NetworkReadyTaintBindingName)) + }) }) Describe("GetValidatingAdmissionPolicies", func() { diff --git a/pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicy.yaml b/pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicy.yaml new file mode 100644 index 0000000000..4c700bc0a2 --- /dev/null +++ b/pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicy.yaml @@ -0,0 +1,33 @@ +# This MutatingAdmissionPolicy taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule +# so that workloads don't schedule onto a node before Calico networking is ready. calico-node removes +# the taint once Felix and BIRD are ready. +# +# This policy is only installed when the network-ready taint feature is enabled in the Installation +# resource. The operator gates installation on that flag and wires the matching tolerations, so this +# must never be applied on its own - without something to remove the taint, nodes would stay tainted. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingAdmissionPolicy +metadata: + name: "networkreadytaint.projectcalico.org" +spec: + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["nodes"] + matchConditions: + # Don't add the taint if it's already there (e.g. the node was registered with it). + - name: taint-absent + expression: "!has(object.spec.taints) || !object.spec.taints.exists(t, t.key == 'node.projectcalico.org/network-not-ready')" + # Never block node registration if the policy fails to evaluate - a node that can't register is + # worse than one that schedules a little early. + failurePolicy: Ignore + reinvocationPolicy: Never + mutations: + - patchType: "JSONPatch" + jsonPatch: + expression: | + has(object.spec.taints) ? + [JSONPatch{op: "add", path: "/spec/taints/-", value: {"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}}] : + [JSONPatch{op: "add", path: "/spec/taints", value: [{"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}]}] diff --git a/pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicybinding.yaml b/pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicybinding.yaml new file mode 100644 index 0000000000..5deaf502b3 --- /dev/null +++ b/pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicybinding.yaml @@ -0,0 +1,13 @@ +# This MutatingAdmissionPolicyBinding binds the network-ready taint mutation to node creates. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: networkreadytaint-binding +spec: + policyName: networkreadytaint.projectcalico.org + matchResources: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["nodes"] diff --git a/pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicy.yaml b/pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicy.yaml new file mode 100644 index 0000000000..4c700bc0a2 --- /dev/null +++ b/pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicy.yaml @@ -0,0 +1,33 @@ +# This MutatingAdmissionPolicy taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule +# so that workloads don't schedule onto a node before Calico networking is ready. calico-node removes +# the taint once Felix and BIRD are ready. +# +# This policy is only installed when the network-ready taint feature is enabled in the Installation +# resource. The operator gates installation on that flag and wires the matching tolerations, so this +# must never be applied on its own - without something to remove the taint, nodes would stay tainted. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingAdmissionPolicy +metadata: + name: "networkreadytaint.projectcalico.org" +spec: + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["nodes"] + matchConditions: + # Don't add the taint if it's already there (e.g. the node was registered with it). + - name: taint-absent + expression: "!has(object.spec.taints) || !object.spec.taints.exists(t, t.key == 'node.projectcalico.org/network-not-ready')" + # Never block node registration if the policy fails to evaluate - a node that can't register is + # worse than one that schedules a little early. + failurePolicy: Ignore + reinvocationPolicy: Never + mutations: + - patchType: "JSONPatch" + jsonPatch: + expression: | + has(object.spec.taints) ? + [JSONPatch{op: "add", path: "/spec/taints/-", value: {"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}}] : + [JSONPatch{op: "add", path: "/spec/taints", value: [{"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}]}] diff --git a/pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicybinding.yaml b/pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicybinding.yaml new file mode 100644 index 0000000000..5deaf502b3 --- /dev/null +++ b/pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicybinding.yaml @@ -0,0 +1,13 @@ +# This MutatingAdmissionPolicyBinding binds the network-ready taint mutation to node creates. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: networkreadytaint-binding +spec: + policyName: networkreadytaint.projectcalico.org + matchResources: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["nodes"] diff --git a/pkg/imports/crds/operator/operator.tigera.io_installations.yaml b/pkg/imports/crds/operator/operator.tigera.io_installations.yaml index 56156e9739..e3a132a901 100644 --- a/pkg/imports/crds/operator/operator.tigera.io_installations.yaml +++ b/pkg/imports/crds/operator/operator.tigera.io_installations.yaml @@ -7303,6 +7303,15 @@ spec: - Disabled type: string type: object + networkReadyTaint: + description: |- + NetworkReadyTaint, when Enabled, taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule + until Calico networking is ready on the node, so that workloads don't schedule onto a node before Calico + can service them. Calico removes the taint once networking is ready. Defaults to Disabled. + enum: + - Enabled + - Disabled + type: string nodeMetricsPort: description: |- NodeMetricsPort specifies which port calico/node serves prometheus metrics on. By default, metrics are not enabled. @@ -16752,6 +16761,15 @@ spec: - Disabled type: string type: object + networkReadyTaint: + description: |- + NetworkReadyTaint, when Enabled, taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule + until Calico networking is ready on the node, so that workloads don't schedule onto a node before Calico + can service them. Calico removes the taint once networking is ready. Defaults to Disabled. + enum: + - Enabled + - Disabled + type: string nodeMetricsPort: description: |- NodeMetricsPort specifies which port calico/node serves prometheus metrics on. By default, metrics are not enabled. diff --git a/pkg/render/common/meta/meta.go b/pkg/render/common/meta/meta.go index 99af4f5690..ba547a335d 100644 --- a/pkg/render/common/meta/meta.go +++ b/pkg/render/common/meta/meta.go @@ -38,6 +38,11 @@ const ( TigeraOperatorCAIssuerPrefix = "tigera-operator-signer" ) +// NetworkReadyTaintKey is the taint Calico applies to nodes until networking is ready. It mirrors +// the constant of the same name in libcalico-go, kept here to avoid a dependency bump before the +// calico-side change merges. +const NetworkReadyTaintKey = "node.projectcalico.org/network-not-ready" + var ( // TolerateControlPlane allows pod to be scheduled on master nodes TolerateControlPlane = []corev1.Toleration{ @@ -117,6 +122,15 @@ var ( Operator: corev1.TolerationOpExists, }, } + + // TolerateNetworkReadyTaint lets a component schedule onto a node that Calico has tainted as + // not-ready. Only host-networked components that don't need pod networking (e.g. Typha) should + // use it, so that calico-node can come up and clear the taint. + TolerateNetworkReadyTaint = corev1.Toleration{ + Key: NetworkReadyTaintKey, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + } ) func DefaultOperatorCASignerName() string { diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 8e082f4656..c2a8eabd5e 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -971,6 +971,10 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) + if c.cfg.Installation.NetworkReadyTaintIsEnabled() { + env = append(env, corev1.EnvVar{Name: "CALICO_MANAGE_NETWORK_READY_TAINT", Value: "true"}) + } + if c.cfg.Installation.Variant.IsEnterprise() { if c.cfg.Tenant != nil { env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) diff --git a/pkg/render/node.go b/pkg/render/node.go index e4c9857e7d..de2383eb6f 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -1573,6 +1573,10 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "CALICO_MANAGE_CNI", Value: "false"}) } + if c.cfg.Installation.NetworkReadyTaintIsEnabled() { + nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "CALICO_MANAGE_NETWORK_READY_TAINT", Value: "true"}) + } + if c.cfg.Installation.CNI != nil && c.cfg.Installation.CNI.Type == operatorv1.PluginAmazonVPC { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}) } diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 264851785f..072b11e1b3 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -433,6 +433,11 @@ func (c *typhaComponent) typhaDeployment() []client.Object { if c.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) } + if c.cfg.Installation.NetworkReadyTaintIsEnabled() { + // Typha is host-networked, so it can run on a node whose networking isn't ready yet. It must + // tolerate the taint so calico-node (which depends on Typha) can come up and clear it. + tolerations = append(tolerations, rmeta.TolerateNetworkReadyTaint) + } deploy := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"},