Add opt-in network-ready taint support

api/v1/installation_types.go

@@ -253,6 +253,26 @@ type InstallationSpec struct {
 	// it installs to protect the Calico components it manages.
 	// +optional
 	NetworkPolicy *NetworkPolicySpec `json:"networkPolicy,omitempty"`
+
+	// NetworkReadyTaint, when Enabled, taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule
+	// until Calico networking is ready on the node, so that workloads don't schedule onto a node before Calico
+	// can service them. Calico removes the taint once networking is ready. Defaults to Disabled.
+	// +kubebuilder:validation:Enum=Enabled;Disabled
+	// +optional
+	NetworkReadyTaint *NetworkReadyTaintType `json:"networkReadyTaint,omitempty"`
+}
+
+// NetworkReadyTaintType specifies whether Calico taints nodes until networking is ready.
+type NetworkReadyTaintType string
+
+const (
+	NetworkReadyTaintEnabled  NetworkReadyTaintType = "Enabled"
+	NetworkReadyTaintDisabled NetworkReadyTaintType = "Disabled"
+)
+
+// NetworkReadyTaintIsEnabled reports whether the network-ready taint feature is enabled.
+func (s InstallationSpec) NetworkReadyTaintIsEnabled() bool {
+	return s.NetworkReadyTaint != nil && *s.NetworkReadyTaint == NetworkReadyTaintEnabled
 }
 
 // BPFNetworkBootstrapType defines how the initial networking configuration is executed.

pkg/controller/installation/core_controller.go

@@ -2401,6 +2401,19 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont
 	}
 
 	desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion)
+	if !install.Spec.NetworkReadyTaintIsEnabled() {
+		// The node-taint policy ships in the embedded set but must only be installed when the feature
+		// is enabled - otherwise nodes get tainted with nothing to remove the taint. Dropping it from
+		// the desired set here also uninstalls it if the feature is later turned off.
+		filtered := desired[:0]
+		for _, obj := range desired {
+			if name := obj.GetName(); name == admission.NetworkReadyTaintPolicyName || name == admission.NetworkReadyTaintBindingName {
+				continue
+			}
+			filtered = append(filtered, obj)
+		}
+		desired = filtered
+	}
 	existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion)
 	if err != nil {
 		r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log)

pkg/imports/admission/admission.go

@@ -66,6 +66,12 @@ const (
 	KindValidatingPolicy = "ValidatingAdmissionPolicy"
 	// KindValidatingBinding is the ValidatingAdmissionPolicyBinding kind.
 	KindValidatingBinding = "ValidatingAdmissionPolicyBinding"
+
+	// NetworkReadyTaintPolicyName and NetworkReadyTaintBindingName identify the node-taint policy,
+	// which ships in the embedded set but is only installed when the network-ready taint feature is
+	// enabled in the Installation (see updateMutatingAdmissionPolicies).
+	NetworkReadyTaintPolicyName  = "networkreadytaint.projectcalico.org"
+	NetworkReadyTaintBindingName = "networkreadytaint-binding"
 )
 
 // PolicyGroupKind is the GroupKind for MutatingAdmissionPolicy. Exposed so the API discovery

pkg/imports/admission/admission_test.go

@@ -29,7 +29,7 @@ var _ = Describe("MutatingAdmissionPolicies", func() {
 	Describe("GetMutatingAdmissionPolicies", func() {
 		It("returns Calico v1beta1 MAPs when v3=true", func() {
 			objs := GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1Beta1)
-			Expect(objs).To(HaveLen(4))
+			Expect(objs).To(HaveLen(6))
 
 			var mapCount, mapbCount int
 			for _, obj := range objs {
@@ -41,13 +41,13 @@ var _ = Describe("MutatingAdmissionPolicies", func() {
 				}
 				Expect(obj.GetLabels()).To(HaveKeyWithValue(ManagedMAPLabel, ManagedMAPLabelValue))
 			}
-			Expect(mapCount).To(Equal(2))
-			Expect(mapbCount).To(Equal(2))
+			Expect(mapCount).To(Equal(3))
+			Expect(mapbCount).To(Equal(3))
 		})
 
 		It("returns Calico v1 MAPs when discovered version is v1", func() {
 			objs := GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1)
-			Expect(objs).To(HaveLen(4))
+			Expect(objs).To(HaveLen(6))
 
 			var mapCount, mapbCount int
 			for _, obj := range objs {
@@ -61,13 +61,13 @@ var _ = Describe("MutatingAdmissionPolicies", func() {
 				}
 				Expect(obj.GetLabels()).To(HaveKeyWithValue(ManagedMAPLabel, ManagedMAPLabelValue))
 			}
-			Expect(mapCount).To(Equal(2))
-			Expect(mapbCount).To(Equal(2))
+			Expect(mapCount).To(Equal(3))
+			Expect(mapbCount).To(Equal(3))
 		})
 
 		It("returns Calico v1alpha1 MAPs when discovered version is v1alpha1", func() {
 			objs := GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1Alpha1)
-			Expect(objs).To(HaveLen(4))
+			Expect(objs).To(HaveLen(6))
 
 			var mapCount, mapbCount int
 			for _, obj := range objs {
@@ -81,13 +81,13 @@ var _ = Describe("MutatingAdmissionPolicies", func() {
 				}
 				Expect(obj.GetLabels()).To(HaveKeyWithValue(ManagedMAPLabel, ManagedMAPLabelValue))
 			}
-			Expect(mapCount).To(Equal(2))
-			Expect(mapbCount).To(Equal(2))
+			Expect(mapCount).To(Equal(3))
+			Expect(mapbCount).To(Equal(3))
 		})
 
 		It("returns Enterprise MAPs at the chosen version", func() {
 			objs := GetMutatingAdmissionPolicies(opv1.CalicoEnterprise, true, VersionV1)
-			Expect(objs).To(HaveLen(4))
+			Expect(objs).To(HaveLen(6))
 
 			var mapCount, mapbCount int
 			for _, obj := range objs {
@@ -98,8 +98,8 @@ var _ = Describe("MutatingAdmissionPolicies", func() {
 					mapbCount++
 				}
 			}
-			Expect(mapCount).To(Equal(2))
-			Expect(mapbCount).To(Equal(2))
+			Expect(mapCount).To(Equal(3))
+			Expect(mapbCount).To(Equal(3))
 		})
 
 		It("returns empty when v3=false", func() {
@@ -117,6 +117,14 @@ var _ = Describe("MutatingAdmissionPolicies", func() {
 				Expect(obj.GetName()).ToNot(BeEmpty())
 			}
 		})
+
+		It("includes the node-taint policy (gating is applied by the controller, not here)", func() {
+			var names []string
+			for _, obj := range GetMutatingAdmissionPolicies(opv1.Calico, true, VersionV1) {
+				names = append(names, obj.GetName())
+			}
+			Expect(names).To(ContainElements(NetworkReadyTaintPolicyName, NetworkReadyTaintBindingName))
+		})
 	})
 
 	Describe("GetValidatingAdmissionPolicies", func() {

pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicy.yaml

@@ -0,0 +1,33 @@
+# This MutatingAdmissionPolicy taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule
+# so that workloads don't schedule onto a node before Calico networking is ready. calico-node removes
+# the taint once Felix and BIRD are ready.
+#
+# This policy is only installed when the network-ready taint feature is enabled in the Installation
+# resource. The operator gates installation on that flag and wires the matching tolerations, so this
+# must never be applied on its own - without something to remove the taint, nodes would stay tainted.
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingAdmissionPolicy
+metadata:
+  name: "networkreadytaint.projectcalico.org"
+spec:
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["nodes"]
+  matchConditions:
+    # Don't add the taint if it's already there (e.g. the node was registered with it).
+    - name: taint-absent
+      expression: "!has(object.spec.taints) || !object.spec.taints.exists(t, t.key == 'node.projectcalico.org/network-not-ready')"
+  # Never block node registration if the policy fails to evaluate - a node that can't register is
+  # worse than one that schedules a little early.
+  failurePolicy: Ignore
+  reinvocationPolicy: Never
+  mutations:
+    - patchType: "JSONPatch"
+      jsonPatch:
+        expression: |
+          has(object.spec.taints) ?
+            [JSONPatch{op: "add", path: "/spec/taints/-", value: {"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}}] :
+            [JSONPatch{op: "add", path: "/spec/taints", value: [{"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}]}]

pkg/imports/admission/calico/nodetaint.mutatingadmissionpolicybinding.yaml

@@ -0,0 +1,13 @@
+# This MutatingAdmissionPolicyBinding binds the network-ready taint mutation to node creates.
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingAdmissionPolicyBinding
+metadata:
+  name: networkreadytaint-binding
+spec:
+  policyName: networkreadytaint.projectcalico.org
+  matchResources:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["nodes"]

pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicy.yaml

@@ -0,0 +1,33 @@
+# This MutatingAdmissionPolicy taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule
+# so that workloads don't schedule onto a node before Calico networking is ready. calico-node removes
+# the taint once Felix and BIRD are ready.
+#
+# This policy is only installed when the network-ready taint feature is enabled in the Installation
+# resource. The operator gates installation on that flag and wires the matching tolerations, so this
+# must never be applied on its own - without something to remove the taint, nodes would stay tainted.
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingAdmissionPolicy
+metadata:
+  name: "networkreadytaint.projectcalico.org"
+spec:
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["nodes"]
+  matchConditions:
+    # Don't add the taint if it's already there (e.g. the node was registered with it).
+    - name: taint-absent
+      expression: "!has(object.spec.taints) || !object.spec.taints.exists(t, t.key == 'node.projectcalico.org/network-not-ready')"
+  # Never block node registration if the policy fails to evaluate - a node that can't register is
+  # worse than one that schedules a little early.
+  failurePolicy: Ignore
+  reinvocationPolicy: Never
+  mutations:
+    - patchType: "JSONPatch"
+      jsonPatch:
+        expression: |
+          has(object.spec.taints) ?
+            [JSONPatch{op: "add", path: "/spec/taints/-", value: {"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}}] :
+            [JSONPatch{op: "add", path: "/spec/taints", value: [{"key": "node.projectcalico.org/network-not-ready", "effect": "NoSchedule"}]}]

pkg/imports/admission/enterprise/nodetaint.mutatingadmissionpolicybinding.yaml

@@ -0,0 +1,13 @@
+# This MutatingAdmissionPolicyBinding binds the network-ready taint mutation to node creates.
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingAdmissionPolicyBinding
+metadata:
+  name: networkreadytaint-binding
+spec:
+  policyName: networkreadytaint.projectcalico.org
+  matchResources:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["nodes"]

pkg/imports/crds/operator/operator.tigera.io_installations.yaml

@@ -7303,6 +7303,15 @@ spec:
                         - Disabled
                       type: string
                   type: object
+                networkReadyTaint:
+                  description: |-
+                    NetworkReadyTaint, when Enabled, taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule
+                    until Calico networking is ready on the node, so that workloads don't schedule onto a node before Calico
+                    can service them. Calico removes the taint once networking is ready. Defaults to Disabled.
+                  enum:
+                    - Enabled
+                    - Disabled
+                  type: string
                 nodeMetricsPort:
                   description: |-
                     NodeMetricsPort specifies which port calico/node serves prometheus metrics on. By default, metrics are not enabled.
@@ -16752,6 +16761,15 @@ spec:
                             - Disabled
                           type: string
                       type: object
+                    networkReadyTaint:
+                      description: |-
+                        NetworkReadyTaint, when Enabled, taints new nodes with node.projectcalico.org/network-not-ready:NoSchedule
+                        until Calico networking is ready on the node, so that workloads don't schedule onto a node before Calico
+                        can service them. Calico removes the taint once networking is ready. Defaults to Disabled.
+                      enum:
+                        - Enabled
+                        - Disabled
+                      type: string
                     nodeMetricsPort:
                       description: |-
                         NodeMetricsPort specifies which port calico/node serves prometheus metrics on. By default, metrics are not enabled.

pkg/render/common/meta/meta.go

@@ -38,6 +38,11 @@ const (
 	TigeraOperatorCAIssuerPrefix = "tigera-operator-signer"
 )
 
+// NetworkReadyTaintKey is the taint Calico applies to nodes until networking is ready. It mirrors
+// the constant of the same name in libcalico-go, kept here to avoid a dependency bump before the
+// calico-side change merges.
+const NetworkReadyTaintKey = "node.projectcalico.org/network-not-ready"
+
 var (
 	// TolerateControlPlane allows pod to be scheduled on master nodes
 	TolerateControlPlane = []corev1.Toleration{
@@ -117,6 +122,15 @@ var (
 			Operator: corev1.TolerationOpExists,
 		},
 	}
+
+	// TolerateNetworkReadyTaint lets a component schedule onto a node that Calico has tainted as
+	// not-ready. Only host-networked components that don't need pod networking (e.g. Typha) should
+	// use it, so that calico-node can come up and clear the taint.
+	TolerateNetworkReadyTaint = corev1.Toleration{
+		Key:      NetworkReadyTaintKey,
+		Operator: corev1.TolerationOpExists,
+		Effect:   corev1.TaintEffectNoSchedule,
+	}
 )
 
 func DefaultOperatorCASignerName() string {

pkg/render/kubecontrollers/kube-controllers.go

@@ -971,6 +971,10 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment {
 
 	env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...)
 
+	if c.cfg.Installation.NetworkReadyTaintIsEnabled() {
+		env = append(env, corev1.EnvVar{Name: "CALICO_MANAGE_NETWORK_READY_TAINT", Value: "true"})
+	}
+
 	if c.cfg.Installation.Variant.IsEnterprise() {
 		if c.cfg.Tenant != nil {
 			env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID})

pkg/render/node.go

@@ -1573,6 +1573,10 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar {
 		nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "CALICO_MANAGE_CNI", Value: "false"})
 	}
 
+	if c.cfg.Installation.NetworkReadyTaintIsEnabled() {
+		nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "CALICO_MANAGE_NETWORK_READY_TAINT", Value: "true"})
+	}
+
 	if c.cfg.Installation.CNI != nil && c.cfg.Installation.CNI.Type == operatorv1.PluginAmazonVPC {
 		nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"})
 	}

pkg/render/typha.go

@@ -433,6 +433,11 @@ func (c *typhaComponent) typhaDeployment() []client.Object {
 	if c.cfg.Installation.KubernetesProvider.IsGKE() {
 		tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule)
 	}
+	if c.cfg.Installation.NetworkReadyTaintIsEnabled() {
+		// Typha is host-networked, so it can run on a node whose networking isn't ready yet. It must
+		// tolerate the taint so calico-node (which depends on Typha) can come up and clear it.
+		tolerations = append(tolerations, rmeta.TolerateNetworkReadyTaint)
+	}
 
 	deploy := &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"},