OCPBUGS-100292: Fix MCO degradation issue when Cluster API Machine resource is missing

cmd/machine-config-controller/start.go

@@ -202,15 +202,14 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 		go pinnedImageSet.Run(ctx, 2)
 
 		// Start the shared factory informers that you need to use in your controller
-		ctrlctx.InformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.KubeInformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.OpenShiftConfigKubeNamespacedInformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.OperatorInformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.KubeNamespacedInformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.KubeMAOSharedInformer.Start(ctrlctx.Stop)
-		ctrlctx.OCLInformerFactory.Start(ctrlctx.Stop)
-		ctrlctx.MCOPodInformerFactory.Start(ctrlctx.Stop)
+		ctrlctx.InformerFactory.Start(ctx.Done())
+		ctrlctx.KubeInformerFactory.Start(ctx.Done())
+		ctrlctx.OpenShiftConfigKubeNamespacedInformerFactory.Start(ctx.Done())
+		ctrlctx.OperatorInformerFactory.Start(ctx.Done())
+		ctrlctx.ConfigInformerFactory.Start(ctx.Done())
+		ctrlctx.KubeNamespacedInformerFactory.Start(ctx.Done())
+		ctrlctx.KubeMAOSharedInformer.Start(ctx.Done())
+		ctrlctx.OCLInformerFactory.Start(ctx.Done())
 
 		close(ctrlctx.InformersStarted)
 
@@ -374,7 +373,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext, inspectionCache *image
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(),
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
 			ctx.KubeInformerFactory.Core().V1().Nodes(),
-			ctx.MCOPodInformerFactory.Core().V1().Pods(),
+			ctx.KubeInformerFactory.Core().V1().Pods(),
 			ctx.OCLInformerFactory.Machineconfiguration().V1().MachineOSConfigs(),
 			ctx.OCLInformerFactory.Machineconfiguration().V1().MachineOSBuilds(),
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigNodes(),

cmd/machine-config-daemon/start.go

@@ -236,15 +236,12 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 	)
 	go pinnedImageSetManager.Run(2, stopCh)
 
-	var mcnScopedInformerStartFunc func(<-chan struct{})
 	if ctrlctx.FeatureGatesHandler.Enabled(features.FeatureGateNoRegistryClusterInstall) {
-		mcnScopedInformer, startFunc := ctrlcommon.NewScopedMachineConfigNodeInformerFromClientBuilder(cb, startOpts.nodeName)
-		mcnScopedInformerStartFunc = startFunc
 		internalReleaseImageManager := internalreleaseimage.New(
 			startOpts.nodeName,
 			ctrlctx.ClientBuilder.MachineConfigClientOrDie(componentName),
 			ctrlctx.InformerFactory.Machineconfiguration().V1().InternalReleaseImages(),
-			mcnScopedInformer,
+			ctrlctx.InformerFactory.Machineconfiguration().V1().MachineConfigNodes(),
 		)
 		go internalReleaseImageManager.Run(1, stopCh)
 	}
@@ -254,9 +251,6 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 	ctrlctx.InformerFactory.Start(stopCh)
 	ctrlctx.OperatorInformerFactory.Start(stopCh)
 	nodeScopedInformerStartFunc(ctrlctx.Stop)
-	if mcnScopedInformerStartFunc != nil {
-		mcnScopedInformerStartFunc(ctrlctx.Stop)
-	}
 	close(ctrlctx.InformersStarted)
 
 	if err := dn.Run(stopCh, exitCh, errCh); err != nil {

cmd/machine-os-builder/start.go

@@ -25,19 +25,13 @@ var (
 	}
 
 	startOpts struct {
-		kubeconfig               string
-		promMetricsListenAddress string
-		tlsCipherSuites          []string
-		tlsMinVersion            string
+		kubeconfig string
 	}
 )
 
 func init() {
 	rootCmd.AddCommand(startCmd)
 	startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)")
-	startCmd.PersistentFlags().StringVar(&startOpts.promMetricsListenAddress, "metrics-listen-address", "127.0.0.1:8797", "Listen address for prometheus metrics listener")
-	startCmd.PersistentFlags().StringSliceVar(&startOpts.tlsCipherSuites, "tls-cipher-suites", nil, "Comma-separated list of cipher suites for the metrics server")
-	startCmd.PersistentFlags().StringVar(&startOpts.tlsMinVersion, "tls-min-version", "VersionTLS12", "Minimum TLS version supported for the metrics server")
 }
 
 func runStartCmd(_ *cobra.Command, _ []string) {
@@ -67,9 +61,6 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 
 		ctrlCtx := ctrlcommon.CreateControllerContext(ctx, cb)
 
-		// Start the metrics listener for OCL telemetry
-		go ctrlcommon.StartMetricsListener(startOpts.promMetricsListenAddress, ctrlCtx.Stop, build.RegisterOCLMetrics, startOpts.tlsMinVersion, startOpts.tlsCipherSuites)
-
 		ctrl := build.NewOSBuildControllerFromControllerContext(ctrlCtx)
 
 		// Wire up our shutdown channel.

install/0000_90_machine-config_01_prometheus-rules.yaml

@@ -15,10 +15,6 @@ spec:
       rules:
         - expr: sum(os_image_url_override)
           record: os_image_url_override:sum
-    - name: ocl-adoption.rules
-      rules:
-        - expr: mco_mosc_count
-          record: mco_mosc_count:sum
     - name: runc-deprecated
       rules:
         - alert: RuncDeprecated
@@ -69,46 +65,7 @@ spec:
             severity: info
           annotations:
             summary: "Boot image skew enforcement is disabled. Scaling operations may not be successful."
-            description: "Boot image skew enforcement mode is set to None. When scaling up, new nodes may be provisioned with older boot images that could introduce compatibility issues. Consider manually updating boot images to match the cluster version. Please refer to docs at https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/machine_configuration/mco-update-boot-skew-mgmt for additional details."
-    - name: ocl-build-failure
-      rules:
-        - alert: OCLBuildFailed
-          expr: |
-            increase(ocl_build_total{state="failed"}[5m]) > 0
-          for: 1m
-          labels:
-            namespace: openshift-machine-config-operator
-            severity: warning
-          annotations:
-            summary: "On-Cluster Layering build failed for pool {{ $labels.pool }}"
-            description: "An OCL image build has failed for MachineConfigPool {{ $labels.pool }}. Nodes in this pool will not receive updated layered images until the build succeeds. Check the machine-os-builder pod logs for details."
-            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/OCLBuildFailed.md
-    - name: ocl-build-degraded
-      rules:
-        - alert: OCLBuildDegraded
-          expr: |
-            ocl_build_state{state="failed"} == 1
-          for: 10m
-          labels:
-            namespace: openshift-machine-config-operator
-            severity: warning
-          annotations:
-            summary: "On-Cluster Layering build is degraded for pool {{ $labels.pool }}"
-            description: "OCL build for MachineConfigPool {{ $labels.pool }} is in a persistent failed state. Check the MachineOSBuild status conditions and machine-os-builder pod logs."
-            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/OCLBuildDegraded.md
-    - name: ocl-image-push-failure
-      rules:
-        - alert: OCLImagePushFailed
-          expr: |
-            increase(ocl_image_push_total{state="failed"}[5m]) > 0
-          for: 10m
-          labels:
-            namespace: openshift-machine-config-operator
-            severity: warning
-          annotations:
-            summary: "On-Cluster Layering image push failed for pool {{ $labels.pool }}"
-            description: "An OCL image push failed for MachineConfigPool {{ $labels.pool }}. The build job completed but the image was not pushed to the registry. Check registry credentials and machine-os-builder pod logs."
-            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/OCLImagePushFailed.md
+            description: "Boot image skew enforcement mode is set to None. When scaling up, new nodes may be provisioned with older boot images that could introduce compatibility issues. Consider manually updating boot images to match the cluster version. Please refer to docs at https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/machine_configuration/mco-update-boot-skew-mgmt for additional details."            
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule

pkg/apihelpers/apihelpers.go

@@ -439,12 +439,7 @@ func AreMCGeneratingSubControllersCompletedForPool(crcLister func(labels.Selecto
 				return fmt.Errorf("status for KubeletConfig %s is being reported for %d, expecting it for %d", mck.ObjectMeta.Name, mck.Status.ObservedGeneration, mck.Generation)
 			}
 
-			if len(mck.Status.Conditions) == 0 {
-				return fmt.Errorf("KubeletConfig has not completed")
-			}
-			lastCondition := mck.Status.Conditions[len(mck.Status.Conditions)-1]
-			if !((lastCondition.Type == mcfgv1.KubeletConfigAccepted && lastCondition.Status == corev1.ConditionTrue) ||
-				lastCondition.Type == mcfgv1.KubeletConfigSuccess) { // backwards compatibility
+			if mck.Status.Conditions[len(mck.Status.Conditions)-1].Type != mcfgv1.KubeletConfigSuccess {
 				return fmt.Errorf("KubeletConfig has not completed")
 			}
 		}

pkg/controller/bootimage/ami.go

@@ -1882,45 +1882,5 @@ var AllowedAMIs = sets.New(
 	"ami-0f4f5b97291d204aa", "ami-0f518858c9c61520f", "ami-0f51e2436e0ed93ef", "ami-0f70a80435f95369f", "ami-0f78698c377b198dd",
 	"ami-0f8259fb88cd731a9", "ami-0f85f0de1b8ffb89f", "ami-0f8a95d604562562b", "ami-0f8e1456e093c2396", "ami-0f9c2cdfa1e965dcc",
 	"ami-0f9d01760c1809e35", "ami-0fa80d356c323288c", "ami-0faffb65920395904", "ami-0fb9513e96a103d58", "ami-0fcbfc54371e9351a",
-	"ami-0fe270d47b42d792f", "ami-0fedec19a27a5eb0d", "ami-0ff0fd0ccdb0d31c2", "ami-0ff7ca07d83e23777", "ami-0001b45ada7d2dc51",
-	"ami-00159303d0698a4e3", "ami-001eff5ce74306328", "ami-002a19eea3002d594", "ami-003f9933b627323b8", "ami-00543bb0095fa7d23",
-	"ami-00667f67a54be771a", "ami-008a7997379c73959", "ami-00aaf7a70b4625181", "ami-00af5f0251f9d393f", "ami-00bceb1d4863de8d5",
-	"ami-00edb1c1d604ca98a", "ami-00f300f51af36e885", "ami-01086887bd3c994e1", "ami-013d71691d6282b85", "ami-01402f6fc7fa6e6dd",
-	"ami-016bcb89d328ec1ac", "ami-016e6fabf9dda07aa", "ami-01734737bbfbe74a1", "ami-018a94206d329a769", "ami-01b8ebd9b3d74d74c",
-	"ami-01c8bfb2576fb7da8", "ami-01d48aa6b50f9d0df", "ami-01f10c1ca2ec66aa1", "ami-02134e8468c69e038", "ami-0221fbcd56aa31a7a",
-	"ami-02264d13cc379c400", "ami-026c3565b2e140a8f", "ami-0286cff0da9d1c1c4", "ami-02d27a05c7b8d13df", "ami-02e98f28520c61cd3",
-	"ami-02ffc396c2432d05f", "ami-03038bbbf3c588c37", "ami-030623e6ced67aefc", "ami-030ca5bf92778d65f", "ami-0315581cb0f47f844",
-	"ami-03362167b2060e845", "ami-034a47eb2ad3e3ffd", "ami-03572bea3f4251a57", "ami-035c3042668af014c", "ami-035e211736d96d4a9",
-	"ami-035fc3102b495e061", "ami-038087b95229af209", "ami-038a860637181b890", "ami-0399f6912ea26e33c", "ami-03b58703d4e9ea61a",
-	"ami-03c5c13a072aae969", "ami-03d56fdb1a727352c", "ami-03e766cc0ac5a9d5e", "ami-03fcff06b8c5ac019", "ami-03fe2813461e69675",
-	"ami-04322409852db2c85", "ami-0450d6616ff432cfd", "ami-0477b3b24fd452453", "ami-04a4d537de220f7e6", "ami-04d475d929cabfc98",
-	"ami-04d6c62a05dc3820a", "ami-04d95d787cc499e7b", "ami-04deeaada42d956a3", "ami-04f22d56acba8a165", "ami-04fe481e7565ae103",
-	"ami-0507f999c00da7b28", "ami-0537c012fb351d6a0", "ami-0540d29d57768211f", "ami-054a256518f6779b1", "ami-05754d266e357951a",
-	"ami-059f37640c6141981", "ami-05b3066122ae9e9ec", "ami-05b5ed005b5082f42", "ami-05b949c2a121f003b", "ami-05c37201878e36dac",
-	"ami-05c6c3d4e9396d633", "ami-061e04994c7f50e3d", "ami-06259340f24c14377", "ami-0625d6961800b66ab", "ami-0648fefdfdfd37772",
-	"ami-065229dcdeea5f9b6", "ami-06593313b37053df5", "ami-066c6f5fba5f747a8", "ami-067f68ab29b40d089", "ami-06837ef5f71b2333b",
-	"ami-06a0a6cdc31d687b4", "ami-06b4716233fe63dc9", "ami-06e5d42796bd6f528", "ami-06fd07583064ed540", "ami-07031017151259e76",
-	"ami-071372b5430c069a2", "ami-0720c264eaffd3628", "ami-0738a052f29ebb8d4", "ami-0754162aa1a6d247e", "ami-0754d7722527cf959",
-	"ami-0757ca703a44448d7", "ami-076ce220d3f44fa4f", "ami-076e084cea4ec4903", "ami-0783bed23a0fc3ca6", "ami-07842bb8f69d670e1",
-	"ami-07c15edaa047a52c1", "ami-07cb628f267b263d1", "ami-07d5097d7ef637209", "ami-07f067e13ec1f3eb6", "ami-083d0fa187efa39e3",
-	"ami-084bd2f45461b2641", "ami-084f80cae5527784a", "ami-0858193e4c8ee5b62", "ami-087614ea800b0b80a", "ami-087ff0fa7fd5aa606",
-	"ami-088ea81e2a08b663e", "ami-08a4fe7bbdf941fc9", "ami-08ae75477cfec863f", "ami-08b9fb58c26d81f5d", "ami-08c4e237a48e8a24d",
-	"ami-08d3e5ac783f96689", "ami-08d5a47fb2f53f23a", "ami-08dd96b9cd7af93b9", "ami-08fabbfdb4f9e6482", "ami-090627c0bd0ce09e3",
-	"ami-090e98376b008c24c", "ami-0918a6de7debcd942", "ami-09493294918f32a0c", "ami-094e87d0b8aab28df", "ami-0980843632a1c3a66",
-	"ami-09d49112a1306f262", "ami-0a44f8809b6269f6d", "ami-0a4c225c885068069", "ami-0a651d826444a4948", "ami-0a7020eb754a9f6fc",
-	"ami-0a7afa68ca22609dd", "ami-0a7b15fc279056718", "ami-0a85236299582ce82", "ami-0a90a0dc9913213aa", "ami-0aba7be9d548d8831",
-	"ami-0ae75c9607a8862d8", "ami-0aee926ea72d3b04c", "ami-0af5f9e96ff66bdf5", "ami-0b5fba6fa0c7197d8", "ami-0b8305344e0fc7d2d",
-	"ami-0ba2883e0cf3e3558", "ami-0bba320960f966f20", "ami-0bd752fae0b2ac2a4", "ami-0bddd3545406608aa", "ami-0be21023c7da97e39",
-	"ami-0beed6ca33a1ae42d", "ami-0bfd310dade0d036f", "ami-0c19d58cb75d733c0", "ami-0c231f4816ba0776a", "ami-0c3faadd1797e3868",
-	"ami-0c500dfe48597e6f6", "ami-0c78f64d9a885c173", "ami-0c8487cb7aca26300", "ami-0cab770415626f477", "ami-0cb61ac0cd757c89c",
-	"ami-0cbb81ed4429c0503", "ami-0ccd894c5c2c7c45c", "ami-0cd956ca21ed98150", "ami-0cf685d7c4e2f115b", "ami-0d05715541d4175e7",
-	"ami-0d1687e928755427e", "ami-0d16b2d6a474b4360", "ami-0d1755bc956801be0", "ami-0d320aa9cc1633024", "ami-0d598411cb2c6a0e5",
-	"ami-0d6093add7a207886", "ami-0d68739d819850b39", "ami-0d6e91c6dae1b3c98", "ami-0d8a13644e176f1f8", "ami-0d9e44cb94e3e2314",
-	"ami-0dc49d202ca3f9193", "ami-0dd185285ca843d25", "ami-0de2f48ef57114708", "ami-0de3372c53819aaaa", "ami-0df26f104866e1a9a",
-	"ami-0e099dce9627ef699", "ami-0e0ac57dadcaf1ab9", "ami-0e0cf94e34dd227cd", "ami-0e0f89624ae6f6561", "ami-0e38d6bd2282d8072",
-	"ami-0e3ce644b0d9b5eab", "ami-0e3f9a41df3b903c6", "ami-0e47406ea13abefd8", "ami-0e651950cf2f78a61", "ami-0e6941714240039fa",
-	"ami-0e6a4ffe9dd1dee56", "ami-0e721586ef04044d2", "ami-0e75465363d8378f0", "ami-0e79be45f2303f422", "ami-0e9a0f9e1a4c49b92",
-	"ami-0ec2f94496976cb5f", "ami-0f186d1a862e47c8f", "ami-0f480726bfb767807", "ami-0f5633a3440704a9d", "ami-0f567edf0859e3b39",
-	"ami-0f5b937ac530500da", "ami-0f752ff59c8b0d39f", "ami-0f7add16535539145", "ami-0f99bc65b81da18b0", "ami-0fb8045529e752a6f",
-	"ami-0fc49ff64e479b1f5", "ami-0fe15e383af1e71d4", "ami-0feb18654dafb55c6", "ami-0fed99e2bf80aea18",
+	"ami-0fe270d47b42d792f", "ami-0fedec19a27a5eb0d", "ami-0ff0fd0ccdb0d31c2", "ami-0ff7ca07d83e23777",
 )

pkg/controller/bootimage/platform_helpers.go

@@ -233,7 +233,7 @@ func reconcileAWSProviderSpec(streamData *stream.Stream, arch string, _ *osconfi
 	return true, false, newProviderSpec, rhcosVersion, nil
 }
 
-func reconcileVSphereProviderSpec(streamData *stream.Stream, arch string, infra *osconfigv1.Infrastructure, providerSpec *machinev1beta1.VSphereMachineProviderSpec, _ string, kubeClient clientset.Interface) (bool, bool, *machinev1beta1.VSphereMachineProviderSpec, string, error) {
+func reconcileVSphereProviderSpec(streamData *stream.Stream, arch string, infra *osconfigv1.Infrastructure, providerSpec *machinev1beta1.VSphereMachineProviderSpec, _ string, secretClient clientset.Interface) (bool, bool, *machinev1beta1.VSphereMachineProviderSpec, string, error) {
 
 	if infra.Spec.PlatformSpec.VSphere == nil {
 		klog.Warningf("Reconcile skipped: VSphere field is nil in PlatformSpec %v", infra.Spec.PlatformSpec)
@@ -253,20 +253,20 @@ func reconcileVSphereProviderSpec(streamData *stream.Stream, arch string, infra
 	newProviderSpec := providerSpec.DeepCopy()
 
 	// Fetch the creds configmap
-	credsSc, err := kubeClient.CoreV1().Secrets("kube-system").Get(context.TODO(), "vsphere-creds", metav1.GetOptions{})
+	credsSc, err := secretClient.CoreV1().Secrets("kube-system").Get(context.TODO(), "vsphere-creds", metav1.GetOptions{})
 	if err != nil {
 		return false, false, nil, "", fmt.Errorf("failed to fetch vsphere-creds Secret during machineset sync: %w", err)
 	}
 
-	newBootImg, patchRequired, err := createNewVMTemplate(streamData, providerSpec, infra, credsSc, kubeClient, arch, artifacts.Release)
+	newBootImg, patchRequired, err := createNewVMTemplate(streamData, providerSpec, infra, credsSc, arch, artifacts.Release)
 	if err != nil {
 		return false, false, nil, "", err
 	}
 
 	// If patch is required, marshal the new providerspec into the machineset
 	if patchRequired {
 		// Ensure the ignition stub is the minimum acceptable spec required for boot image updates
-		if err := upgradeStubIgnitionIfRequired(providerSpec.UserDataSecret.Name, kubeClient); err != nil {
+		if err := upgradeStubIgnitionIfRequired(providerSpec.UserDataSecret.Name, secretClient); err != nil {
 			return false, false, nil, "", err
 		}
 		newProviderSpec.Template = newBootImg