feat(scheduling): phase 1 - swift-nic capacity scheduling, pending reservations, provision shard pinning

Builds on phase 0 (ready/not-ready mirroring). Adds capacity-aware placement of
HostedControlPlanes onto management clusters based on SWIFT-NIC capacity, plus
provision-shard pinning at Cluster Service creation.

Per eligible management cluster, available swift NICs are computed as:
  ScaleCeiling.Capacity[swift-nic]
    - ObservedResources.Usage[swift-nic]
    - (non-empty NotReadyResourceIDs)*3
    - (non-nil PendingAssignedClusters)*3
and an HCP fits when available >= 3 (swiftNICsPerHCP; a conservative flat
per-HCP cost that never overbooks). Nil/empty list entries do not correspond to
a real HCP and never reserve capacity.

API:
- coreapi: add ServiceProviderCluster.Spec.ManagementClusterResourceID
  (scheduler intent; drift from the observed Status placement is logged for
  investigation but NOT auto-corrected), regenerate deepcopy.
- fleetapi: add ManagementClusterScheduling.Status.PendingAssignedClusters
  (transient reservations), regenerate deepcopy.

Controllers:
- placement: new PlacementController (cluster-keyed, single worker). Backfills
  Spec from an already-observed Status placement during rollout; otherwise runs
  a pure, capacity-aware selection (fit filter + lowest-available bin-pack
  tie-break), reserves capacity in the chosen MC's PendingAssignedClusters, then
  records Spec. Transient write/conflict failures return an error so the
  workqueue retries with backoff.
- placement: new PendingCleanupController (MC-keyed periodic sweep) removes stale
  pending reservations. Effective placement favors the SPC's observed Status over
  Spec; an entry is kept when it points here or is still resolving (nil) and
  removed when it points elsewhere or the SPC is gone. SPCs are read via the
  shared lister/cache; transient failures return an error for workqueue backoff.
- placement: ManagementClusterPlacementSync reconciles Status from Cluster
  Service only while the observed placement is still unknown; once Status is set
  it skips the CS lookup. Spec/Status drift is logged, never auto-corrected.
- fleet CapacityReportingController: in the same read-modify-write, drops
  pending reservations that are now observed (Ready or NotReady).
- creation: ClusterPendingClusterServiceIDAssign gates on
  Spec.ManagementClusterResourceID != nil; ClusterClusterServiceCreate pins the
  provision shard via ClusterBuilder.ProvisionShardID resolved from the placed
  management cluster.
- wire PlacementController and PendingCleanupController into backend (single
  worker each; FleetDBClient + listers).

Tests: pure selection combos + lowest-available tie-break; capacity formula
edges incl. nil/empty entries; capacity-report pending cleanup; pending-cleanup
stale-entry matrix (Status-favored); rollout backfill; placement-sync
skip-when-Status-set and log-only drift; gated needsWork; provision shard
pinning.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: redhat-chai-bot

backend/pkg/app/backend.go

@@ -462,6 +462,9 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context,
 	_, serviceProviderClusterLister := backendInformers.ServiceProviderClusters()
 	_, serviceProviderNodePoolLister := backendInformers.ServiceProviderNodePools()
 
+	// Aggregate placement-state gauge recomputed periodically from the SPC cache.
+	placementMetricsController := metrics.NewPlacementMetricsController(b.options.MetricsRegisterer, serviceProviderClusterLister)
+
 	subscriptionNonClusterDataDumpController := datadump.NewSubscriptionNonClusterDataDumpController(b.options.ResourcesDBClient, backendInformers)
 	clusterRecursiveDataDumpController := datadump.NewClusterRecursiveDataDumpController(b.options.ResourcesDBClient, b.options.KubeApplierDBClients, managementClusterLister, activeOperationLister, backendInformers, unionKubeApplierInformers)
 	csStateDumpController := datadump.NewCSStateDumpController(b.options.ResourcesDBClient, activeOperationLister, backendInformers, unionKubeApplierInformers, b.options.ClustersServiceClient)
@@ -881,6 +884,18 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context,
 		backendInformers,
 		unionKubeApplierInformers,
 	)
+	placementController := clusterplacement.NewPlacementController(
+		b.options.ResourcesDBClient,
+		b.options.FleetDBClient,
+		managementClusterLister,
+		backendInformers,
+		unionKubeApplierInformers,
+	)
+	pendingCleanupController := clusterplacement.NewPendingCleanupController(
+		b.options.FleetDBClient,
+		serviceProviderClusterLister,
+		fleetInformers,
+	)
 
 	nodePoolClusterServiceCreateController := nodepoolcreation.NewNodePoolClusterServiceCreateController(
 		b.options.ResourcesDBClient,
@@ -952,6 +967,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context,
 	clusterClusterServiceCreateController := clustercreation.NewClusterClusterServiceCreateController(
 		b.options.ResourcesDBClient,
 		b.options.ClustersServiceClient,
+		managementClusterLister,
 		backendInformers,
 	)
 
@@ -1126,7 +1142,10 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context,
 				go nodePoolMetricsController.Run(ctx, 1)
 				go externalAuthMetricsController.Run(ctx, 1)
 				go clusterInfoMetricsController.Run(ctx, 1)
+				go placementMetricsController.Run(ctx, 1)
 				go placementSyncController.Run(ctx, 20)
+				go placementController.Run(ctx, 1)      // single worker: capacity selection reads/reserves across MCs and must not race itself
+				go pendingCleanupController.Run(ctx, 1) // single worker: sweeps pending reservations per management cluster
 				go cosmosMigrationController.Run(ctx, 5)
 				go virtualMachineResourceSKUsCachedReaderController.Run(ctx, 20)
 				go backupScheduleController.Run(ctx, 20)

backend/pkg/controllers/cluster/creation/cluster_cluster_service_create_controller.go

@@ -29,31 +29,35 @@ import (
 	"github.com/Azure/ARO-HCP/internal/database/cosmosstorage/cosmosstorageutils"
 	"github.com/Azure/ARO-HCP/internal/database/informers/coreinformers"
 	"github.com/Azure/ARO-HCP/internal/database/listers/corelisters"
+	"github.com/Azure/ARO-HCP/internal/database/listers/fleetlisters"
 	"github.com/Azure/ARO-HCP/internal/ocm"
 	"github.com/Azure/ARO-HCP/internal/utils"
 )
 
 type clusterClusterServiceCreateSyncer struct {
-	resourcesDBClient     corecosmosstorage.ResourcesDBClient
-	clusterLister         corelisters.ClusterLister
-	subscriptionLister    corelisters.SubscriptionLister
-	clustersServiceClient ocm.ClusterServiceClientSpec
+	resourcesDBClient       corecosmosstorage.ResourcesDBClient
+	clusterLister           corelisters.ClusterLister
+	subscriptionLister      corelisters.SubscriptionLister
+	managementClusterLister fleetlisters.ManagementClusterLister
+	clustersServiceClient   ocm.ClusterServiceClientSpec
 }
 
 var _ controllerutils.ClusterSyncer = (*clusterClusterServiceCreateSyncer)(nil)
 
 func NewClusterClusterServiceCreateController(
 	resourcesDBClient corecosmosstorage.ResourcesDBClient,
 	clustersServiceClient ocm.ClusterServiceClientSpec,
+	managementClusterLister fleetlisters.ManagementClusterLister,
 	backendInformers coreinformers.BackendInformers,
 ) controllerutils.Controller {
 	_, clusterLister := backendInformers.Clusters()
 	_, subscriptionLister := backendInformers.Subscriptions()
 	syncer := &clusterClusterServiceCreateSyncer{
-		resourcesDBClient:     resourcesDBClient,
-		clusterLister:         clusterLister,
-		subscriptionLister:    subscriptionLister,
-		clustersServiceClient: clustersServiceClient,
+		resourcesDBClient:       resourcesDBClient,
+		clusterLister:           clusterLister,
+		subscriptionLister:      subscriptionLister,
+		managementClusterLister: managementClusterLister,
+		clustersServiceClient:   clustersServiceClient,
 	}
 
 	return controllerutils.NewClusterWatchingController(
@@ -235,10 +239,18 @@ func (c *clusterClusterServiceCreateSyncer) csClustersMatchingClusterByAzureInfo
 func (c *clusterClusterServiceCreateSyncer) createClusterServiceCluster(ctx context.Context, cluster *coreapi.HCPOpenShiftCluster, serviceProviderCluster *coreapi.ServiceProviderCluster, tenantID string) (*arohcpv1alpha1.Cluster, error) {
 	logger := utils.LoggerFromContext(ctx)
 
+	provisionShardID, err := c.provisionShardID(ctx, serviceProviderCluster)
+	if err != nil {
+		return nil, utils.TrackError(err)
+	}
+
 	csClusterBuilder, err := ocm.BuildCSCluster(cluster.ID, tenantID, cluster, nil, nil, serviceProviderCluster)
 	if err != nil {
 		return nil, utils.TrackError(fmt.Errorf("failed to build CS cluster: %w", err))
 	}
+	// Pin the CS provision shard for the scheduler-selected management cluster via
+	// the SDK builder method.
+	csClusterBuilder.ProvisionShardID(provisionShardID)
 	clusterServiceUID := cluster.ServiceProviderProperties.PendingClusterServiceID.ClusterID()
 
 	logger.Info("Creating cluster in Cluster Service", "version", serviceProviderCluster.Spec.ControlPlaneVersion.DesiredVersion.String())
@@ -253,3 +265,36 @@ func (c *clusterClusterServiceCreateSyncer) createClusterServiceCluster(ctx cont
 
 	return result, nil
 }
+
+// provisionShardID resolves the Cluster Service provision shard ID for the
+// management cluster the scheduler pinned on
+// ServiceProviderCluster.Spec.ManagementClusterResourceID. The caller sets it on
+// the CS cluster via ClusterBuilder.ProvisionShardID so the new CS cluster is
+// created on the correct provision shard.
+func (c *clusterClusterServiceCreateSyncer) provisionShardID(ctx context.Context, serviceProviderCluster *coreapi.ServiceProviderCluster) (string, error) {
+	managementClusterResourceID := serviceProviderCluster.Spec.ManagementClusterResourceID
+	if managementClusterResourceID == nil {
+		return "", fmt.Errorf("ServiceProviderCluster has no Spec.ManagementClusterResourceID; placement is not resolved")
+	}
+	// A management cluster is a singleton within a stamp, so its resource ID is
+	// .../stamps/<stampIdentifier>/managementClusters/default and the lister is
+	// keyed by the stamp identifier (the parent segment's name).
+	if managementClusterResourceID.Parent == nil {
+		return "", fmt.Errorf("management cluster resource ID %q has no parent stamp", managementClusterResourceID.String())
+	}
+	stampIdentifier := managementClusterResourceID.Parent.Name
+
+	managementCluster, err := c.managementClusterLister.Get(ctx, stampIdentifier)
+	if cosmosstorageutils.IsNotFoundError(err) {
+		return "", fmt.Errorf("management cluster %q not found", managementClusterResourceID.String())
+	}
+	if err != nil {
+		return "", utils.TrackError(fmt.Errorf("failed to get management cluster %q: %w", managementClusterResourceID.String(), err))
+	}
+
+	if managementCluster.Status.ClusterServiceProvisionShardID == nil {
+		return "", fmt.Errorf("management cluster %q has no ClusterServiceProvisionShardID", managementClusterResourceID.String())
+	}
+
+	return managementCluster.Status.ClusterServiceProvisionShardID.ID(), nil
+}

backend/pkg/controllers/cluster/creation/cluster_cluster_service_create_controller_test.go

@@ -34,10 +34,12 @@ import (
 
 	"github.com/Azure/ARO-HCP/backend/pkg/utils/controllerutils"
 	"github.com/Azure/ARO-HCP/internal/api/coreapi"
+	"github.com/Azure/ARO-HCP/internal/api/fleetapi"
 	"github.com/Azure/ARO-HCP/internal/api/metadataapi"
 	"github.com/Azure/ARO-HCP/internal/apitesting/coreapitesting"
 	"github.com/Azure/ARO-HCP/internal/database/cosmosstoragetesting/corecosmosstoragetesting"
 	"github.com/Azure/ARO-HCP/internal/database/listertesting/corelistertesting"
+	"github.com/Azure/ARO-HCP/internal/database/listertesting/fleetlistertesting"
 	"github.com/Azure/ARO-HCP/internal/ocm"
 	"github.com/Azure/ARO-HCP/internal/utils"
 )
@@ -52,8 +54,30 @@ const (
 	testClusterUID          = "00000000-0000-0000-0000-000000000000"
 	// testManagedResourceGroup must match what coreapitesting.MinimumValidClusterTestCase() sets.
 	testManagedResourceGroup = "testManagedResourceGroup"
+	// testStampIdentifier / testProvisionShardID drive the management-cluster
+	// placement + provision-shard-pinning fixtures.
+	testStampIdentifier  = "1"
+	testProvisionShardID = "shard-abc123"
 )
 
+// testManagementClusterResourceID returns the resource ID of the placed management cluster.
+func testManagementClusterResourceID() *azcorearm.ResourceID {
+	return metadataapi.Must(fleetapi.ToManagementClusterResourceID(testStampIdentifier))
+}
+
+// newTestManagementCluster returns a management cluster carrying the CS provision
+// shard used by the provision-shard-pinning tests.
+func newTestManagementCluster() *fleetapi.ManagementCluster {
+	resourceID := testManagementClusterResourceID()
+	return &fleetapi.ManagementCluster{
+		CosmosMetadata: coreapi.CosmosMetadata{ResourceID: resourceID, PartitionKey: testStampIdentifier},
+		ResourceID:     resourceID,
+		Status: fleetapi.ManagementClusterStatus{
+			ClusterServiceProvisionShardID: ptr.To(metadataapi.Must(metadataapi.NewInternalID("/api/aro_hcp/v1alpha1/provision_shards/" + testProvisionShardID))),
+		},
+	}
+}
+
 // testClusterResourceID builds the ARM resource ID for the test cluster.
 func testClusterResourceID() *azcorearm.ResourceID {
 	return metadataapi.Must(azcorearm.ParseResourceID(
@@ -128,6 +152,7 @@ func TestClusterClusterServiceCreate_SyncOnce(t *testing.T) {
 		listCluster                    *coreapi.HCPOpenShiftCluster    // cluster seeded into the lister (nil = not found)
 		dbCluster                      *coreapi.HCPOpenShiftCluster    // cluster stored in the DB
 		existingServiceProviderCluster *coreapi.ServiceProviderCluster // nil = not pre-seeded; controller get-or-creates
+		managementClusters             []*fleetapi.ManagementCluster   // seeded into the fleet lister
 		setupMockCS                    func(ctrl *gomock.Controller) ocm.ClusterServiceClientSpec
 		expectError                    bool
 		verifyDB                       func(t *testing.T, ctx context.Context, db *corecosmosstoragetesting.MockResourcesDBClient)
@@ -142,7 +167,9 @@ func TestClusterClusterServiceCreate_SyncOnce(t *testing.T) {
 			}),
 			existingServiceProviderCluster: newTestSPC(func(spc *coreapi.ServiceProviderCluster) {
 				spc.Spec.ControlPlaneVersion.DesiredVersion = desiredVersion
+				spc.Spec.ManagementClusterResourceID = testManagementClusterResourceID()
 			}),
+			managementClusters: []*fleetapi.ManagementCluster{newTestManagementCluster()},
 			setupMockCS: func(ctrl *gomock.Controller) ocm.ClusterServiceClientSpec {
 				mockCS := ocm.NewMockClusterServiceClientSpec(ctrl)
 				mockCS.EXPECT().
@@ -154,6 +181,7 @@ func TestClusterClusterServiceCreate_SyncOnce(t *testing.T) {
 						built, buildErr := builder.Build()
 						require.NoError(t, buildErr)
 						assert.Equal(t, pendingClusterServiceID.ID(), built.ID(), "PostCluster should use the final segment of PendingClusterServiceID")
+						assert.Equal(t, testProvisionShardID, built.ProvisionShardID(), "PostCluster should pin the provision shard from the placed management cluster")
 						csCluster, err := arohcpv1alpha1.NewCluster().
 							ID(pendingClusterServiceID.ID()).
 							HREF(testClusterServiceIDStr).
@@ -232,7 +260,9 @@ func TestClusterClusterServiceCreate_SyncOnce(t *testing.T) {
 			}),
 			existingServiceProviderCluster: newTestSPC(func(spc *coreapi.ServiceProviderCluster) {
 				spc.Spec.ControlPlaneVersion.DesiredVersion = desiredVersion
+				spc.Spec.ManagementClusterResourceID = testManagementClusterResourceID()
 			}),
+			managementClusters: []*fleetapi.ManagementCluster{newTestManagementCluster()},
 			setupMockCS: func(ctrl *gomock.Controller) ocm.ClusterServiceClientSpec {
 				mockCS := ocm.NewMockClusterServiceClientSpec(ctrl)
 				// Build the CS cluster with Azure fields matching the test cluster so it
@@ -284,10 +314,11 @@ func TestClusterClusterServiceCreate_SyncOnce(t *testing.T) {
 				listerClusters = []*coreapi.HCPOpenShiftCluster{tt.listCluster}
 			}
 			syncer := &clusterClusterServiceCreateSyncer{
-				resourcesDBClient:     mockDB,
-				clusterLister:         &corelistertesting.SliceClusterLister{Clusters: listerClusters},
-				subscriptionLister:    &corelistertesting.SliceSubscriptionLister{Subscriptions: []*coreapi.Subscription{subscription}},
-				clustersServiceClient: mockCS,
+				resourcesDBClient:       mockDB,
+				clusterLister:           &corelistertesting.SliceClusterLister{Clusters: listerClusters},
+				subscriptionLister:      &corelistertesting.SliceSubscriptionLister{Subscriptions: []*coreapi.Subscription{subscription}},
+				managementClusterLister: &fleetlistertesting.SliceManagementClusterLister{ManagementClusters: tt.managementClusters},
+				clustersServiceClient:   mockCS,
 			}
 
 			key := controllerutils.HCPClusterKey{

backend/pkg/controllers/cluster/creation/cluster_pending_cluster_service_id_assign_controller.go

@@ -31,8 +31,9 @@ import (
 )
 
 type clusterPendingClusterServiceIDAssignSyncer struct {
-	clusterLister     corelisters.ClusterLister
-	resourcesDBClient corecosmosstorage.ResourcesDBClient
+	clusterLister                corelisters.ClusterLister
+	serviceProviderClusterLister corelisters.ServiceProviderClusterLister
+	resourcesDBClient            corecosmosstorage.ResourcesDBClient
 }
 
 var _ controllerutils.ClusterSyncer = (*clusterPendingClusterServiceIDAssignSyncer)(nil)
@@ -41,9 +42,11 @@ const ClusterPendingClusterServiceIDAssignControllerName = "ClusterPendingCluste
 
 func NewClusterPendingClusterServiceIDAssignController(resourcesDBClient corecosmosstorage.ResourcesDBClient, backendInformers coreinformers.BackendInformers) controllerutils.Controller {
 	_, clusterLister := backendInformers.Clusters()
+	_, serviceProviderClusterLister := backendInformers.ServiceProviderClusters()
 	syncer := &clusterPendingClusterServiceIDAssignSyncer{
-		clusterLister:     clusterLister,
-		resourcesDBClient: resourcesDBClient,
+		clusterLister:                clusterLister,
+		serviceProviderClusterLister: serviceProviderClusterLister,
+		resourcesDBClient:            resourcesDBClient,
 	}
 
 	return controllerutils.NewClusterWatchingController(
@@ -56,11 +59,19 @@ func NewClusterPendingClusterServiceIDAssignController(resourcesDBClient corecos
 	)
 }
 
-func (c *clusterPendingClusterServiceIDAssignSyncer) needsWork(cluster *coreapi.HCPOpenShiftCluster) bool {
+// needsWork reports whether a PendingClusterServiceID should be assigned. In
+// addition to the cluster not yet having a (pending or resolved) Cluster Service
+// ID and not being deleted, placement must already be resolved: the
+// ServiceProviderCluster must have Spec.ManagementClusterResourceID set by the
+// PlacementController. This gates Cluster Service creation on a management
+// cluster having been chosen first.
+func (c *clusterPendingClusterServiceIDAssignSyncer) needsWork(cluster *coreapi.HCPOpenShiftCluster, serviceProviderCluster *coreapi.ServiceProviderCluster) bool {
 	return cluster.ServiceProviderProperties.DeletionTimestamp == nil &&
 		cluster.ServiceProviderProperties.PendingClusterServiceID == nil &&
 		(cluster.ServiceProviderProperties.ClusterServiceID == nil ||
-			len(cluster.ServiceProviderProperties.ClusterServiceID.String()) == 0)
+			len(cluster.ServiceProviderProperties.ClusterServiceID.String()) == 0) &&
+		serviceProviderCluster != nil &&
+		serviceProviderCluster.Spec.ManagementClusterResourceID != nil
 }
 
 func (c *clusterPendingClusterServiceIDAssignSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPClusterKey) error {
@@ -74,7 +85,16 @@ func (c *clusterPendingClusterServiceIDAssignSyncer) SyncOnce(ctx context.Contex
 		return utils.TrackError(err)
 	}
 
-	if !c.needsWork(cluster) {
+	serviceProviderCluster, err := c.serviceProviderClusterLister.Get(ctx, key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName)
+	if cosmosstorageutils.IsNotFoundError(err) {
+		// Placement has not produced a ServiceProviderCluster yet; wait.
+		return nil
+	}
+	if err != nil {
+		return utils.TrackError(err)
+	}
+
+	if !c.needsWork(cluster, serviceProviderCluster) {
 		return nil
 	}