diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index a19e39bc4ae..0b3c7a13653 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -484,6 +484,13 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, backendInformers, ) + fetchMSIIdentitiesInfoController := controllers.NewFetchMSIIdentitiesInfoController( + b.options.CosmosDBClient, + activeOperationLister, + backendInformers, + b.options.FPAMIDataplaneClientBuilder, + ) + le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ Lock: b.options.LeaderElectionLock, LeaseDuration: leaderElectionLeaseDuration, @@ -527,6 +534,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, go maestroReadAndPersistReadonlyBundlesContentController.Run(ctx, 20) go maestroDeleteOrphanedReadonlyBundlesController.Run(ctx, 20) go triggerNodePoolUpgradeController.Run(ctx, 20) + go fetchMSIIdentitiesInfoController.Run(ctx, 20) }, OnStoppedLeading: func() { // This needs to be defined even though it does nothing. diff --git a/backend/pkg/controllers/fetch_msi_identities_info.go b/backend/pkg/controllers/fetch_msi_identities_info.go new file mode 100644 index 00000000000..21674ede720 --- /dev/null +++ b/backend/pkg/controllers/fetch_msi_identities_info.go @@ -0,0 +1,183 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "k8s.io/apimachinery/pkg/api/equality" + + "github.com/Azure/msi-dataplane/pkg/dataplane" + + azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/informers" + "github.com/Azure/ARO-HCP/backend/pkg/listers" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/utils" +) + +// fetchMSIIdentitiesInfoSyncer is a controller that fetches the Client ID and Principal ID of the MSI-based managed identities +// associated to the cluster and stores them in Cosmos. The MSI-based managed identities are the Cluster's control plane operators +// managed identities and the Cluster's service managed identity. +type fetchMSIIdentitiesInfoSyncer struct { + cooldownChecker controllerutils.CooldownChecker + + cosmosClient database.DBClient + + fpaMIdataplaneClientBuilder azureclient.FPAMIDataplaneClientBuilder +} + +var _ controllerutils.ClusterSyncer = (*fetchMSIIdentitiesInfoSyncer)(nil) + +func NewFetchMSIIdentitiesInfoController( + cosmosClient database.DBClient, + activeOperationLister listers.ActiveOperationLister, + backendInformers informers.BackendInformers, + fpaMIdataplaneClientBuilder azureclient.FPAMIDataplaneClientBuilder, +) controllerutils.Controller { + + syncer := &fetchMSIIdentitiesInfoSyncer{ + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + fpaMIdataplaneClientBuilder: fpaMIdataplaneClientBuilder, + } + + controller := controllerutils.NewClusterWatchingController( + "FetchMSIIdentitiesInfo", + cosmosClient, + backendInformers, + 1*time.Minute, + syncer, + ) + + return controller +} + +func (c *fetchMSIIdentitiesInfoSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPClusterKey) error { + existingCluster, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).Get(ctx, key.HCPClusterName) + if database.IsResponseError(err, http.StatusNotFound) { + return nil // cluster doesn't exist, no work to do + } + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + } + + // TODO do we need to check if existingCluster.Identity is nil or are we guaranteed that after Frontend stores to cosmos + // that section is not nil? + // TODO do we need to check if existingCluster.Identity.UserAssignedIdentities is nil or are we guaranteed that after Frontend stores to cosmos + // that section is not nil? + var identitiesToSync []string + for identityResourceIDStr, identity := range existingCluster.Identity.UserAssignedIdentities { + if identity.ClientID == nil || len(*identity.ClientID) == 0 { + identitiesToSync = append(identitiesToSync, identityResourceIDStr) + } + + if identity.PrincipalID == nil || len(*identity.PrincipalID) == 0 { + identitiesToSync = append(identitiesToSync, identityResourceIDStr) + } + } + + if len(identitiesToSync) == 0 { + return nil + } + + // As a relevant note, on environments where the real Managed Identities Data Plane service is not available a + // fake implementation of the Managed Identities Data Plane client is used, which always returns the information and + // same set of credentials for all requests. The returned information is the information associated to the "mock MSI" identity. + fpaMIDataplaneClient, err := c.fpaMIdataplaneClientBuilder.ManagedIdentitiesDataplane(existingCluster.ServiceProviderProperties.ManagedIdentitiesDataPlaneIdentityURL) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Managed Identities Data Plane Client: %w", err)) + } + + // We get all the Managed Identities information in a single Managed Identities Data Plane Credentials request because + // we have been told to minimize calls to the Managed Identities Data Plane Service. + fpaMIDataplaneCredentialsRequest := dataplane.UserAssignedIdentitiesRequest{ + IdentityIDs: identitiesToSync, + } + fpaMIDataplaneCredentials, err := fpaMIDataplaneClient.GetUserAssignedIdentitiesCredentials(ctx, fpaMIDataplaneCredentialsRequest) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Managed Identities Data Plane Credentials: %w", err)) + } + + // TODO at some point we will also have to implement logic that retrieves the initial set of credentials for the + // control plane operators managed identities and for the service managed identity and store it in the Managed + // Identities Key Vault (a Management Cluster scoped resource). Do we want to do it here at the same time because + // we are already calling the Managed Identities Data Plane Service and getting credentials here? As relevant context, + // these set of initial credentials should be stored in the Managed Identities Key Vault before creating the HostedCluster + // and those credentials have a limited lifespan (unknown which without investigating further). + + if len(fpaMIDataplaneCredentials.ExplicitIdentities) == 0 { + return utils.TrackError(fmt.Errorf("returned number of Managed Identities Data Plane Credentials is 0")) + } + + if len(fpaMIDataplaneCredentials.ExplicitIdentities) != len(identitiesToSync) { + return utils.TrackError(fmt.Errorf("unexpected number of Managed Identities Data Plane Credentials. Expected: %d, Received: %d", len(identitiesToSync), len(fpaMIDataplaneCredentials.ExplicitIdentities))) + } + + desiredMSIIdentities := make(map[string]*arm.UserAssignedIdentity) + var syncErrors []error + for i, fpaMIDataplaneCredential := range fpaMIDataplaneCredentials.ExplicitIdentities { + if fpaMIDataplaneCredential.ResourceID == nil || len(*fpaMIDataplaneCredential.ResourceID) == 0 { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("unexpected Managed Identities Data Plane Credential %s Resource ID is nil or empty", identitiesToSync[i]))) + continue + } + desiredMSIIdentities[*fpaMIDataplaneCredential.ResourceID] = &arm.UserAssignedIdentity{} + currentDesiredMSIIdentity := desiredMSIIdentities[*fpaMIDataplaneCredential.ResourceID] + + if fpaMIDataplaneCredential.ClientID != nil && len(*fpaMIDataplaneCredential.ClientID) > 0 { + currentDesiredMSIIdentity.ClientID = fpaMIDataplaneCredential.ClientID + } else { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("unexpected Managed Identities Data Plane Credential %s Client ID is nil or empty", identitiesToSync[i]))) + } + + if fpaMIDataplaneCredential.ObjectID != nil && len(*fpaMIDataplaneCredential.ObjectID) > 0 { + currentDesiredMSIIdentity.PrincipalID = fpaMIDataplaneCredential.ObjectID + } else { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("unexpected Managed Identities Data Plane Credential %s Principal ID is nil or empty", identitiesToSync[i]))) + } + } + + // TODO are we ok with storing this directly in the HCPCluster resource, as it needs to be set anyway because it + // is returned as part of the API of the Cluster to end-users? + if !equality.Semantic.DeepEqual(existingCluster.Identity.UserAssignedIdentities, desiredMSIIdentities) && len(desiredMSIIdentities) > 0 { + // TODO should we check if existingCluster.Identity is nil and initialize it? or are we guaranteed that after Frontend stores to cosmos + // that section is not nil? + + for desiredIdentityResourceIDStr, desiredIdentity := range desiredMSIIdentities { + if desiredIdentity.ClientID != nil && len(*desiredIdentity.ClientID) > 0 { + existingCluster.Identity.UserAssignedIdentities[desiredIdentityResourceIDStr].ClientID = desiredIdentity.ClientID + } + if desiredIdentity.PrincipalID != nil && len(*desiredIdentity.PrincipalID) > 0 { + existingCluster.Identity.UserAssignedIdentities[desiredIdentityResourceIDStr].PrincipalID = desiredIdentity.PrincipalID + } + } + + _, err := c.cosmosClient.HCPClusters(existingCluster.ID.SubscriptionID, existingCluster.ID.ResourceGroupName).Replace(ctx, existingCluster, nil) + if err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to replace HCPCluster: %w", err))) + } + } + + return errors.Join(syncErrors...) +} + +func (c *fetchMSIIdentitiesInfoSyncer) CooldownChecker() controllerutils.CooldownChecker { + return c.cooldownChecker +} diff --git a/test-integration/go.mod b/test-integration/go.mod index e08ca522efa..b46c9f2c218 100644 --- a/test-integration/go.mod +++ b/test-integration/go.mod @@ -27,10 +27,17 @@ require ( require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-kusto-go v0.16.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 // indirect + github.com/Azure/msi-dataplane v0.4.3 // indirect github.com/Azure/retry v0.0.0-20250221010952-92c9290cea0f // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -40,6 +47,7 @@ require ( github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/getsentry/sentry-go v0.20.0 // indirect github.com/go-json-experiment/json v0.0.0-20250517221953-25912455fbc8 // indirect diff --git a/test-integration/go.sum b/test-integration/go.sum index 5a9d5fe8b9e..a9c83ec2260 100644 --- a/test-integration/go.sum +++ b/test-integration/go.sum @@ -16,8 +16,14 @@ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.4.1 h1:ToPLhnXvatKVN4Zkcx github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.4.1/go.mod h1:Krtog/7tz27z75TwM5cIS8bxEH4dcBUezcq+kGVeZEo= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0/go.mod h1:Ms6gYEy0+A2knfKrwdatsggTXYA2+ICKug8w7STorFw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 h1:HYGD75g0bQ3VO/Omedm54v4LrD3B1cGImuRF3AJ5wLo= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0/go.mod h1:ulHyBFJOI0ONiRL4vcJTmS7rx18jQQlEPmAgo80cRdM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0 h1:bYq3jfB2x36hslKMHyge3+esWzROtJNk/4dCjsKlrl4= @@ -26,8 +32,18 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 h1:guyQA4b8XB2sbJZXzUnOF9mn0WDBv/ZT7me9wTipKtE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 h1:RTTsXUJWn0jumeX62Mb153wYXykqnrzYBYDeHp0kiuk= github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0/go.mod h1:k4MMjrPHIEK+umaMGk1GNLgjEybJZ9mHSRDZ+sDFv3Y= +github.com/Azure/msi-dataplane v0.4.3 h1:dWPWzY4b54tLIR9T1Q014Xxd/1DxOsMIp6EjRFAJlQY= +github.com/Azure/msi-dataplane v0.4.3/go.mod h1:yAfxdJyvcnvSDfSyOFV9qm4fReEQDl+nZLGeH2ZWSmw= github.com/Azure/retry v0.0.0-20250221010952-92c9290cea0f h1:XjKfallhRhddiRmBG0u2gs+Rd75QjvAlPVDy3ZWLjPg= github.com/Azure/retry v0.0.0-20250221010952-92c9290cea0f/go.mod h1:4FpEaBWwrdI8kVPeNESpqzIYAZipu7K6MCGCCC6bJ/A= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= @@ -36,6 +52,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -66,6 +84,8 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=