diff --git a/backend/cmd/root.go b/backend/cmd/root.go index d77e4ee132e..a8d423acb8d 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -253,7 +253,12 @@ func (f *BackendRootCmdFlags) ToBackendOptions(ctx context.Context, cmd *cobra.C return nil, utils.TrackError(fmt.Errorf("failed to create Azure configuration: %w", err)) } - fpaClientBuilder, err := app.NewFirstPartyApplicationClientBuilder(ctx, f.AzureFirstPartyApplicationCertificateBundlePath, f.AzureFirstPartyApplicationClientID, azureConfig) + fpaTokenCredRetriever, err := app.NewFirstPartyApplicationTokenCredentialRetriever(ctx, f.AzureFirstPartyApplicationCertificateBundlePath, f.AzureFirstPartyApplicationClientID, azureConfig) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to create FPA token credential retriever: %w", err)) + } + + fpaClientBuilder, err := app.NewFirstPartyApplicationClientBuilder(fpaTokenCredRetriever, azureConfig) if err != nil { return nil, utils.TrackError(fmt.Errorf("failed to create FPA client builder: %w", err)) } @@ -263,6 +268,16 @@ func (f *BackendRootCmdFlags) ToBackendOptions(ctx context.Context, cmd *cobra.C return nil, utils.TrackError(fmt.Errorf("failed to create backend identity azure clients: %w", err)) } + fpaMIDataplaneClientBuilder, err := app.NewFirstPartyApplicationManagedIdentitiesDataplaneClientBuilder( + fpaTokenCredRetriever, + f.InsecureAzureManagedIdentityMockCertificateBundlePath, f.InsecureAzureManagedIdentityMockClientID, f.InsecureAzureManagedIdentityMockServicePrincipalID, f.InsecureAzureManagedIdentityMockTenantID, + azureConfig, + ) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("error getting FPA MI dataplane client builder: %w", err)) + } + smiClientBuilder := app.NewServiceManagedIdentityClientBuilder(fpaMIDataplaneClientBuilder, azureConfig) + cosmosDBClient, err := app.NewCosmosDBClient( ctx, f.AzureCosmosDBURL, f.AzureCosmosDBName, *azureConfig.CloudEnvironment.AZCoreClientOptions(), @@ -290,6 +305,8 @@ func (f *BackendRootCmdFlags) ToBackendOptions(ctx context.Context, cmd *cobra.C FPAClientBuilder: fpaClientBuilder, BackendIdentityAzureClients: backendIdentityAzureClients, ExitOnPanic: f.ExitOnPanic, + FPAMIDataplaneClientBuilder: fpaMIDataplaneClientBuilder, + SMIClientBuilder: smiClientBuilder, } return backendOptions, nil diff --git a/backend/go.mod b/backend/go.mod index 2562a96160a..3713b7b6ca1 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -7,9 +7,11 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.4.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 + github.com/Azure/msi-dataplane v0.4.3 github.com/blang/semver/v4 v4.0.0 github.com/go-logr/logr v1.4.3 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 @@ -42,8 +44,13 @@ require ( dario.cat/mergo v1.0.1 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect @@ -56,6 +63,7 @@ require ( github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/getsentry/sentry-go v0.20.0 // indirect github.com/go-errors/errors v1.5.1 // indirect @@ -119,6 +127,7 @@ require ( go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.47.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.40.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index c75dda9c2d6..0a7fba90895 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -16,22 +16,38 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDo github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0/go.mod h1:Ms6gYEy0+A2knfKrwdatsggTXYA2+ICKug8w7STorFw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0 h1:bYq3jfB2x36hslKMHyge3+esWzROtJNk/4dCjsKlrl4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0/go.mod h1:fewgRjNVE84QVVh798sIMFb7gPXPp7NmnekGnboSnXk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 h1:guyQA4b8XB2sbJZXzUnOF9mn0WDBv/ZT7me9wTipKtE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 h1:RTTsXUJWn0jumeX62Mb153wYXykqnrzYBYDeHp0kiuk= github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0/go.mod h1:k4MMjrPHIEK+umaMGk1GNLgjEybJZ9mHSRDZ+sDFv3Y= +github.com/Azure/msi-dataplane v0.4.3 h1:dWPWzY4b54tLIR9T1Q014Xxd/1DxOsMIp6EjRFAJlQY= +github.com/Azure/msi-dataplane v0.4.3/go.mod h1:yAfxdJyvcnvSDfSyOFV9qm4fReEQDl+nZLGeH2ZWSmw= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -61,6 +77,8 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ= @@ -292,6 +310,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index dfc1749774d..d58627f9c2a 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -69,6 +69,8 @@ type BackendOptions struct { FPAClientBuilder azureclient.FirstPartyApplicationClientBuilder BackendIdentityAzureClients *azureclient.BackendIdentityAzureClients ExitOnPanic bool + FPAMIDataplaneClientBuilder azureclient.FPAMIDataplaneClientBuilder + SMIClientBuilder azureclient.ServiceManagedIdentityClientBuilder } func (o *BackendOptions) RunBackend(ctx context.Context) error { @@ -371,13 +373,18 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, b.options.CosmosDBClient, backendInformers, ) - azureClusterResourceGroupExistenceValidationController := validationcontrollers.NewClusterValidationController( validations.NewAzureClusterResourceGroupExistenceValidation(b.options.FPAClientBuilder), activeOperationLister, b.options.CosmosDBClient, backendInformers, ) + azureClusterManagedIdentitiesExistenceValidationController := validationcontrollers.NewClusterValidationController( + validations.NewAzureClusterManagedIdentitiesExistenceValidation(b.options.SMIClientBuilder), + activeOperationLister, + b.options.CosmosDBClient, + backendInformers, + ) nodePoolVersionController := upgradecontrollers.NewNodePoolVersionController( b.options.CosmosDBClient, @@ -420,6 +427,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, go identityMigrationController.Run(ctx, 20) go azureRPRegistrationValidationController.Run(ctx, 20) go azureClusterResourceGroupExistenceValidationController.Run(ctx, 20) + go azureClusterManagedIdentitiesExistenceValidationController.Run(ctx, 20) go nodePoolVersionController.Run(ctx, 20) }, OnStoppedLeading: func() { diff --git a/backend/pkg/app/fpa_wiring.go b/backend/pkg/app/fpa_wiring.go index 138a0cff72d..32834a80193 100644 --- a/backend/pkg/app/fpa_wiring.go +++ b/backend/pkg/app/fpa_wiring.go @@ -16,7 +16,9 @@ package app import ( "context" + "encoding/base64" "fmt" + "os" "time" azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" @@ -24,10 +26,10 @@ import ( "github.com/Azure/ARO-HCP/internal/fpa" ) -func NewFirstPartyApplicationClientBuilder( - ctx context.Context, fpaCertBundlePath string, fpaClientID string, - azureConfig *azureconfig.AzureConfig, -) (azureclient.FirstPartyApplicationClientBuilder, error) { +func NewFirstPartyApplicationTokenCredentialRetriever( + ctx context.Context, fpaCertBundlePath string, + fpaClientID string, azureConfig *azureconfig.AzureConfig, +) (fpa.FirstPartyApplicationTokenCredentialRetriever, error) { if len(fpaCertBundlePath) == 0 || len(fpaClientID) == 0 { return nil, nil } @@ -42,10 +44,6 @@ func NewFirstPartyApplicationClientBuilder( return nil, fmt.Errorf("failed to create certificate reader: %w", err) } - // We create the FPA token credential retriever here. Then we pass it to the cluster inflights controller, - // which then is used to instantiate a validation that uses the FPA token credential retriever. And then the - // validations uses the retriever to retrieve a token credential based on the information associated to the - // cluster(the tenant of the cluster, the subscription id, ...) fpaTokenCredRetriever, err := fpa.NewFirstPartyApplicationTokenCredentialRetriever( fpaClientID, certReader, @@ -55,9 +53,58 @@ func NewFirstPartyApplicationClientBuilder( return nil, fmt.Errorf("failed to create FPA token credential retriever: %w", err) } + return fpaTokenCredRetriever, nil +} + +func NewFirstPartyApplicationClientBuilder(fpaTokenCredRetriever fpa.FirstPartyApplicationTokenCredentialRetriever, azureConfig *azureconfig.AzureConfig) (azureclient.FirstPartyApplicationClientBuilder, error) { fpaClientBuilder := azureclient.NewFirstPartyApplicationClientBuilder( fpaTokenCredRetriever, azureConfig.CloudEnvironment.ARMClientOptions(), ) return fpaClientBuilder, nil } + +func NewFirstPartyApplicationManagedIdentitiesDataplaneClientBuilder( + fpaTokenCredRetriever fpa.FirstPartyApplicationTokenCredentialRetriever, + azureMIMockCertBundlePath string, azureMIMockClientID string, azureMIMockPrincipalID string, azureMIMockTenantID string, + azureConfig *azureconfig.AzureConfig, +) (azureclient.FPAMIDataplaneClientBuilder, error) { + if len(azureMIMockCertBundlePath) > 0 && len(azureMIMockClientID) > 0 && len(azureMIMockPrincipalID) > 0 && len(azureMIMockTenantID) > 0 { + // TODO this can be improved at some point to support detecting when + // the cert bundle path content changes. We could use a file watcher similar + // to the one used in the fpa token credential retriever, and pass the retriever + // to the client builder. + bundle, err := os.ReadFile(azureMIMockCertBundlePath) + if err != nil { + return nil, fmt.Errorf("failed to read bundle file: %w", err) + } + bundleBase64Encoded := base64.StdEncoding.EncodeToString(bundle) + hardcodedIdentity := &azureclient.HardcodedIdentity{ + ClientID: azureMIMockClientID, + ClientSecret: bundleBase64Encoded, + PrincipalID: azureMIMockPrincipalID, + TenantID: azureMIMockTenantID, + } + hardcodedIdentityFPAMIDataplaneClientBuilder := azureclient.NewHardcodedIdentityFPAMIDataplaneClientBuilder( + azureConfig.CloudEnvironment.CloudConfiguration(), + hardcodedIdentity, + ) + return hardcodedIdentityFPAMIDataplaneClientBuilder, nil + } + + fpaMIdataplaneClientBuilder := azureclient.NewFPAMIDataplaneClientBuilder( + azureConfig.AzureRuntimeConfig.ServiceTenantID, + fpaTokenCredRetriever, + azureConfig.AzureRuntimeConfig.ManagedIdentitiesDataPlaneAudienceResource, + azureConfig.CloudEnvironment.AZCoreClientOptions(), + ) + + return fpaMIdataplaneClientBuilder, nil +} + +func NewServiceManagedIdentityClientBuilder(fpaMIdataplaneClientBuilder azureclient.FPAMIDataplaneClientBuilder, azureConfig *azureconfig.AzureConfig) azureclient.ServiceManagedIdentityClientBuilder { + return azureclient.NewServiceManagedIdentityClientBuilder( + fpaMIdataplaneClientBuilder, + azureConfig.CloudEnvironment.ARMClientOptions(), + ) +} diff --git a/backend/pkg/azure/client/errors.go b/backend/pkg/azure/client/errors.go index a8ef594dc41..3e177c48f97 100644 --- a/backend/pkg/azure/client/errors.go +++ b/backend/pkg/azure/client/errors.go @@ -25,3 +25,12 @@ func IsResourceGroupNotFoundErr(err error) bool { var azErr *azcore.ResponseError return errors.As(err, &azErr) && azErr.ErrorCode == "ResourceGroupNotFound" } + +// IsResourceNotFoundErr is used to determine if we are failing to find a resource within azure. +// *WARNING* Not all azure API operations return the `ResourceNotFound` error code when the resource +// is not found, and more specific error codes are returned for some of them e.g `RoleAssignmentNotFound` +// is returned when a role assignement is not found +func IsResourceNotFoundErr(err error) bool { + var azErr *azcore.ResponseError + return errors.As(err, &azErr) && azErr.ErrorCode == "ResourceNotFound" +} diff --git a/backend/pkg/azure/client/fpa_mi_dataplane_client_builder.go b/backend/pkg/azure/client/fpa_mi_dataplane_client_builder.go new file mode 100644 index 00000000000..6a48076b502 --- /dev/null +++ b/backend/pkg/azure/client/fpa_mi_dataplane_client_builder.go @@ -0,0 +1,85 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/msi-dataplane/pkg/dataplane" + + "github.com/Azure/ARO-HCP/internal/fpa" +) + +// FPAClientBuilderType is a type that represents the type of the MIDataplaneClientBuilder +// interface. It is used to ensure that that interface is incompatible +// with other client builder interfaces that might have the same set of +// methods +type FPAMIDataplaneClientBuilderType string + +const ( + // FPAClientBuilderTypeValue is the value of the FPABuilderType type that + // represents the FPA client builder. + FPAMIDataplaneClientBuilderTypeValue FPAMIDataplaneClientBuilderType = "FPA-MIDP" +) + +// FPAMIDataplaneClientBuilder offers the ability to create Managed Identity Data Plane clients +// authenticating as the the First Party Application (FPA) identity. +type FPAMIDataplaneClientBuilder interface { + BuilderType() FPAMIDataplaneClientBuilderType + // ManagedIdentitiesDataplane returns a new Managed Identity Data Plane client using the given identity URL. + ManagedIdentitiesDataplane(identityURL string) (ManagedIdentitiesDataplaneClient, error) +} + +type fpaMIdataplaneClientBuilder struct { + serviceTenantID string + audience string + fpaTokenCredRetriever fpa.FirstPartyApplicationTokenCredentialRetriever + options *azcore.ClientOptions +} + +var _ FPAMIDataplaneClientBuilder = (*fpaMIdataplaneClientBuilder)(nil) + +func (b *fpaMIdataplaneClientBuilder) BuilderType() FPAMIDataplaneClientBuilderType { + return FPAMIDataplaneClientBuilderTypeValue +} + +func (b *fpaMIdataplaneClientBuilder) ManagedIdentitiesDataplane(identityURL string) (ManagedIdentitiesDataplaneClient, error) { + creds, err := b.fpaTokenCredRetriever.RetrieveCredential( + b.serviceTenantID, + // The MI dataplane client receives tenant from the bearer challenge, we use a widlcard * so as + // to not limit the allowed tenants in the credential. This was taken from + // https://github.com/Azure/ARO-RP/blob/9719391dd5d2213abb1b895e9b9471925f5aec0d/pkg/cluster/cluster.go#L329 + // which was added as part of needed fixes to make Managed Identity work in MSFT Canary env + // in https://github.com/Azure/ARO-RP/pull/3957 + "*", + ) + if err != nil { + return nil, err + } + + dpClientFactory := dataplane.NewClientFactory(creds, b.audience, b.options) + return dpClientFactory.NewClient(identityURL) +} + +// NewFPAMIDataplaneClientBuilder provides a new instance of +// FPAMIDataplaneClientBuilder that allows to retrieve Managed Identities Data Plane clients +// authenticating as the the First Party Application (FPA) identity. +func NewFPAMIDataplaneClientBuilder(serviceTenantID string, fpaTokenCredRetriever fpa.FirstPartyApplicationTokenCredentialRetriever, audience string, options *azcore.ClientOptions) FPAMIDataplaneClientBuilder { + return &fpaMIdataplaneClientBuilder{ + serviceTenantID: serviceTenantID, + fpaTokenCredRetriever: fpaTokenCredRetriever, + audience: audience, + options: options, + } +} diff --git a/backend/pkg/azure/client/hardcoded_identity_fpa_mi_dataplane_client_builder.go b/backend/pkg/azure/client/hardcoded_identity_fpa_mi_dataplane_client_builder.go new file mode 100644 index 00000000000..9d609d78f6b --- /dev/null +++ b/backend/pkg/azure/client/hardcoded_identity_fpa_mi_dataplane_client_builder.go @@ -0,0 +1,52 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + +// hardcodedIdentityFPAMIDataplaneClientBuilder is used to +// create Managed Identity Data Plane clients based on the +// hardcoded identity implementation of the Managed Identities +// Data Plane client hardcodedIdentityManagedIdentitiesDataplaneClient. +type hardcodedIdentityFPAMIDataplaneClientBuilder struct { + cloudConfiguration *cloud.Configuration + hardcodedIdentity *HardcodedIdentity +} + +var _ FPAMIDataplaneClientBuilder = (*hardcodedIdentityFPAMIDataplaneClientBuilder)(nil) + +func (b *hardcodedIdentityFPAMIDataplaneClientBuilder) BuilderType() FPAMIDataplaneClientBuilderType { + return FPAMIDataplaneClientBuilderTypeValue +} + +// ManagedIdentitiesDataplane returns a new Managed Identity Data Plane client +// based on the hardcoded identity implementation of the Managed Identities +// Data Plane client hardcodedIdentityManagedIdentitiesDataplaneClient. +// The identity URL parameter is not used in the hardcoded identity implementation +// of the managed identities dataplane clientso we ignore it. +func (b *hardcodedIdentityFPAMIDataplaneClientBuilder) ManagedIdentitiesDataplane(_ string) (ManagedIdentitiesDataplaneClient, error) { + return newHardcodedIdentityManagedIdentitiesDataPlaneClient(b.cloudConfiguration, b.hardcodedIdentity), nil +} + +// NewHardcodedIdentityFPAMIDataplaneClientBuilder provides a new instance of +// FPAMIDataplaneClientBuilder that allows to retrieve Managed Identities Data Plane clients +// based on the hardcoded identity implementation of the Managed Identities Data Plane client +// hardcodedIdentityManagedIdentitiesDataplaneClient. +func NewHardcodedIdentityFPAMIDataplaneClientBuilder(cloudConfiguration *cloud.Configuration, hardcodedIdentity *HardcodedIdentity) FPAMIDataplaneClientBuilder { + return &hardcodedIdentityFPAMIDataplaneClientBuilder{ + cloudConfiguration: cloudConfiguration, + hardcodedIdentity: hardcodedIdentity, + } +} diff --git a/backend/pkg/azure/client/hardcoded_identity_mi_dataplane_client.go b/backend/pkg/azure/client/hardcoded_identity_mi_dataplane_client.go new file mode 100644 index 00000000000..00fc757e961 --- /dev/null +++ b/backend/pkg/azure/client/hardcoded_identity_mi_dataplane_client.go @@ -0,0 +1,126 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "time" + + "k8s.io/utils/ptr" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/msi-dataplane/pkg/dataplane" +) + +// HardcodedIdentity represents the information of an Azure identity +// that will be returned when leveraging the hardcodedIdentityManagedIdentitiesDataPlaneClient +// Data Plane client. HardcodedIdentity support is limited to the following +// Azure identities +// - Azure User-Assigned Managed Identity +// - Azure Service Principal +// It is often the case that what we really want to hardcode is the information +// of an Azure Service Principal instead of a User-Assigned Managed Identity. +type HardcodedIdentity struct { + // ClientID is the Client ID of a valid identity. + ClientID string + // ClientSecret is The base64 encoded bundle + // certificate (public + private key) of the identity. + // The identity is a valid identity credential associated to the identity + // identified by ClientID. + ClientSecret string + // PrincipalID is the Principal ID of the identity identified by ClientID. + // For User-Assigned Managed identities this is the Object (principal) ID + // of the Managed Identity. This is, the Object ID of the service principal + // backing the Managed Identity. + // For Service Principals it is the Object ID of the service principal. + PrincipalID string + // TenantID is the Tenant ID of the identity identified by ClientID. + TenantID string +} + +// hardcodedIdentityManagedIdentitiesDataplaneClient is a mock implementation of the +// ManagedIdentitiesDataplaneClient interface. The Managed Identities Data +// Plane service is only available in Azure tenants where Microsoft's +// First Party Application (FPA) integration is available. For the environments +// where the FPA integration is not enabled we cannot communicate with the +// Managed Identities Data Plane service so instead we use this mock implementation +// of the client, where all requests made with it return a single +// Azure Service Principal identity, disguised as a Managed Identity from the +// point of view of the consumers of the client. We commonly refer to this +// identity as the "mock MSI" (also known as mi mock) identity. +type hardcodedIdentityManagedIdentitiesDataplaneClient struct { + cloudConfiguration *cloud.Configuration + + // hardcodedIdentity represents part of the identity information that will + // be hardcoded and returned in all responses provided by the client. + hardcodedIdentity *HardcodedIdentity +} + +var _ ManagedIdentitiesDataplaneClient = (*hardcodedIdentityManagedIdentitiesDataplaneClient)(nil) + +// GetUserAssignedIdentities returns the User Assigned Managed Identities associated +// The returned results will have the stubbed data provided during construction of the client +// for the client id, client secret and principal id attributes. +func (c *hardcodedIdentityManagedIdentitiesDataplaneClient) GetUserAssignedIdentitiesCredentials(ctx context.Context, request dataplane.UserAssignedIdentitiesRequest) (*dataplane.ManagedIdentityCredentials, error) { + now := time.Now().UTC() + aHundredYearsFromNow := now.AddDate(100, 0, 0).Format(time.RFC3339) + aDayAgo := now.AddDate(0, 0, -1).Format(time.RFC3339) + managedIdentityCredentials := dataplane.ManagedIdentityCredentials{ + AuthenticationEndpoint: ptr.To(c.cloudConfiguration.ActiveDirectoryAuthorityHost), + NotBefore: ptr.To(aDayAgo), + CannotRenewAfter: ptr.To(aHundredYearsFromNow), + RenewAfter: ptr.To(aHundredYearsFromNow), + NotAfter: ptr.To(aHundredYearsFromNow), + } + + placeholder := "placeholder" + identities := make([]dataplane.UserAssignedIdentityCredentials, len(request.IdentityIDs)) + for i, miResourceID := range request.IdentityIDs { + identity := dataplane.UserAssignedIdentityCredentials{ + ClientID: ptr.To(c.hardcodedIdentity.ClientID), + ClientSecret: ptr.To(c.hardcodedIdentity.ClientSecret), + TenantID: ptr.To(c.hardcodedIdentity.TenantID), + ResourceID: ptr.To(miResourceID), + AuthenticationEndpoint: ptr.To(c.cloudConfiguration.ActiveDirectoryAuthorityHost), + ClientSecretURL: &placeholder, + MtlsAuthenticationEndpoint: &placeholder, + NotBefore: ptr.To(aDayAgo), + CannotRenewAfter: ptr.To(aHundredYearsFromNow), + RenewAfter: ptr.To(aHundredYearsFromNow), + NotAfter: ptr.To(aHundredYearsFromNow), + CustomClaims: &dataplane.CustomClaims{ + XMSAzNwperimid: []string{placeholder}, + XMSAzTm: &placeholder, + }, + // In this specific context Object ID is equivalent to Principal ID + ObjectID: ptr.To(c.hardcodedIdentity.PrincipalID), + } + + identities[i] = identity + } + + managedIdentityCredentials.ExplicitIdentities = identities + return &managedIdentityCredentials, nil +} + +// newHardcodedIdentityManagedIdentitiesDataPlaneClient provides a new instance of +// ManagedIdentitiesDataplaneClient based on the hardcoded identity implementation +// of the Managed Identities Data Plane client hardcodedIdentityManagedIdentitiesDataplaneClient. +func newHardcodedIdentityManagedIdentitiesDataPlaneClient(cloudConfiguration *cloud.Configuration, hardcodedIdentity *HardcodedIdentity) ManagedIdentitiesDataplaneClient { + return &hardcodedIdentityManagedIdentitiesDataplaneClient{ + cloudConfiguration: cloudConfiguration, + hardcodedIdentity: hardcodedIdentity, + } +} diff --git a/backend/pkg/azure/client/mi_dataplane_client.go b/backend/pkg/azure/client/mi_dataplane_client.go new file mode 100644 index 00000000000..40f3fab42e3 --- /dev/null +++ b/backend/pkg/azure/client/mi_dataplane_client.go @@ -0,0 +1,38 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + + "github.com/Azure/msi-dataplane/pkg/dataplane" +) + +// ManagedIdentitiesDataplaneClient is the interface to interact with Azure's Managed Identity +// Data Plane service. The Managed Identities Data Plane service is only +// available in Azure tenants where Microsoft's First Party Application (FPA) +// integration is available. For the environments where the FPA integration is not available +// we cannot communicate with the Managed Identities Data Plane service, so +// instead we use a mock implementation of the ManagedIdentitiesDataplaneClient that +// always returns a single Azure Service Principal identity representing +// a Managed Identity. This mock implementation and details on it can be found +// in the hardcodedIdentityManagedIdentitiesDataplaneClient Go type. +// This client is different than Azure Go SDK's armmsi.UserAssignedIdentitiesClient/armmsiSystemAssignedIdentitiesClient +// clients, which are used to interact with the control plane side of the Managed Identities service. +type ManagedIdentitiesDataplaneClient interface { + GetUserAssignedIdentitiesCredentials(ctx context.Context, request dataplane.UserAssignedIdentitiesRequest) (*dataplane.ManagedIdentityCredentials, error) +} + +var _ ManagedIdentitiesDataplaneClient = (dataplane.Client)(nil) diff --git a/backend/pkg/azure/client/smi_client_builder.go b/backend/pkg/azure/client/smi_client_builder.go new file mode 100644 index 00000000000..122db6f209d --- /dev/null +++ b/backend/pkg/azure/client/smi_client_builder.go @@ -0,0 +1,100 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "fmt" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + "github.com/Azure/msi-dataplane/pkg/dataplane" + + "github.com/Azure/ARO-HCP/internal/utils" +) + +// ServiceManagedIdentityClientBuilderType is a type that represents the type of the +// ServiceManagedIdentityClientBuilder interface. It is used to ensure that +// that interface is incompatible with other client builder interfaces that +// might have the same set of methods +type ServiceManagedIdentityClientBuilderType string + +const ( + // ServiceManagedIdentityClientBuilderTypeValue is the value of the ServiceManagedIdentityClientBuilderType type that + // represents the SMI client builder. + ServiceManagedIdentityClientBuilderTypeValue ServiceManagedIdentityClientBuilderType = "SMI" +) + +// ServiceManagedIdentityClientBuilder offers the ability tocreate Azure clients +// authenticating as the the Cluster's Service Managed Identity, which is +// a cluster-scoped identity. +type ServiceManagedIdentityClientBuilder interface { + BuilderType() ServiceManagedIdentityClientBuilderType + // UserAssignedIdentitiesClient returns a new User Assigned Identities client. + UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) +} + +type serviceManagedIdentityClientBuilder struct { + fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder + azCoreARMClientOptions *azcorearm.ClientOptions +} + +var _ ServiceManagedIdentityClientBuilder = (*serviceManagedIdentityClientBuilder)(nil) + +func (b *serviceManagedIdentityClientBuilder) BuilderType() ServiceManagedIdentityClientBuilderType { + return ServiceManagedIdentityClientBuilderTypeValue +} + +func (b *serviceManagedIdentityClientBuilder) UserAssignedIdentitiesClient(ctx context.Context, clusterIdentityURL string, smiResourceID *azcorearm.ResourceID, subscriptionID string) (UserAssignedIdentitiesClient, error) { + // We obtain the Managed Identity Data Plane client using the Cluster's Identity URL. + miDataplaneClient, err := b.fpaMIdataplaneClientBuilder.ManagedIdentitiesDataplane(clusterIdentityURL) + if err != nil { + return nil, err + } + + // We then use the Managed Identity Data Plane client to get + // credentials associated to the Cluster's Service Managed Identity. + dataplaneRequest := dataplane.UserAssignedIdentitiesRequest{ + IdentityIDs: []string{smiResourceID.String()}, + } + resp, err := miDataplaneClient.GetUserAssignedIdentitiesCredentials(ctx, dataplaneRequest) + if err != nil { + return nil, err + } + if len(resp.ExplicitIdentities) == 0 { + return nil, + utils.TrackError(fmt.Errorf("managed identities data plane returned no credentials for the cluster's service managed identity '%s", smiResourceID.String())) + } + + // We convert the received UserAssignedIdentityCredentials result into + // an azidentity.ClientCertificateCredential, which Azure Go SDK's uses + // to instantiate a UserAssignedIdentitiesClient. + userAssignedIdentityCredential := resp.ExplicitIdentities[0] + creds, err := dataplane.GetCredential(b.azCoreARMClientOptions.ClientOptions, userAssignedIdentityCredential) + if err != nil { + return nil, err + } + + // We finally instantiate the UserAssignedIdentitiesClient using the + // the credentials we obtained from the Managed Identities Data Plane Service. + return armmsi.NewUserAssignedIdentitiesClient(subscriptionID, creds, b.azCoreARMClientOptions) +} + +func NewServiceManagedIdentityClientBuilder(fpaMIdataplaneClientBuilder FPAMIDataplaneClientBuilder, options *azcorearm.ClientOptions) ServiceManagedIdentityClientBuilder { + return &serviceManagedIdentityClientBuilder{ + fpaMIdataplaneClientBuilder: fpaMIdataplaneClientBuilder, + azCoreARMClientOptions: options, + } +} diff --git a/backend/pkg/azure/client/user_assigned_identities_client.go b/backend/pkg/azure/client/user_assigned_identities_client.go new file mode 100644 index 00000000000..e37649eee35 --- /dev/null +++ b/backend/pkg/azure/client/user_assigned_identities_client.go @@ -0,0 +1,42 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" +) + +// UserAssignedIdentitiesClient is an interface that defines the +// methods that we want to use from the UserAssignedIdentitiesClient type in +// the Azure Go SDK (https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/resourcemanager/msi/armmsi). +// The aim is to only contain methods that are defined in the Azure Go SDK +// UserAssignedIdentitiesClient client. +// If you need to use a method provided by the Azure Go SDK UserAssignedIdentitiesClient +// client but it is not defined in this interface then it has to be added here and all +// the types implementing this interface have to implement the new method. +type UserAssignedIdentitiesClient interface { + CreateOrUpdate(ctx context.Context, resourceGroupName string, resourceName string, parameters armmsi.Identity, options *armmsi.UserAssignedIdentitiesClientCreateOrUpdateOptions) (armmsi.UserAssignedIdentitiesClientCreateOrUpdateResponse, error) + + Delete(ctx context.Context, resourceGroupName string, resourceName string, options *armmsi.UserAssignedIdentitiesClientDeleteOptions) (armmsi.UserAssignedIdentitiesClientDeleteResponse, error) + + Get(ctx context.Context, resourceGroupName string, resourceName string, options *armmsi.UserAssignedIdentitiesClientGetOptions) (armmsi.UserAssignedIdentitiesClientGetResponse, error) +} + +// interface guard to ensure that all methods defined in the UserAssignedIdentitiesClient +// interface are implemented by the real Azure Go SDK UserAssignedIdentitiesClient +// client. This interface guard should always compile +var _ UserAssignedIdentitiesClient = (*armmsi.UserAssignedIdentitiesClient)(nil) diff --git a/backend/pkg/azure/config/azure_cloud_environment.go b/backend/pkg/azure/config/azure_cloud_environment.go index 3d597d78b86..8f80daa2a41 100644 --- a/backend/pkg/azure/config/azure_cloud_environment.go +++ b/backend/pkg/azure/config/azure_cloud_environment.go @@ -133,3 +133,7 @@ func (a AzureCloudEnvironment) ARMClientOptions() *azcorearm.ClientOptions { ClientOptions: *a.clientOptions, } } + +func (a AzureCloudEnvironment) CloudConfiguration() *cloud.Configuration { + return a.configuration +} diff --git a/backend/pkg/controllers/validationcontrollers/validations/azure_cluster_mis_existence_validation.go b/backend/pkg/controllers/validationcontrollers/validations/azure_cluster_mis_existence_validation.go new file mode 100644 index 00000000000..382e356f38a --- /dev/null +++ b/backend/pkg/controllers/validationcontrollers/validations/azure_cluster_mis_existence_validation.go @@ -0,0 +1,96 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "context" + "fmt" + "strings" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + azureclient "github.com/Azure/ARO-HCP/backend/pkg/azure/client" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/utils" +) + +// AzureClusterManagedIdentitiesExistenceValidation validates the existence of all managed identities defined in the cluster. +// It assumes all identities present are for recognized operators. +type AzureClusterManagedIdentitiesExistenceValidation struct { + smiClientBuilder azureclient.ServiceManagedIdentityClientBuilder +} + +func NewAzureClusterManagedIdentitiesExistenceValidation( + smiClientBuilder azureclient.ServiceManagedIdentityClientBuilder, +) *AzureClusterManagedIdentitiesExistenceValidation { + return &AzureClusterManagedIdentitiesExistenceValidation{ + smiClientBuilder: smiClientBuilder, + } +} + +func (v *AzureClusterManagedIdentitiesExistenceValidation) Name() string { + return "AzureClusterManagedIdentitiesExistenceValidation" +} + +func (v *AzureClusterManagedIdentitiesExistenceValidation) Validate(ctx context.Context, clusterSubscription *arm.Subscription, cluster *api.HCPOpenShiftCluster) error { + smiResourceID := cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ServiceManagedIdentity + clusterIdentityURL := cluster.ServiceProviderProperties.ManagedIdentitiesDataPlaneIdentityURL + // We check the existence of the Cluster's Service Managed Identity by + // attempting to retrieve the user assigned identities client using the + // service managed identity's identity credentials, which we obtain by + // requesting them via the Managed Identities Data Plane Service. If the + // service managed identity does not exist the request will fail. + uaisClient, err := v.smiClientBuilder.UserAssignedIdentitiesClient(ctx, clusterIdentityURL, smiResourceID, cluster.ID.SubscriptionID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get user assigned identities client: %w", err)) + } + + clusterUAIsProfile := &cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities + clusterOperatorsMIsResourceIDs := v.clusterOperatorsManagedIdentities(clusterUAIsProfile) + + var notFoundMIsStrs []string + for _, resourceID := range clusterOperatorsMIsResourceIDs { + _, err := uaisClient.Get(ctx, resourceID.ResourceGroupName, resourceID.Name, nil) + if azureclient.IsResourceNotFoundErr(err) { + notFoundMIsStrs = append(notFoundMIsStrs, resourceID.String()) + } + if err != nil { + // TODO is it ok to error when one of them fails to get when the error is not a resource not found error? + return utils.TrackError(fmt.Errorf("failed to get managed identity '%s': %w", resourceID, err)) + } + } + + if len(notFoundMIsStrs) > 0 { + return utils.TrackError(fmt.Errorf("managed identities not found: %s", strings.Join(notFoundMIsStrs, ", "))) + } + + return nil +} + +// clusterOperatorsManagedIdentities returns a list of the control and data plane identities defined in the cluster. +func (v *AzureClusterManagedIdentitiesExistenceValidation) clusterOperatorsManagedIdentities( + clusterUAIsProfile *api.UserAssignedIdentitiesProfile) []*azcorearm.ResourceID { + var resourceIDs []*azcorearm.ResourceID + + for _, miResourceID := range clusterUAIsProfile.ControlPlaneOperators { + resourceIDs = append(resourceIDs, miResourceID) + } + for _, miResourceID := range clusterUAIsProfile.DataPlaneOperators { + resourceIDs = append(resourceIDs, miResourceID) + } + + return resourceIDs +}