diff --git a/internal/ocm/convert.go b/internal/ocm/convert.go index 9b63f26c7c1..e19f404639b 100644 --- a/internal/ocm/convert.go +++ b/internal/ocm/convert.go @@ -224,6 +224,17 @@ func convertEnableEncryptionAtHostToCSBuilder(in coreapi.NodePoolPlatformProfile return arohcpv1alpha1.NewAzureNodePoolEncryptionAtHost().State(state) } +func buildCSOsDisk(osDisk coreapi.OSDiskProfile, storageAccountType, persistence string) *arohcpv1alpha1.AzureNodePoolOsDiskBuilder { + builder := arohcpv1alpha1.NewAzureNodePoolOsDisk(). + SizeGibibytes(int(*osDisk.SizeGiB)). + StorageAccountType(storageAccountType). + Persistence(persistence) + if osDisk.EncryptionSetID != nil { + builder.SseEncryptionSetResourceId(osDisk.EncryptionSetID.String()) + } + return builder +} + func convertClusterImageRegistryStateRPToCS(in coreapi.ClusterImageRegistryProfile) (string, error) { switch in.State { case metadataapi.ClusterImageRegistryStateDisabled: @@ -629,10 +640,7 @@ func BuildCSNodePool(ctx context.Context, nodePool *coreapi.HCPOpenShiftClusterN ResourceName(strings.ToLower(nodePool.Name)). VMSize(nodePool.Properties.Platform.VMSize). EncryptionAtHost(convertEnableEncryptionAtHostToCSBuilder(nodePool.Properties.Platform)). - OsDisk(arohcpv1alpha1.NewAzureNodePoolOsDisk(). - SizeGibibytes(int(*nodePool.Properties.Platform.OSDisk.SizeGiB)). - StorageAccountType(csDiskStorageAccountType). - Persistence(csPersistence))). + OsDisk(buildCSOsDisk(nodePool.Properties.Platform.OSDisk, csDiskStorageAccountType, csPersistence))). AvailabilityZone(nodePool.Properties.Platform.AvailabilityZone). AutoRepair(nodePool.Properties.AutoRepair) } diff --git a/internal/ocm/convert_test.go b/internal/ocm/convert_test.go index b5aae7fffc2..19938987d23 100644 --- a/internal/ocm/convert_test.go +++ b/internal/ocm/convert_test.go @@ -440,6 +440,35 @@ func TestBuildCSNodePool(t *testing.T) { ), ), }, + { + name: "passes disk encryption set ID to CS", + hcpNodePool: getHCPNodePoolResource( + func(hsc *coreapi.HCPOpenShiftClusterNodePool) { + hsc.Properties.Platform.OSDisk.EncryptionSetID = metadataapi.Must(azcorearm.ParseResourceID( + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-rg/providers/Microsoft.Compute/diskEncryptionSets/test-des")) + }, + ), + expectedCSNodePool: getBaseCSNodePoolBuilder(). + AzureNodePool(arohcpv1alpha1.NewAzureNodePool(). + ResourceName(""). + VMSize(""). + EncryptionAtHost( + arohcpv1alpha1.NewAzureNodePoolEncryptionAtHost(). + State(csEncryptionAtHostStateDisabled), + ). + OsDisk(arohcpv1alpha1.NewAzureNodePoolOsDisk(). + SizeGibibytes(64). + StorageAccountType(string(metadataapi.DiskStorageAccountTypePremium_LRS)). + Persistence("persistent"). + SseEncryptionSetResourceId("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-rg/providers/Microsoft.Compute/diskEncryptionSets/test-des"), + ), + ), + }, + { + name: "nil disk encryption set ID does not set SSE field", + hcpNodePool: getHCPNodePoolResource(), + expectedCSNodePool: getBaseCSNodePoolBuilder(), + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/test/e2e-setup/bicep/modules/customer-infra.bicep b/test/e2e-setup/bicep/modules/customer-infra.bicep index 23fcb43a557..b58dab507f5 100644 --- a/test/e2e-setup/bicep/modules/customer-infra.bicep +++ b/test/e2e-setup/bicep/modules/customer-infra.bicep @@ -22,6 +22,15 @@ param privateKeyVault bool = false @description('Assign Key Vault Crypto Officer role to the deployer on the customer KeyVault that contains etcd encryption key') param assignKeyVaultCryptoOfficer bool = false +@description('Enable soft delete on the customer Key Vault') +param enableKeyVaultSoftDelete bool = false + +@description('Enable purge protection on the customer Key Vault (requires soft delete)') +param enableKeyVaultPurgeProtection bool = false + +@description('Soft delete retention in days') +param keyVaultSoftDeleteRetentionInDays int = 7 + // // Variables // @@ -99,7 +108,9 @@ resource customerKeyVault 'Microsoft.KeyVault/vaults@2024-12-01-preview' = { location: resourceGroup().location properties: { enableRbacAuthorization: true - enableSoftDelete: false + enableSoftDelete: enableKeyVaultSoftDelete ? true : null + enablePurgeProtection: enableKeyVaultPurgeProtection ? true : null + softDeleteRetentionInDays: enableKeyVaultSoftDelete ? keyVaultSoftDeleteRetentionInDays : null tenantId: subscription().tenantId publicNetworkAccess: privateKeyVault ? 'Disabled' : 'Enabled' sku: { diff --git a/test/e2e/nodepool_osdisk_encryption.go b/test/e2e/nodepool_osdisk_encryption.go new file mode 100644 index 00000000000..e65b1c88cf0 --- /dev/null +++ b/test/e2e/nodepool_osdisk_encryption.go @@ -0,0 +1,175 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + + hcpsdk20251223preview "github.com/Azure/ARO-HCP/test/sdk/v20251223preview/resourcemanager/redhatopenshifthcp/armredhatopenshifthcp" + "github.com/Azure/ARO-HCP/test/util/framework" + "github.com/Azure/ARO-HCP/test/util/labels" + "github.com/Azure/ARO-HCP/test/util/verifiers" +) + +var _ = Describe("Nodepool OS Disk Encryption", func() { + It("should create a nodepool with customer-managed disk encryption via DES", + labels.RequireNothing, + labels.Critical, + labels.Positive, + labels.AroRpApiCompatible, + labels.MIContainers(1), + func(ctx context.Context) { + const ( + customerClusterName = "des-encrypt" + customerNodePoolName = "des-np" + ) + + tc := framework.NewTestContext() + + if tc.UsePooledIdentities() { + err := tc.AssignIdentityContainers(ctx, 1, framework.IdentityContainerAssignmentRetryInterval) + Expect(err).NotTo(HaveOccurred(), "failed to assign pooled identity containers") + } + + By("creating a resource group") + resourceGroup, err := tc.NewResourceGroup(ctx, "des-encrypt", tc.Location()) + Expect(err).NotTo(HaveOccurred(), "failed to create resource group") + + By("creating cluster parameters") + clusterParams := framework.NewDefaultClusterParams20251223() + clusterParams.ClusterName = customerClusterName + managedResourceGroupName := framework.SuffixName(*resourceGroup.Name, "-managed", 64) + clusterParams.ManagedResourceGroupName = managedResourceGroupName + + By("creating customer resources (infrastructure and managed identities)") + clusterParams, err = tc.CreateClusterCustomerResources20251223(ctx, + resourceGroup, + clusterParams, + map[string]interface{}{ + "assignKeyVaultCryptoOfficer": true, + "enableKeyVaultSoftDelete": true, + "enableKeyVaultPurgeProtection": true, + "keyVaultSoftDeleteRetentionInDays": 7, + }, + TestArtifactsFS, + framework.RBACScopeResourceGroup, + ) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster customer resources") + + By("resolving service managed identity principal ID") + Expect(clusterParams.UserAssignedIdentitiesProfile).NotTo(BeNil(), "cluster params missing UserAssignedIdentitiesProfile") + Expect(clusterParams.UserAssignedIdentitiesProfile.ServiceManagedIdentity).NotTo(BeNil(), "cluster params missing ServiceManagedIdentity resource ID") + + serviceMIResourceID, err := azcorearm.ParseResourceID(*clusterParams.UserAssignedIdentitiesProfile.ServiceManagedIdentity) + Expect(err).NotTo(HaveOccurred(), "failed to parse service managed identity resource ID") + + subscriptionID, err := tc.SubscriptionID(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID") + + creds, err := tc.AzureCredential() + Expect(err).NotTo(HaveOccurred(), "failed to get Azure credentials") + + msiClientFactory, err := armmsi.NewClientFactory(subscriptionID, creds, nil) + Expect(err).NotTo(HaveOccurred(), "failed to create MSI client factory") + + serviceMI, err := msiClientFactory.NewUserAssignedIdentitiesClient().Get(ctx, serviceMIResourceID.ResourceGroupName, serviceMIResourceID.Name, nil) + Expect(err).NotTo(HaveOccurred(), "failed to get service managed identity") + Expect(serviceMI.Properties.PrincipalID).NotTo(BeNil(), "service managed identity has no principal ID") + + By("resolving cluster-api-azure managed identity principal ID") + Expect(clusterParams.UserAssignedIdentitiesProfile.ControlPlaneOperators).NotTo(BeNil(), "cluster params missing ControlPlaneOperators") + clusterAPIAzureResourceIDStr := clusterParams.UserAssignedIdentitiesProfile.ControlPlaneOperators["cluster-api-azure"] + Expect(clusterAPIAzureResourceIDStr).NotTo(BeNil(), "cluster params missing cluster-api-azure identity") + clusterAPIAzureResourceID, err := azcorearm.ParseResourceID(*clusterAPIAzureResourceIDStr) + Expect(err).NotTo(HaveOccurred(), "failed to parse cluster-api-azure resource ID") + clusterAPIAzureMI, err := msiClientFactory.NewUserAssignedIdentitiesClient().Get(ctx, clusterAPIAzureResourceID.ResourceGroupName, clusterAPIAzureResourceID.Name, nil) + Expect(err).NotTo(HaveOccurred(), "failed to get cluster-api-azure managed identity") + Expect(clusterAPIAzureMI.Properties.PrincipalID).NotTo(BeNil(), "cluster-api-azure managed identity has no principal ID") + + By("creating disk encryption set backed by KeyVault") + desResourceID, err := tc.CreateDiskEncryptionSet(ctx, *resourceGroup.Name, clusterParams.KeyVaultName, customerClusterName, tc.Location(), *serviceMI.Properties.PrincipalID, *clusterAPIAzureMI.Properties.PrincipalID) + Expect(err).NotTo(HaveOccurred(), "failed to create disk encryption set") + + By("creating the HCP cluster") + err = tc.CreateHCPClusterFromParam20251223(ctx, + GinkgoLogr, + *resourceGroup.Name, + clusterParams, + nil, + framework.ClusterCreationTimeout, + ) + Expect(err).NotTo(HaveOccurred(), "failed to create HCP cluster %s", customerClusterName) + + By("creating the nodepool with disk encryption set") + nodePoolParams := framework.NewDefaultNodePoolParams20251223() + nodePoolParams.ClusterName = customerClusterName + nodePoolParams.NodePoolName = customerNodePoolName + nodePoolParams.EncryptionSetID = desResourceID + + err = tc.CreateNodePoolFromParam20251223(ctx, + GinkgoLogr, + *resourceGroup.Name, + managedResourceGroupName, + customerClusterName, + nodePoolParams, + framework.NodePoolCreationTimeout, + ) + Expect(err).NotTo(HaveOccurred(), "failed to create nodepool %s with DES", customerNodePoolName) + + By("verifying nodepool ARM resource has encryptionSetId") + created, err := framework.GetNodePool20251223(ctx, + tc.Get20251223ClientFactoryOrDie(ctx).NewNodePoolsClient(), + *resourceGroup.Name, + customerClusterName, + customerNodePoolName, + ) + Expect(err).NotTo(HaveOccurred(), "failed to get nodepool %s", customerNodePoolName) + Expect(created.Properties).ToNot(BeNil(), "nodepool Properties was nil") + Expect(created.Properties.ProvisioningState).ToNot(BeNil(), "nodepool ProvisioningState was nil") + Expect(*created.Properties.ProvisioningState).To(Equal(hcpsdk20251223preview.ProvisioningStateSucceeded), "nodepool %s should be Succeeded", customerNodePoolName) + Expect(created.Properties.Platform).ToNot(BeNil(), "nodepool Platform was nil") + Expect(created.Properties.Platform.OSDisk).ToNot(BeNil(), "nodepool OSDisk was nil") + Expect(created.Properties.Platform.OSDisk.EncryptionSetID).ToNot(BeNil(), + "nodepool OSDisk.EncryptionSetID should be set") + Expect(strings.EqualFold(*created.Properties.Platform.OSDisk.EncryptionSetID, desResourceID)).To(BeTrue(), + "nodepool OSDisk.EncryptionSetID should match the DES resource ID") + + By("getting credentials to verify cluster health") + adminRESTConfig, err := tc.GetAdminRESTConfigForHCPCluster20240610( + ctx, + tc.Get20240610ClientFactoryOrDie(ctx).NewHcpOpenShiftClustersClient(), + *resourceGroup.Name, + customerClusterName, + framework.GetAdminRESTConfigTimeout, + ) + Expect(err).NotTo(HaveOccurred(), "failed to get admin REST config") + + By("verifying cluster health, node readiness, and VM OS disk encryption in parallel") + computeFactory := tc.GetARMComputeClientFactoryOrDie(ctx) + err = verifiers.VerifyHCPCluster(ctx, adminRESTConfig, + verifiers.VerifyNodeCount(customerClusterName, int(nodePoolParams.Replicas)), + verifiers.VerifyNodesReady(), + verifiers.VerifyVMOSDiskCustomerEncryption(computeFactory, managedResourceGroupName, customerNodePoolName, desResourceID), + ) + Expect(err).NotTo(HaveOccurred(), "cluster verification failed") + }) +}) diff --git a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_dev_cd_check_paralleldev_cd_check_parallel.txt b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_dev_cd_check_paralleldev_cd_check_parallel.txt index ce1526889e7..bc94de22e3d 100644 --- a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_dev_cd_check_paralleldev_cd_check_parallel.txt +++ b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_dev_cd_check_paralleldev_cd_check_parallel.txt @@ -83,6 +83,7 @@ Engineering should be able to retrieve kusto logs for a cluster and services Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled Customer should be able to update node pool labels and taints +Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES Customer should be able to update nodepool replicas and autoscaling Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest diff --git a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_integration_parallelintegration_parallel.txt b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_integration_parallelintegration_parallel.txt index 40e335122a9..1db17105bbb 100644 --- a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_integration_parallelintegration_parallel.txt +++ b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_integration_parallelintegration_parallel.txt @@ -79,6 +79,7 @@ Customer should be able to create a cluster with default autoscaling and a nodep Customer should respect cluster-wide node limits with nodepool autoscaling Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled Customer should be able to update node pool labels and taints +Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest diff --git a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_prod_parallelprod_parallel.txt b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_prod_parallelprod_parallel.txt index 59e13455a25..6fcf7a15577 100644 --- a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_prod_parallelprod_parallel.txt +++ b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_prod_parallelprod_parallel.txt @@ -79,6 +79,7 @@ Customer should be able to create a cluster with default autoscaling and a nodep Customer should respect cluster-wide node limits with nodepool autoscaling Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled Customer should be able to update node pool labels and taints +Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest diff --git a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallel_01rp_api_compat_all_parallel_development.txt b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallel_01rp_api_compat_all_parallel_development.txt index d5f1d6a92df..8e39cc971ba 100644 --- a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallel_01rp_api_compat_all_parallel_development.txt +++ b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallel_01rp_api_compat_all_parallel_development.txt @@ -88,6 +88,7 @@ Customer should be able to create a cluster with default autoscaling and a nodep Customer should respect cluster-wide node limits with nodepool autoscaling Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled Customer should be able to update node pool labels and taints +Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES Customer should be able to update nodepool replicas and autoscaling Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest diff --git a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallelrp_api_compat_all_parallel.txt b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallelrp_api_compat_all_parallel.txt index cd93de9e223..a110ba946f1 100644 --- a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallelrp_api_compat_all_parallel.txt +++ b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallelrp_api_compat_all_parallel.txt @@ -76,6 +76,7 @@ Customer should be able to create a cluster with default autoscaling and a nodep Customer should respect cluster-wide node limits with nodepool autoscaling Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled Customer should be able to update node pool labels and taints +Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest diff --git a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_stage_parallelstage_parallel.txt b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_stage_parallelstage_parallel.txt index 59e13455a25..6fcf7a15577 100644 --- a/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_stage_parallelstage_parallel.txt +++ b/test/testdata/zz_fixture_TestMainListSuitesForEachSuite_stage_parallelstage_parallel.txt @@ -79,6 +79,7 @@ Customer should be able to create a cluster with default autoscaling and a nodep Customer should respect cluster-wide node limits with nodepool autoscaling Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled Customer should be able to update node pool labels and taints +Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest diff --git a/test/util/framework/disk_encryption_set_helper.go b/test/util/framework/disk_encryption_set_helper.go new file mode 100644 index 00000000000..70db33fc033 --- /dev/null +++ b/test/util/framework/disk_encryption_set_helper.go @@ -0,0 +1,152 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys" +) + +const ( + desKeyName = "des-encryption-key" + + kvCryptoServiceEncryptionUserRoleID = "e147488a-f6f5-4113-8e2d-b22465e65bf6" + readerRoleID = "acdd72a7-3385-48ef-bd42-f606fba81ae7" +) + +func (tc *perItOrDescribeTestContext) CreateDiskEncryptionSet(ctx context.Context, resourceGroupName, keyVaultName, clusterName, location string, readerPrincipalIDs ...string) (string, error) { + startTime := time.Now() + defer func() { + tc.RecordTestStep("Create disk encryption set", startTime, time.Now()) + }() + + subscriptionID, err := tc.SubscriptionID(ctx) + if err != nil { + return "", fmt.Errorf("failed to get subscription ID: %w", err) + } + + creds, err := tc.AzureCredential() + if err != nil { + return "", fmt.Errorf("failed to get Azure credentials: %w", err) + } + + keyVaultURL := fmt.Sprintf("https://%s.vault.azure.net/", keyVaultName) + keyVaultResourceID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.KeyVault/vaults/%s", subscriptionID, resourceGroupName, keyVaultName) + + keyClient, err := azkeys.NewClient(keyVaultURL, creds, nil) + if err != nil { + return "", fmt.Errorf("failed to create Key Vault keys client: %w", err) + } + + createKeyResp, err := keyClient.CreateKey(ctx, desKeyName, azkeys.CreateKeyParameters{ + Kty: to.Ptr(azkeys.KeyTypeRSA), + KeySize: to.Ptr(int32(2048)), + }, nil) + if err != nil { + return "", fmt.Errorf("failed to create DES encryption key: %w", err) + } + if createKeyResp.Key == nil || createKeyResp.Key.KID == nil { + return "", fmt.Errorf("created key response or KID was nil") + } + + keyURL := string(*createKeyResp.Key.KID) + ginkgo.GinkgoLogr.Info("Created DES encryption key", "keyVaultName", keyVaultName, "keyName", desKeyName, "keyURL", keyURL) + + desName := fmt.Sprintf("%s-des", clusterName) + desClient := tc.GetARMComputeClientFactoryOrDie(ctx).NewDiskEncryptionSetsClient() + + poller, err := desClient.BeginCreateOrUpdate(ctx, resourceGroupName, desName, armcompute.DiskEncryptionSet{ + Location: &location, + Identity: &armcompute.EncryptionSetIdentity{ + Type: to.Ptr(armcompute.DiskEncryptionSetIdentityTypeSystemAssigned), + }, + Properties: &armcompute.EncryptionSetProperties{ + ActiveKey: &armcompute.KeyForDiskEncryptionSet{ + KeyURL: &keyURL, + SourceVault: &armcompute.SourceVault{ + ID: &keyVaultResourceID, + }, + }, + EncryptionType: to.Ptr(armcompute.DiskEncryptionSetTypeEncryptionAtRestWithCustomerKey), + }, + }, nil) + if err != nil { + return "", fmt.Errorf("failed to begin creating disk encryption set: %w", err) + } + + desResult, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return "", fmt.Errorf("failed to create disk encryption set: %w", err) + } + + desResourceID := *desResult.ID + ginkgo.GinkgoLogr.Info("Created disk encryption set", "name", desName, "resourceID", desResourceID) + + if desResult.Identity == nil || desResult.Identity.PrincipalID == nil { + return "", fmt.Errorf("disk encryption set has no system-assigned identity principal ID") + } + + roleAssignmentsClient, err := armauthorization.NewRoleAssignmentsClient(subscriptionID, creds, nil) + if err != nil { + return "", fmt.Errorf("failed to create role assignments client: %w", err) + } + + kvCryptoRoleDefID := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/%s", subscriptionID, kvCryptoServiceEncryptionUserRoleID) + kvRoleAssignmentName := guid(keyVaultResourceID, *desResult.Identity.PrincipalID, kvCryptoRoleDefID) + + kvRoleResult, err := roleAssignmentsClient.Create(ctx, keyVaultResourceID, kvRoleAssignmentName, armauthorization.RoleAssignmentCreateParameters{ + Properties: &armauthorization.RoleAssignmentProperties{ + PrincipalID: desResult.Identity.PrincipalID, + RoleDefinitionID: &kvCryptoRoleDefID, + PrincipalType: to.Ptr(armauthorization.PrincipalTypeServicePrincipal), + }, + }, nil) + if err != nil { + return "", fmt.Errorf("failed to assign Key Vault Crypto Service Encryption User to DES identity: %w", err) + } + tc.trackRoleAssignment(*kvRoleResult.ID) + ginkgo.GinkgoLogr.Info("Assigned KV Crypto Service Encryption User to DES identity", "scope", keyVaultResourceID, "principalID", *desResult.Identity.PrincipalID) + + readerRoleDefID := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/%s", subscriptionID, readerRoleID) + for _, principalID := range readerPrincipalIDs { + if principalID == "" { + continue + } + pid := principalID + desReaderAssignmentName := guid(desResourceID, pid, readerRoleDefID) + desReaderResult, err := roleAssignmentsClient.Create(ctx, desResourceID, desReaderAssignmentName, armauthorization.RoleAssignmentCreateParameters{ + Properties: &armauthorization.RoleAssignmentProperties{ + PrincipalID: &pid, + RoleDefinitionID: &readerRoleDefID, + PrincipalType: to.Ptr(armauthorization.PrincipalTypeServicePrincipal), + }, + }, nil) + if err != nil { + return "", fmt.Errorf("failed to assign Reader to principal %s on DES: %w", pid, err) + } + tc.trackRoleAssignment(*desReaderResult.ID) + ginkgo.GinkgoLogr.Info("Assigned Reader to principal on DES", "scope", desResourceID, "principalID", pid) + } + + return desResourceID, nil +} diff --git a/test/util/framework/helpers_v20240610preview.go b/test/util/framework/helpers_v20240610preview.go index 04a79a017a7..f28da6fbab0 100644 --- a/test/util/framework/helpers_v20240610preview.go +++ b/test/util/framework/helpers_v20240610preview.go @@ -93,6 +93,7 @@ type NodePoolParams20240610 struct { AutoScaling *NodePoolAutoScalingParams AvailabilityZone string Tags map[string]*string + EncryptionSetID string } // ======================================================================== @@ -1261,6 +1262,10 @@ func BuildNodePoolFromParams20240610( }, } + if parameters.EncryptionSetID != "" { + nodePool.Properties.Platform.OSDisk.EncryptionSetID = to.Ptr(parameters.EncryptionSetID) + } + if parameters.AutoScaling != nil { nodePool.Properties.AutoScaling = &hcpsdk20240610preview.NodePoolAutoScaling{ Min: to.Ptr(parameters.AutoScaling.Min), diff --git a/test/util/framework/helpers_v20251223preview.go b/test/util/framework/helpers_v20251223preview.go index fc67bafd436..b71b7e885d0 100644 --- a/test/util/framework/helpers_v20251223preview.go +++ b/test/util/framework/helpers_v20251223preview.go @@ -88,6 +88,7 @@ type NodePoolParams20251223 struct { AvailabilityZone string AutoRepair bool Tags map[string]*string + EncryptionSetID string } // --------------------------------------------------------------------------- @@ -433,6 +434,10 @@ func BuildNodePoolFromParams20251223( }, } + if parameters.EncryptionSetID != "" { + nodePool.Properties.Platform.OSDisk.EncryptionSetID = to.Ptr(parameters.EncryptionSetID) + } + if parameters.AutoScaling != nil { nodePool.Properties.AutoScaling = &hcpsdk20251223preview.NodePoolAutoScaling{ Min: to.Ptr(parameters.AutoScaling.Min), diff --git a/test/util/framework/helpers_v20260630preview.go b/test/util/framework/helpers_v20260630preview.go index ebe6e0aea9b..0e4eefdb171 100644 --- a/test/util/framework/helpers_v20260630preview.go +++ b/test/util/framework/helpers_v20260630preview.go @@ -90,6 +90,7 @@ type NodePoolParams20260630 struct { AvailabilityZone string AutoRepair bool Tags map[string]*string + EncryptionSetID string } // --------------------------------------------------------------------------- @@ -685,6 +686,10 @@ func BuildNodePoolFromParams20260630( }, } + if parameters.EncryptionSetID != "" { + nodePool.Properties.Platform.OSDisk.EncryptionSetID = to.Ptr(parameters.EncryptionSetID) + } + if parameters.AutoScaling != nil { nodePool.Properties.AutoScaling = &hcpsdk20260630preview.NodePoolAutoScaling{ Min: to.Ptr(parameters.AutoScaling.Min), diff --git a/test/util/framework/helpers_v20260901preview.go b/test/util/framework/helpers_v20260901preview.go index 98e4c52495e..5cca9ccaef5 100644 --- a/test/util/framework/helpers_v20260901preview.go +++ b/test/util/framework/helpers_v20260901preview.go @@ -95,6 +95,7 @@ type NodePoolParams20260901 struct { AvailabilityZone string AutoRepair bool Tags map[string]*string + EncryptionSetID string } // --- Functions from deployment_params.go --- @@ -662,6 +663,10 @@ func BuildNodePoolFromParams20260901( }, } + if parameters.EncryptionSetID != "" { + nodePool.Properties.Platform.OSDisk.EncryptionSetID = to.Ptr(parameters.EncryptionSetID) + } + if parameters.AutoScaling != nil { nodePool.Properties.AutoScaling = &hcpsdk20260901preview.NodePoolAutoScaling{ Min: to.Ptr(parameters.AutoScaling.Min), diff --git a/test/util/framework/per_test_framework.go b/test/util/framework/per_test_framework.go index 0f48e2f65f4..3e6cdc2e298 100644 --- a/test/util/framework/per_test_framework.go +++ b/test/util/framework/per_test_framework.go @@ -268,6 +268,12 @@ func (tc *perItOrDescribeTestContext) deleteCreatedResources(ctx context.Context tc.contextLock.RUnlock() ginkgo.GinkgoLogr.Info("deleting created resources") + if subscriptionID, err := tc.SubscriptionID(ctx); err != nil { + ginkgo.GinkgoLogr.Error(err, "failed to get subscription ID for role assignment cleanup") + } else if err := tc.cleanupRoleAssignments(ctx, subscriptionID); err != nil { + ginkgo.GinkgoLogr.Error(err, "failed to cleanup role assignments before resource group deletion") + } + opts := CleanupResourceGroupsOptions{ ResourceGroupNames: resourceGroupNames, Timeout: 60 * time.Minute, @@ -804,6 +810,11 @@ func (tc *perItOrDescribeTestContext) purgeDeletedKeyVaultsInResourceGroup(ctx c if !strings.Contains(strings.ToLower(*deleted.Properties.VaultID), rgMarker) { continue } + if keyVaultPurgeProtected(deleted.Properties) { + ginkgo.GinkgoLogr.Info("skipping purge of soft-deleted key vault with purge protection enabled", + "keyVault", *deleted.Name, "resourceGroup", resourceGroupName) + continue + } ginkgo.GinkgoLogr.Info("purging soft-deleted key vault", "keyVault", *deleted.Name, "location", *deleted.Properties.Location, "resourceGroup", resourceGroupName) poller, err := vaultsClient.BeginPurgeDeleted(ctx, *deleted.Name, *deleted.Properties.Location, nil) @@ -829,6 +840,12 @@ func (tc *perItOrDescribeTestContext) purgeDeletedKeyVaultsInResourceGroup(ctx c } } +// keyVaultPurgeProtected reports whether a soft-deleted vault has purge +// protection enabled, meaning the purge API will reject any purge attempt. +func keyVaultPurgeProtected(props *armkeyvault.DeletedVaultProperties) bool { + return props != nil && props.PurgeProtectionEnabled != nil && *props.PurgeProtectionEnabled +} + // isKeyVaultNotFound reports whether err is an Azure 404 response, which for a // purge means the vault is already gone (already purged or soft-delete window // expired) and can be treated as a successful no-op. diff --git a/test/util/framework/per_test_framework_test.go b/test/util/framework/per_test_framework_test.go index 75ac146f0d0..85494ff83cf 100644 --- a/test/util/framework/per_test_framework_test.go +++ b/test/util/framework/per_test_framework_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" ) func TestIsResourceGroupNotFoundError(t *testing.T) { @@ -148,6 +149,48 @@ func TestIsKeyVaultNotFound(t *testing.T) { } } +func TestKeyVaultPurgeProtected(t *testing.T) { + t.Parallel() + + trueVal := true + falseVal := false + + tests := []struct { + name string + props *armkeyvault.DeletedVaultProperties + want bool + }{ + { + name: "nil properties", + props: nil, + want: false, + }, + { + name: "nil PurgeProtectionEnabled", + props: &armkeyvault.DeletedVaultProperties{PurgeProtectionEnabled: nil}, + want: false, + }, + { + name: "purge protection disabled", + props: &armkeyvault.DeletedVaultProperties{PurgeProtectionEnabled: &falseVal}, + want: false, + }, + { + name: "purge protection enabled", + props: &armkeyvault.DeletedVaultProperties{PurgeProtectionEnabled: &trueVal}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := keyVaultPurgeProtected(tt.props) + assert.Equal(t, tt.want, got) + }) + } +} + func TestIsIgnorableResourceGroupCleanupError(t *testing.T) { t.Parallel() diff --git a/test/util/verifiers/osdisk_encryption.go b/test/util/verifiers/osdisk_encryption.go new file mode 100644 index 00000000000..2eca045d08c --- /dev/null +++ b/test/util/verifiers/osdisk_encryption.go @@ -0,0 +1,120 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package verifiers + +import ( + "context" + "errors" + "fmt" + "strings" + + "k8s.io/client-go/rest" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" +) + +type verifyVMOSDiskCustomerEncryption struct { + computeFactory *armcompute.ClientFactory + managedResourceGroup string + nodePoolName string + expectedDESResourceID string +} + +func (v verifyVMOSDiskCustomerEncryption) Name() string { + return fmt.Sprintf("VerifyVMOSDiskCustomerEncryption(nodePool=%s)", v.nodePoolName) +} + +func (v verifyVMOSDiskCustomerEncryption) Verify(ctx context.Context, _ *rest.Config) error { + vmClient := v.computeFactory.NewVirtualMachinesClient() + disksClient := v.computeFactory.NewDisksClient() + + var vms []*armcompute.VirtualMachine + pager := vmClient.NewListPager(v.managedResourceGroup, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return fmt.Errorf("failed to list VMs in managed resource group %q: %w", v.managedResourceGroup, err) + } + vms = append(vms, page.Value...) + } + + var workerVMs []*armcompute.VirtualMachine + for _, vm := range vms { + if vm.Name != nil && strings.Contains(*vm.Name, v.nodePoolName) { + workerVMs = append(workerVMs, vm) + } + } + if len(workerVMs) == 0 { + return fmt.Errorf("no VMs found for nodepool %s in managed resource group %s", v.nodePoolName, v.managedResourceGroup) + } + + var errs []error + for _, vm := range workerVMs { + if err := v.verifyVM(ctx, disksClient, vm); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return fmt.Errorf("OS disk encryption verification failed for %d/%d VMs: %w", len(errs), len(workerVMs), errors.Join(errs...)) + } + return nil +} + +func (v verifyVMOSDiskCustomerEncryption) verifyVM(ctx context.Context, disksClient *armcompute.DisksClient, vm *armcompute.VirtualMachine) error { + if vm.Name == nil { + return fmt.Errorf("VM has no name") + } + vmName := *vm.Name + + if vm.Properties == nil || vm.Properties.StorageProfile == nil || vm.Properties.StorageProfile.OSDisk == nil || vm.Properties.StorageProfile.OSDisk.ManagedDisk == nil { + return fmt.Errorf("VM %s missing storage profile or managed disk", vmName) + } + + osDiskName := vm.Properties.StorageProfile.OSDisk.Name + if osDiskName == nil { + return fmt.Errorf("VM %s OS disk has no name", vmName) + } + + disk, err := disksClient.Get(ctx, v.managedResourceGroup, *osDiskName, nil) + if err != nil { + return fmt.Errorf("failed to get disk %s for VM %s: %w", *osDiskName, vmName, err) + } + + if disk.Properties == nil || disk.Properties.Encryption == nil { + return fmt.Errorf("disk %s for VM %s has no encryption properties", *osDiskName, vmName) + } + if disk.Properties.Encryption.Type == nil { + return fmt.Errorf("disk %s for VM %s has no encryption type", *osDiskName, vmName) + } + if *disk.Properties.Encryption.Type != armcompute.EncryptionTypeEncryptionAtRestWithCustomerKey { + return fmt.Errorf("disk %s for VM %s has encryption type %s, expected EncryptionAtRestWithCustomerKey", *osDiskName, vmName, *disk.Properties.Encryption.Type) + } + if disk.Properties.Encryption.DiskEncryptionSetID == nil { + return fmt.Errorf("disk %s for VM %s has no DiskEncryptionSetID", *osDiskName, vmName) + } + if !strings.EqualFold(*disk.Properties.Encryption.DiskEncryptionSetID, v.expectedDESResourceID) { + return fmt.Errorf("disk %s for VM %s DiskEncryptionSetID mismatch: got %s, expected %s", *osDiskName, vmName, *disk.Properties.Encryption.DiskEncryptionSetID, v.expectedDESResourceID) + } + return nil +} + +func VerifyVMOSDiskCustomerEncryption(computeFactory *armcompute.ClientFactory, managedResourceGroup, nodePoolName, expectedDESResourceID string) HostedClusterVerifier { + return verifyVMOSDiskCustomerEncryption{ + computeFactory: computeFactory, + managedResourceGroup: managedResourceGroup, + nodePoolName: nodePoolName, + expectedDESResourceID: expectedDESResourceID, + } +}