feat: add KSM HCP E2E test with rendered config support

Makefile

@@ -197,6 +197,7 @@ e2e-local/setup:
 
 e2e-local/run: $(ARO_HCP_TESTS)
 	export LOCATION="$${LOCATION:-westus3}"; \
+	export RENDERED_CONFIG="$${RENDERED_CONFIG:-$(CURDIR)/config/rendered/dev/$(DEPLOY_ENV)/$$LOCATION.yaml}"; \
 	export AROHCP_ENV="development"; \
 	export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \
 	export ARTIFACT_DIR=$${ARTIFACT_DIR:-_artifacts}; \

nonlocal-e2e-specs.txt

@@ -8,6 +8,7 @@
   "Successfully lists clusters filtered by resource group name",
   "creates a cluster and fails to update its name with a PATCH request",
   "creates and deletes a GPU nodepool in a single cluster",
+  "kube_node_info metrics should be present in Azure Monitor for the happy-path cluster",
   "should allow pods with images from allowed registries and have a valid allowlist",
   "should be able to create an HCP cluster then delete it by deleting the customer resource group",
   "should be able to delete an HCP cluster whose managed identities were deleted first",

test/E2ELocal.mk

@@ -23,6 +23,7 @@ SNAPSHOT_RENDERED_CONFIG := $(shell mktemp)
 e2e-local/run-test: $(ARO_HCP_TESTS)
 	$(MAKE) -C $(DIR) -f $(THIS) .e2e-local/setup
 	export LOCATION="$${LOCATION:-${REGION}}"; \
+	export RENDERED_CONFIG="$${RENDERED_CONFIG:-$(DIR)/../config/rendered/dev/$(DEPLOY_ENV)/$$LOCATION.yaml}"; \
 	export AROHCP_ENV="development"; \
 	export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \
 	export AZURE_TENANT_ID="$$(az account show --output tsv --query 'tenantId')"; \

test/cmd/aro-hcp-tests/custom-link-tools/options.go

@@ -33,7 +33,7 @@ import (
 	"k8s.io/utils/clock"
 
 	"github.com/Azure/ARO-HCP/internal/utils"
-	"github.com/Azure/ARO-HCP/test/cmd/aro-hcp-tests/internal/testutil"
+	"github.com/Azure/ARO-HCP/test/util/testconfig"
 	"github.com/Azure/ARO-HCP/test/util/timing"
 	"github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
@@ -133,32 +133,32 @@ func (o *RawOptions) Validate() (*ValidatedOptions, error) {
 }
 
 func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) {
-	cfg, err := testutil.LoadRenderedConfig(o.RenderedConfig)
+	cfg, err := testconfig.LoadRenderedConfig(o.RenderedConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	svcClusterName, err := testutil.ConfigGetString(cfg, "svc.aks.name")
+	svcClusterName, err := testconfig.ConfigGetString(cfg, "svc.aks.name")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get svc cluster name from config: %w", err)
 	}
-	mgmtClusterName, err := testutil.ConfigGetString(cfg, "mgmt.aks.name")
+	mgmtClusterName, err := testconfig.ConfigGetString(cfg, "mgmt.aks.name")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get mgmt cluster name from config: %w", err)
 	}
-	kustoName, err := testutil.ConfigGetString(cfg, "kusto.kustoName")
+	kustoName, err := testconfig.ConfigGetString(cfg, "kusto.kustoName")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get kusto name from config: %w", err)
 	}
-	kustoRegion, err := testutil.ConfigGetString(cfg, "kusto.location")
+	kustoRegion, err := testconfig.ConfigGetString(cfg, "kusto.location")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get kusto location from config: %w", err)
 	}
-	serviceLogsDB, err := testutil.ConfigGetString(cfg, "kusto.serviceLogsDatabase")
+	serviceLogsDB, err := testconfig.ConfigGetString(cfg, "kusto.serviceLogsDatabase")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get service logs database from config: %w", err)
 	}
-	hcpLogsDB, err := testutil.ConfigGetString(cfg, "kusto.hostedControlPlaneLogsDatabase")
+	hcpLogsDB, err := testconfig.ConfigGetString(cfg, "kusto.hostedControlPlaneLogsDatabase")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get hosted control plane logs database from config: %w", err)
 	}

test/cmd/aro-hcp-tests/gather-observability/options.go

@@ -39,8 +39,9 @@ import (
 
 	"github.com/Azure/ARO-HCP/internal/api/metadataapi"
 	"github.com/Azure/ARO-HCP/internal/utils"
-	"github.com/Azure/ARO-HCP/test/cmd/aro-hcp-tests/internal/testutil"
 	"github.com/Azure/ARO-HCP/test/util/junit"
+	promutil "github.com/Azure/ARO-HCP/test/util/prometheus"
+	"github.com/Azure/ARO-HCP/test/util/testconfig"
 	"github.com/Azure/ARO-HCP/test/util/timing"
 )
 
@@ -130,20 +131,20 @@ func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) {
 		return nil, fmt.Errorf("failed to create output directory %s: %w", o.OutputDir, err)
 	}
 
-	cfg, err := testutil.LoadRenderedConfig(o.RenderedConfig)
+	cfg, err := testconfig.LoadRenderedConfig(o.RenderedConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	regionRG, err := testutil.ConfigGetString(cfg, "regionRG")
+	regionRG, err := testconfig.ConfigGetString(cfg, "regionRG")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get regionRG from config: %w", err)
 	}
-	svcWorkspace, err := testutil.ConfigGetString(cfg, "monitoring.svcWorkspaceName")
+	svcWorkspace, err := testconfig.ConfigGetString(cfg, "monitoring.svcWorkspaceName")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get monitoring.svcWorkspaceName from config: %w", err)
 	}
-	hcpWorkspace, err := testutil.ConfigGetString(cfg, "monitoring.hcpWorkspaceName")
+	hcpWorkspace, err := testconfig.ConfigGetString(cfg, "monitoring.hcpWorkspaceName")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get monitoring.hcpWorkspaceName from config: %w", err)
 	}
@@ -152,7 +153,7 @@ func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) {
 	// the name from frontend.cosmosDB.name (see region.bicep / output-region.bicep).
 	// Its platform metrics (NormalizedRUConsumption, AutoscaledRU, ...) are queried
 	// via the Azure Monitor metrics API rather than Prometheus.
-	cosmosDBName, err := testutil.ConfigGetString(cfg, "frontend.cosmosDB.name")
+	cosmosDBName, err := testconfig.ConfigGetString(cfg, "frontend.cosmosDB.name")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get frontend.cosmosDB.name from config: %w", err)
 	}
@@ -248,13 +249,13 @@ func buildCosmosAutoscaleMaxLookup(cfg configtypes.Configuration) (autoscaleMaxL
 	}
 	byContainer := make(map[string]float64, len(fixed))
 	for container, path := range fixed {
-		v, err := testutil.ConfigGetInt(cfg, path)
+		v, err := testconfig.ConfigGetInt(cfg, path)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get %s from config: %w", path, err)
 		}
 		byContainer[container] = float64(v)
 	}
-	manifestsMax, err := testutil.ConfigGetInt(cfg, "kubeApplier.cosmosContainerMaxScale")
+	manifestsMax, err := testconfig.ConfigGetInt(cfg, "kubeApplier.cosmosContainerMaxScale")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get kubeApplier.cosmosContainerMaxScale from config: %w", err)
 	}
@@ -453,7 +454,7 @@ func (o Options) runQueries(ctx context.Context, workspaces map[string]*workspac
 
 				logger.Info("executing PromQL query", "panel", panel.Title, "title", q.Title, "workspace", q.Workspace)
 
-				resp, err := queryRange(ctx, httpClient, o.cred, endpoint, q.Query, o.TimeWindow.Start, o.TimeWindow.End, q.Step)
+				resp, err := promutil.QueryRange(ctx, httpClient, o.cred, endpoint, q.Query, o.TimeWindow.Start, o.TimeWindow.End, q.Step)
 				if err != nil {
 					logger.Error(err, "PromQL query failed", "title", q.Title)
 					queryErr = err.Error()

test/cmd/aro-hcp-tests/gather-observability/promql.go

@@ -15,25 +15,15 @@
 package gatherobservability
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"slices"
-	"strconv"
 	"strings"
-	"time"
 
 	_ "embed"
 
 	"sigs.k8s.io/yaml"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor"
+	promutil "github.com/Azure/ARO-HCP/test/util/prometheus"
 )
 
 // QueriesConfig holds panels of grouped PromQL queries to run against Azure
@@ -141,24 +131,13 @@ type QuerySpec struct {
 }
 
 // PrometheusResponse is the top-level Prometheus HTTP API response.
-type PrometheusResponse struct {
-	Status    string         `json:"status"`
-	Data      PrometheusData `json:"data"`
-	ErrorType string         `json:"errorType,omitempty"`
-	Error     string         `json:"error,omitempty"`
-}
+type PrometheusResponse = promutil.Response
 
 // PrometheusData holds the result set from a query_range call.
-type PrometheusData struct {
-	ResultType string             `json:"resultType"`
-	Result     []PrometheusResult `json:"result"`
-}
+type PrometheusData = promutil.Data
 
 // PrometheusResult is a single timeseries returned by query_range.
-type PrometheusResult struct {
-	Metric map[string]string `json:"metric"`
-	Values [][]any           `json:"values"` // each element is [unix_timestamp_float, "string_value"]
-}
+type PrometheusResult = promutil.Result
 
 //go:embed queries.yaml
 var defaultQueriesYAML []byte
@@ -276,77 +255,3 @@ func sortedMetricResources() []string {
 	slices.Sort(out)
 	return out
 }
-
-// lookupPrometheusEndpoint retrieves the Prometheus query endpoint for an
-// Azure Monitor workspace using the ARM SDK.
-func lookupPrometheusEndpoint(ctx context.Context, cred azcore.TokenCredential, subscriptionID, resourceGroup, workspaceName string) (string, error) {
-	client, err := armmonitor.NewAzureMonitorWorkspacesClient(subscriptionID, cred, nil)
-	if err != nil {
-		return "", fmt.Errorf("failed to create monitor workspaces client: %w", err)
-	}
-	resp, err := client.Get(ctx, resourceGroup, workspaceName, nil)
-	if err != nil {
-		return "", fmt.Errorf("failed to get workspace %s: %w", workspaceName, err)
-	}
-	if resp.Properties == nil || resp.Properties.Metrics == nil || resp.Properties.Metrics.PrometheusQueryEndpoint == nil {
-		return "", fmt.Errorf("workspace %s has no Prometheus query endpoint", workspaceName)
-	}
-	return *resp.Properties.Metrics.PrometheusQueryEndpoint, nil
-}
-
-// queryRange executes a Prometheus query_range request against an Azure Monitor
-// Prometheus endpoint using bearer token authentication. The caller should pass
-// a shared *http.Client to amortize connection setup across multiple queries.
-func queryRange(ctx context.Context, httpClient *http.Client, cred azcore.TokenCredential, endpoint, query string, start, end time.Time, step string) (*PrometheusResponse, error) {
-	token, err := cred.GetToken(ctx, policy.TokenRequestOptions{
-		Scopes: []string{"https://prometheus.monitor.azure.com/.default"},
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to get Prometheus token: %w", err)
-	}
-
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse endpoint URL %q: %w", endpoint, err)
-	}
-	u.Path = strings.TrimRight(u.Path, "/") + "/api/v1/query_range"
-
-	params := url.Values{}
-	params.Set("query", query)
-	params.Set("start", strconv.FormatInt(start.Unix(), 10))
-	params.Set("end", strconv.FormatInt(end.Unix(), 10))
-	params.Set("step", step)
-	u.RawQuery = params.Encode()
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request: %w", err)
-	}
-	req.Header.Set("Authorization", "Bearer "+token.Token)
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("prometheus query_range request failed: %w", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("prometheus query_range returned %d: %s", resp.StatusCode, string(body))
-	}
-
-	var promResp PrometheusResponse
-	dec := json.NewDecoder(bytes.NewReader(body))
-	dec.UseNumber()
-	if err := dec.Decode(&promResp); err != nil {
-		return nil, fmt.Errorf("failed to parse Prometheus response: %w", err)
-	}
-	if promResp.Status != "success" {
-		return nil, fmt.Errorf("prometheus query error (%s): %s", promResp.ErrorType, promResp.Error)
-	}
-	return &promResp, nil
-}

test/cmd/aro-hcp-tests/gather-observability/workspace.go

@@ -25,6 +25,8 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+
+	promutil "github.com/Azure/ARO-HCP/test/util/prometheus"
 )
 
 const (
@@ -46,7 +48,7 @@ func fetchWorkspaceData(ctx context.Context, cred azcore.TokenCredential, wsType
 		return nil, fmt.Errorf("logger not found in context: %w", err)
 	}
 
-	promEndpoint, err := lookupPrometheusEndpoint(ctx, cred, workspaceResourceID.SubscriptionID, workspaceResourceID.ResourceGroupName, workspaceResourceID.Name)
+	promEndpoint, err := promutil.LookupPrometheusEndpoint(ctx, cred, workspaceResourceID.SubscriptionID, workspaceResourceID.ResourceGroupName, workspaceResourceID.Name)
 	if err != nil {
 		return nil, fmt.Errorf("failed to look up Prometheus endpoint: %w", err)
 	}

test/e2e/ksm_hcp_metrics.go

@@ -0,0 +1,80 @@
+// Copyright 2026 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/Azure/ARO-HCP/test/util/framework"
+	"github.com/Azure/ARO-HCP/test/util/labels"
+	promutil "github.com/Azure/ARO-HCP/test/util/prometheus"
+	"github.com/Azure/ARO-HCP/test/util/testconfig"
+)
+
+var _ = Describe("KSM HCP Metrics", func() {
+	It("kube_node_info metrics should be present in Azure Monitor for the happy-path cluster",
+		labels.RequireHappyPathInfra,
+		labels.Medium,
+		labels.Positive,
+		labels.MIContainers(0),
+		func(ctx context.Context) {
+			tc := framework.NewTestContext()
+
+			By("Loading rendered config")
+			cfg, err := tc.RenderedConfig()
+			Expect(err).NotTo(HaveOccurred(), "failed to load rendered config — is RENDERED_CONFIG set?")
+
+			regionRG, err := testconfig.ConfigGetString(cfg, "regionRG")
+			Expect(err).NotTo(HaveOccurred(), "failed to get regionRG from rendered config")
+
+			hcpWorkspaceName, err := testconfig.ConfigGetString(cfg, "monitoring.hcpWorkspaceName")
+			Expect(err).NotTo(HaveOccurred(), "failed to get monitoring.hcpWorkspaceName from rendered config")
+
+			subscriptionID, err := tc.SubscriptionID(ctx)
+			Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID")
+
+			cred, err := tc.AzureCredential()
+			Expect(err).NotTo(HaveOccurred(), "failed to get Azure credential")
+
+			By("Resolving HCP workspace Prometheus endpoint")
+			endpoint, err := promutil.LookupPrometheusEndpoint(ctx, cred, subscriptionID, regionRG, hcpWorkspaceName)
+			Expect(err).NotTo(HaveOccurred(), "failed to look up HCP Prometheus endpoint")
+
+			clusterName := e2eSetup.Cluster.Name
+			query := fmt.Sprintf(`kube_node_info{hostedcontrolplane=~".*%s.*"}`, regexp.QuoteMeta(clusterName))
+
+			httpClient := &http.Client{Timeout: 30 * time.Second}
+
+			By("Polling Azure Monitor for kube_node_info metrics")
+			// Azure Monitor Prometheus ingestion latency for new metric series can exceed 10 minutes.
+			Eventually(func(g Gomega) {
+				now := time.Now()
+				start := now.Add(-5 * time.Minute)
+
+				resp, err := promutil.QueryRange(ctx, httpClient, cred, endpoint, query, start, now, "60s")
+				g.Expect(err).NotTo(HaveOccurred(), "Prometheus query_range failed")
+				g.Expect(resp.Data.Result).NotTo(BeEmpty(),
+					"expected kube_node_info metrics for cluster %q but got no results", clusterName)
+			}).WithTimeout(15*time.Minute).WithPolling(30*time.Second).WithContext(ctx).Should(Succeed(),
+				"kube_node_info metrics never appeared in Azure Monitor for cluster %q", clusterName)
+		})
+})