feat: add KSM HCP E2E test with rendered config support

nonlocal-e2e-specs.txt

@@ -8,6 +8,7 @@
   "Successfully lists clusters filtered by resource group name",
   "creates a cluster and fails to update its name with a PATCH request",
   "creates and deletes a GPU nodepool in a single cluster",
+  "kube_node_info metrics should be present in Azure Monitor for the happy-path cluster",
   "should allow pods with images from allowed registries and have a valid allowlist",
   "should be able to create an HCP cluster then delete it by deleting the customer resource group",
   "should be able to delete an HCP cluster whose managed identities were deleted first",

test/E2ELocal.mk

@@ -23,6 +23,10 @@ SNAPSHOT_RENDERED_CONFIG := $(shell mktemp)
 e2e-local/run-test: $(ARO_HCP_TESTS)
 	$(MAKE) -C $(DIR) -f $(THIS) .e2e-local/setup
 	export LOCATION="$${LOCATION:-${REGION}}"; \
+	export REGION_RG="$${REGION_RG:-${REGION_RG}}"; \
+	export HCP_WORKSPACE_NAME="$${HCP_WORKSPACE_NAME:-${HCP_WORKSPACE_NAME}}"; \
+	export KUSTO_NAME="$${KUSTO_NAME:-${KUSTO_NAME}}"; \
+	export KUSTO_REGION="$${KUSTO_REGION:-${KUSTO_REGION}}"; \
 	export AROHCP_ENV="development"; \
 	export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \
 	export AZURE_TENANT_ID="$$(az account show --output tsv --query 'tenantId')"; \

test/Env.mk

@@ -1,2 +1,6 @@
 SVC_CLUSTER ?= {{ .svc.aks.name }}
-REGION ?= {{ .region }}
+REGION ?= {{ .region }}
+REGION_RG ?= {{ .regionRG }}
+HCP_WORKSPACE_NAME ?= {{ .monitoring.hcpWorkspaceName }}
+KUSTO_NAME ?= {{ .kusto.kustoName }}
+KUSTO_REGION ?= {{ .kusto.location }}

test/e2e/ksm_hcp_metrics.go

@@ -0,0 +1,76 @@
+// Copyright 2026 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"regexp"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/Azure/ARO-HCP/test/util/framework"
+	"github.com/Azure/ARO-HCP/test/util/labels"
+	promutil "github.com/Azure/ARO-HCP/test/util/prometheus"
+)
+
+var _ = Describe("KSM HCP Metrics", func() {
+	It("kube_node_info metrics should be present in Azure Monitor for the happy-path cluster",
+		labels.RequireHappyPathInfra,
+		labels.Medium,
+		labels.Positive,
+		labels.MIContainers(0),
+		func(ctx context.Context) {
+			tc := framework.NewTestContext()
+
+			regionRG := os.Getenv("REGION_RG")
+			Expect(regionRG).NotTo(BeEmpty(), "REGION_RG environment variable must be set")
+
+			hcpWorkspaceName := os.Getenv("HCP_WORKSPACE_NAME")
+			Expect(hcpWorkspaceName).NotTo(BeEmpty(), "HCP_WORKSPACE_NAME environment variable must be set")
+
+			subscriptionID, err := tc.SubscriptionID(ctx)
+			Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID")
+
+			cred, err := tc.AzureCredential()
+			Expect(err).NotTo(HaveOccurred(), "failed to get Azure credential")
+
+			By("Resolving HCP workspace Prometheus endpoint")
+			endpoint, err := promutil.LookupPrometheusEndpoint(ctx, cred, subscriptionID, regionRG, hcpWorkspaceName)
+			Expect(err).NotTo(HaveOccurred(), "failed to look up HCP Prometheus endpoint")
+
+			clusterName := e2eSetup.Cluster.Name
+			query := fmt.Sprintf(`kube_node_info{hostedcontrolplane=~".*%s.*"}`, regexp.QuoteMeta(clusterName))
+
+			httpClient := &http.Client{Timeout: 30 * time.Second}
+
+			By("Polling Azure Monitor for kube_node_info metrics")
+			// Azure Monitor Prometheus ingestion latency for new metric series can exceed 10 minutes.
+			Eventually(func(g Gomega) {
+				now := time.Now()
+				start := now.Add(-5 * time.Minute)
+
+				resp, err := promutil.QueryRange(ctx, httpClient, cred, endpoint, query, start, now, "60s")
+				g.Expect(err).NotTo(HaveOccurred(), "Prometheus query_range failed")
+				g.Expect(resp.Data.Result).NotTo(BeEmpty(),
+					"expected kube_node_info metrics for cluster %q but got no results", clusterName)
+			}).WithTimeout(15*time.Minute).WithPolling(30*time.Second).WithContext(ctx).Should(Succeed(),
+				"kube_node_info metrics never appeared in Azure Monitor for cluster %q", clusterName)
+		})
+})

test/util/prometheus/prometheus.go

@@ -0,0 +1,126 @@
+// Copyright 2026 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor"
+)
+
+// Response is the top-level Prometheus HTTP API response.
+type Response struct {
+	Status    string `json:"status"`
+	Data      Data   `json:"data"`
+	ErrorType string `json:"errorType,omitempty"`
+	Error     string `json:"error,omitempty"`
+}
+
+// Data holds the result set from a query_range call.
+type Data struct {
+	ResultType string   `json:"resultType"`
+	Result     []Result `json:"result"`
+}
+
+// Result is a single timeseries returned by query_range.
+type Result struct {
+	Metric map[string]string `json:"metric"`
+	Values [][]any           `json:"values"` // each element is [unix_timestamp_float, "string_value"]
+}
+
+// LookupPrometheusEndpoint retrieves the Prometheus query endpoint for an
+// Azure Monitor workspace using the ARM SDK.
+func LookupPrometheusEndpoint(ctx context.Context, cred azcore.TokenCredential, subscriptionID, resourceGroup, workspaceName string) (string, error) {
+	client, err := armmonitor.NewAzureMonitorWorkspacesClient(subscriptionID, cred, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create monitor workspaces client: %w", err)
+	}
+	resp, err := client.Get(ctx, resourceGroup, workspaceName, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to get workspace %s: %w", workspaceName, err)
+	}
+	if resp.Properties == nil || resp.Properties.Metrics == nil || resp.Properties.Metrics.PrometheusQueryEndpoint == nil {
+		return "", fmt.Errorf("workspace %s has no Prometheus query endpoint", workspaceName)
+	}
+	return *resp.Properties.Metrics.PrometheusQueryEndpoint, nil
+}
+
+// QueryRange executes a Prometheus query_range request against an Azure Monitor
+// Prometheus endpoint using bearer token authentication. The caller should pass
+// a shared *http.Client to amortize connection setup across multiple queries.
+func QueryRange(ctx context.Context, httpClient *http.Client, cred azcore.TokenCredential, endpoint, query string, start, end time.Time, step string) (*Response, error) {
+	token, err := cred.GetToken(ctx, policy.TokenRequestOptions{
+		Scopes: []string{"https://prometheus.monitor.azure.com/.default"},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get Prometheus token: %w", err)
+	}
+
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse endpoint URL %q: %w", endpoint, err)
+	}
+	u.Path = strings.TrimRight(u.Path, "/") + "/api/v1/query_range"
+
+	params := url.Values{}
+	params.Set("query", query)
+	params.Set("start", strconv.FormatInt(start.Unix(), 10))
+	params.Set("end", strconv.FormatInt(end.Unix(), 10))
+	params.Set("step", step)
+	u.RawQuery = params.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Authorization", "Bearer "+token.Token)
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("prometheus query_range request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("prometheus query_range returned %d: %s", resp.StatusCode, string(body))
+	}
+
+	var promResp Response
+	dec := json.NewDecoder(bytes.NewReader(body))
+	dec.UseNumber()
+	if err := dec.Decode(&promResp); err != nil {
+		return nil, fmt.Errorf("failed to parse Prometheus response: %w", err)
+	}
+	if promResp.Status != "success" {
+		return nil, fmt.Errorf("prometheus query error (%s): %s", promResp.ErrorType, promResp.Error)
+	}
+	return &promResp, nil
+}