From 78ff716bd5fc34257c8a5684e658624afbaf8368 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Tue, 25 Aug 2026 09:10:02 +0000 Subject: [PATCH 1/3] feat: add KSM HCP E2E test with rendered config and shared Prometheus utilities Extract Prometheus query utilities and config loading helpers from internal packages into shared locations (test/util/prometheus/ and test/util/testconfig/) so E2E tests can access them. Wire rendered config into the E2E framework via RENDERED_CONFIG env var with lazy-loaded access through tc.RenderedConfig(). Add a new E2E test verifying that kube-state-metrics (KSM) metrics deployed by the ksmhcp controller are present in Azure Monitor for running HCP clusters by querying kube_node_info. Also remove hardcoded kusto cluster/region values from the must-gather verifiers, reading them from rendered config instead. Ref: AROSLSRE-1869 Co-Authored-By: Claude Opus 4.6 --- nonlocal-e2e-specs.txt | 1 + .../custom-link-tools/options.go | 16 +-- .../gather-observability/options.go | 19 +-- .../gather-observability/promql.go | 103 +------------- .../gather-observability/workspace.go | 4 +- test/e2e/ksm_hcp_metrics.go | 80 +++++++++++ test/e2e/kusto_logs_present.go | 15 ++- .../framework/per_invocation_framework.go | 25 ++++ test/util/framework/per_test_framework.go | 7 + test/util/prometheus/prometheus.go | 126 ++++++++++++++++++ .../testutil => util/testconfig}/config.go | 4 +- test/util/verifiers/kusto.go | 8 +- test/util/verifiers/must_gather_cli.go | 15 ++- 13 files changed, 290 insertions(+), 133 deletions(-) create mode 100644 test/e2e/ksm_hcp_metrics.go create mode 100644 test/util/prometheus/prometheus.go rename test/{cmd/aro-hcp-tests/internal/testutil => util/testconfig}/config.go (97%) diff --git a/nonlocal-e2e-specs.txt b/nonlocal-e2e-specs.txt index c99a228d1a0..483ffeef68a 100644 --- a/nonlocal-e2e-specs.txt +++ b/nonlocal-e2e-specs.txt @@ -8,6 +8,7 @@ "Successfully lists clusters filtered by resource group name", "creates a cluster and fails to update its name with a PATCH request", "creates and deletes a GPU nodepool in a single cluster", + "kube_node_info metrics should be present in Azure Monitor for the happy-path cluster", "should allow pods with images from allowed registries and have a valid allowlist", "should be able to create an HCP cluster then delete it by deleting the customer resource group", "should be able to delete an HCP cluster whose managed identities were deleted first", diff --git a/test/cmd/aro-hcp-tests/custom-link-tools/options.go b/test/cmd/aro-hcp-tests/custom-link-tools/options.go index c516a9e2c83..c0f5c5a0359 100644 --- a/test/cmd/aro-hcp-tests/custom-link-tools/options.go +++ b/test/cmd/aro-hcp-tests/custom-link-tools/options.go @@ -33,7 +33,7 @@ import ( "k8s.io/utils/clock" "github.com/Azure/ARO-HCP/internal/utils" - "github.com/Azure/ARO-HCP/test/cmd/aro-hcp-tests/internal/testutil" + "github.com/Azure/ARO-HCP/test/util/testconfig" "github.com/Azure/ARO-HCP/test/util/timing" "github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto" "github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline" @@ -133,32 +133,32 @@ func (o *RawOptions) Validate() (*ValidatedOptions, error) { } func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) { - cfg, err := testutil.LoadRenderedConfig(o.RenderedConfig) + cfg, err := testconfig.LoadRenderedConfig(o.RenderedConfig) if err != nil { return nil, err } - svcClusterName, err := testutil.ConfigGetString(cfg, "svc.aks.name") + svcClusterName, err := testconfig.ConfigGetString(cfg, "svc.aks.name") if err != nil { return nil, fmt.Errorf("failed to get svc cluster name from config: %w", err) } - mgmtClusterName, err := testutil.ConfigGetString(cfg, "mgmt.aks.name") + mgmtClusterName, err := testconfig.ConfigGetString(cfg, "mgmt.aks.name") if err != nil { return nil, fmt.Errorf("failed to get mgmt cluster name from config: %w", err) } - kustoName, err := testutil.ConfigGetString(cfg, "kusto.kustoName") + kustoName, err := testconfig.ConfigGetString(cfg, "kusto.kustoName") if err != nil { return nil, fmt.Errorf("failed to get kusto name from config: %w", err) } - kustoRegion, err := testutil.ConfigGetString(cfg, "kusto.location") + kustoRegion, err := testconfig.ConfigGetString(cfg, "kusto.location") if err != nil { return nil, fmt.Errorf("failed to get kusto location from config: %w", err) } - serviceLogsDB, err := testutil.ConfigGetString(cfg, "kusto.serviceLogsDatabase") + serviceLogsDB, err := testconfig.ConfigGetString(cfg, "kusto.serviceLogsDatabase") if err != nil { return nil, fmt.Errorf("failed to get service logs database from config: %w", err) } - hcpLogsDB, err := testutil.ConfigGetString(cfg, "kusto.hostedControlPlaneLogsDatabase") + hcpLogsDB, err := testconfig.ConfigGetString(cfg, "kusto.hostedControlPlaneLogsDatabase") if err != nil { return nil, fmt.Errorf("failed to get hosted control plane logs database from config: %w", err) } diff --git a/test/cmd/aro-hcp-tests/gather-observability/options.go b/test/cmd/aro-hcp-tests/gather-observability/options.go index 6ed42f1af77..c71d06b5488 100644 --- a/test/cmd/aro-hcp-tests/gather-observability/options.go +++ b/test/cmd/aro-hcp-tests/gather-observability/options.go @@ -39,8 +39,9 @@ import ( "github.com/Azure/ARO-HCP/internal/api/metadataapi" "github.com/Azure/ARO-HCP/internal/utils" - "github.com/Azure/ARO-HCP/test/cmd/aro-hcp-tests/internal/testutil" "github.com/Azure/ARO-HCP/test/util/junit" + promutil "github.com/Azure/ARO-HCP/test/util/prometheus" + "github.com/Azure/ARO-HCP/test/util/testconfig" "github.com/Azure/ARO-HCP/test/util/timing" ) @@ -130,20 +131,20 @@ func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) { return nil, fmt.Errorf("failed to create output directory %s: %w", o.OutputDir, err) } - cfg, err := testutil.LoadRenderedConfig(o.RenderedConfig) + cfg, err := testconfig.LoadRenderedConfig(o.RenderedConfig) if err != nil { return nil, err } - regionRG, err := testutil.ConfigGetString(cfg, "regionRG") + regionRG, err := testconfig.ConfigGetString(cfg, "regionRG") if err != nil { return nil, fmt.Errorf("failed to get regionRG from config: %w", err) } - svcWorkspace, err := testutil.ConfigGetString(cfg, "monitoring.svcWorkspaceName") + svcWorkspace, err := testconfig.ConfigGetString(cfg, "monitoring.svcWorkspaceName") if err != nil { return nil, fmt.Errorf("failed to get monitoring.svcWorkspaceName from config: %w", err) } - hcpWorkspace, err := testutil.ConfigGetString(cfg, "monitoring.hcpWorkspaceName") + hcpWorkspace, err := testconfig.ConfigGetString(cfg, "monitoring.hcpWorkspaceName") if err != nil { return nil, fmt.Errorf("failed to get monitoring.hcpWorkspaceName from config: %w", err) } @@ -152,7 +153,7 @@ func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) { // the name from frontend.cosmosDB.name (see region.bicep / output-region.bicep). // Its platform metrics (NormalizedRUConsumption, AutoscaledRU, ...) are queried // via the Azure Monitor metrics API rather than Prometheus. - cosmosDBName, err := testutil.ConfigGetString(cfg, "frontend.cosmosDB.name") + cosmosDBName, err := testconfig.ConfigGetString(cfg, "frontend.cosmosDB.name") if err != nil { return nil, fmt.Errorf("failed to get frontend.cosmosDB.name from config: %w", err) } @@ -248,13 +249,13 @@ func buildCosmosAutoscaleMaxLookup(cfg configtypes.Configuration) (autoscaleMaxL } byContainer := make(map[string]float64, len(fixed)) for container, path := range fixed { - v, err := testutil.ConfigGetInt(cfg, path) + v, err := testconfig.ConfigGetInt(cfg, path) if err != nil { return nil, fmt.Errorf("failed to get %s from config: %w", path, err) } byContainer[container] = float64(v) } - manifestsMax, err := testutil.ConfigGetInt(cfg, "kubeApplier.cosmosContainerMaxScale") + manifestsMax, err := testconfig.ConfigGetInt(cfg, "kubeApplier.cosmosContainerMaxScale") if err != nil { return nil, fmt.Errorf("failed to get kubeApplier.cosmosContainerMaxScale from config: %w", err) } @@ -453,7 +454,7 @@ func (o Options) runQueries(ctx context.Context, workspaces map[string]*workspac logger.Info("executing PromQL query", "panel", panel.Title, "title", q.Title, "workspace", q.Workspace) - resp, err := queryRange(ctx, httpClient, o.cred, endpoint, q.Query, o.TimeWindow.Start, o.TimeWindow.End, q.Step) + resp, err := promutil.QueryRange(ctx, httpClient, o.cred, endpoint, q.Query, o.TimeWindow.Start, o.TimeWindow.End, q.Step) if err != nil { logger.Error(err, "PromQL query failed", "title", q.Title) queryErr = err.Error() diff --git a/test/cmd/aro-hcp-tests/gather-observability/promql.go b/test/cmd/aro-hcp-tests/gather-observability/promql.go index d9b290ebbe2..590a2ce0412 100644 --- a/test/cmd/aro-hcp-tests/gather-observability/promql.go +++ b/test/cmd/aro-hcp-tests/gather-observability/promql.go @@ -15,25 +15,15 @@ package gatherobservability import ( - "bytes" - "context" - "encoding/json" "fmt" - "io" - "net/http" - "net/url" "slices" - "strconv" "strings" - "time" _ "embed" "sigs.k8s.io/yaml" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" + promutil "github.com/Azure/ARO-HCP/test/util/prometheus" ) // QueriesConfig holds panels of grouped PromQL queries to run against Azure @@ -141,24 +131,13 @@ type QuerySpec struct { } // PrometheusResponse is the top-level Prometheus HTTP API response. -type PrometheusResponse struct { - Status string `json:"status"` - Data PrometheusData `json:"data"` - ErrorType string `json:"errorType,omitempty"` - Error string `json:"error,omitempty"` -} +type PrometheusResponse = promutil.Response // PrometheusData holds the result set from a query_range call. -type PrometheusData struct { - ResultType string `json:"resultType"` - Result []PrometheusResult `json:"result"` -} +type PrometheusData = promutil.Data // PrometheusResult is a single timeseries returned by query_range. -type PrometheusResult struct { - Metric map[string]string `json:"metric"` - Values [][]any `json:"values"` // each element is [unix_timestamp_float, "string_value"] -} +type PrometheusResult = promutil.Result //go:embed queries.yaml var defaultQueriesYAML []byte @@ -276,77 +255,3 @@ func sortedMetricResources() []string { slices.Sort(out) return out } - -// lookupPrometheusEndpoint retrieves the Prometheus query endpoint for an -// Azure Monitor workspace using the ARM SDK. -func lookupPrometheusEndpoint(ctx context.Context, cred azcore.TokenCredential, subscriptionID, resourceGroup, workspaceName string) (string, error) { - client, err := armmonitor.NewAzureMonitorWorkspacesClient(subscriptionID, cred, nil) - if err != nil { - return "", fmt.Errorf("failed to create monitor workspaces client: %w", err) - } - resp, err := client.Get(ctx, resourceGroup, workspaceName, nil) - if err != nil { - return "", fmt.Errorf("failed to get workspace %s: %w", workspaceName, err) - } - if resp.Properties == nil || resp.Properties.Metrics == nil || resp.Properties.Metrics.PrometheusQueryEndpoint == nil { - return "", fmt.Errorf("workspace %s has no Prometheus query endpoint", workspaceName) - } - return *resp.Properties.Metrics.PrometheusQueryEndpoint, nil -} - -// queryRange executes a Prometheus query_range request against an Azure Monitor -// Prometheus endpoint using bearer token authentication. The caller should pass -// a shared *http.Client to amortize connection setup across multiple queries. -func queryRange(ctx context.Context, httpClient *http.Client, cred azcore.TokenCredential, endpoint, query string, start, end time.Time, step string) (*PrometheusResponse, error) { - token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ - Scopes: []string{"https://prometheus.monitor.azure.com/.default"}, - }) - if err != nil { - return nil, fmt.Errorf("failed to get Prometheus token: %w", err) - } - - u, err := url.Parse(endpoint) - if err != nil { - return nil, fmt.Errorf("failed to parse endpoint URL %q: %w", endpoint, err) - } - u.Path = strings.TrimRight(u.Path, "/") + "/api/v1/query_range" - - params := url.Values{} - params.Set("query", query) - params.Set("start", strconv.FormatInt(start.Unix(), 10)) - params.Set("end", strconv.FormatInt(end.Unix(), 10)) - params.Set("step", step) - u.RawQuery = params.Encode() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Authorization", "Bearer "+token.Token) - - resp, err := httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("prometheus query_range request failed: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("prometheus query_range returned %d: %s", resp.StatusCode, string(body)) - } - - var promResp PrometheusResponse - dec := json.NewDecoder(bytes.NewReader(body)) - dec.UseNumber() - if err := dec.Decode(&promResp); err != nil { - return nil, fmt.Errorf("failed to parse Prometheus response: %w", err) - } - if promResp.Status != "success" { - return nil, fmt.Errorf("prometheus query error (%s): %s", promResp.ErrorType, promResp.Error) - } - return &promResp, nil -} diff --git a/test/cmd/aro-hcp-tests/gather-observability/workspace.go b/test/cmd/aro-hcp-tests/gather-observability/workspace.go index 820e4fa6beb..73cf187b309 100644 --- a/test/cmd/aro-hcp-tests/gather-observability/workspace.go +++ b/test/cmd/aro-hcp-tests/gather-observability/workspace.go @@ -25,6 +25,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + promutil "github.com/Azure/ARO-HCP/test/util/prometheus" ) const ( @@ -46,7 +48,7 @@ func fetchWorkspaceData(ctx context.Context, cred azcore.TokenCredential, wsType return nil, fmt.Errorf("logger not found in context: %w", err) } - promEndpoint, err := lookupPrometheusEndpoint(ctx, cred, workspaceResourceID.SubscriptionID, workspaceResourceID.ResourceGroupName, workspaceResourceID.Name) + promEndpoint, err := promutil.LookupPrometheusEndpoint(ctx, cred, workspaceResourceID.SubscriptionID, workspaceResourceID.ResourceGroupName, workspaceResourceID.Name) if err != nil { return nil, fmt.Errorf("failed to look up Prometheus endpoint: %w", err) } diff --git a/test/e2e/ksm_hcp_metrics.go b/test/e2e/ksm_hcp_metrics.go new file mode 100644 index 00000000000..fdbfbbf731a --- /dev/null +++ b/test/e2e/ksm_hcp_metrics.go @@ -0,0 +1,80 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "net/http" + "regexp" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/Azure/ARO-HCP/test/util/framework" + "github.com/Azure/ARO-HCP/test/util/labels" + promutil "github.com/Azure/ARO-HCP/test/util/prometheus" + "github.com/Azure/ARO-HCP/test/util/testconfig" +) + +var _ = Describe("KSM HCP Metrics", func() { + It("kube_node_info metrics should be present in Azure Monitor for the happy-path cluster", + labels.RequireHappyPathInfra, + labels.Medium, + labels.Positive, + labels.MIContainers(0), + func(ctx context.Context) { + tc := framework.NewTestContext() + + By("Loading rendered config") + cfg, err := tc.RenderedConfig() + Expect(err).NotTo(HaveOccurred(), "failed to load rendered config — is RENDERED_CONFIG set?") + + regionRG, err := testconfig.ConfigGetString(cfg, "regionRG") + Expect(err).NotTo(HaveOccurred(), "failed to get regionRG from rendered config") + + hcpWorkspaceName, err := testconfig.ConfigGetString(cfg, "monitoring.hcpWorkspaceName") + Expect(err).NotTo(HaveOccurred(), "failed to get monitoring.hcpWorkspaceName from rendered config") + + subscriptionID, err := tc.SubscriptionID(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID") + + cred, err := tc.AzureCredential() + Expect(err).NotTo(HaveOccurred(), "failed to get Azure credential") + + By("Resolving HCP workspace Prometheus endpoint") + endpoint, err := promutil.LookupPrometheusEndpoint(ctx, cred, subscriptionID, regionRG, hcpWorkspaceName) + Expect(err).NotTo(HaveOccurred(), "failed to look up HCP Prometheus endpoint") + + clusterName := e2eSetup.Cluster.Name + query := fmt.Sprintf(`kube_node_info{hostedcontrolplane=~".*%s.*"}`, regexp.QuoteMeta(clusterName)) + + httpClient := &http.Client{Timeout: 30 * time.Second} + + By("Polling Azure Monitor for kube_node_info metrics") + // Azure Monitor Prometheus ingestion latency for new metric series can exceed 10 minutes. + Eventually(func(g Gomega) { + now := time.Now() + start := now.Add(-5 * time.Minute) + + resp, err := promutil.QueryRange(ctx, httpClient, cred, endpoint, query, start, now, "60s") + g.Expect(err).NotTo(HaveOccurred(), "Prometheus query_range failed") + g.Expect(resp.Data.Result).NotTo(BeEmpty(), + "expected kube_node_info metrics for cluster %q but got no results", clusterName) + }).WithTimeout(15*time.Minute).WithPolling(30*time.Second).WithContext(ctx).Should(Succeed(), + "kube_node_info metrics never appeared in Azure Monitor for cluster %q", clusterName) + }) +}) diff --git a/test/e2e/kusto_logs_present.go b/test/e2e/kusto_logs_present.go index 132ba6928e9..fb583cec4cc 100644 --- a/test/e2e/kusto_logs_present.go +++ b/test/e2e/kusto_logs_present.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/ARO-HCP/test/util/framework" "github.com/Azure/ARO-HCP/test/util/labels" + "github.com/Azure/ARO-HCP/test/util/testconfig" "github.com/Azure/ARO-HCP/test/util/verifiers" ) @@ -88,15 +89,23 @@ var _ = Describe("Engineering", func() { Expect(err).NotTo(HaveOccurred(), "failed to create HCP cluster for kusto logs test") subscriptionID, err := tc.SubscriptionID(ctx) Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID") - By("verifying kusto logs are present") + By("loading kusto config from rendered config") + cfg, err := tc.RenderedConfig() + Expect(err).NotTo(HaveOccurred(), "failed to load rendered config — is RENDERED_CONFIG set?") + kustoName, err := testconfig.ConfigGetString(cfg, "kusto.kustoName") + Expect(err).NotTo(HaveOccurred(), "failed to get kusto.kustoName from rendered config") + kustoRegion, err := testconfig.ConfigGetString(cfg, "kusto.location") + Expect(err).NotTo(HaveOccurred(), "failed to get kusto.location from rendered config") + + By("verifying kusto logs are present") Eventually(func() error { - return verifiers.VerifyMustGatherLogs(subscriptionID, *resourceGroup.Name).Verify(ctx) + return verifiers.VerifyMustGatherLogs(subscriptionID, *resourceGroup.Name, kustoName, kustoRegion).Verify(ctx) }, 10*time.Minute, 60*time.Second).Should(Succeed()) By("verifying must-gather CLI works") Eventually(func() error { - return verifiers.VerifyMustGatherCLI(subscriptionID, *resourceGroup.Name).Verify(ctx) + return verifiers.VerifyMustGatherCLI(subscriptionID, *resourceGroup.Name, kustoName, kustoRegion).Verify(ctx) }, 10*time.Minute, 60*time.Second).Should(Succeed()) }) diff --git a/test/util/framework/per_invocation_framework.go b/test/util/framework/per_invocation_framework.go index be2aaff45cf..c614a3c4395 100644 --- a/test/util/framework/per_invocation_framework.go +++ b/test/util/framework/per_invocation_framework.go @@ -34,6 +34,7 @@ import ( "github.com/onsi/ginkgo/v2/types" "golang.org/x/net/http2" + configtypes "github.com/Azure/ARO-Tools/config/types" "github.com/Azure/azure-sdk-for-go/sdk/azcore" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" @@ -43,6 +44,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" "github.com/Azure/ARO-HCP/internal/azsdk" + "github.com/Azure/ARO-HCP/test/util/testconfig" ) type perBinaryInvocationTestContext struct { @@ -62,6 +64,8 @@ type perBinaryInvocationTestContext struct { pooledIdentities bool compressTimingMetadata bool + renderedConfigPath string + contextLock sync.RWMutex subscriptionID string azureCredentials azcore.TokenCredential @@ -72,6 +76,10 @@ type perBinaryInvocationTestContext struct { // target the same location, so a suite-wide cache avoids repeatedly burning // per-test timeout budget on the same ARM call. virtualMachineResourceSKUsByLocation map[string][]*armcompute.ResourceSKU + + renderedConfigOnce sync.Once + renderedConfig configtypes.Configuration + renderedConfigErr error } type CleanupFunc func(ctx context.Context) error @@ -122,6 +130,7 @@ func invocationContext() *perBinaryInvocationTestContext { skipCleanup: skipCleanup(), pooledIdentities: pooledIdentities(), compressTimingMetadata: compressTimingMetadata(), + renderedConfigPath: renderedConfigPath(), defaultTransport: defaultHTTPTransport(), virtualMachineResourceSKUsByLocation: make(map[string][]*armcompute.ResourceSKU), } @@ -467,6 +476,22 @@ func IsDevelopmentEnvironment() bool { return strings.ToLower(os.Getenv("AROHCP_ENV")) == "development" } +// renderedConfigPath returns the value of RENDERED_CONFIG environment variable +func renderedConfigPath() string { + return os.Getenv("RENDERED_CONFIG") +} + +func (tc *perBinaryInvocationTestContext) getRenderedConfig() (configtypes.Configuration, error) { + tc.renderedConfigOnce.Do(func() { + if tc.renderedConfigPath == "" { + tc.renderedConfigErr = fmt.Errorf("RENDERED_CONFIG environment variable is not set") + return + } + tc.renderedConfig, tc.renderedConfigErr = testconfig.LoadRenderedConfig(tc.renderedConfigPath) + }) + return tc.renderedConfig, tc.renderedConfigErr +} + // Must is a generic function that takes a value of type T and an error. // If the error is not nil, it panics with the error. // Otherwise, it returns the value of type T. diff --git a/test/util/framework/per_test_framework.go b/test/util/framework/per_test_framework.go index 0f48e2f65f4..a68fe9a8b78 100644 --- a/test/util/framework/per_test_framework.go +++ b/test/util/framework/per_test_framework.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/yaml" + configtypes "github.com/Azure/ARO-Tools/config/types" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" @@ -1275,6 +1276,12 @@ func (tc *perItOrDescribeTestContext) AzureCredential() (azcore.TokenCredential, return tc.perBinaryInvocationTestContext.getAzureCredentials() } +// RenderedConfig returns the parsed rendered configuration. +// Requires the RENDERED_CONFIG environment variable to be set to a valid config YAML path. +func (tc *perItOrDescribeTestContext) RenderedConfig() (configtypes.Configuration, error) { + return tc.perBinaryInvocationTestContext.getRenderedConfig() +} + func (tc *perItOrDescribeTestContext) TenantID() string { return tc.perBinaryInvocationTestContext.tenantID } diff --git a/test/util/prometheus/prometheus.go b/test/util/prometheus/prometheus.go new file mode 100644 index 00000000000..13167952af8 --- /dev/null +++ b/test/util/prometheus/prometheus.go @@ -0,0 +1,126 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" +) + +// Response is the top-level Prometheus HTTP API response. +type Response struct { + Status string `json:"status"` + Data Data `json:"data"` + ErrorType string `json:"errorType,omitempty"` + Error string `json:"error,omitempty"` +} + +// Data holds the result set from a query_range call. +type Data struct { + ResultType string `json:"resultType"` + Result []Result `json:"result"` +} + +// Result is a single timeseries returned by query_range. +type Result struct { + Metric map[string]string `json:"metric"` + Values [][]any `json:"values"` // each element is [unix_timestamp_float, "string_value"] +} + +// LookupPrometheusEndpoint retrieves the Prometheus query endpoint for an +// Azure Monitor workspace using the ARM SDK. +func LookupPrometheusEndpoint(ctx context.Context, cred azcore.TokenCredential, subscriptionID, resourceGroup, workspaceName string) (string, error) { + client, err := armmonitor.NewAzureMonitorWorkspacesClient(subscriptionID, cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create monitor workspaces client: %w", err) + } + resp, err := client.Get(ctx, resourceGroup, workspaceName, nil) + if err != nil { + return "", fmt.Errorf("failed to get workspace %s: %w", workspaceName, err) + } + if resp.Properties == nil || resp.Properties.Metrics == nil || resp.Properties.Metrics.PrometheusQueryEndpoint == nil { + return "", fmt.Errorf("workspace %s has no Prometheus query endpoint", workspaceName) + } + return *resp.Properties.Metrics.PrometheusQueryEndpoint, nil +} + +// QueryRange executes a Prometheus query_range request against an Azure Monitor +// Prometheus endpoint using bearer token authentication. The caller should pass +// a shared *http.Client to amortize connection setup across multiple queries. +func QueryRange(ctx context.Context, httpClient *http.Client, cred azcore.TokenCredential, endpoint, query string, start, end time.Time, step string) (*Response, error) { + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://prometheus.monitor.azure.com/.default"}, + }) + if err != nil { + return nil, fmt.Errorf("failed to get Prometheus token: %w", err) + } + + u, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint URL %q: %w", endpoint, err) + } + u.Path = strings.TrimRight(u.Path, "/") + "/api/v1/query_range" + + params := url.Values{} + params.Set("query", query) + params.Set("start", strconv.FormatInt(start.Unix(), 10)) + params.Set("end", strconv.FormatInt(end.Unix(), 10)) + params.Set("step", step) + u.RawQuery = params.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("prometheus query_range request failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("prometheus query_range returned %d: %s", resp.StatusCode, string(body)) + } + + var promResp Response + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + if err := dec.Decode(&promResp); err != nil { + return nil, fmt.Errorf("failed to parse Prometheus response: %w", err) + } + if promResp.Status != "success" { + return nil, fmt.Errorf("prometheus query error (%s): %s", promResp.ErrorType, promResp.Error) + } + return &promResp, nil +} diff --git a/test/cmd/aro-hcp-tests/internal/testutil/config.go b/test/util/testconfig/config.go similarity index 97% rename from test/cmd/aro-hcp-tests/internal/testutil/config.go rename to test/util/testconfig/config.go index 9e92de33ba3..10a87ff379e 100644 --- a/test/cmd/aro-hcp-tests/internal/testutil/config.go +++ b/test/util/testconfig/config.go @@ -1,4 +1,4 @@ -// Copyright 2025 Microsoft Corporation +// Copyright 2026 Microsoft Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package testutil +package testconfig import ( "encoding/json" diff --git a/test/util/verifiers/kusto.go b/test/util/verifiers/kusto.go index 83dcf2df63b..132201b6b4c 100644 --- a/test/util/verifiers/kusto.go +++ b/test/util/verifiers/kusto.go @@ -138,11 +138,11 @@ func (v verifyMustGatherLogsImpl) Verify(ctx context.Context) error { return nil } -// VerifyMustGatherLogs creates a new must-gather logs verifier with default configuration -func VerifyMustGatherLogs(subscriptionID, rgName string) verifyMustGatherLogsImpl { +// VerifyMustGatherLogs creates a new must-gather logs verifier with default configuration. +func VerifyMustGatherLogs(subscriptionID, rgName, kustoCluster, kustoRegion string) verifyMustGatherLogsImpl { config := mustGatherVerifierConfig{ - KustoCluster: "hcp-dev-us-2", - KustoRegion: "eastus2", + KustoCluster: kustoCluster, + KustoRegion: kustoRegion, SubscriptionID: subscriptionID, ResourceGroup: rgName, QueryTimeout: 5 * time.Minute, diff --git a/test/util/verifiers/must_gather_cli.go b/test/util/verifiers/must_gather_cli.go index a19a4659cf4..fdd0041a147 100644 --- a/test/util/verifiers/must_gather_cli.go +++ b/test/util/verifiers/must_gather_cli.go @@ -26,9 +26,6 @@ import ( ) const ( - kustoCluster = "hcp-dev-us-2" - kustoRegion = "eastus2" - serviceDir = "service" hostedControlPlaneDir = "hosted-control-plane" clusterDir = "cluster" @@ -44,6 +41,8 @@ type mustGatherCLITestCase struct { type mustGatherCLIVerifier struct { subscriptionID string resourceGroup string + kustoCluster string + kustoRegion string } func (v mustGatherCLIVerifier) Name() string { @@ -127,7 +126,7 @@ func (v mustGatherCLIVerifier) Verify(ctx context.Context) error { var errors []string for _, tc := range testCases { logger.Info("Running must-gather CLI test case", "name", tc.name) - if err := runTestCase(ctx, binary, tc); err != nil { + if err := runTestCase(ctx, binary, tc, v.kustoCluster, v.kustoRegion); err != nil { errors = append(errors, fmt.Sprintf("%s: %v", tc.name, err)) logger.Error(err, "Test case failed", "name", tc.name) } else { @@ -141,7 +140,7 @@ func (v mustGatherCLIVerifier) Verify(ctx context.Context) error { return nil } -func runTestCase(ctx context.Context, binary string, tc mustGatherCLITestCase) error { +func runTestCase(ctx context.Context, binary string, tc mustGatherCLITestCase, kustoCluster, kustoRegion string) error { outputDir, err := os.MkdirTemp("", "must-gather-cli-e2e-"+tc.name+"-") if err != nil { return fmt.Errorf("failed to create temp dir: %w", err) @@ -213,10 +212,12 @@ func infraClusterNames() (svcCluster, mgmtCluster string) { return svcCluster, mgmtCluster } -// VerifyMustGatherCLI creates a verifier that tests the hcpctl must-gather CLI -func VerifyMustGatherCLI(subscriptionID, resourceGroup string) mustGatherCLIVerifier { +// VerifyMustGatherCLI creates a verifier that tests the hcpctl must-gather CLI. +func VerifyMustGatherCLI(subscriptionID, resourceGroup, kustoCluster, kustoRegion string) mustGatherCLIVerifier { return mustGatherCLIVerifier{ subscriptionID: subscriptionID, resourceGroup: resourceGroup, + kustoCluster: kustoCluster, + kustoRegion: kustoRegion, } } From 522f45348af966609d94d707be3f7405af988ccf Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Tue, 25 Aug 2026 10:16:20 +0000 Subject: [PATCH 2/3] feat: default RENDERED_CONFIG in e2e-local/run target Provides a sane default for RENDERED_CONFIG based on DEPLOY_ENV and LOCATION so local E2E runs can pick up the rendered config without requiring the variable to be set explicitly. Co-Authored-By: Claude Opus 4.6 --- Makefile | 1 + test/E2ELocal.mk | 1 + 2 files changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 71ac3bf457d..503cf7a26ac 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,7 @@ e2e-local/setup: e2e-local/run: $(ARO_HCP_TESTS) export LOCATION="$${LOCATION:-westus3}"; \ + export RENDERED_CONFIG="$${RENDERED_CONFIG:-$(CURDIR)/config/rendered/dev/$(DEPLOY_ENV)/$$LOCATION.yaml}"; \ export AROHCP_ENV="development"; \ export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \ export ARTIFACT_DIR=$${ARTIFACT_DIR:-_artifacts}; \ diff --git a/test/E2ELocal.mk b/test/E2ELocal.mk index 42b748427ea..a47d31c3ddf 100644 --- a/test/E2ELocal.mk +++ b/test/E2ELocal.mk @@ -23,6 +23,7 @@ SNAPSHOT_RENDERED_CONFIG := $(shell mktemp) e2e-local/run-test: $(ARO_HCP_TESTS) $(MAKE) -C $(DIR) -f $(THIS) .e2e-local/setup export LOCATION="$${LOCATION:-${REGION}}"; \ + export RENDERED_CONFIG="$${RENDERED_CONFIG:-$(DIR)/../config/rendered/dev/$(DEPLOY_ENV)/$$LOCATION.yaml}"; \ export AROHCP_ENV="development"; \ export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \ export AZURE_TENANT_ID="$$(az account show --output tsv --query 'tenantId')"; \ From 7a93dd5851f582b9fb50f30c5951fe2f48d805ce Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Tue, 25 Aug 2026 14:15:58 +0000 Subject: [PATCH 3/3] refactor: replace rendered config loading with environment variables Replace the RENDERED_CONFIG-based config loading approach with direct environment variables (REGION_RG, HCP_WORKSPACE_NAME, KUSTO_NAME, KUSTO_REGION) defined in test/Env.mk and resolved via templatize. This removes the rendered config infrastructure from the E2E framework (RenderedConfig(), getRenderedConfig(), renderedConfigPath) and instead has tests read values directly from env vars, matching the existing pattern for LOCATION, SVC_CLUSTER, etc. Co-Authored-By: Claude Opus 4.6 --- Makefile | 1 - test/E2ELocal.mk | 5 +- test/Env.mk | 6 +- .../custom-link-tools/options.go | 16 +-- .../gather-observability/options.go | 19 ++-- .../gather-observability/promql.go | 103 +++++++++++++++++- .../gather-observability/workspace.go | 4 +- .../internal/testutil}/config.go | 4 +- test/e2e/ksm_hcp_metrics.go | 14 +-- test/e2e/kusto_logs_present.go | 15 +-- .../framework/per_invocation_framework.go | 25 ----- test/util/framework/per_test_framework.go | 7 -- test/util/verifiers/kusto.go | 8 +- test/util/verifiers/must_gather_cli.go | 15 ++- 14 files changed, 147 insertions(+), 95 deletions(-) rename test/{util/testconfig => cmd/aro-hcp-tests/internal/testutil}/config.go (97%) diff --git a/Makefile b/Makefile index 503cf7a26ac..71ac3bf457d 100644 --- a/Makefile +++ b/Makefile @@ -197,7 +197,6 @@ e2e-local/setup: e2e-local/run: $(ARO_HCP_TESTS) export LOCATION="$${LOCATION:-westus3}"; \ - export RENDERED_CONFIG="$${RENDERED_CONFIG:-$(CURDIR)/config/rendered/dev/$(DEPLOY_ENV)/$$LOCATION.yaml}"; \ export AROHCP_ENV="development"; \ export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \ export ARTIFACT_DIR=$${ARTIFACT_DIR:-_artifacts}; \ diff --git a/test/E2ELocal.mk b/test/E2ELocal.mk index a47d31c3ddf..476ca4e1312 100644 --- a/test/E2ELocal.mk +++ b/test/E2ELocal.mk @@ -23,7 +23,10 @@ SNAPSHOT_RENDERED_CONFIG := $(shell mktemp) e2e-local/run-test: $(ARO_HCP_TESTS) $(MAKE) -C $(DIR) -f $(THIS) .e2e-local/setup export LOCATION="$${LOCATION:-${REGION}}"; \ - export RENDERED_CONFIG="$${RENDERED_CONFIG:-$(DIR)/../config/rendered/dev/$(DEPLOY_ENV)/$$LOCATION.yaml}"; \ + export REGION_RG="$${REGION_RG:-${REGION_RG}}"; \ + export HCP_WORKSPACE_NAME="$${HCP_WORKSPACE_NAME:-${HCP_WORKSPACE_NAME}}"; \ + export KUSTO_NAME="$${KUSTO_NAME:-${KUSTO_NAME}}"; \ + export KUSTO_REGION="$${KUSTO_REGION:-${KUSTO_REGION}}"; \ export AROHCP_ENV="development"; \ export CUSTOMER_SUBSCRIPTION="$$(az account show --output tsv --query 'name')"; \ export AZURE_TENANT_ID="$$(az account show --output tsv --query 'tenantId')"; \ diff --git a/test/Env.mk b/test/Env.mk index 75eacb8ce01..c0fac80a23e 100644 --- a/test/Env.mk +++ b/test/Env.mk @@ -1,2 +1,6 @@ SVC_CLUSTER ?= {{ .svc.aks.name }} -REGION ?= {{ .region }} \ No newline at end of file +REGION ?= {{ .region }} +REGION_RG ?= {{ .regionRG }} +HCP_WORKSPACE_NAME ?= {{ .monitoring.hcpWorkspaceName }} +KUSTO_NAME ?= {{ .kusto.kustoName }} +KUSTO_REGION ?= {{ .kusto.location }} \ No newline at end of file diff --git a/test/cmd/aro-hcp-tests/custom-link-tools/options.go b/test/cmd/aro-hcp-tests/custom-link-tools/options.go index c0f5c5a0359..c516a9e2c83 100644 --- a/test/cmd/aro-hcp-tests/custom-link-tools/options.go +++ b/test/cmd/aro-hcp-tests/custom-link-tools/options.go @@ -33,7 +33,7 @@ import ( "k8s.io/utils/clock" "github.com/Azure/ARO-HCP/internal/utils" - "github.com/Azure/ARO-HCP/test/util/testconfig" + "github.com/Azure/ARO-HCP/test/cmd/aro-hcp-tests/internal/testutil" "github.com/Azure/ARO-HCP/test/util/timing" "github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto" "github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline" @@ -133,32 +133,32 @@ func (o *RawOptions) Validate() (*ValidatedOptions, error) { } func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) { - cfg, err := testconfig.LoadRenderedConfig(o.RenderedConfig) + cfg, err := testutil.LoadRenderedConfig(o.RenderedConfig) if err != nil { return nil, err } - svcClusterName, err := testconfig.ConfigGetString(cfg, "svc.aks.name") + svcClusterName, err := testutil.ConfigGetString(cfg, "svc.aks.name") if err != nil { return nil, fmt.Errorf("failed to get svc cluster name from config: %w", err) } - mgmtClusterName, err := testconfig.ConfigGetString(cfg, "mgmt.aks.name") + mgmtClusterName, err := testutil.ConfigGetString(cfg, "mgmt.aks.name") if err != nil { return nil, fmt.Errorf("failed to get mgmt cluster name from config: %w", err) } - kustoName, err := testconfig.ConfigGetString(cfg, "kusto.kustoName") + kustoName, err := testutil.ConfigGetString(cfg, "kusto.kustoName") if err != nil { return nil, fmt.Errorf("failed to get kusto name from config: %w", err) } - kustoRegion, err := testconfig.ConfigGetString(cfg, "kusto.location") + kustoRegion, err := testutil.ConfigGetString(cfg, "kusto.location") if err != nil { return nil, fmt.Errorf("failed to get kusto location from config: %w", err) } - serviceLogsDB, err := testconfig.ConfigGetString(cfg, "kusto.serviceLogsDatabase") + serviceLogsDB, err := testutil.ConfigGetString(cfg, "kusto.serviceLogsDatabase") if err != nil { return nil, fmt.Errorf("failed to get service logs database from config: %w", err) } - hcpLogsDB, err := testconfig.ConfigGetString(cfg, "kusto.hostedControlPlaneLogsDatabase") + hcpLogsDB, err := testutil.ConfigGetString(cfg, "kusto.hostedControlPlaneLogsDatabase") if err != nil { return nil, fmt.Errorf("failed to get hosted control plane logs database from config: %w", err) } diff --git a/test/cmd/aro-hcp-tests/gather-observability/options.go b/test/cmd/aro-hcp-tests/gather-observability/options.go index c71d06b5488..6ed42f1af77 100644 --- a/test/cmd/aro-hcp-tests/gather-observability/options.go +++ b/test/cmd/aro-hcp-tests/gather-observability/options.go @@ -39,9 +39,8 @@ import ( "github.com/Azure/ARO-HCP/internal/api/metadataapi" "github.com/Azure/ARO-HCP/internal/utils" + "github.com/Azure/ARO-HCP/test/cmd/aro-hcp-tests/internal/testutil" "github.com/Azure/ARO-HCP/test/util/junit" - promutil "github.com/Azure/ARO-HCP/test/util/prometheus" - "github.com/Azure/ARO-HCP/test/util/testconfig" "github.com/Azure/ARO-HCP/test/util/timing" ) @@ -131,20 +130,20 @@ func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) { return nil, fmt.Errorf("failed to create output directory %s: %w", o.OutputDir, err) } - cfg, err := testconfig.LoadRenderedConfig(o.RenderedConfig) + cfg, err := testutil.LoadRenderedConfig(o.RenderedConfig) if err != nil { return nil, err } - regionRG, err := testconfig.ConfigGetString(cfg, "regionRG") + regionRG, err := testutil.ConfigGetString(cfg, "regionRG") if err != nil { return nil, fmt.Errorf("failed to get regionRG from config: %w", err) } - svcWorkspace, err := testconfig.ConfigGetString(cfg, "monitoring.svcWorkspaceName") + svcWorkspace, err := testutil.ConfigGetString(cfg, "monitoring.svcWorkspaceName") if err != nil { return nil, fmt.Errorf("failed to get monitoring.svcWorkspaceName from config: %w", err) } - hcpWorkspace, err := testconfig.ConfigGetString(cfg, "monitoring.hcpWorkspaceName") + hcpWorkspace, err := testutil.ConfigGetString(cfg, "monitoring.hcpWorkspaceName") if err != nil { return nil, fmt.Errorf("failed to get monitoring.hcpWorkspaceName from config: %w", err) } @@ -153,7 +152,7 @@ func (o *ValidatedOptions) Complete(ctx context.Context) (*Options, error) { // the name from frontend.cosmosDB.name (see region.bicep / output-region.bicep). // Its platform metrics (NormalizedRUConsumption, AutoscaledRU, ...) are queried // via the Azure Monitor metrics API rather than Prometheus. - cosmosDBName, err := testconfig.ConfigGetString(cfg, "frontend.cosmosDB.name") + cosmosDBName, err := testutil.ConfigGetString(cfg, "frontend.cosmosDB.name") if err != nil { return nil, fmt.Errorf("failed to get frontend.cosmosDB.name from config: %w", err) } @@ -249,13 +248,13 @@ func buildCosmosAutoscaleMaxLookup(cfg configtypes.Configuration) (autoscaleMaxL } byContainer := make(map[string]float64, len(fixed)) for container, path := range fixed { - v, err := testconfig.ConfigGetInt(cfg, path) + v, err := testutil.ConfigGetInt(cfg, path) if err != nil { return nil, fmt.Errorf("failed to get %s from config: %w", path, err) } byContainer[container] = float64(v) } - manifestsMax, err := testconfig.ConfigGetInt(cfg, "kubeApplier.cosmosContainerMaxScale") + manifestsMax, err := testutil.ConfigGetInt(cfg, "kubeApplier.cosmosContainerMaxScale") if err != nil { return nil, fmt.Errorf("failed to get kubeApplier.cosmosContainerMaxScale from config: %w", err) } @@ -454,7 +453,7 @@ func (o Options) runQueries(ctx context.Context, workspaces map[string]*workspac logger.Info("executing PromQL query", "panel", panel.Title, "title", q.Title, "workspace", q.Workspace) - resp, err := promutil.QueryRange(ctx, httpClient, o.cred, endpoint, q.Query, o.TimeWindow.Start, o.TimeWindow.End, q.Step) + resp, err := queryRange(ctx, httpClient, o.cred, endpoint, q.Query, o.TimeWindow.Start, o.TimeWindow.End, q.Step) if err != nil { logger.Error(err, "PromQL query failed", "title", q.Title) queryErr = err.Error() diff --git a/test/cmd/aro-hcp-tests/gather-observability/promql.go b/test/cmd/aro-hcp-tests/gather-observability/promql.go index 590a2ce0412..d9b290ebbe2 100644 --- a/test/cmd/aro-hcp-tests/gather-observability/promql.go +++ b/test/cmd/aro-hcp-tests/gather-observability/promql.go @@ -15,15 +15,25 @@ package gatherobservability import ( + "bytes" + "context" + "encoding/json" "fmt" + "io" + "net/http" + "net/url" "slices" + "strconv" "strings" + "time" _ "embed" "sigs.k8s.io/yaml" - promutil "github.com/Azure/ARO-HCP/test/util/prometheus" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" ) // QueriesConfig holds panels of grouped PromQL queries to run against Azure @@ -131,13 +141,24 @@ type QuerySpec struct { } // PrometheusResponse is the top-level Prometheus HTTP API response. -type PrometheusResponse = promutil.Response +type PrometheusResponse struct { + Status string `json:"status"` + Data PrometheusData `json:"data"` + ErrorType string `json:"errorType,omitempty"` + Error string `json:"error,omitempty"` +} // PrometheusData holds the result set from a query_range call. -type PrometheusData = promutil.Data +type PrometheusData struct { + ResultType string `json:"resultType"` + Result []PrometheusResult `json:"result"` +} // PrometheusResult is a single timeseries returned by query_range. -type PrometheusResult = promutil.Result +type PrometheusResult struct { + Metric map[string]string `json:"metric"` + Values [][]any `json:"values"` // each element is [unix_timestamp_float, "string_value"] +} //go:embed queries.yaml var defaultQueriesYAML []byte @@ -255,3 +276,77 @@ func sortedMetricResources() []string { slices.Sort(out) return out } + +// lookupPrometheusEndpoint retrieves the Prometheus query endpoint for an +// Azure Monitor workspace using the ARM SDK. +func lookupPrometheusEndpoint(ctx context.Context, cred azcore.TokenCredential, subscriptionID, resourceGroup, workspaceName string) (string, error) { + client, err := armmonitor.NewAzureMonitorWorkspacesClient(subscriptionID, cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create monitor workspaces client: %w", err) + } + resp, err := client.Get(ctx, resourceGroup, workspaceName, nil) + if err != nil { + return "", fmt.Errorf("failed to get workspace %s: %w", workspaceName, err) + } + if resp.Properties == nil || resp.Properties.Metrics == nil || resp.Properties.Metrics.PrometheusQueryEndpoint == nil { + return "", fmt.Errorf("workspace %s has no Prometheus query endpoint", workspaceName) + } + return *resp.Properties.Metrics.PrometheusQueryEndpoint, nil +} + +// queryRange executes a Prometheus query_range request against an Azure Monitor +// Prometheus endpoint using bearer token authentication. The caller should pass +// a shared *http.Client to amortize connection setup across multiple queries. +func queryRange(ctx context.Context, httpClient *http.Client, cred azcore.TokenCredential, endpoint, query string, start, end time.Time, step string) (*PrometheusResponse, error) { + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://prometheus.monitor.azure.com/.default"}, + }) + if err != nil { + return nil, fmt.Errorf("failed to get Prometheus token: %w", err) + } + + u, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint URL %q: %w", endpoint, err) + } + u.Path = strings.TrimRight(u.Path, "/") + "/api/v1/query_range" + + params := url.Values{} + params.Set("query", query) + params.Set("start", strconv.FormatInt(start.Unix(), 10)) + params.Set("end", strconv.FormatInt(end.Unix(), 10)) + params.Set("step", step) + u.RawQuery = params.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("prometheus query_range request failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("prometheus query_range returned %d: %s", resp.StatusCode, string(body)) + } + + var promResp PrometheusResponse + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + if err := dec.Decode(&promResp); err != nil { + return nil, fmt.Errorf("failed to parse Prometheus response: %w", err) + } + if promResp.Status != "success" { + return nil, fmt.Errorf("prometheus query error (%s): %s", promResp.ErrorType, promResp.Error) + } + return &promResp, nil +} diff --git a/test/cmd/aro-hcp-tests/gather-observability/workspace.go b/test/cmd/aro-hcp-tests/gather-observability/workspace.go index 73cf187b309..820e4fa6beb 100644 --- a/test/cmd/aro-hcp-tests/gather-observability/workspace.go +++ b/test/cmd/aro-hcp-tests/gather-observability/workspace.go @@ -25,8 +25,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - - promutil "github.com/Azure/ARO-HCP/test/util/prometheus" ) const ( @@ -48,7 +46,7 @@ func fetchWorkspaceData(ctx context.Context, cred azcore.TokenCredential, wsType return nil, fmt.Errorf("logger not found in context: %w", err) } - promEndpoint, err := promutil.LookupPrometheusEndpoint(ctx, cred, workspaceResourceID.SubscriptionID, workspaceResourceID.ResourceGroupName, workspaceResourceID.Name) + promEndpoint, err := lookupPrometheusEndpoint(ctx, cred, workspaceResourceID.SubscriptionID, workspaceResourceID.ResourceGroupName, workspaceResourceID.Name) if err != nil { return nil, fmt.Errorf("failed to look up Prometheus endpoint: %w", err) } diff --git a/test/util/testconfig/config.go b/test/cmd/aro-hcp-tests/internal/testutil/config.go similarity index 97% rename from test/util/testconfig/config.go rename to test/cmd/aro-hcp-tests/internal/testutil/config.go index 10a87ff379e..9e92de33ba3 100644 --- a/test/util/testconfig/config.go +++ b/test/cmd/aro-hcp-tests/internal/testutil/config.go @@ -1,4 +1,4 @@ -// Copyright 2026 Microsoft Corporation +// Copyright 2025 Microsoft Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package testconfig +package testutil import ( "encoding/json" diff --git a/test/e2e/ksm_hcp_metrics.go b/test/e2e/ksm_hcp_metrics.go index fdbfbbf731a..43f219fe992 100644 --- a/test/e2e/ksm_hcp_metrics.go +++ b/test/e2e/ksm_hcp_metrics.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "os" "regexp" "time" @@ -27,7 +28,6 @@ import ( "github.com/Azure/ARO-HCP/test/util/framework" "github.com/Azure/ARO-HCP/test/util/labels" promutil "github.com/Azure/ARO-HCP/test/util/prometheus" - "github.com/Azure/ARO-HCP/test/util/testconfig" ) var _ = Describe("KSM HCP Metrics", func() { @@ -39,15 +39,11 @@ var _ = Describe("KSM HCP Metrics", func() { func(ctx context.Context) { tc := framework.NewTestContext() - By("Loading rendered config") - cfg, err := tc.RenderedConfig() - Expect(err).NotTo(HaveOccurred(), "failed to load rendered config — is RENDERED_CONFIG set?") + regionRG := os.Getenv("REGION_RG") + Expect(regionRG).NotTo(BeEmpty(), "REGION_RG environment variable must be set") - regionRG, err := testconfig.ConfigGetString(cfg, "regionRG") - Expect(err).NotTo(HaveOccurred(), "failed to get regionRG from rendered config") - - hcpWorkspaceName, err := testconfig.ConfigGetString(cfg, "monitoring.hcpWorkspaceName") - Expect(err).NotTo(HaveOccurred(), "failed to get monitoring.hcpWorkspaceName from rendered config") + hcpWorkspaceName := os.Getenv("HCP_WORKSPACE_NAME") + Expect(hcpWorkspaceName).NotTo(BeEmpty(), "HCP_WORKSPACE_NAME environment variable must be set") subscriptionID, err := tc.SubscriptionID(ctx) Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID") diff --git a/test/e2e/kusto_logs_present.go b/test/e2e/kusto_logs_present.go index fb583cec4cc..132ba6928e9 100644 --- a/test/e2e/kusto_logs_present.go +++ b/test/e2e/kusto_logs_present.go @@ -23,7 +23,6 @@ import ( "github.com/Azure/ARO-HCP/test/util/framework" "github.com/Azure/ARO-HCP/test/util/labels" - "github.com/Azure/ARO-HCP/test/util/testconfig" "github.com/Azure/ARO-HCP/test/util/verifiers" ) @@ -89,23 +88,15 @@ var _ = Describe("Engineering", func() { Expect(err).NotTo(HaveOccurred(), "failed to create HCP cluster for kusto logs test") subscriptionID, err := tc.SubscriptionID(ctx) Expect(err).NotTo(HaveOccurred(), "failed to get subscription ID") - - By("loading kusto config from rendered config") - cfg, err := tc.RenderedConfig() - Expect(err).NotTo(HaveOccurred(), "failed to load rendered config — is RENDERED_CONFIG set?") - kustoName, err := testconfig.ConfigGetString(cfg, "kusto.kustoName") - Expect(err).NotTo(HaveOccurred(), "failed to get kusto.kustoName from rendered config") - kustoRegion, err := testconfig.ConfigGetString(cfg, "kusto.location") - Expect(err).NotTo(HaveOccurred(), "failed to get kusto.location from rendered config") - By("verifying kusto logs are present") + Eventually(func() error { - return verifiers.VerifyMustGatherLogs(subscriptionID, *resourceGroup.Name, kustoName, kustoRegion).Verify(ctx) + return verifiers.VerifyMustGatherLogs(subscriptionID, *resourceGroup.Name).Verify(ctx) }, 10*time.Minute, 60*time.Second).Should(Succeed()) By("verifying must-gather CLI works") Eventually(func() error { - return verifiers.VerifyMustGatherCLI(subscriptionID, *resourceGroup.Name, kustoName, kustoRegion).Verify(ctx) + return verifiers.VerifyMustGatherCLI(subscriptionID, *resourceGroup.Name).Verify(ctx) }, 10*time.Minute, 60*time.Second).Should(Succeed()) }) diff --git a/test/util/framework/per_invocation_framework.go b/test/util/framework/per_invocation_framework.go index c614a3c4395..be2aaff45cf 100644 --- a/test/util/framework/per_invocation_framework.go +++ b/test/util/framework/per_invocation_framework.go @@ -34,7 +34,6 @@ import ( "github.com/onsi/ginkgo/v2/types" "golang.org/x/net/http2" - configtypes "github.com/Azure/ARO-Tools/config/types" "github.com/Azure/azure-sdk-for-go/sdk/azcore" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" @@ -44,7 +43,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" "github.com/Azure/ARO-HCP/internal/azsdk" - "github.com/Azure/ARO-HCP/test/util/testconfig" ) type perBinaryInvocationTestContext struct { @@ -64,8 +62,6 @@ type perBinaryInvocationTestContext struct { pooledIdentities bool compressTimingMetadata bool - renderedConfigPath string - contextLock sync.RWMutex subscriptionID string azureCredentials azcore.TokenCredential @@ -76,10 +72,6 @@ type perBinaryInvocationTestContext struct { // target the same location, so a suite-wide cache avoids repeatedly burning // per-test timeout budget on the same ARM call. virtualMachineResourceSKUsByLocation map[string][]*armcompute.ResourceSKU - - renderedConfigOnce sync.Once - renderedConfig configtypes.Configuration - renderedConfigErr error } type CleanupFunc func(ctx context.Context) error @@ -130,7 +122,6 @@ func invocationContext() *perBinaryInvocationTestContext { skipCleanup: skipCleanup(), pooledIdentities: pooledIdentities(), compressTimingMetadata: compressTimingMetadata(), - renderedConfigPath: renderedConfigPath(), defaultTransport: defaultHTTPTransport(), virtualMachineResourceSKUsByLocation: make(map[string][]*armcompute.ResourceSKU), } @@ -476,22 +467,6 @@ func IsDevelopmentEnvironment() bool { return strings.ToLower(os.Getenv("AROHCP_ENV")) == "development" } -// renderedConfigPath returns the value of RENDERED_CONFIG environment variable -func renderedConfigPath() string { - return os.Getenv("RENDERED_CONFIG") -} - -func (tc *perBinaryInvocationTestContext) getRenderedConfig() (configtypes.Configuration, error) { - tc.renderedConfigOnce.Do(func() { - if tc.renderedConfigPath == "" { - tc.renderedConfigErr = fmt.Errorf("RENDERED_CONFIG environment variable is not set") - return - } - tc.renderedConfig, tc.renderedConfigErr = testconfig.LoadRenderedConfig(tc.renderedConfigPath) - }) - return tc.renderedConfig, tc.renderedConfigErr -} - // Must is a generic function that takes a value of type T and an error. // If the error is not nil, it panics with the error. // Otherwise, it returns the value of type T. diff --git a/test/util/framework/per_test_framework.go b/test/util/framework/per_test_framework.go index a68fe9a8b78..0f48e2f65f4 100644 --- a/test/util/framework/per_test_framework.go +++ b/test/util/framework/per_test_framework.go @@ -43,7 +43,6 @@ import ( "sigs.k8s.io/yaml" - configtypes "github.com/Azure/ARO-Tools/config/types" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" @@ -1276,12 +1275,6 @@ func (tc *perItOrDescribeTestContext) AzureCredential() (azcore.TokenCredential, return tc.perBinaryInvocationTestContext.getAzureCredentials() } -// RenderedConfig returns the parsed rendered configuration. -// Requires the RENDERED_CONFIG environment variable to be set to a valid config YAML path. -func (tc *perItOrDescribeTestContext) RenderedConfig() (configtypes.Configuration, error) { - return tc.perBinaryInvocationTestContext.getRenderedConfig() -} - func (tc *perItOrDescribeTestContext) TenantID() string { return tc.perBinaryInvocationTestContext.tenantID } diff --git a/test/util/verifiers/kusto.go b/test/util/verifiers/kusto.go index 132201b6b4c..83dcf2df63b 100644 --- a/test/util/verifiers/kusto.go +++ b/test/util/verifiers/kusto.go @@ -138,11 +138,11 @@ func (v verifyMustGatherLogsImpl) Verify(ctx context.Context) error { return nil } -// VerifyMustGatherLogs creates a new must-gather logs verifier with default configuration. -func VerifyMustGatherLogs(subscriptionID, rgName, kustoCluster, kustoRegion string) verifyMustGatherLogsImpl { +// VerifyMustGatherLogs creates a new must-gather logs verifier with default configuration +func VerifyMustGatherLogs(subscriptionID, rgName string) verifyMustGatherLogsImpl { config := mustGatherVerifierConfig{ - KustoCluster: kustoCluster, - KustoRegion: kustoRegion, + KustoCluster: "hcp-dev-us-2", + KustoRegion: "eastus2", SubscriptionID: subscriptionID, ResourceGroup: rgName, QueryTimeout: 5 * time.Minute, diff --git a/test/util/verifiers/must_gather_cli.go b/test/util/verifiers/must_gather_cli.go index fdd0041a147..a19a4659cf4 100644 --- a/test/util/verifiers/must_gather_cli.go +++ b/test/util/verifiers/must_gather_cli.go @@ -26,6 +26,9 @@ import ( ) const ( + kustoCluster = "hcp-dev-us-2" + kustoRegion = "eastus2" + serviceDir = "service" hostedControlPlaneDir = "hosted-control-plane" clusterDir = "cluster" @@ -41,8 +44,6 @@ type mustGatherCLITestCase struct { type mustGatherCLIVerifier struct { subscriptionID string resourceGroup string - kustoCluster string - kustoRegion string } func (v mustGatherCLIVerifier) Name() string { @@ -126,7 +127,7 @@ func (v mustGatherCLIVerifier) Verify(ctx context.Context) error { var errors []string for _, tc := range testCases { logger.Info("Running must-gather CLI test case", "name", tc.name) - if err := runTestCase(ctx, binary, tc, v.kustoCluster, v.kustoRegion); err != nil { + if err := runTestCase(ctx, binary, tc); err != nil { errors = append(errors, fmt.Sprintf("%s: %v", tc.name, err)) logger.Error(err, "Test case failed", "name", tc.name) } else { @@ -140,7 +141,7 @@ func (v mustGatherCLIVerifier) Verify(ctx context.Context) error { return nil } -func runTestCase(ctx context.Context, binary string, tc mustGatherCLITestCase, kustoCluster, kustoRegion string) error { +func runTestCase(ctx context.Context, binary string, tc mustGatherCLITestCase) error { outputDir, err := os.MkdirTemp("", "must-gather-cli-e2e-"+tc.name+"-") if err != nil { return fmt.Errorf("failed to create temp dir: %w", err) @@ -212,12 +213,10 @@ func infraClusterNames() (svcCluster, mgmtCluster string) { return svcCluster, mgmtCluster } -// VerifyMustGatherCLI creates a verifier that tests the hcpctl must-gather CLI. -func VerifyMustGatherCLI(subscriptionID, resourceGroup, kustoCluster, kustoRegion string) mustGatherCLIVerifier { +// VerifyMustGatherCLI creates a verifier that tests the hcpctl must-gather CLI +func VerifyMustGatherCLI(subscriptionID, resourceGroup string) mustGatherCLIVerifier { return mustGatherCLIVerifier{ subscriptionID: subscriptionID, resourceGroup: resourceGroup, - kustoCluster: kustoCluster, - kustoRegion: kustoRegion, } }