feat: add MC AKS node health metrics to gather-observability

test/cmd/aro-hcp-tests/gather-observability/chart.go

@@ -59,12 +59,14 @@ type parsedSeries struct {
 	data   []opts.LineData
 }
 
-func (s parsedSeries) peakValue() float64 {
+func seriesPeakValue(series []parsedSeries) float64 {
 	var peak float64
-	for _, d := range s.data {
-		if arr, ok := d.Value.([]any); ok && len(arr) == 2 {
-			if v, ok := arr[1].(float64); ok && v > peak {
-				peak = v
+	for _, s := range series {
+		for _, d := range s.data {
+			if arr, ok := d.Value.([]any); ok && len(arr) == 2 {
+				if v, ok := arr[1].(float64); ok && v > peak {
+					peak = v
+				}
 			}
 		}
 	}
@@ -216,17 +218,17 @@ func buildLineChartData(q QuerySpec, resourceID string, series []parsedSeries, t
 	for i := range series {
 		series[i].data = insertGapMarkers(series[i].data)
 	}
-	// Sort by peak value descending for consistent legend ordering
-	slices.SortFunc(series, func(a, b parsedSeries) int {
-		return cmp.Compare(b.peakValue(), a.peakValue())
-	})
 	subtitle := fmt.Sprintf("Window: %s — %s", tw.Start.UTC().Format(time.RFC3339), tw.End.UTC().Format(time.RFC3339))
 
 	// Build labels: strip label keys that are the same across all series
 	commonLabels := findCommonLabels(series)
 	for i := range series {
 		series[i].label = compactMetricLabel(series[i].metric, commonLabels)
 	}
+	// Sort by label for consistent color assignment across charts
+	slices.SortFunc(series, func(a, b parsedSeries) int {
+		return cmp.Compare(a.label, b.label)
+	})
 
 	// Adjust chart height for legend when many series
 	seriesLabels := make([]string, len(series))
@@ -265,12 +267,22 @@ func buildLineChartData(q QuerySpec, resourceID string, series []parsedSeries, t
 			Min:  tw.Start.UnixMilli(),
 			Max:  tw.End.UnixMilli(),
 		}),
-		charts.WithYAxisOpts(opts.YAxis{
-			Type:         "value",
-			Name:         q.Unit,
-			NameLocation: "middle",
-			NameGap:      50,
-		}),
+		charts.WithYAxisOpts(func() opts.YAxis {
+			axis := opts.YAxis{
+				Type:         "value",
+				Name:         q.Unit,
+				NameLocation: "middle",
+				NameGap:      50,
+			}
+			if q.Unit == "percent" {
+				axis.Min = 0
+				maxVal := seriesPeakValue(series)
+				if maxVal <= 100 {
+					axis.Max = 100
+				}
+			}
+			return axis
+		}()),
 		charts.WithGridOpts(opts.Grid{
 			Left:   "80",
 			Right:  "40",

test/cmd/aro-hcp-tests/gather-observability/queries.yaml

@@ -320,3 +320,126 @@ panels:
     unit: /s
     workspace: svc
     step: "60s"
+- title: "MC AKS Metrics"
+  queries:
+  - title: "MC User Node Count by Phase"
+    description: "User node count per management cluster, stacked by phase: Ready (healthy, age >10m), NotReady (unhealthy, age >10m), Joining (any status, age ≤10m). Excludes system and infra pool nodes. Each sub-chart shows one management cluster."
+    query: |
+      label_replace(
+        count by (cluster) (
+          (max by (node, cluster, condition, status) (
+            kube_node_status_condition{cluster=~".*-mgmt-.*", node!~"aks-system-.*|aks-infra.*-.*", condition="Ready", status="true"}
+          ) == 1)
+          and on (node, cluster)
+          (time() - max by (node, cluster) (kube_node_created{cluster=~".*-mgmt-.*"}) > 600)
+        ),
+        "phase", "Ready", "", ""
+      )
+      or
+      label_replace(
+        count by (cluster) (
+          (max by (node, cluster, condition, status) (
+            kube_node_status_condition{cluster=~".*-mgmt-.*", node!~"aks-system-.*|aks-infra.*-.*", condition="Ready", status!="true"}
+          ) == 1)
+          and on (node, cluster)
+          (time() - max by (node, cluster) (kube_node_created{cluster=~".*-mgmt-.*"}) > 600)
+        ),
+        "phase", "NotReady", "", ""
+      )
+      or
+      label_replace(
+        count by (cluster) (
+          max by (node, cluster) (
+            kube_node_status_condition{cluster=~".*-mgmt-.*", node!~"aks-system-.*|aks-infra.*-.*", condition="Ready"}
+          )
+          and on (node, cluster)
+          (time() - max by (node, cluster) (kube_node_created{cluster=~".*-mgmt-.*"}) <= 600)
+        ),
+        "phase", "Joining", "", ""
+      )
+    unit: nodes
+    workspace: svc
+    step: "60s"
+    chartType: faceted-stacked-area
+    facetBy: cluster
+    stackBy: phase
+    colors:
+      Ready: "#4CAF50"
+      NotReady: "#F44336"
+      Joining: "#2196F3"
+  - title: "MC User Node Unhealthy Conditions"
+    description: "Count of user nodes with unhealthy conditions per management cluster. Only non-zero conditions are shown. Excludes Ready (shown separately) and VMEventScheduled (informational). Each sub-chart shows one management cluster."
+    query: |
+      count by (cluster, condition) (
+        max by (node, cluster, condition, status) (
+          kube_node_status_condition{
+            cluster=~".*-mgmt-.*",
+            node!~"aks-system-.*|aks-infra.*-.*",
+            condition!~"Ready|VMEventScheduled",
+            status="true"
+          }
+        ) == 1
+      )
+    unit: nodes
+    workspace: svc
+    step: "60s"
+    chartType: faceted-stacked-area
+    facetBy: cluster
+    stackBy: condition
+  - title: "MC User Node CPU Usage"
+    description: "Maximum CPU usage percentage across user nodes per management cluster. Shows the busiest node at each point. Sustained values above 80% indicate CPU pressure that may affect pod scheduling and kubelet health."
+    query: |
+      max by (cluster) (
+        (1 - avg by (instance, cluster) (
+          max without (prometheus_replica) (
+            rate(node_cpu_seconds_total{cluster=~".*-mgmt-.*", instance!~"aks-system-.*|aks-infra.*-.*", mode="idle"}[5m])
+          )
+        ))
+      ) * 100
+    unit: percent
+    workspace: svc
+    step: "60s"
+  - title: "MC User Node Memory Usage"
+    description: "Maximum memory usage percentage across user nodes per management cluster. Computed as (total - available) / total. Sustained high values indicate memory pressure that may trigger OOM kills or kubelet evictions."
+    query: |
+      max by (cluster) (
+        (1 - (
+          max by (instance, cluster) (node_memory_MemAvailable_bytes{cluster=~".*-mgmt-.*", instance!~"aks-system-.*|aks-infra.*-.*"})
+          /
+          max by (instance, cluster) (node_memory_MemTotal_bytes{cluster=~".*-mgmt-.*", instance!~"aks-system-.*|aks-infra.*-.*"})
+        ))
+      ) * 100
+    unit: percent
+    workspace: svc
+    step: "60s"
+  - title: "MC User Node Disk Usage"
+    description: "Maximum root filesystem usage percentage across user nodes per management cluster. Computed as (size - available) / size for the root mount. High values trigger DiskPressure node condition and pod evictions."
+    query: |
+      max by (cluster) (
+        (1 - (
+          max by (instance, cluster) (node_filesystem_avail_bytes{cluster=~".*-mgmt-.*", instance!~"aks-system-.*|aks-infra.*-.*", mountpoint="/", fstype!="tmpfs"})
+          /
+          max by (instance, cluster) (node_filesystem_size_bytes{cluster=~".*-mgmt-.*", instance!~"aks-system-.*|aks-infra.*-.*", mountpoint="/", fstype!="tmpfs"})
+        ))
+      ) * 100
+    unit: percent
+    workspace: svc
+    step: "60s"
+  - title: "MC User Node Pod Capacity"
+    description: "Overall pod utilization across user nodes per management cluster. Computed as total running pods / total pod capacity. Values approaching 100% mean the cluster is running out of pod capacity."
+    query: |
+      sum by (cluster) (
+        max by (instance, cluster) (kubelet_running_pods{cluster=~".*-mgmt-.*", instance!~"aks-system-.*|aks-infra.*-.*"})
+      )
+      /
+      sum by (cluster) (
+        max by (instance, cluster) (
+          label_replace(
+            kube_node_status_capacity{cluster=~".*-mgmt-.*", node!~"aks-system-.*|aks-infra.*-.*", resource="pods"},
+            "instance", "$1", "node", "(.*)"
+          )
+        )
+      ) * 100
+    unit: percent
+    workspace: svc
+    step: "60s"