diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep index 83dad6db77c..f92977df648 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep @@ -375,6 +375,85 @@ resource arohcpClusterProvisionSloErrorAlerts 'Microsoft.AlertsManagement/promet } } +resource rpUserJourneyClusterUpgradeMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'rp-user-journey-cluster-upgrade-monitor-rules' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeStuckInDesired' + enabled: true + labels: { + component: 'slo' + severity: '4' + slo: 'cluster-upgrade-reach-partial' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeStuckInDesired/{{ $labels.cluster }}{{ $labels.resource_id }}/{{ $labels.version }}' + description: '''Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. With the alert pending for 5 minutes, paging starts after ~25 minutes. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + info: '''Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. With the alert pending for 5 minutes, paging starts after ~25 minutes. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '{{ $labels.cluster }}: Cluster upgrade on {{ $labels.resource_id }} to version {{ $labels.version }} not progressing' + title: '{{ $labels.cluster }}: Cluster upgrade on {{ $labels.resource_id }} to version {{ $labels.version }} not progressing' + } + expression: 'hosted_control_plane_upgrade:duration_in_desired:seconds > 1200' + for: 'PT5M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeStuckInProgress' + enabled: true + labels: { + component: 'slo' + severity: '4' + slo: 'cluster-upgrade-complete' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeStuckInProgress/{{ $labels.cluster }}{{ $labels.resource_id }}/{{ $labels.version }}' + description: '''Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. With the alert pending for 5 minutes, paging starts after ~35 minutes. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + info: '''Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. With the alert pending for 5 minutes, paging starts after ~35 minutes. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '{{ $labels.cluster }}: Cluster upgrade on {{ $labels.resource_id }} to version {{ $labels.version }} stuck' + title: '{{ $labels.cluster }}: Cluster upgrade on {{ $labels.resource_id }} to version {{ $labels.version }} stuck' + } + expression: 'hosted_control_plane_upgrade:duration_in_progress:seconds > 1800' + for: 'PT5M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + ] + scopes: [ + azureMonitoring + ] + } +} + resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { name: 'arohcp_nodepool_slo_error_alerts' location: location diff --git a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep index 0ec3bfe63fe..ca8bc7002e0 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep @@ -53,3 +53,37 @@ resource arohcpClusterProvisionSloRecordingRules 'Microsoft.AlertsManagement/pro ] } } + +resource arohcpUserJourneyClusterUpgradeRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'arohcp_user_journey_cluster_upgrade_recording_rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ + { + record: 'hosted_control_plane_upgrade:upgrade_eligible:info' + expression: '((count by (cluster, resource_id, subscription_id, cluster_uuid) (count by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info == 1)) >= 2) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1) >= 1)) * 0 + 1' + } + { + record: 'hosted_control_plane_upgrade:version_state_first_seen:timestamp' + expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ((hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1))) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (max by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1))))' + } + { + record: 'hosted_control_plane_upgrade:in_progress:count' + expression: 'count by (cluster) (count by (cluster, resource_id) ((max by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (backend_cluster_version_info{state=~"desired|partial"} == 1) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) max by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1))) >= 1 and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or 0 * count by (cluster) (backend_cluster_version_info)' + } + { + record: 'hosted_control_plane_upgrade:duration_in_desired:seconds' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (max by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (backend_cluster_version_info{state="desired"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (max by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="partial"} == 1 or backend_cluster_version_info{state="completed"} == 1)))) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' + } + { + record: 'hosted_control_plane_upgrade:duration_in_progress:seconds' + expression: '((time() - min without (state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp{state=~"desired|partial"})) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (max by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1))) * on (cluster, resource_id, subscription_id, cluster_uuid, version) group_left (state) (max by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (backend_cluster_version_info{state="partial"} == 1 or (backend_cluster_version_info{state="desired"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) max by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="partial"} == 1))))' + } + ] + } +} diff --git a/observability/alerts-rp-services.yaml b/observability/alerts-rp-services.yaml index 951826805f6..24a6afcae1b 100644 --- a/observability/alerts-rp-services.yaml +++ b/observability/alerts-rp-services.yaml @@ -2,6 +2,7 @@ prometheusRules: rulesFolders: - alerts/access-cluster-slo-prometheusRule.yaml - alerts/cluster-provision-slo-prometheusRule.yaml + - alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml - alerts/nodepool-slo-prometheusRule.yaml untestedRules: [] testDependencies: diff --git a/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml new file mode 100644 index 00000000000..284a9a8c1f2 --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml @@ -0,0 +1,57 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: user-journey-cluster-upgrade-alerts + namespace: monitoring +spec: + groups: + # Per-upgrade stuck alerts for cluster upgrade SLOs. Sev 4. + # At current upgrade volumes, alert directly on duration instead of fleet burn-rate. + # + # The ControlPlaneActiveVersion controller derives upgrade Partial/Completed state from + # HostedCluster version history (preferring status.controlPlaneVersion.history, else + # status.version.history), which feeds backend_cluster_version_info and these alerts. + # Control plane version history reports for versions >= 4.20, so completeness tracks the + # control plane independently of the data plane. While 4.19 is still around, those + # clusters fall back to status.version.history, where upgrade completeness depends on + # data plane state — e.g. no worker nodes means the upgrade will not reach completed; + # missing console component in external auth prevents CVO from marking the upgrade + # complete. The complete alert may therefore fire for 4.19 clusters due to missing + # data plane configuration rather than control plane failure. + - name: rp-user-journey-cluster-upgrade-monitor-rules + rules: + - alert: userJourneyClusterUpgradeStuckInDesired + expr: | + hosted_control_plane_upgrade:duration_in_desired:seconds > 1200 + for: 5m + labels: + severity: "4" + slo: cluster-upgrade-reach-partial + component: slo + annotations: + summary: "{{ $labels.cluster }}: Cluster upgrade on {{ $labels.resource_id }} to version {{ $labels.version }} not progressing" + description: | + Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. With the alert pending for 5 minutes, paging starts after ~25 minutes. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeStuckInDesired/{{ $labels.cluster }}{{ $labels.resource_id }}/{{ $labels.version }}" + - alert: userJourneyClusterUpgradeStuckInProgress + expr: | + hosted_control_plane_upgrade:duration_in_progress:seconds > 1800 + for: 5m + labels: + severity: "4" + slo: cluster-upgrade-complete + component: slo + annotations: + summary: "{{ $labels.cluster }}: Cluster upgrade on {{ $labels.resource_id }} to version {{ $labels.version }} stuck" + description: | + Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. With the alert pending for 5 minutes, paging starts after ~35 minutes. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeStuckInProgress/{{ $labels.cluster }}{{ $labels.resource_id }}/{{ $labels.version }}" diff --git a/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml new file mode 100644 index 00000000000..cca55109125 --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml @@ -0,0 +1,130 @@ +rule_files: +- userJourneyClusterUpgrade-recordingRule.yaml +- userJourneyClusterUpgradeMonitor-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Reach partial: successful — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 25m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] +# ======================================== +# Reach partial: stuck in desired >20m — alert pending for 5m, fires after ~25m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 20m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] + - eval_time: 30m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: + - exp_labels: + alertname: userJourneyClusterUpgradeStuckInDesired + cluster: "svc-1" + cluster_uuid: "uuid-2" + resource_id: "/sub/2/rg/c2" + subscription_id: "sub-2" + version: "4.19.21" + state: "desired" + severity: "4" + slo: cluster-upgrade-reach-partial + component: "slo" + exp_annotations: + summary: "svc-1: Cluster upgrade on /sub/2/rg/c2 to version 4.19.21 not progressing" + description: | + Cluster upgrade target version 4.19.21 has been in desired for over 20 minutes without reaching partial. With the alert pending for 5 minutes, paging starts after ~25 minutes. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeStuckInDesired/svc-1/sub/2/rg/c2/4.19.21" +# ======================================== +# Reach partial: install only — not upgrade eligible, no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 50m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] +# ======================================== +# Upgrade complete: successful — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] +# ======================================== +# Upgrade complete: stuck in progress >30m — alert pending for 5m, fires after ~35m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1x10 0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="partial"}' + values: "0x10 1x50" + alert_rule_test: + - eval_time: 30m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: + - exp_labels: + alertname: userJourneyClusterUpgradeStuckInProgress + cluster: "svc-1" + cluster_uuid: "uuid-2" + resource_id: "/sub/2/rg/c2" + subscription_id: "sub-2" + version: "4.19.21" + state: "partial" + severity: "4" + slo: cluster-upgrade-complete + component: "slo" + exp_annotations: + summary: "svc-1: Cluster upgrade on /sub/2/rg/c2 to version 4.19.21 stuck" + description: | + Cluster upgrade target version 4.19.21 has been in progress for over 30 minutes without reaching completed. With the alert pending for 5 minutes, paging starts after ~35 minutes. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeStuckInProgress/svc-1/sub/2/rg/c2/4.19.21" +# ======================================== +# Upgrade complete: install only — not upgrade eligible, no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 50m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] diff --git a/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json new file mode 100644 index 00000000000..a87aa8b5e06 --- /dev/null +++ b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json @@ -0,0 +1,1446 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### Cluster Upgrade SLOs\n\nPer-upgrade observability from `backend_cluster_version_info` via `hosted_control_plane_upgrade:*` recording rules.\n\n| SLO | Target | Alert |\n|-----|--------|-------|\n| **Reach partial** | `partial` within 20m | `userJourneyClusterUpgradeStuckInDesired` (>20m in `desired`, `for: 5m` \u2192 pages after ~25m) |\n| **Upgrade complete** | `completed` within 30m | `userJourneyClusterUpgradeStuckInProgress` (>30m in progress, `for: 5m` \u2192 pages after ~35m) |\n\n**Upgrade SLOs** \u2014 in-progress and stuck upgrade health against the 20m / 30m thresholds (alerts add a 5-minute pending window before paging).\n\n**Upgrade and Version Visualization Across the Fleet** \u2014 share of clusters with an upgrade in progress, clusters by latest active z-stream, success rate, stuck rate by z-stream (threshold via Stuck Minutes), and latest z-stream per minor (ControlPlaneDesiredVersion gateway selection).\n\n**Runbook:** [https://aka.ms/arohcp-runbook-cluster-upgrade](https://aka.ms/arohcp-runbook-cluster-upgrade)", + "mode": "markdown" + }, + "pluginVersion": "12.4.3", + "title": "Dashboard Info", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 11, + "panels": [], + "title": "Upgrade SLOs", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Upgrade-eligible clusters with a version upgrade currently in progress (desired or partial, not yet completed).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "Count of upgrades in progress", + "refId": "A" + } + ], + "title": "Count of Upgrades In Progress", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Upgrades stuck in desired for over 20 minutes without reaching partial (SLO threshold). Alert userJourneyClusterUpgradeStuckInDesired adds for: 5m and pages after ~25m.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200) or (0 * hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"})) or vector(0)", + "instant": true, + "legendFormat": "Stuck in desired", + "refId": "A" + } + ], + "title": "Stuck in Desired (>20m)", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Upgrades stuck in desired or partial for over 30 minutes without reaching completed (SLO threshold). Alert userJourneyClusterUpgradeStuckInProgress adds for: 5m and pages after ~35m.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800) or (0 * hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"})) or vector(0)", + "instant": true, + "legendFormat": "Stuck in progress", + "refId": "A" + } + ], + "title": "Stuck in Progress (>30m)", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Upgrade-eligible clusters with a version upgrade currently in progress (desired or partial, not yet completed).", + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time In Progress (Minutes)" + }, + "properties": [ + { + "id": "custom.filterable", + "value": false + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "showHeader": true + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "In-Progress Upgrade Durations", + "type": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "resource_id", + "state", + "version", + "Value" + ] + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^cluster$", + "renamePattern": "Service Cluster" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^resource_id$", + "renamePattern": "Resource ID" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version$", + "renamePattern": "Target Version" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^state$", + "renamePattern": "Current State" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^Value$", + "renamePattern": "Time In Progress (Minutes)" + } + } + ] + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Clusters with upgrades stuck past the 20 minute reach-partial window.", + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "showHeader": true + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Desired Detail", + "type": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "resource_id", + "state", + "version", + "Value" + ] + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^cluster$", + "renamePattern": "Service Cluster" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^resource_id$", + "renamePattern": "Resource ID" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version$", + "renamePattern": "Target Version" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^state$", + "renamePattern": "Current State" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^Value$", + "renamePattern": "Time In Desired (Minutes)" + } + } + ] + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Clusters with upgrades stuck past the 30 minute upgrade-complete window.", + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time In Progress (Minutes)" + }, + "properties": [ + { + "id": "custom.filterable", + "value": false + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 7, + "options": { + "showHeader": true + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Progress Detail", + "type": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "resource_id", + "state", + "version", + "Value" + ] + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^cluster$", + "renamePattern": "Service Cluster" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^resource_id$", + "renamePattern": "Resource ID" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version$", + "renamePattern": "Target Version" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^state$", + "renamePattern": "Current State" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^Value$", + "renamePattern": "Time In Progress (Minutes)" + } + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 12, + "panels": [], + "title": "Upgrade and Version Visualization Across the Fleet", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Percentage of clusters with a version upgrade currently in progress (desired or partial, not completed), divided by total clusters (backend_cluster_created_time_seconds).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 8, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n (sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0))\n /\n (count(backend_cluster_created_time_seconds{cluster=~\"$cluster\"}) or vector(1))\n) * 100", + "instant": true, + "legendFormat": "Percentage of clusters upgrading", + "refId": "A" + } + ], + "title": "Percentage of Clusters with Upgrade In Progress", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Percentage of clusters at each z-stream, using each cluster's latest active version (highest semver among partial|completed). Percentages sum to 100% across bars.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "decimals": 1, + "max": 100, + "min": 0, + "unit": "percent", + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 13, + "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "max by (resource_id, version) (\n max without (prometheus_replica, state) (\n backend_cluster_version_info{cluster=~\"$cluster\", state=~\"partial|completed\"} == 1\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Percentage of Clusters by Latest Active Z-Stream", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "/^(resource_id|version|Value.*)$/" + } + } + }, + { + "id": "extractFields", + "options": { + "source": "version", + "format": "regexp", + "regex": "^(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)$", + "replace": false + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "major", + "destinationType": "number" + }, + { + "targetField": "minor", + "destinationType": "number" + }, + { + "targetField": "z", + "destinationType": "number" + } + ], + "fields": {} + } + }, + { + "id": "calculateField", + "options": { + "alias": "_m1", + "mode": "binary", + "binary": { + "left": "major", + "operator": "*", + "right": 1000000 + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "_m2", + "mode": "binary", + "binary": { + "left": "minor", + "operator": "*", + "right": 1000 + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "_m3", + "mode": "binary", + "binary": { + "left": "_m1", + "operator": "+", + "right": "_m2" + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "joinKey", + "mode": "binary", + "binary": { + "left": "_m3", + "operator": "+", + "right": "z" + }, + "replaceFields": false + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "joinKey", + "desc": true + } + ] + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "resource_id": { + "aggregations": [], + "operation": "groupby" + }, + "version": { + "aggregations": [ + "first" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version \\(first\\)$", + "renamePattern": "Version" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Version": { + "aggregations": [], + "operation": "groupby" + }, + "resource_id": { + "aggregations": [ + "count" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^resource_id \\(count\\)$", + "renamePattern": "Clusters" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "Clusters", + "destinationType": "number" + } + ], + "fields": {} + } + }, + { + "id": "calculateField", + "options": { + "alias": "Percentage", + "mode": "binary", + "binary": { + "left": "Clusters", + "operator": "/", + "right": "Clusters" + }, + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "Percentage", + "mode": "binary", + "binary": { + "left": "Percentage", + "operator": "*", + "right": 100 + }, + "replaceFields": true + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Version", + "Percentage" + ] + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "Percentage", + "destinationType": "number" + } + ], + "fields": {} + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Percentage", + "desc": true + } + ] + } + }, + { + "id": "rowsToFields", + "options": { + "mappings": [ + { + "fieldName": "Version", + "handlerKey": "field.name" + }, + { + "fieldName": "Percentage", + "handlerKey": "field.value" + } + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Percentage of upgrades to each z-stream level that have succeeded.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": 90 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "unit": "percent", + "custom": { + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 9, + "options": { + "showHeader": true + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n (\n count by (version) (\n count by (version, resource_id) (\n max without (prometheus_replica) (\n backend_cluster_version_info{cluster=~\"$cluster\", state=\"completed\"} == 1\n )\n and on (resource_id)\n hosted_control_plane_upgrade:upgrade_eligible:info{cluster=~\"$cluster\"} == 1\n )\n )\n or\n (\n count by (version) (\n count by (version, resource_id) (\n max without (prometheus_replica) (\n backend_cluster_version_info{cluster=~\"$cluster\", state=~\"desired|partial|completed\"} == 1\n )\n and on (resource_id)\n hosted_control_plane_upgrade:upgrade_eligible:info{cluster=~\"$cluster\"} == 1\n )\n ) * 0\n )\n )\n /\n count by (version) (\n count by (version, resource_id) (\n max without (prometheus_replica) (\n backend_cluster_version_info{cluster=~\"$cluster\", state=~\"desired|partial|completed\"} == 1\n )\n and on (resource_id)\n hosted_control_plane_upgrade:upgrade_eligible:info{cluster=~\"$cluster\"} == 1\n )\n )\n) * 100", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Upgrade Success Rate by Z-Stream", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "/^(version|Value.*)$/" + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version$", + "renamePattern": "Version" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*$", + "renamePattern": "Percentage" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Percentage of upgrades to each z-stream that are currently stuck (desired or partial, not completed) for more than ${stuckMinutes} minutes. Change the Stuck Minutes dashboard variable to adjust the threshold. Denominator is upgrade-eligible clusters with that version in desired|partial|completed. Uses duration_in_progress (excludes completed).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "percent", + "custom": { + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 14, + "options": { + "showHeader": true + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n (\n count by (version) (\n count by (version, resource_id) (\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > (${stuckMinutes} * 60)\n )\n )\n or\n (\n count by (version) (\n count by (version, resource_id) (\n max without (prometheus_replica) (\n backend_cluster_version_info{cluster=~\"$cluster\", state=~\"desired|partial|completed\"} == 1\n )\n and on (resource_id)\n hosted_control_plane_upgrade:upgrade_eligible:info{cluster=~\"$cluster\"} == 1\n )\n ) * 0\n )\n )\n /\n count by (version) (\n count by (version, resource_id) (\n max without (prometheus_replica) (\n backend_cluster_version_info{cluster=~\"$cluster\", state=~\"desired|partial|completed\"} == 1\n )\n and on (resource_id)\n hosted_control_plane_upgrade:upgrade_eligible:info{cluster=~\"$cluster\"} == 1\n )\n )\n) * 100", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck Upgrade Rate by Z-Stream (>${stuckMinutes}m)", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "/^(version|Value.*)$/" + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version$", + "renamePattern": "Version" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*$", + "renamePattern": "Percentage" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Highest x.y.z present in the fleet for each minor (x.y). Fleet versions are driven by the ARO-HCP ControlPlaneDesiredVersion controller, which selects the newest Cincinnati candidate in that minor that is a gateway to the next Y-stream (has an upgrade path into x.(y+1)), not necessarily Cincinnati's absolute tip in the channel.", + "fieldConfig": { + "defaults": { + "custom": { + "filterable": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Minor" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latest Z-Stream" + }, + "properties": [ + { + "id": "custom.filterable", + "value": false + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "12.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "count by (version, Minor) (\n label_replace(\n max without (prometheus_replica, state) (\n backend_cluster_version_info{cluster=~\"$cluster\"}\n ),\n \"Minor\", \"$1.$2\", \"version\", \"([0-9]+)[.]([0-9]+)[.][0-9]+\"\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Latest Z-Stream per Y-Stream", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "/^(Minor|version|Value.*)$/" + } + } + }, + { + "id": "extractFields", + "options": { + "source": "version", + "format": "regexp", + "regex": "^(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)$", + "replace": false + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "z", + "destinationType": "number" + }, + { + "targetField": "maj", + "destinationType": "number" + }, + { + "targetField": "min", + "destinationType": "number" + } + ], + "fields": {} + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "z", + "desc": true + } + ] + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Minor": { + "aggregations": [], + "operation": "groupby" + }, + "version": { + "aggregations": [ + "first" + ], + "operation": "aggregate" + }, + "maj": { + "aggregations": [ + "max" + ], + "operation": "aggregate" + }, + "min": { + "aggregations": [ + "max" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "maj (max)", + "desc": true + }, + { + "field": "min (max)", + "desc": true + } + ] + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "/^(Minor|version \\(first\\))$/" + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^version \\(first\\)$", + "renamePattern": "Latest Z-Stream" + } + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [ + "cluster-upgrade", + "aro-hcp", + "slo", + "user-journey" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "^Managed_Prometheus_services-.*$", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(backend_cluster_version_info, cluster)", + "includeAll": true, + "multi": true, + "name": "cluster", + "query": "label_values(backend_cluster_version_info, cluster)", + "refresh": 2, + "type": "query", + "label": "Service Cluster" + }, + { + "allowCustomValue": true, + "current": { + "text": "30", + "value": "30" + }, + "description": "Minutes an upgrade may remain in desired/partial (not completed) before counting as stuck. Default 30 matches the upgrade-complete SLO.", + "hide": 0, + "includeAll": false, + "label": "Stuck Minutes", + "multi": false, + "name": "stuckMinutes", + "options": [ + { + "selected": false, + "text": "20", + "value": "20" + }, + { + "selected": true, + "text": "30", + "value": "30" + }, + { + "selected": false, + "text": "45", + "value": "45" + }, + { + "selected": false, + "text": "60", + "value": "60" + }, + { + "selected": false, + "text": "90", + "value": "90" + }, + { + "selected": false, + "text": "120", + "value": "120" + } + ], + "query": "20,30,45,60,90,120", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Cluster Upgrade SLO", + "uid": "cluster-upgrade-slo", + "version": 1 +} diff --git a/observability/recording-rules-services.yaml b/observability/recording-rules-services.yaml index 9c418ac40b1..81beeb2bd05 100644 --- a/observability/recording-rules-services.yaml +++ b/observability/recording-rules-services.yaml @@ -2,5 +2,6 @@ prometheusRules: rulesFolders: - recording-rules/access-cluster-slo-recordingRule.yaml - recording-rules/cluster-provision-slo-recordingRule.yaml + - recording-rules/userJourneyClusterUpgrade-recordingRule.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep diff --git a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml new file mode 100644 index 00000000000..aaef63a62e3 --- /dev/null +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml @@ -0,0 +1,142 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: recording-rules + name: user-journey-cluster-upgrade-recording-rules + namespace: monitoring +spec: + groups: + # ======================================== + # Cluster Upgrade recording rules + # ======================================== + # Eligibility, first-seen timestamps, in-progress fleet counts, and per-upgrade + # duration metrics used by stuck alerts and the cluster-upgrade SLO dashboard. + # + # upgrade_eligible:info must be evaluated before version_state_first_seen (same cycle). + # Eligibility: per resource_id, at least two distinct version labels and at least one completed. + # Excludes initial install. Cluster create bounds when install ends (operation_cluster_create.go): + # https://github.com/Azure/ARO-HCP/blob/main/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go#L316-L320 + # + # Naming: level:metric:operations per https://prometheus.io/docs/practices/rules/#naming + - name: arohcp_user_journey_cluster_upgrade_recording_rules + interval: 1m + rules: + - record: hosted_control_plane_upgrade:upgrade_eligible:info + expr: | + ( + ( + count by (cluster, resource_id, subscription_id, cluster_uuid) ( + count by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info == 1 + ) + ) >= 2 + ) + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + ) >= 1 + ) + ) * 0 + 1 + # When each upgrade target version first entered desired or partial (start time for SLO durations). + # Stop recording once the version reaches completed so latched timestamps do not persist forever. + - record: hosted_control_plane_upgrade:version_state_first_seen:timestamp + expr: | + min without(prometheus_replica) ( + min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( + ( + hosted_control_plane_upgrade:version_state_first_seen:timestamp + or + ( + timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + ) + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + max by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) + ) + ) + ) + # Count of clusters with an upgrade currently in progress (desired or partial on a target version, without being completed). + - record: hosted_control_plane_upgrade:in_progress:count + expr: | + count by (cluster) ( + count by (cluster, resource_id) ( + ( + max by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) + max by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) + ) + ) >= 1 + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + ) + or + 0 * count by (cluster) (backend_cluster_version_info) + # Seconds the target version has been in desired without reaching partial or completed. + - record: hosted_control_plane_upgrade:duration_in_desired:seconds + expr: | + ( + time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} + ) + and on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + max by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( + backend_cluster_version_info{state="desired"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + max by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="partial"} == 1 + or + backend_cluster_version_info{state="completed"} == 1 + ) + ) + ) + ) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + # Seconds since the target version was first seen as desired or partial until it completes. + - record: hosted_control_plane_upgrade:duration_in_progress:seconds + expr: | + ( + ( + time() + - + min without(state) ( + hosted_control_plane_upgrade:version_state_first_seen:timestamp{state=~"desired|partial"} + ) + ) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + max by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) + ) + ) + * on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left(state) + ( + max by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( + backend_cluster_version_info{state="partial"} == 1 + or + ( + backend_cluster_version_info{state="desired"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) + max by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="partial"} == 1 + ) + ) + ) + ) diff --git a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml new file mode 100644 index 00000000000..3efbd119a09 --- /dev/null +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml @@ -0,0 +1,279 @@ +rule_files: +- userJourneyClusterUpgrade-recordingRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Test 1: Reach partial — desired transitions to partial within 20m (upgrade) +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/1/rg/c1"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1"}' + value: 1 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="partial"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="partial"}' + value: 900 +# ======================================== +# Test 1b: Partial with stale desired — must not count as stuck in desired +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] +# ======================================== +# Test 1c: Completed with stale desired — must not count as stuck in desired or in progress +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 25m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 2: Stuck in desired — 1 of 1 active upgrades stuck >20m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/2/rg/c2"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2"}' + value: 1 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="desired"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="desired"}' + value: 2100 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 +# ======================================== +# Test 3: Install only — desired without completed is excluded from SLO +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/3/rg/c4"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/3/rg/c4", version="4.19.20", state="desired"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 4: Complete — desired transitions to completed within 30m (upgrade) +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="desired"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 4b: Completed with stale partial — must not count as in progress or emit duration +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "1+0x35" + promql_expr_test: + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 35m + exp_samples: [] +# ======================================== +# Test 5: Stuck in progress — partial without completed after 35m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1x10 0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="partial"}' + values: "0x10 1x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="partial"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="partial"}' + value: 2100 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 +# ======================================== +# Test 6: Two versions without completed — not upgrade eligible +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/c5", subscription_id="sub-4", cluster_uuid="uuid-5", version="4.19.19", state="partial"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/c5", subscription_id="sub-4", cluster_uuid="uuid-5", version="4.19.20", state="desired"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/4/rg/c5"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/4/rg/c5", version="4.19.20", state="desired"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 7: Single completed version only — not upgrade eligible +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/c6", subscription_id="sub-5", cluster_uuid="uuid-6", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/5/rg/c6"} + eval_time: 35m + exp_samples: [] +# ======================================== +# Test 8: HA prometheus_replica duplicates must not break binary matching +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed", prometheus_replica="prom-agent-prometheus-0"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed", prometheus_replica="prom-agent-prometheus-1"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired", prometheus_replica="prom-agent-prometheus-0"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired", prometheus_replica="prom-agent-prometheus-1"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial", prometheus_replica="prom-agent-prometheus-0"}' + values: "0x10 1x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial", prometheus_replica="prom-agent-prometheus-1"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="partial"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="partial"}' + value: 900 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 +# ======================================== +# Test 9: Partial without prior desired — clock from partial first-seen +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "1+0x35" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="partial"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="partial"}' + value: 900 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: []