operator: check config reload happened before completing reconcile

api/operator/v1beta1/vmagent_types.go

@@ -283,11 +283,26 @@ func (cr *VMAgent) GetReloadURL(host string) string {
 	return BuildLocalURL(reloadAuthKeyFlag, host, cr.Spec.Port, reloadPath, cr.Spec.ExtraArgs)
 }
 
+// GetMetricsURL implements configReloadWaitable interface
+func (cr *VMAgent) GetMetricsURL(host string) string {
+	return BuildLocalURL("", host, cr.Spec.Port, metricsPath, cr.Spec.ExtraArgs)
+}
+
 // GetReloaderParams implements reloadable interface
 func (cr *VMAgent) GetReloaderParams() *CommonConfigReloaderParams {
 	return &cr.Spec.CommonConfigReloaderParams
 }
 
+// GetConfigReloadMetricNames returns vmagent's config-reload-completion metrics for the given assets,
+// or for all assets if none are given.
+func (cr *VMAgent) GetConfigReloadMetricNames(assets ...ConfigReloadAsset) []string {
+	return configReloadMetricNames(map[ConfigReloadAsset]string{
+		ConfigReloadAssetMain:       "vm_promscrape_config_last_reload_success_timestamp_seconds",
+		ConfigReloadAssetRelabel:    "vmagent_relabel_config_last_reload_success_timestamp_seconds",
+		ConfigReloadAssetStreamAggr: "vmagent_streamaggr_config_reload_success_timestamp_seconds",
+	}, assets)
+}
+
 // UseProxyProtocol implements build.probeCRD interface
 func (cr *VMAgent) UseProxyProtocol() bool {
 	return UseProxyProtocol(cr.Spec.ExtraArgs)
@@ -710,6 +725,27 @@ func (cr *VMAgent) GetAdditionalService() *AdditionalServiceSpec {
 	return cr.Spec.ServiceSpec
 }
 
+// HasRemoteWriteSecrets reports whether any remoteWrite entry references a credential secret.
+func (cr *VMAgent) HasRemoteWriteSecrets() bool {
+	for _, rw := range cr.Spec.RemoteWrite {
+		if rw.BasicAuth != nil && len(rw.BasicAuth.Password.Name) > 0 {
+			return true
+		}
+		if rw.BearerTokenSecret != nil && rw.BearerTokenSecret.Name != "" {
+			return true
+		}
+		if rw.OAuth2 != nil && rw.OAuth2.ClientSecret != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// HasConfigReloader reports whether vmagent's pod spec includes a config-reloader sidecar.
+func (cr *VMAgent) HasConfigReloader() bool {
+	return !ptr.Deref(cr.Spec.IngestOnlyMode, false) || cr.HasAnyRelabellingConfigs() || cr.HasAnyStreamAggrRule() || cr.HasRemoteWriteSecrets()
+}
+
 func checkRelabelConfigs(src []*RelabelConfig) error {
 	// TODO: restore check when issue will be fixed at golang
 	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6911

api/operator/v1beta1/vmalert_types.go

@@ -168,11 +168,24 @@ func (cr *VMAlert) GetReloadURL(host string) string {
 	return BuildLocalURL(reloadAuthKeyFlag, host, cr.Spec.Port, reloadPath, cr.Spec.ExtraArgs)
 }
 
+// GetMetricsURL implements configReloadWaitable interface
+func (cr *VMAlert) GetMetricsURL(host string) string {
+	return BuildLocalURL("", host, cr.Spec.Port, metricsPath, cr.Spec.ExtraArgs)
+}
+
 // GetReloaderParams implements reloadable interface
 func (cr *VMAlert) GetReloaderParams() *CommonConfigReloaderParams {
 	return &cr.Spec.CommonConfigReloaderParams
 }
 
+// GetConfigReloadMetricNames returns vmalert's config-reload-completion metric.
+// vmalert has only one config asset, so assets is accepted but ignored.
+func (cr *VMAlert) GetConfigReloadMetricNames(_ ...ConfigReloadAsset) []string {
+	return []string{
+		"vmalert_config_last_reload_success_timestamp_seconds",
+	}
+}
+
 // UseProxyProtocol implements build.probeCRD interface
 func (cr *VMAlert) UseProxyProtocol() bool {
 	return UseProxyProtocol(cr.Spec.ExtraArgs)
@@ -530,6 +543,11 @@ func (cr *VMAlert) IsUnmanaged() bool {
 	return !cr.Spec.SelectAllByDefault && cr.Spec.RuleSelector == nil && cr.Spec.RuleNamespaceSelector == nil
 }
 
+// HasConfigReloader reports whether vmalert's pod spec includes a config-reloader sidecar.
+func (cr *VMAlert) HasConfigReloader() bool {
+	return !cr.IsUnmanaged()
+}
+
 // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated
 func (cr *VMAlert) LastSpecUpdated() bool {
 	updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec)

api/operator/v1beta1/vmalertmanager_types.go

@@ -238,11 +238,29 @@ func (cr *VMAlertmanager) GetReloadURL(host string) string {
 	return localReloadURL.String()
 }
 
+// GetMetricsURL implements configReloadWaitable interface
+func (cr *VMAlertmanager) GetMetricsURL(host string) string {
+	localMetricsURL := &url.URL{
+		Scheme: cr.accessScheme(),
+		Host:   fmt.Sprintf("%s:%s", host, cr.Port()),
+		Path:   cr.GetMetricsPath(),
+	}
+	return localMetricsURL.String()
+}
+
 // GetReloaderParams implements reloadable interface
 func (cr *VMAlertmanager) GetReloaderParams() *CommonConfigReloaderParams {
 	return &cr.Spec.CommonConfigReloaderParams
 }
 
+// GetConfigReloadMetricNames returns alertmanager's config-reload-completion metric.
+// Alertmanager has only one config asset, so assets is accepted but ignored.
+func (cr *VMAlertmanager) GetConfigReloadMetricNames(_ ...ConfigReloadAsset) []string {
+	return []string{
+		"alertmanager_config_last_reload_success_timestamp_seconds",
+	}
+}
+
 // UseProxyProtocol implements build.probeCRD interface
 func (cr *VMAlertmanager) UseProxyProtocol() bool {
 	return false

api/operator/v1beta1/vmauth_types.go

@@ -772,18 +772,42 @@ func (cr *VMAuth) IsUnmanaged() bool {
 
 // GetReloadURL implements reloadable interface
 func (cr *VMAuth) GetReloadURL(host string) string {
-	port := cr.Spec.Port
+	return BuildLocalURL(reloadAuthKeyFlag, host, cr.metricsPort(), reloadPath, cr.Spec.ExtraArgs)
+}
+
+// metricsPort returns the port vmauth serves /metrics (and other internal
+// routes) on: the internal port if configured, else the main port.
+func (cr *VMAuth) metricsPort() string {
 	if len(cr.Spec.InternalListenPort) > 0 {
-		port = cr.Spec.InternalListenPort
+		return cr.Spec.InternalListenPort
 	}
-	return BuildLocalURL(reloadAuthKeyFlag, host, port, reloadPath, cr.Spec.ExtraArgs)
+	return cr.Spec.Port
+}
+
+// GetMetricsURL implements configReloadWaitable interface
+func (cr *VMAuth) GetMetricsURL(host string) string {
+	return BuildLocalURL("", host, cr.metricsPort(), metricsPath, cr.Spec.ExtraArgs)
 }
 
 // GetReloaderParams implements reloadable interface
 func (cr *VMAuth) GetReloaderParams() *CommonConfigReloaderParams {
 	return &cr.Spec.CommonConfigReloaderParams
 }
 
+// GetConfigReloadMetricNames returns vmauth's config-reload-completion metric.
+// vmauth has only one config asset, so assets is accepted but ignored.
+func (cr *VMAuth) GetConfigReloadMetricNames(_ ...ConfigReloadAsset) []string {
+	return []string{
+		"vmauth_config_last_reload_success_timestamp_seconds",
+	}
+}
+
+// HasConfigReloader reports whether vmauth's pod spec includes a config-reloader sidecar -
+// only true when the operator itself owns and writes the auth config (no SecretRef/LocalPath).
+func (cr *VMAuth) HasConfigReloader() bool {
+	return cr.Spec.SecretRef == nil && cr.Spec.LocalPath == ""
+}
+
 // UseProxyProtocol implements build.probeCRD interface
 func (cr *VMAuth) UseProxyProtocol() bool {
 	hasInternalPorts := len(cr.Spec.InternalListenPort) == 0

api/operator/v1beta1/vmextra_types.go

@@ -71,6 +71,40 @@ const (
 	MetadataStrategyMergePromPriority MetadataStrategy = "merge-prometheus-priority"
 )
 
+// ConfigReloadAsset identifies a distinct config asset that a CR's config-reloader
+// sidecar watches and reloads independently, each tracked by its own
+// config-reload-completion metric.
+type ConfigReloadAsset string
+
+const (
+	// ConfigReloadAssetMain identifies the primary config asset (e.g. scrape config,
+	// alerting/recording rules, vmauth config, alertmanager config).
+	ConfigReloadAssetMain ConfigReloadAsset = "main"
+	// ConfigReloadAssetRelabel identifies the global/per-target relabeling config asset.
+	ConfigReloadAssetRelabel ConfigReloadAsset = "relabel"
+	// ConfigReloadAssetStreamAggr identifies the stream aggregation config asset.
+	ConfigReloadAssetStreamAggr ConfigReloadAsset = "streamAggr"
+)
+
+// configReloadMetricNames selects metric names from all for the requested assets.
+// With no assets requested, metric names for every known asset are returned.
+func configReloadMetricNames(all map[ConfigReloadAsset]string, assets []ConfigReloadAsset) []string {
+	if len(assets) == 0 {
+		names := make([]string, 0, len(all))
+		for _, name := range all {
+			names = append(names, name)
+		}
+		return names
+	}
+	names := make([]string, 0, len(assets))
+	for _, asset := range assets {
+		if name, ok := all[asset]; ok {
+			names = append(names, name)
+		}
+	}
+	return names
+}
+
 const (
 	httpPathPrefixFlag       = "http.pathPrefix"
 	httpUseProxyProtocolFlag = "httpListenAddr.useProxyProtocol"
@@ -1387,7 +1421,7 @@ type CommonConfigReloaderParams struct {
 	// UseVMConfigReloader replaces prometheus-like config-reloader
 	// with vm one. It uses secrets watch instead of file watch
 	// which greatly increases speed of config updates
-	// Removed since v0.67.0: this property is ignored and no longer needed
+	// +notes={removed_in: "v0.67.0"}
 	// +optional
 	UseVMConfigReloader *bool `json:"useVMConfigReloader,omitempty"`
 	// ConfigReloaderImageTag defines image:tag for config-reloader container
@@ -1408,9 +1442,16 @@ type CommonConfigReloaderParams struct {
 	ConfigReloaderExtraArgs map[string]string `json:"configReloaderExtraArgs,omitempty"`
 	// ConfigReloadAuthKeySecret defines optional secret reference authKey for /-/reload API requests.
 	// Given secret reference will be added to the application and vm-config-reloader as volume
-	// available since v0.57.0 version
+	// +notes={available_from: "v0.57.0"}
 	// +optional
 	ConfigReloadAuthKeySecret *corev1.SecretKeySelector `json:"configReloadAuthKeySecret,omitempty"`
+	// WaitForConfigReload instructs the operator to verify, via the
+	// application's own config-reload-completion metric, that every replica
+	// has actually reloaded before marking dependent objects as applied.
+	// No-op for application types that don't report such a metric.
+	// +optional
+	// +notes={available_from: "v0.74.0"}
+	WaitForConfigReload *bool `json:"waitForConfigReload,omitempty"`
 }
 
 // CommonAppsParams defines common params

api/operator/v1beta1/vmsingle_types.go

@@ -126,6 +126,11 @@ func (cr *VMSingle) HasAnyRelabellingConfigs() bool {
 	return cr.Spec.HasAnyRelabellingConfigs()
 }
 
+// HasConfigReloader reports whether vmsingle's pod spec includes a config-reloader sidecar.
+func (cr *VMSingle) HasConfigReloader() bool {
+	return !ptr.Deref(cr.Spec.IngestOnlyMode, true) || cr.HasAnyRelabellingConfigs() || cr.HasAnyStreamAggrRule()
+}
+
 // ScrapeSelectors gets object and namespace sepectors
 func (cr *VMSingle) ScrapeSelectors(scrape client.Object) (*metav1.LabelSelector, *metav1.LabelSelector) {
 	return cr.Spec.ScrapeSelectors(scrape)
@@ -168,11 +173,26 @@ func (cr *VMSingle) GetReloadURL(host string) string {
 	return BuildLocalURL(reloadAuthKeyFlag, host, cr.Spec.Port, reloadPath, cr.Spec.ExtraArgs)
 }
 
+// GetMetricsURL implements configReloadWaitable interface
+func (cr *VMSingle) GetMetricsURL(host string) string {
+	return BuildLocalURL("", host, cr.Spec.Port, metricsPath, cr.Spec.ExtraArgs)
+}
+
 // GetReloaderParams implements reloadable interface
 func (cr *VMSingle) GetReloaderParams() *CommonConfigReloaderParams {
 	return &cr.Spec.CommonConfigReloaderParams
 }
 
+// GetConfigReloadMetricNames returns vmsingle's config-reload-completion metrics for the given assets,
+// or for all assets if none are given.
+func (cr *VMSingle) GetConfigReloadMetricNames(assets ...ConfigReloadAsset) []string {
+	return configReloadMetricNames(map[ConfigReloadAsset]string{
+		ConfigReloadAssetMain:       "vm_promscrape_config_last_reload_success_timestamp_seconds",
+		ConfigReloadAssetRelabel:    "vm_relabel_config_last_reload_success_timestamp_seconds",
+		ConfigReloadAssetStreamAggr: "vminsert_streamagg_config_last_reload_success_timestamp_seconds",
+	}, assets)
+}
+
 // UseProxyProtocol implements build.probeCRD interface
 func (cr *VMSingle) UseProxyProtocol() bool {
 	return UseProxyProtocol(cr.Spec.ExtraArgs)