Add two initial metrics to the indexer

crates/spk-cli/cmd-env/Cargo.toml

@@ -25,6 +25,7 @@ clap = { workspace = true }
 spfs = { workspace = true }
 spfs-cli-common = { workspace = true }
 spk-cli-common = { workspace = true }
+spk-config = { workspace = true }
 spk-exec = { workspace = true }
 spk-solve = { workspace = true }
 statsd = { version = "0.15.0", optional = true }

crates/spk-cli/cmd-env/src/cmd_env.rs

@@ -10,9 +10,9 @@ use miette::{Context, Result};
 use spfs::tracking::SpecFile;
 use spfs_cli_common::Progress;
 use spk_cli_common::{CommandArgs, Run, build_required_packages, flags};
-use spk_exec::setup_runtime_with_reporter;
 #[cfg(feature = "statsd")]
-use spk_solve::{SPK_RUN_TIME_METRIC, get_metrics_client};
+use spk_config::{SPK_RUN_TIME_METRIC, get_metrics_client};
+use spk_exec::setup_runtime_with_reporter;
 use spk_solve::{Solver, SolverMut};
 
 /// Resolve and run an environment on-the-fly

crates/spk-cli/cmd-repo/src/cmd_repo.rs

@@ -121,6 +121,10 @@ impl RepoCommand {
                 };
                 let repos = vec![(repo_to_index.name().to_string(), repo_to_index.clone())];
 
+                // This is not running an indexer, so there isn't a
+                // metric name..
+                let no_metric_name: Option<String> = None;
+
                 if !update.is_empty() {
                     // Update the existing index for the given package/version
                     let start = Instant::now();
@@ -156,7 +160,9 @@ impl RepoCommand {
                     let mut was_full_index = String::from("");
                     let result = match FlatBufferRepoIndex::from_repo_file(&repo_to_index).await {
                         Ok(current_index) => {
-                            current_index.update_packages(&repo_to_index, &idents).await
+                            current_index
+                                .update_packages(&repo_to_index, &idents, &no_metric_name)
+                                .await
                         }
                         Err(storage::Error::IndexOpenError(err)) => {
                             // There isn't an existing index, so generate one from scratch that
@@ -165,7 +171,7 @@ impl RepoCommand {
                             tracing::warn!("No current index to update. Creating a full index ...");
                             was_full_index =
                                 " [no previous index, so a full index was created]".to_string();
-                            FlatBufferRepoIndex::index_repo(&repos).await
+                            FlatBufferRepoIndex::index_repo(&repos, &no_metric_name).await
                         }
                         Err(err) => {
                             return Err(err.into());
@@ -194,7 +200,7 @@ impl RepoCommand {
                 } else {
                     // Generate a full index from scratch
                     let start = Instant::now();
-                    FlatBufferRepoIndex::index_repo(&repos).await?;
+                    FlatBufferRepoIndex::index_repo(&repos, &no_metric_name).await?;
 
                     tracing::info!(
                         "Index generation for '{}' repo completed in: {} secs",

crates/spk-cli/common/src/flags.rs

@@ -20,6 +20,9 @@ use solve::{
     SolverImpl,
     SolverMut,
 };
+#[cfg(unix)]
+#[cfg(feature = "statsd")]
+use spk_config::{SPK_RUN_TIME_METRIC, get_metrics_client};
 use spk_schema::foundation::format::FormatIdent;
 use spk_schema::foundation::ident_build::Build;
 use spk_schema::foundation::ident_component::Component;
@@ -52,9 +55,6 @@ use spk_schema::{
     VariantExt,
 };
 use spk_solve::{self as solve};
-#[cfg(unix)]
-#[cfg(feature = "statsd")]
-use spk_solve::{SPK_RUN_TIME_METRIC, get_metrics_client};
 use spk_storage as storage;
 use spk_storage::IndexedRepository;
 use spk_workspace::{FindOrLoadPackageTemplateError, FindPackageTemplateError};

crates/spk-config/Cargo.toml

@@ -13,6 +13,7 @@ description = { workspace = true }
 workspace = true
 
 [features]
+statsd = ["dep:statsd"]
 
 [dependencies]
 config = { workspace = true }
@@ -22,6 +23,7 @@ once_cell = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 spfs = { workspace = true }
+statsd = { version = "0.15.0", optional = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }

crates/spk-config/src/config.rs

@@ -375,6 +375,21 @@ pub struct Indexer {
     /// Indexer's kafka broker maximum polling interval in milliseconds
     #[serde(default = "default_indexer_max_polling_interval_ms")]
     pub max_polling_interval_ms: u64,
+
+    /// Name of the metric to update whenever the indexer sends an
+    /// index event message (including heartbeats). With suitable
+    /// external systems, this can be used to monitor an indexer's
+    /// workload and whether it is healthy. If it is not set the, or
+    /// statsd is not configured, metric will not be updated.
+    #[serde(default)]
+    pub index_event_metric_name: Option<String>,
+
+    /// Name of the metric to update whenever an indexer starts
+    /// updating an index. With suitable external systems, this can be
+    /// used to see how often an index is updated. If it is not set,
+    /// or statsd is not configured, the metric will not be updated.
+    #[serde(default)]
+    pub index_update_start_metric_name: Option<String>,
 }
 
 /// Configuration values for spk.

crates/spk-config/src/error.rs

@@ -27,6 +27,11 @@ pub enum Error {
     Json(#[from] serde_json::Error),
     #[error(transparent)]
     Config(#[from] config::ConfigError),
+
+    #[error(
+        "Unsupported statsd metric format: {0}. Please specify SPK_STATSD_FORMAT as one of: {1}"
+    )]
+    UnsupportedMetric(String, String),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;

crates/spk-config/src/lib.rs

@@ -4,7 +4,22 @@
 
 mod config;
 mod error;
+#[cfg(feature = "statsd")]
+mod metrics;
 
 pub use error::{Error, Result};
+#[cfg(feature = "statsd")]
+pub use metrics::{
+    MetricsClient,
+    SPK_ERROR_COUNT_METRIC,
+    SPK_RUN_COUNT_METRIC,
+    SPK_RUN_TIME_METRIC,
+    SPK_SOLUTION_PACKAGE_COUNT_METRIC,
+    SPK_SOLVER_INITIAL_REQUESTS_COUNT_METRIC,
+    SPK_SOLVER_RUN_COUNT_METRIC,
+    SPK_SOLVER_RUN_TIME_METRIC,
+    SPK_SOLVER_SOLUTION_SIZE_METRIC,
+    get_metrics_client,
+};
 
 pub use self::config::*;

crates/spk-solve/src/metrics.rs → crates/spk-config/src/metrics.rs

@@ -21,7 +21,7 @@ const STATSD_FORMAT: &str = "statsd";
 const LIBRATO_FORMAT: &str = "statsd-exporter-librato";
 
 static METRICS_CLIENT: Lazy<Option<MetricsClient>> = Lazy::new(|| {
-    let Ok(config) = spk_config::get_config() else {
+    let Ok(config) = crate::config::get_config() else {
         return None;
     };
     let statsd_config = &config.statsd;
@@ -123,9 +123,7 @@ impl FromStr for StatsdFormat {
                     .map(ToString::to_string)
                     .collect::<Vec<String>>()
                     .join(", ");
-                Err(Error::String(format!(
-                    "Unsupported statsd metric format: {input}. Please specify SPK_STATSD_FORMAT as one of: {valid_values}"
-                )))
+                Err(Error::UnsupportedMetric(input.to_string(), valid_values))
             }
         }
     }

crates/spk-solve/src/io.rs

@@ -20,6 +20,15 @@ use futures::stream::FuturesUnordered;
 use futures::{Stream, StreamExt};
 use itertools::Itertools;
 use once_cell::sync::Lazy;
+#[cfg(feature = "statsd")]
+use spk_config::{
+    SPK_SOLUTION_PACKAGE_COUNT_METRIC,
+    SPK_SOLVER_INITIAL_REQUESTS_COUNT_METRIC,
+    SPK_SOLVER_RUN_COUNT_METRIC,
+    SPK_SOLVER_RUN_TIME_METRIC,
+    SPK_SOLVER_SOLUTION_SIZE_METRIC,
+    get_metrics_client,
+};
 use spk_schema::foundation::format::{
     FormatChange,
     FormatChangeOptions,
@@ -43,15 +52,6 @@ use spk_solve_graph::{
 use crate::solvers::step::ErrorFreq;
 use crate::solvers::{StepSolver, StepSolverRuntime};
 use crate::{Error, Result, Solution, Solver, StatusLine, show_search_space_stats};
-#[cfg(feature = "statsd")]
-use crate::{
-    SPK_SOLUTION_PACKAGE_COUNT_METRIC,
-    SPK_SOLVER_INITIAL_REQUESTS_COUNT_METRIC,
-    SPK_SOLVER_RUN_COUNT_METRIC,
-    SPK_SOLVER_RUN_TIME_METRIC,
-    SPK_SOLVER_SOLUTION_SIZE_METRIC,
-    get_metrics_client,
-};
 
 const STOP_ON_BLOCK_FLAG: &str = "--stop-on-block";
 const BY_USER: &str = "by user";

crates/spk-solve/src/lib.rs

@@ -4,8 +4,6 @@
 
 mod error;
 mod io;
-#[cfg(feature = "statsd")]
-mod metrics;
 mod search_space;
 mod solver;
 mod solvers;
@@ -18,19 +16,6 @@ pub use io::{
     DecisionFormatterBuilder,
     MultiSolverKind,
 };
-#[cfg(feature = "statsd")]
-pub use metrics::{
-    MetricsClient,
-    SPK_ERROR_COUNT_METRIC,
-    SPK_RUN_COUNT_METRIC,
-    SPK_RUN_TIME_METRIC,
-    SPK_SOLUTION_PACKAGE_COUNT_METRIC,
-    SPK_SOLVER_INITIAL_REQUESTS_COUNT_METRIC,
-    SPK_SOLVER_RUN_COUNT_METRIC,
-    SPK_SOLVER_RUN_TIME_METRIC,
-    SPK_SOLVER_SOLUTION_SIZE_METRIC,
-    get_metrics_client,
-};
 pub(crate) use search_space::show_search_space_stats;
 pub use serde;
 pub use serde_json;

crates/spk-storage/Cargo.toml

@@ -17,6 +17,7 @@ sentry = [
     "dep:sentry",
     "dep:sentry-miette",
 ]
+statsd = ["dep:statsd"]
 
 [dependencies]
 arc-swap = { workspace = true }
@@ -60,6 +61,7 @@ spfs = { workspace = true }
 spk-config = { workspace = true }
 spk-proto = { workspace = true }
 spk-schema = { workspace = true }
+statsd = { version = "0.15.0", optional = true }
 strum = { workspace = true }
 sys-info = "0.9.0"
 tar = { workspace = true }