fix(sync): surface an unreachable or silent upstream peer

crates/core/src/config.rs

@@ -945,6 +945,8 @@ pub struct MinibfConfig {
     pub url: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     max_scan_items: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    max_tip_age_sec: Option<u64>,
 }
 
 impl MinibfConfig {
@@ -955,6 +957,7 @@ impl MinibfConfig {
             token_registry_url: None,
             url: None,
             max_scan_items: None,
+            max_tip_age_sec: None,
         }
     }
 
@@ -975,6 +978,22 @@ impl MinibfConfig {
     pub fn max_scan_items(&self) -> u64 {
         self.max_scan_items.unwrap_or(default_max_scan_items())
     }
+
+    pub fn with_max_tip_age_sec(mut self, max_tip_age_sec: u64) -> Self {
+        self.max_tip_age_sec = Some(max_tip_age_sec);
+        self
+    }
+
+    /// Age, in seconds, past which the node's tip is considered too stale to
+    /// serve and `/health` starts answering `503`.
+    ///
+    /// Unset by default: a node that has fallen behind still reports healthy,
+    /// because a node catching up from a bootstrap is hours behind by design.
+    /// `/health/tip` reports `tip_age_seconds` whenever the tip is readable, so
+    /// staleness is observable whether or not an operator opts into the gate.
+    pub fn max_tip_age_sec(&self) -> Option<u64> {
+        self.max_tip_age_sec
+    }
 }
 
 #[derive(Deserialize, Serialize, Clone)]

crates/minibf/src/lib.rs

@@ -369,9 +369,10 @@ where
     let permissive_cors = facade.config.permissive_cors();
     let app = Router::new()
         .route("/", get(routes::root::<D>))
-        .route("/health", get(routes::health::naked))
+        .route("/health", get(routes::health::naked::<D>))
         .route("/metrics", get(routes::metrics::metrics::<D>))
         .route("/health/clock", get(routes::health::clock))
+        .route("/health/tip", get(routes::health::tip::<D>))
         .route("/genesis", get(routes::genesis::naked::<D>))
         .route("/network", get(routes::network::naked::<D>))
         .route("/network/eras", get(routes::network::eras::<D>))

crates/minibf/src/routes/health.rs

@@ -1,15 +1,107 @@
-use axum::{http::StatusCode, Json};
+use axum::{extract::State, http::StatusCode, Json};
+use dolos_core::Domain;
 use serde::{Deserialize, Serialize};
 use std::time::{SystemTime, UNIX_EPOCH};
 
+use crate::Facade;
+
+/// Blockfrost's `/health` body, and deliberately nothing more.
+///
+/// The conformance suite compares this response with `toStrictEqual`, so an
+/// extra field here is a conformance failure. The staleness detail a caller
+/// needs lives on [`TipResponse`] instead; what `/health` carries is the
+/// verdict, which is the part a load balancer reads.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct RootResponse {
     pub is_healthy: bool,
 }
 
-pub async fn naked() -> Result<Json<RootResponse>, StatusCode> {
-    // TODO: Relate this value to sync status. If not in tip, then unhealthy.
-    Ok(Json(RootResponse { is_healthy: true }))
+/// `/health/tip` — how current the node's own chain is.
+///
+/// A sibling of `/health/clock`, which already establishes that this server
+/// answers time-correctness questions about itself.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TipResponse {
+    /// Slot of the node's own tip.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tip_slot: Option<u64>,
+
+    /// Wall-clock seconds between the tip's slot and now.
+    ///
+    /// The whole point of the endpoint: it lets a caller tell a current node
+    /// from one that has been serving the same block for hours, without
+    /// holding a second opinion about where the chain actually is.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tip_age_seconds: Option<u64>,
+
+    /// The configured threshold, echoed so a caller can see what verdict it is
+    /// being given against. `None` when the operator set none.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_tip_age_seconds: Option<u64>,
+
+    /// Whether `tip_age_seconds` is past `max_tip_age_seconds`. Absent when no
+    /// threshold is configured, or when the tip could not be measured.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub is_stale: Option<bool>,
+}
+
+/// Age of the node's tip in wall-clock seconds, or `None` when the tip or the
+/// era summary cannot be read.
+///
+/// Staleness reporting is strictly additive to liveness, so a failure to
+/// measure it must not turn a reachable node into an unhealthy one: every
+/// error here degrades to "unknown age" rather than propagating.
+fn tip_age_seconds<D: Domain>(domain: &Facade<D>) -> Option<(u64, u64)> {
+    let tip_slot = domain.get_tip_slot().ok()?;
+    let summary = domain.get_chain_summary().ok()?;
+
+    let tip_time = summary.slot_time(tip_slot);
+
+    let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs();
+
+    Some((tip_slot, now.saturating_sub(tip_time)))
+}
+
+/// Whether the node's tip is past the configured staleness threshold.
+///
+/// `None` means "no verdict": either the operator configured no threshold, or
+/// the tip could not be measured. A node that has fallen behind is only
+/// *unhealthy* if someone said how far behind is too far — a node catching up
+/// from a bootstrap is hours behind by design, so defaulting to a threshold
+/// would pull every such node out of its load balancer mid-sync.
+fn is_stale<D: Domain>(domain: &Facade<D>, tip_age: Option<u64>) -> Option<bool> {
+    match (domain.config.max_tip_age_sec(), tip_age) {
+        (Some(max), Some(age)) => Some(age > max),
+        _ => None,
+    }
+}
+
+pub async fn naked<D: Domain>(State(domain): State<Facade<D>>) -> (StatusCode, Json<RootResponse>) {
+    let tip_age = tip_age_seconds(&domain).map(|(_, age)| age);
+
+    let stale = is_stale(&domain, tip_age).unwrap_or(false);
+
+    let status = if stale {
+        StatusCode::SERVICE_UNAVAILABLE
+    } else {
+        StatusCode::OK
+    };
+
+    (status, Json(RootResponse { is_healthy: !stale }))
+}
+
+pub async fn tip<D: Domain>(State(domain): State<Facade<D>>) -> Json<TipResponse> {
+    let (tip_slot, tip_age) = match tip_age_seconds(&domain) {
+        Some((slot, age)) => (Some(slot), Some(age)),
+        None => (None, None),
+    };
+
+    Json(TipResponse {
+        tip_slot,
+        tip_age_seconds: tip_age,
+        max_tip_age_seconds: domain.config.max_tip_age_sec(),
+        is_stale: is_stale(&domain, tip_age),
+    })
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -30,3 +122,105 @@ impl Default for ClockResponse {
 pub async fn clock() -> Result<Json<ClockResponse>, StatusCode> {
     Ok(Json(ClockResponse::default()))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_support::{TestApp, TestFault};
+
+    async fn get_json<T: serde::de::DeserializeOwned>(
+        app: &TestApp,
+        path: &str,
+    ) -> (StatusCode, T) {
+        let (status, bytes) = app.get_bytes(path).await;
+        let parsed = serde_json::from_slice(&bytes).unwrap_or_else(|err| {
+            panic!(
+                "failed to parse {path} response ({err}): {}",
+                String::from_utf8_lossy(&bytes)
+            )
+        });
+
+        (status, parsed)
+    }
+
+    /// The synthetic chain sits at a slot whose wall-clock time is years in the
+    /// past, so it is stale by any threshold an operator would set — which is
+    /// what makes it a usable fixture for the staleness cases below.
+    #[tokio::test]
+    async fn health_stays_blockfrost_exact() {
+        let app = TestApp::new();
+        let (status, bytes) = app.get_bytes("/health").await;
+
+        assert_eq!(status, StatusCode::OK);
+
+        // Parsed as raw JSON, not into `RootResponse`, which would accept an
+        // extra field silently and let the conformance regression through.
+        let body: serde_json::Value = serde_json::from_slice(&bytes).expect("valid json");
+        assert_eq!(body, serde_json::json!({ "is_healthy": true }));
+    }
+
+    #[tokio::test]
+    async fn health_fails_once_the_tip_is_older_than_the_threshold() {
+        let app = TestApp::new_with_minibf_config(|cfg| cfg.with_max_tip_age_sec(60));
+        let (status, body) = get_json::<RootResponse>(&app, "/health").await;
+
+        assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
+        assert!(!body.is_healthy);
+    }
+
+    #[tokio::test]
+    async fn health_passes_when_the_tip_is_within_the_threshold() {
+        let app = TestApp::new_with_minibf_config(|cfg| cfg.with_max_tip_age_sec(u64::MAX));
+        let (status, body) = get_json::<RootResponse>(&app, "/health").await;
+
+        assert_eq!(status, StatusCode::OK);
+        assert!(body.is_healthy);
+    }
+
+    #[tokio::test]
+    async fn tip_reports_age_without_a_threshold() {
+        let app = TestApp::new();
+        let (status, body) = get_json::<TipResponse>(&app, "/health/tip").await;
+
+        assert_eq!(status, StatusCode::OK);
+
+        assert_eq!(body.is_stale, None);
+        assert_eq!(body.max_tip_age_seconds, None);
+
+        assert!(body.tip_slot.is_some(), "tip slot should be reported");
+        assert!(
+            body.tip_age_seconds.is_some_and(|age| age > 0),
+            "a tip in the past should report a non-zero age, got {:?}",
+            body.tip_age_seconds
+        );
+    }
+
+    #[tokio::test]
+    async fn tip_judges_against_a_configured_threshold() {
+        let app = TestApp::new_with_minibf_config(|cfg| cfg.with_max_tip_age_sec(60));
+        let (status, body) = get_json::<TipResponse>(&app, "/health/tip").await;
+
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body.is_stale, Some(true));
+        assert_eq!(body.max_tip_age_seconds, Some(60));
+        assert!(body.tip_age_seconds.is_some_and(|age| age > 60));
+    }
+
+    /// Staleness reporting is additive to liveness: a node whose tip cannot be
+    /// read is unmeasurable, not unhealthy, so `/health` keeps its original
+    /// meaning rather than inventing a failure.
+    #[tokio::test]
+    async fn an_unreadable_tip_is_unmeasurable_not_unhealthy() {
+        let app = TestApp::new_with_fault(Some(TestFault::StateStoreError));
+
+        let (status, body) = get_json::<RootResponse>(&app, "/health").await;
+        assert_eq!(status, StatusCode::OK);
+        assert!(body.is_healthy);
+
+        let (status, body) = get_json::<TipResponse>(&app, "/health/tip").await;
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body.tip_slot, None);
+        assert_eq!(body.tip_age_seconds, None);
+        assert_eq!(body.is_stale, None);
+    }
+}

crates/minibf/src/test_support.rs

@@ -121,6 +121,18 @@ impl TestApp {
         Self::from_domain(domain, vectors, fault)
     }
 
+    /// Same as [`TestApp::new`], with the minibf config adjusted before the
+    /// router is built — for routes whose behaviour is config-driven.
+    pub fn new_with_minibf_config(tweak: impl FnOnce(MinibfConfig) -> MinibfConfig) -> Self {
+        let cfg = SyntheticBlockConfig {
+            block_count: 5,
+            txs_per_block: 3,
+            ..Default::default()
+        };
+        let (domain, vectors) = TestDomainBuilder::new_with_synthetic(cfg).finish();
+        Self::from_domain_with_config(domain, vectors, None, tweak)
+    }
+
     pub fn new_with_cfg_and_setup(
         cfg: SyntheticBlockConfig,
         setup: impl FnOnce(&ToyDomain, &SyntheticVectors),
@@ -131,12 +143,23 @@ impl TestApp {
     }
 
     fn from_domain(domain: ToyDomain, vectors: SyntheticVectors, fault: Option<TestFault>) -> Self {
+        Self::from_domain_with_config(domain, vectors, fault, |cfg| cfg)
+    }
+
+    fn from_domain_with_config(
+        domain: ToyDomain,
+        vectors: SyntheticVectors,
+        fault: Option<TestFault>,
+        tweak: impl FnOnce(MinibfConfig) -> MinibfConfig,
+    ) -> Self {
         let domain = match fault {
             Some(fault) => dolos_testing::faults::FaultyToyDomain::new(domain, fault),
             None => dolos_testing::faults::FaultyToyDomain::new(domain, TestFault::None),
         };
 
-        let cfg = MinibfConfig::new("[::]:0".parse().expect("invalid listen address"));
+        let cfg = tweak(MinibfConfig::new(
+            "[::]:0".parse().expect("invalid listen address"),
+        ));
 
         let facade = Facade {
             inner: domain.clone(),

docs/content/apis/minibf.mdx

@@ -73,12 +73,14 @@ The `serve.minibf` section controls the options for the MiniBF endpoint that can
 | token_registry_url | string  | "https://token-registry.io"     |
 | url                | string  | "https://minibf.local"          |
 | max_scan_items     | integer | 3000                             |
+| max_tip_age_sec    | integer | 300                              |
 
 - `listen_address`: the local address (`IP:PORT`) to listen for incoming connections (`[::]` represents any IP address).
 - `permissive_cors`: allow cross-origin requests from any origin.
 - `token_registry_url`: optional token registry base URL used for off-chain asset metadata. 
 - `url`: optional public URL used in the `/` root response.
 - `max_scan_items`: caps page-based scans for heavy endpoints (defaults to 3000 if unset).
+- `max_tip_age_sec`: opts into the health gate described under "Health and staleness" below — past this many seconds of tip age, `/health` answers `503`. Unset by default, in which case `/health` keeps reporting healthy however far behind the node has fallen.
 
 This is an example of the `serve.minibf` fragment with a `dolos.toml` configuration file.
 
@@ -89,6 +91,7 @@ permissive_cors = true
 token_registry_url = "https://token-registry.io"
 url = "https://minibf.local"
 max_scan_items = 3000
+max_tip_age_sec = 300
 ```
 
 Check the [Configuration Schema](../configuration/schema) for detailed info on how to set this up.
@@ -102,6 +105,7 @@ Dolos provides many, but not all of the Blockfrost endpoints. The following list
 | `/`                                        | Service info (version, revision)                         |
 | `/health`                                  | Health check                                             |
 | `/health/clock`                            | Clock health                                             |
+| `/health/tip`                              | Tip age and staleness verdict                            |
 | `/metrics`                                 | Prometheus metrics                                       |
 | `/accounts/{stake_address}`                | Get account information for a stake address              |
 | `/accounts/{stake_address}/addresses`      | Get addresses for a stake address                        |
@@ -167,3 +171,41 @@ Dolos provides many, but not all of the Blockfrost endpoints. The following list
 | `/txs/{tx_hash}/stakes`                    | Get stakes for a specific transaction                    |
 | `/txs/{tx_hash}/utxos`                     | Get UTXOs for a specific transaction                     |
 | `/txs/{tx_hash}/withdrawals`               | Get withdrawals for a specific transaction               |
+
+## Health and staleness
+
+`/health` answers the Blockfrost question — is this node fit to serve? — and
+nothing else:
+
+```json
+{ "is_healthy": true }
+```
+
+`/health/tip` answers the more useful one: how current is it?
+
+```json
+{
+  "tip_slot": 84916321,
+  "tip_age_seconds": 34,
+  "max_tip_age_seconds": 300,
+  "is_stale": false
+}
+```
+
+`tip_age_seconds` is the wall-clock gap between the node's tip and now. It is
+the field that distinguishes a node tracking the chain from one that has been
+serving the same block for hours — a distinction a caller otherwise cannot make
+without a second, independent opinion about where the chain actually is.
+
+By default dolos measures but does not judge: `is_stale` and
+`max_tip_age_seconds` are omitted, and `/health` keeps answering `200` however
+far behind the node has fallen, since a node catching up from a bootstrap is
+hours behind by design. Set `serve.minibf.max_tip_age_sec` to opt into a gate:
+past that many seconds `/health` answers `503` with `is_healthy: false`, which
+is what lets a load balancer or supervisor take a stale node out of rotation,
+while `/health/tip` reports `is_stale: true` alongside the age that earned the
+verdict.
+
+`tip_slot` and `tip_age_seconds` are omitted when the tip cannot be read at
+all — an unmeasurable node is reported as unmeasurable rather than as unhealthy,
+so `/health` keeps its original meaning.