feat(slasher): add slasher metrics

Author: Rexagon

scripts/gen-dashboard.py

@@ -2763,6 +2763,70 @@ def validator() -> RowPanel:
     return create_row("Validator", metrics)
 
 
+def slasher() -> RowPanel:
+    metrics = [
+        timeseries_panel(
+            targets=[
+                target(
+                    'tycho_slasher_blocks_batch_size{instance=~"$instance"} and \
+                    on(instance, job) tycho_slasher_enabled{instance=~"$instance"} == 1',
+                    legend_format="{{instance}}",
+                )
+            ],
+            title="Slasher blocks batch size",
+            unit=UNITS.NUMBER_FORMAT,
+        ),
+        create_gauge_panel("tycho_slasher_active_sessions", "Slasher active sessions"),
+        create_gauge_panel(
+            "tycho_slasher_session_init_queue_len", "Slasher session init queue"
+        ),
+        create_gauge_panel(
+            "tycho_slasher_batch_delivery_tasks", "Slasher batch delivery tasks"
+        ),
+        create_gauge_panel(
+            "tycho_slasher_pending_messages", "Slasher pending messages"
+        ),
+        create_counter_panel(
+            "tycho_slasher_blocks_batch_send_attempts_total",
+            "Slasher batch send attempts",
+        ),
+        create_counter_panel(
+            "tycho_slasher_blocks_batch_send_results_total",
+            "Slasher batch send results",
+            by_labels=["instance", "result"],
+            legend_format="{{instance}} {{result}}",
+        ),
+        create_counter_panel(
+            "tycho_slasher_blocks_batches_submitted_total",
+            "Slasher submitted batches",
+            by_labels=["instance", "origin"],
+            legend_format="{{instance}} {{origin}}",
+        ),
+        create_counter_panel(
+            "tycho_slasher_contract_event_decode_errors_total",
+            "Slasher contract event decode errors",
+        ),
+        create_counter_panel(
+            "tycho_slasher_vset_reports_total", "Slasher vset reports"
+        ),
+        create_counter_panel(
+            "tycho_slasher_accusations_total",
+            "Slasher accusations",
+            by_labels=["instance", "pubkey"],
+            legend_format="{{instance}} {{pubkey}}",
+            legend_placement="bottom",
+        ),
+        create_heatmap_panel(
+            "tycho_slasher_handle_state_time", "Slasher handle state time"
+        ),
+        create_heatmap_panel(
+            "tycho_slasher_blocks_batch_delivery_time",
+            "Slasher blocks batch delivery time",
+        ),
+    ]
+    return create_row("Slasher", metrics)
+
+
 def mempool_rounds() -> RowPanel:
     metrics = [
         create_gauge_panel(
@@ -3682,6 +3746,7 @@ def templates() -> Templating:
         collator_misc_operations_metrics(),
         collator_commit_block_metrics(),
         validator(),
+        slasher(),
         mempool_rounds(),
         mempool_payload_rates(),
         mempool_engine_rates(),

slasher/src/bc/mod.rs

@@ -16,6 +16,8 @@ use crate::util::BitSet;
 
 mod contract;
 
+const METRIC_PENDING_MESSAGES: &str = "tycho_slasher_pending_messages";
+
 #[derive(Clone, Copy)]
 pub struct EncodeBlocksBatchMessage<'a> {
     pub address: &'a StdAddr,
@@ -82,6 +84,7 @@ impl ContractSubscription {
         match self.pending_messages.entry(*msg_hash) {
             Entry::Vacant(entry) => {
                 entry.insert(PendingMessage { expire_at, tx });
+                self.report_pending_messages();
                 Ok(rx)
             }
             Entry::Occupied(_) => anyhow::bail!("duplicate external message: {msg_hash}"),
@@ -100,6 +103,7 @@ impl ContractSubscription {
 
         if let Some((_, pending)) = self.pending_messages.remove(msg_hash) {
             pending.tx.send(MessageDelivered { tx_hash: *tx_hash }).ok();
+            self.report_pending_messages();
             return Ok(true);
         }
         Ok(false)
@@ -115,6 +119,15 @@ impl ContractSubscription {
         if dropped > 0 {
             tracing::warn!(dropped, "dropped pending messages");
         }
+        self.report_pending_messages();
+    }
+
+    pub fn reset_pending_messages_metrics() {
+        metrics::gauge!(METRIC_PENDING_MESSAGES).set(0);
+    }
+
+    pub fn report_pending_messages(&self) {
+        metrics::gauge!(METRIC_PENDING_MESSAGES).set(self.pending_messages.len() as f64);
     }
 }
 

slasher/src/collector/validator_events.rs

@@ -15,6 +15,8 @@ use tycho_util::{DashMapEntry, FastDashMap};
 use crate::bc::BlocksBatch;
 
 const INIT_QUEUE_CAPACITY: usize = 3;
+const METRIC_ACTIVE_SESSIONS: &str = "tycho_slasher_active_sessions";
+const METRIC_SESSION_INIT_QUEUE_LEN: &str = "tycho_slasher_session_init_queue_len";
 
 pub trait BlockBatchesStore {
     fn known_batch_size(&self) -> AtomicU32;
@@ -55,6 +57,8 @@ impl ValidatorEventsCollector {
     pub fn new(default_batch_size: NonZeroU32) -> Self {
         let init_queue_capacity = INIT_QUEUE_CAPACITY;
         let init_queue = Mutex::new(VecDeque::with_capacity(init_queue_capacity));
+        metrics::gauge!(METRIC_ACTIVE_SESSIONS).set(0);
+        metrics::gauge!(METRIC_SESSION_INIT_QUEUE_LEN).set(0);
 
         Self {
             default_batch_size: AtomicU32::new(default_batch_size.get()),
@@ -71,7 +75,9 @@ impl ValidatorEventsCollector {
         {
             return None;
         }
-        queue.pop_front()
+        let result = queue.pop_front();
+        metrics::gauge!(METRIC_SESSION_INIT_QUEUE_LEN).set(queue.len() as f64);
+        result
     }
 
     fn push_session_to_init(&self, info: ValidatorSessionInfo) {
@@ -85,6 +91,7 @@ impl ValidatorEventsCollector {
             );
         }
         items.push_back(info);
+        metrics::gauge!(METRIC_SESSION_INIT_QUEUE_LEN).set(items.len() as f64);
     }
 
     pub fn set_default_batch_size(&self, batch_size: NonZeroU32) {
@@ -119,7 +126,11 @@ impl ValidatorEventsCollector {
     }
 
     pub fn skip_session(&self, session_id: ValidationSessionId) -> bool {
-        self.sessions.remove(&session_id).is_some()
+        let removed = self.sessions.remove(&session_id).is_some();
+        if removed {
+            metrics::gauge!(METRIC_ACTIVE_SESSIONS).set(self.sessions.len() as f64);
+        }
+        removed
     }
 }
 
@@ -163,6 +174,7 @@ impl ValidatorEventsListener for ValidatorEventsCollector {
                 own_validator_idx,
                 validators,
             });
+            metrics::gauge!(METRIC_ACTIVE_SESSIONS).set(self.sessions.len() as f64);
         } else {
             tracing::warn!("duplicate session");
         }
@@ -176,6 +188,7 @@ impl ValidatorEventsListener for ValidatorEventsCollector {
         {
             tracing::warn!("failed to commit blocks batch on finish: {e:?}");
         }
+        metrics::gauge!(METRIC_ACTIVE_SESSIONS).set(self.sessions.len() as f64);
     }
 
     #[instrument(skip_all, fields(session_id = ?session_id))]