fix(persistence): latch a degraded state when the data dir vanishes — no per-tick ENOENT error loop (#366)

CHANGELOG.md

@@ -6,6 +6,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- **Data-dir deleted under a running server now latches a degraded state
+  instead of error-looping the persistence tick (#366).** When `--dir`
+  vanishes (operator `rm -rf`, tmp-cleaner sweep, mount loss), the per-shard
+  1ms WAL/checkpoint tick used to retry its file operations on `ENOENT`
+  forever — observed in the wild at ~667% CPU on an idle 4-shard server,
+  with a wedged `PING`. The disk monitor's 5s poll now distinguishes
+  `statvfs` `ENOENT`/`ENOTDIR` on the monitored path and latches `dir_lost`
+  (one loud `ERROR` log): write commands are refused with `MOONERR
+  dirmissing` (reads and `PING` keep serving, mirroring the diskfull guard),
+  and the persistence tick skips WAL flush / checkpoint / WAL-overflow scan
+  / auto-save entirely — zero per-tick syscalls and zero per-tick log lines
+  while latched. The latch self-heals: the next poll that sees the directory
+  again resumes writes (with an explicit warning that data accepted since
+  the deletion is not guaranteed durable until restart). Only engages when
+  persistence or disk-offload is configured, and works with
+  `--disk-free-min-pct 0`. Two hardening layers close the shallow-heal hole
+  (adversarial review): WAL segment rotation re-creates a missing parent
+  directory (`mkdir -p <dir>` after an incident no longer leaves the nested
+  `wal-v3/` dir dead), and any tick-flush failure arms a 1s retry backoff so
+  no flush error class can ever loop at the 1ms tick cadence again.
+
 ### Performance
 - **Disk-offload: shard event loop no longer pays manifest-commit fsyncs
   (task #59 levers 1+2).** Under spill load, `apply_completion_vec` ran one

src/main.rs

@@ -1025,10 +1025,19 @@ fn main() -> anyhow::Result<()> {
 
     // MA12: Initialise disk free-space monitor.
     // Monitors the WAL/persistence volume. When disk_free_min_pct == 0, the
-    // monitor is inactive (poll_global is a no-op, is_write_paused always false).
+    // free-space guard is inactive, but a persistence-enabled server still
+    // polls so the dir-lost latch (#366) can refuse writes and quiesce the
+    // persistence tick if the data directory is deleted out from under it.
     {
         let monitor_path = persistence_dir.as_deref().unwrap_or(&config.dir);
-        moon::shard::disk_monitor::init_global(config.disk_free_min_pct, monitor_path);
+        // Arm the dir-lost latch for any config that writes to the data dir:
+        // AOF/save persistence, OR disk-offload (spill + vector warm tier
+        // write there even with appendonly=no).
+        moon::shard::disk_monitor::init_global(
+            config.disk_free_min_pct,
+            monitor_path,
+            persistence_dir.is_some() || config.disk_offload_enabled(),
+        );
     }
 
     // Wave 3: Initialise proactive RSS memory watchdog ("mem-full guard").

src/persistence/wal_v3/segment.rs

@@ -353,6 +353,10 @@ pub struct WalWriterV3 {
     /// Set when agent spawn failed once — never retried (inline fsync
     /// fallback, logged once).
     sync_agent_unavailable: bool,
+    /// Backoff deadline after a tick-flush failure (#366): the 1ms tick must
+    /// never retry a failing flush at tick cadence (syscall+log error loop).
+    /// `None` while healthy.
+    flush_backoff_until: Option<std::time::Instant>,
 }
 
 /// Bound on every blocking durability wait (checkpoint ordering gates,
@@ -393,6 +397,7 @@ impl WalWriterV3 {
             max_wal_bytes: DEFAULT_MAX_WAL_BYTES,
             sync_agent: None,
             sync_agent_unavailable: false,
+            flush_backoff_until: None,
         };
 
         writer.open_new_segment()?;
@@ -555,6 +560,31 @@ impl WalWriterV3 {
         }
     }
 
+    /// True while the tick-flush is backing off after a failure (#366).
+    ///
+    /// Healthy path cost: one `Option` discriminant check — `Instant::now()`
+    /// only runs while a backoff is actually armed.
+    #[inline]
+    pub fn flush_backing_off(&self) -> bool {
+        self.flush_backoff_until
+            .is_some_and(|until| std::time::Instant::now() < until)
+    }
+
+    /// Arm a 1s tick-flush backoff after a flush failure (#366): the 1ms
+    /// tick must never retry a failing flush at tick cadence.
+    pub fn note_flush_failure(&mut self) {
+        self.flush_backoff_until =
+            Some(std::time::Instant::now() + std::time::Duration::from_secs(1));
+    }
+
+    /// Clear the backoff after a successful flush.
+    #[inline]
+    pub fn clear_flush_backoff(&mut self) {
+        if self.flush_backoff_until.is_some() {
+            self.flush_backoff_until = None;
+        }
+    }
+
     /// Fsync without writing — call after `flush_write` / `flush_if_needed`
     /// to make all previously written records durable.
     pub fn sync_data(&mut self) -> std::io::Result<()> {
@@ -808,6 +838,11 @@ impl WalWriterV3 {
 
     /// Open a new segment file and write its 64-byte header.
     fn open_new_segment(&mut self) -> std::io::Result<()> {
+        // Self-repair a vanished parent (#366 heal path): `create(true)`
+        // only creates the LEAF file, so a shallow `mkdir -p <dir>` after a
+        // dir-lost incident would leave this rotation failing ENOENT on
+        // every retry. Rotations are rare — one extra mkdir syscall is free.
+        fs::create_dir_all(&self.wal_dir)?;
         let path = WalSegment::segment_path(&self.wal_dir, self.current_sequence);
         let mut file = OpenOptions::new()
             .create(true)
@@ -1084,6 +1119,66 @@ mod tests {
         assert_eq!(WalSegment::segment_name(0), "000000000000.wal");
     }
 
+    /// #366 review blocker B1: after a dir-lost incident an operator does a
+    /// shallow `mkdir -p <dir>` — the nested `wal-v3/` directory is still
+    /// gone. Segment rotation must self-repair the parent instead of failing
+    /// ENOENT on every retry (which would re-create the tick error loop the
+    /// dir-lost latch exists to kill, this time UNLATCHED because the
+    /// top-level path exists again).
+    #[test]
+    fn test_rotation_self_repairs_deleted_wal_dir() {
+        let tmp = tempfile::tempdir().unwrap();
+        let wal_dir = tmp.path().join("wal");
+        // Tiny segment so a handful of appends forces rotation.
+        let mut writer = WalWriterV3::new(0, &wal_dir, 256).unwrap();
+
+        // The shallow-heal state: the whole wal dir vanishes while the
+        // writer's current-segment fd stays open (unlinked inode).
+        fs::remove_dir_all(&wal_dir).unwrap();
+
+        // Push enough bytes through append+flush to cross segment_size and
+        // force rotate_segment() -> open_new_segment().
+        for i in 0..16u32 {
+            writer.append(
+                WalRecordType::Command,
+                format!("SET key{i} {}", "v".repeat(64)).as_bytes(),
+            );
+        }
+        writer
+            .flush_write()
+            .expect("rotation must recreate the missing wal dir, not fail ENOENT");
+
+        assert!(wal_dir.exists(), "wal dir must have been re-created");
+        assert!(
+            writer.current_segment_sequence() > 1,
+            "rotation must actually have happened for this test to prove anything"
+        );
+        // And the new segment file is a real on-disk file.
+        let seg = WalSegment::segment_path(&wal_dir, writer.current_segment_sequence());
+        assert!(seg.exists(), "post-rotation segment must exist on disk");
+    }
+
+    /// #366: the tick flush must never retry a failing flush at 1ms tick
+    /// cadence — a failure arms a 1s backoff, success clears it.
+    #[test]
+    fn test_flush_backoff_arms_and_clears() {
+        let tmp = tempfile::tempdir().unwrap();
+        let wal_dir = tmp.path().join("wal");
+        let mut writer = WalWriterV3::new(0, &wal_dir, DEFAULT_SEGMENT_SIZE).unwrap();
+
+        assert!(
+            !writer.flush_backing_off(),
+            "healthy writer never backs off"
+        );
+        writer.note_flush_failure();
+        assert!(writer.flush_backing_off(), "failure must arm the backoff");
+        writer.clear_flush_backoff();
+        assert!(
+            !writer.flush_backing_off(),
+            "success must clear the backoff"
+        );
+    }
+
     #[test]
     fn test_writer_creates_segment() {
         let tmp = tempfile::tempdir().unwrap();

src/server/conn/handler_monoio/dispatch.rs

@@ -814,7 +814,12 @@ pub(super) fn try_enforce_readonly(
 #[inline]
 pub(super) fn try_enforce_disk_full(cmd: &[u8], responses: &mut Vec<Frame>) -> bool {
     if metadata::is_write(cmd) && crate::shard::segment_stall::is_any_write_stall_active() {
-        let msg: &'static [u8] = if crate::shard::disk_monitor::is_write_paused() {
+        // dir-lost first: it also sets `is_write_paused`, and "diskfull"
+        // would send the operator hunting free space instead of the missing
+        // data directory (#366).
+        let msg: &'static [u8] = if crate::shard::disk_monitor::is_dir_lost() {
+            b"MOONERR dirmissing: data directory was removed; writes refused until it is restored"
+        } else if crate::shard::disk_monitor::is_write_paused() {
             b"MOONERR diskfull: writes paused until free space recovers"
         } else if crate::shard::mem_monitor::is_write_paused() {
             b"MOONERR memfull: writes paused until memory pressure recovers"

src/server/conn/handler_sharded/dispatch.rs

@@ -779,8 +779,12 @@ pub(super) fn try_enforce_readonly(
 #[inline]
 pub(super) fn try_enforce_disk_full(cmd: &[u8], responses: &mut Vec<Frame>) -> bool {
     if metadata::is_write(cmd) && crate::shard::segment_stall::is_any_write_stall_active() {
-        // Distinguish the stall source for operator clarity.
-        let msg: &'static [u8] = if crate::shard::disk_monitor::is_write_paused() {
+        // Distinguish the stall source for operator clarity. dir-lost first:
+        // it also sets `is_write_paused`, and "diskfull" would send the
+        // operator hunting free space instead of the missing data dir (#366).
+        let msg: &'static [u8] = if crate::shard::disk_monitor::is_dir_lost() {
+            b"MOONERR dirmissing: data directory was removed; writes refused until it is restored"
+        } else if crate::shard::disk_monitor::is_write_paused() {
             b"MOONERR diskfull: writes paused until free space recovers"
         } else if crate::shard::mem_monitor::is_write_paused() {
             b"MOONERR memfull: writes paused until memory pressure recovers"