perf(vector): move COLD-segment reload off the shard event loop (audit #18)

CHANGELOG.md

@@ -6,6 +6,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Performance — Vector: COLD-segment reload moved off the shard event loop (PR #251)
+
+- Under `--disk-offload`, an idle vector segment is demoted to the COLD tier
+  (`UnloadedSegment` stub, everything in-memory dropped). The first FT.SEARCH to
+  touch it must reload it — `WarmSearchSegment::from_files`: mmap + page-in (+
+  optional mlock) of a whole segment. That reload ran **inline on the
+  single-threaded shard event loop** (`promote_unloaded` at the top of the
+  yielding search capture), so it stalled EVERY other connection on the shard for
+  the full reload (hundreds of ms–seconds on real NVMe/EBS with a cold page
+  cache) — a multi-tenancy violation (audit finding 18).
+- A new process-global `SegmentReloadPool` (`src/vector/reload_pool.rs`, auto-on;
+  `MOON_VEC_RELOAD_WORKERS=0` disables) now performs the reload I/O on dedicated
+  worker threads. The yielding capture path calls `submit_unloaded_reloads`
+  (non-blocking): it submits each COLD stub off-loop and stashes the `flume`
+  receivers in the `SearchSnapshot`; the handler `await`s them
+  (`await_pending_reloads`) before scanning, which parks only the triggering
+  query's **task** (not the OS thread) and then splices the reloaded segments
+  into that query's own snapshot. Result: the dense-KNN query that triggered the
+  reload still sees **full recall**, while sibling connections on the shard keep
+  running. Reloads are **single-flight** (concurrent first-touches of one segment
+  share a job) and cached until the next capture installs them into WARM
+  (reclaiming the stub's memory).
+- Design-for-failure: a reload that errors or whose worker panics/dies replies
+  `Err`/disconnect to every waiter — the query answers with degraded recall
+  (segment stays COLD, retried next touch) rather than hanging or crashing. When
+  the pool is disabled, `submit_unloaded_reloads` falls back to the blocking
+  `promote_unloaded`, preserving exact pre-#18 behavior. Non-yielding sync search
+  paths are unchanged (still block); HYBRID/SPARSE/RANGE/non-default-field
+  queries do not take the off-loop path (accepted follow-up).
+
 ### Docs — engineering roadmap suite under `docs/roadmap/` (PR #254)
 
 - New planning docs (excluded from the published docs site via `mkdocs.yml`

src/command/vector_search/ft_search/dispatch.rs

@@ -667,11 +667,15 @@ fn capture_dense_knn_snapshot(
     let filter_strategy =
         crate::vector::filter::selectivity::select_strategy(filter_bitmap.as_ref(), total_vectors);
 
-    // WS3 round 2: the yielding (worker-pool) search path captures its
+    // WS3 round 2 + #18: the yielding (worker-pool) search path captures its
     // segment snapshot here, separately from `SegmentHolder::search_filtered`'s
-    // own promote-before-search call -- must reload any COLD segments before
-    // this capture too, or a query on this path would silently skip them.
-    idx.segments.promote_unloaded();
+    // own promote-before-search call. Rather than BLOCK the shard event loop on
+    // COLD-segment reloads, submit them off-loop and stash the receivers in the
+    // snapshot; the handler awaits them (parking only this query's task) before
+    // scanning, so siblings never stall. When the reload pool is disabled this
+    // falls back to the blocking `promote_unloaded` (no receivers), exactly as
+    // before, so a COLD segment is never silently skipped.
+    let pending_reloads = idx.segments.submit_unloaded_reloads();
     let segments = idx.segments.load_full();
     let mutable_len = segments.mutable.len();
 
@@ -696,6 +700,7 @@ fn capture_dense_knn_snapshot(
             rerank_mult: idx.meta.rerank_mult,
             exact_beam: idx.meta.exact_beam,
         },
+        pending_reloads,
     })
 }
 

src/main.rs

@@ -488,6 +488,19 @@ fn main() -> anyhow::Result<()> {
         info!("FT.SEARCH worker pool: {ft_workers} threads");
     }
 
+    // #18: off-loop COLD-segment reload pool. Moves the mmap+page-in of an
+    // idle-demoted vector segment off the shard event loop so the first
+    // FT.SEARCH that touches it doesn't stall sibling connections. Auto-on with
+    // a small worker count (reloads are rare + single-flight); disable with
+    // MOON_VEC_RELOAD_WORKERS=0 to fall back to the blocking reload.
+    let reload_workers = moon::vector::reload_pool::default_workers(num_shards);
+    moon::vector::reload_pool::init_global(reload_workers);
+    if reload_workers > 0 {
+        info!("vector COLD-segment reload pool: {reload_workers} threads");
+    } else {
+        info!("vector COLD-segment reload pool: disabled (blocking reload)");
+    }
+
     // Checked cast: --shards is bounded by clap's value_parser, but `as u16`
     // would silently wrap for values > 65535. Fail loudly instead.
     // ALLOW: panic is appropriate here — this is `main`, not library code.

src/server/conn/handler_monoio/ft.rs

@@ -641,6 +641,10 @@ pub(super) async fn try_handle_ft_command(
                     offset,
                     count,
                 } => {
+                    // #18: await any off-loop COLD→WARM reloads submitted at
+                    // capture (parks this task, not the shard thread) so the
+                    // scan below sees full recall while siblings keep running.
+                    snapshot.await_pending_reloads().await;
                     let results =
                         crate::vector::segment::holder::SegmentHolder::search_mvcc_yielding(
                             &mut *snapshot,

src/server/conn/handler_sharded/ft.rs

@@ -523,6 +523,10 @@ pub(super) async fn try_handle_ft_command(
                 offset,
                 count,
             } => {
+                // #18: await any off-loop COLD→WARM reloads submitted at
+                // capture (parks this task, not the shard thread) so the scan
+                // below sees full recall while siblings keep running.
+                snapshot.await_pending_reloads().await;
                 let results = crate::vector::segment::holder::SegmentHolder::search_mvcc_yielding(
                     &mut *snapshot,
                     crate::vector::segment::holder::ft_search_yield_budget(),

src/vector/mod.rs

@@ -13,6 +13,7 @@ pub mod keymap;
 pub mod metrics;
 pub mod mvcc;
 pub mod persistence;
+pub mod reload_pool;
 pub mod search_pool;
 pub mod segment;
 pub mod sparse;