perf(shard): re-pin auxiliary threads off pinned shard cores (O5)

CHANGELOG.md

@@ -55,6 +55,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   Both flags are CLI-only: a `moon.conf` value cannot reach jemalloc
   (its config is read at process start, before the conf file is parsed)
   and now triggers a loud startup warning instead of a silent no-op.
+- **Auxiliary threads no longer contend with pinned shard cores (O5).**
+  `manifest-sync-N`, `spill-N`, and `moon-wal-sync-N` are spawned from
+  inside the shard's own (pinned) event-loop thread and, on Linux,
+  inherit that thread's exact single-core affinity mask at
+  `pthread_create` time — they weren't merely sharing the shard's core,
+  they were born wearing its mask and could never leave it (confirmed via
+  `ps -eLo psr`: `manifest-sync-N`/`spill-N` co-located on shard N's
+  core; the main accept/dispatch thread also floated onto a shard core,
+  ~12% CPU steal observed under load). When `--shards` leaves a
+  remainder of un-pinned cores, every auxiliary thread (main
+  accept/dispatch, `manifest-sync-N`, `spill-N`, `moon-wal-sync-N`,
+  the AOF writer(s), auto-save) now re-pins itself to that non-shard
+  core set (round-robin) as the first act of its closure — except the
+  main thread itself, which re-pins at the LAST moment before entering
+  the accept loop, because children spawned from main inherit its mask
+  and the vector pools must keep the full-machine mask. Adversarial
+  review widened coverage to every OTHER pinned-parent spawn site in
+  the tree: `moon-heap-orphan-sweep-N` (a ~40s crash-recovery I/O burst
+  previously confined to its shard's own core), the lazily-spawned
+  `moon-cold-read-N` pool (previously born on whichever ONE shard core
+  served the first cold read and stuck there for the process lifetime,
+  serving all shards), `moon-vec-snapshot-N`, `moon-vec-idx-gc`, and
+  the `moon-vec-compact-N` pool (its old last-core-downward heuristic
+  landed workers on shard cores whenever shards ≈ cores; it now prefers
+  the non-shard set via `aux_core_at`, keeping the heuristic as
+  fallback). cpuset caveat: core IDs come from the machine-wide online
+  list, so under a restrictive cgroup cpuset the pin can fail — the
+  thread then keeps its inherited mask (debug-logged), never worse than
+  before. No-op when
+  `--shards >= cores` (no remainder to place them on) or on non-Linux.
+  `MOON_NO_AUX_PIN=1` disables it. The vector segment-reload pool
+  (`moon-vec-reload-N`) is deliberately left scheduler-placed — reloads
+  are latency-critical and the cold-start/crash-recovery reload storm
+  wants every core while shard threads are idle; confining it to the
+  small non-shard remainder risks 3x worse cold-reload latency, the
+  regression PR #237 fixed. jemalloc's background-reclaim threads are
+  unaffected — jemalloc spawns them internally with no Rust-side
+  affinity hook. A/B on GCE t2a-standard-8 (8 dedicated ARM vCPUs,
+  shards=4, AOF on, 4 interleaved rounds vs `MOON_NO_AUX_PIN=1`):
+  SET median +1.5%, GET median −1.9% (inside noise; GET run-to-run
+  spread tightened 23%→7% under pinning), with `ps -eLo psr` placement
+  proof — pinned legs show all 12 aux threads on the non-shard cores
+  4-7, unpinned legs show `manifest-sync-N`/`spill-N` sitting exactly
+  on shard cores 0-3.
 - **Shard offload paths are precomputed at shard init — the recurring
   tick paths no longer allocate (#45).** The 100ms eviction tick, the
   memory-pressure cascade, the 10s warm-transition check, and the

src/main.rs

@@ -543,6 +543,19 @@ fn main() -> anyhow::Result<()> {
 
     info!("Starting with {} shards", num_shards);
 
+    // O5: record the non-shard core set BEFORE any auxiliary thread spawns
+    // (AOF writers, the vector pools below, and — indirectly — the
+    // manifest-sync/spill/wal-sync threads each shard spawns internally).
+    //
+    // Deliberately NOT re-pinning the main thread here: threads spawned from
+    // main inherit its affinity mask at spawn time, and the vector
+    // search/reload pools just below must stay scheduler-placed across ALL
+    // cores (see src/vector/reload_pool.rs) — pinning main first would have
+    // them born wearing one aux core's mask with no escape. Main re-pins
+    // itself as "main-listener" at the LAST moment instead, right before it
+    // enters the accept loop, after every child thread has been spawned.
+    moon::shard::numa::init_aux_pinning(num_shards);
+
     // FT.SEARCH intra-query worker pool: fan per-segment HNSW searches of one
     // query across workers (threads spawn eagerly here — the pool is tiny and
     // parks on its channel when vector search is unused). Default OFF: each
@@ -820,6 +833,9 @@ fn main() -> anyhow::Result<()> {
                 std::thread::Builder::new()
                     .name(thread_name)
                     .spawn(move || {
+                        // O5: escape the shard-core mask this thread would
+                        // otherwise inherit from its spawning thread.
+                        moon::shard::numa::pin_current_aux_thread(&thread_name_inner);
                         RuntimeFactoryImpl::block_on_local(
                             thread_name_inner,
                             aof::per_shard_aof_writer_task(
@@ -866,6 +882,9 @@ fn main() -> anyhow::Result<()> {
             std::thread::Builder::new()
                 .name("aof-writer".to_string())
                 .spawn(move || {
+                    // O5: escape the shard-core mask this thread would
+                    // otherwise inherit from its spawning thread.
+                    moon::shard::numa::pin_current_aux_thread("aof-writer");
                     RuntimeFactoryImpl::block_on_local(
                         "aof-writer".to_string(),
                         aof::aof_writer_task(
@@ -1792,6 +1811,12 @@ fn main() -> anyhow::Result<()> {
             .build()
             .expect("failed to build listener runtime");
 
+        // O5: every child thread is spawned by now — safe to confine the
+        // accept/dispatch loop (this thread) to the non-shard core set. The
+        // tokio::spawn calls inside the block_on run on this same
+        // current-thread runtime, not on new OS threads.
+        moon::shard::numa::pin_current_aux_thread("main-listener");
+
         listener_rt.block_on(async {
             // Set up auto-save timer if save rules are configured (sharded mode)
             if config.save.is_some() {
@@ -1903,6 +1928,9 @@ fn main() -> anyhow::Result<()> {
                 std::thread::Builder::new()
                     .name("auto-save".to_string())
                     .spawn(move || {
+                        // O5: escape the shard-core mask this thread would
+                        // otherwise inherit from its spawning thread.
+                        moon::shard::numa::pin_current_aux_thread("auto-save");
                         RuntimeFactoryImpl::block_on_local(
                             "auto-save".to_string(),
                             moon::persistence::auto_save::run_auto_save_sharded(
@@ -1925,6 +1953,9 @@ fn main() -> anyhow::Result<()> {
         // kernel distributes connections across all bound sockets, per-shard handles some
         // directly (no MPSC hop), central forwards the rest via conn_txs.
         let per_shard_accept = false;
+        // O5: every child thread is spawned by now — safe to confine the
+        // accept/dispatch loop (this thread) to the non-shard core set.
+        moon::shard::numa::pin_current_aux_thread("main-listener");
         RuntimeFactoryImpl::block_on_local("listener".to_string(), async move {
             if let Err(e) = server::listener::run_sharded(
                 config,

src/persistence/manifest_sync.rs

@@ -96,6 +96,11 @@ impl ManifestSyncAgent {
         match std::thread::Builder::new()
             .name(format!("manifest-sync-{shard_id}"))
             .spawn(move || {
+                // O5: this thread is spawned from the shard's own (pinned)
+                // event-loop thread and would otherwise inherit its exact
+                // single-core mask — re-pin to the non-shard core set as the
+                // first act, before anything else runs.
+                crate::shard::numa::pin_current_aux_thread(&format!("manifest-sync-{shard_id}"));
                 // The spawner sends the io immediately after a successful
                 // spawn; Err here means the agent was dropped first — exit.
                 let Ok(io) = io_rx.recv() else { return };

src/persistence/wal_v3/sync_agent.rs

@@ -116,6 +116,11 @@ impl WalSyncAgent {
         let thread = std::thread::Builder::new()
             .name(format!("moon-wal-sync-{shard_id}"))
             .spawn(move || {
+                // O5: this thread is spawned from the shard's own (pinned)
+                // event-loop thread and would otherwise inherit its exact
+                // single-core mask — re-pin to the non-shard core set as the
+                // first act, before anything else runs.
+                crate::shard::numa::pin_current_aux_thread(&format!("moon-wal-sync-{shard_id}"));
                 // Requests arrive in LSN order (single producer); fetch_max
                 // in publish() guards the watermark even if that ever
                 // changes.

src/server/embedded.rs

@@ -134,6 +134,13 @@ pub async fn run_embedded(
         num_shards, config.bind, config.port
     );
 
+    // O5: record the non-shard core set for this process before any
+    // auxiliary thread spawns below (AOF writer, and — indirectly — the
+    // manifest-sync/spill/wal-sync threads each shard spawns internally).
+    // Unlike main.rs, the CALLING thread here belongs to the embedding
+    // application, not to us — it is deliberately left untouched.
+    crate::shard::numa::init_aux_pinning(num_shards);
+
     // One-time global init that the production binary normally performs.
     crate::admin::metrics_setup::init_global_slowlog(
         config.slowlog_max_len,
@@ -165,6 +172,9 @@ pub async fn run_embedded(
         let handle = std::thread::Builder::new()
             .name("embedded-moon-aof".to_string())
             .spawn(move || {
+                // O5: escape the shard-core mask this thread would otherwise
+                // inherit from its spawning thread.
+                crate::shard::numa::pin_current_aux_thread("embedded-moon-aof");
                 RuntimeFactoryImpl::block_on_local(
                     "embedded-moon-aof".to_string(),
                     aof::aof_writer_task(rx, aof_file_path, fsync, aof_token, None),

src/shard/event_loop.rs

@@ -666,6 +666,13 @@ impl super::Shard {
                 std::thread::Builder::new()
                     .name(format!("moon-heap-orphan-sweep-{shard_id}"))
                     .spawn(move || {
+                        // O5: spawned from the pinned shard thread — escape
+                        // the inherited single-core mask (this sweep can run
+                        // ~40s of I/O; on the shard's own core it would
+                        // contend with command processing the whole time).
+                        crate::shard::numa::pin_current_aux_thread(&format!(
+                            "moon-heap-orphan-sweep-{shard_id}"
+                        ));
                         for path in &pending_heap_orphans {
                             crate::storage::tiered::kv_spill::remove_orphan_heap_file(path);
                         }