MILAB-6705: disk-persist the tree mirror (middle layer)

.changeset/disk-persist-tree-codec.md

@@ -0,0 +1,5 @@
+---
+"@milaboratories/pl-tree": minor
+---
+
+Snapshot codec for persisting a tree mirror to disk, plus capture and restore. Bodies are stored against global ids with signatures in a side table, so a snapshot survives the signatures it was taken with.

.changeset/disk-persist-tree-middle-layer.md

@@ -0,0 +1,6 @@
+---
+"@milaboratories/pl-middle-layer": minor
+"@milaboratories/pl-tree": minor
+---
+
+Persist project tree mirrors to disk and restore them on open, so reopening a project transfers what changed rather than the whole tree. On by default, with a kill switch in `treeSnapshotOps`.

lib/node/pl-middle-layer/build.node.config.js

@@ -1,5 +1,46 @@
 import { createRolldownNodeConfig } from "@milaboratories/ts-builder/configs/utils/createRolldownNodeConfig.js";
+import { execFileSync } from "node:child_process";
+
+/**
+ * Identifies this build for the persisted-tree cache key, see `src/middle_layer/build_stamp.ts`.
+ *
+ * A clean worktree stamps its commit; a dirty one stamps the build time too, so editing the
+ * tree pruning or traversal rules locally cannot hit a snapshot written under the old ones.
+ * `git status` covers the whole repo, which over-invalidates rather than under-.
+ *
+ * Note that release builds take the dirty path as well: CI runs `version-packages` before
+ * building and commits the bump afterwards, so the worktree always carries the version edits at
+ * build time. Harmless, because each published artifact bakes in one stamp and only has to be
+ * stable within itself, but it does mean a rebuild of identical code produces a different one.
+ */
+function buildStamp() {
+  const git = (args) =>
+    execFileSync("git", args, { stdio: ["ignore", "pipe", "ignore"] })
+      .toString()
+      .trim();
+  try {
+    const sha = git(["rev-parse", "--short=12", "HEAD"]);
+    const dirty = git(["status", "--porcelain"]).length > 0;
+    return dirty ? `${sha}-dirty-${Date.now()}` : sha;
+  } catch {
+    // No git available (a published tarball being rebuilt, for instance). Falling back to the
+    // build time keeps the stamp honest: it cannot claim to be a commit it does not know.
+    return `nogit-${Date.now()}`;
+  }
+}
 
 export default createRolldownNodeConfig({
   entry: ["./src/index.ts", "./src/worker/worker.ts"],
-});
+}).map((config) => ({
+  ...config,
+  // Note `transform.define`, not a top-level `define`: rolldown ignores the latter without
+  // complaining, which leaves the identifier in the output and the cache permanently cold.
+  // The spread preserves `transform.target` from the shared config.
+  transform: {
+    ...config.transform,
+    define: {
+      ...config.transform?.define,
+      __PL_ML_BUILD_STAMP__: JSON.stringify(buildStamp()),
+    },
+  },
+}));

lib/node/pl-middle-layer/src/middle_layer/build_stamp.ts

@@ -0,0 +1,36 @@
+/** Injected by rolldown at build time, see `build.node.config.js`. Absent when the package is
+ *  consumed straight from sources (`USE_SOURCES=1`), because no build step runs then. */
+declare const __PL_ML_BUILD_STAMP__: string | undefined;
+
+function injectedStamp(): string | undefined {
+  try {
+    // Read inside a try: with no build step the identifier is an undeclared global, and
+    // reading it throws a ReferenceError rather than yielding undefined.
+    return __PL_ML_BUILD_STAMP__;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Identifies the build of this package, and through it the rules that shape what a persisted
+ * tree mirror contains: the pruning function, the field filter and the traversal stop rules,
+ * all of which live in this package. (The finality predicate comes from pl-client and is NOT
+ * covered, which is harmless: finality is recomputed on restore, so it is the one rule that
+ * cannot poison a stored file.)
+ *
+ * Used as a cache-key component, so a change to those rules invalidates every snapshot, costing
+ * one cold open. Each built artifact bakes in one stamp, so reopens stay warm across restarts
+ * of an installed version. A build from a dirty worktree includes the build time, so editing
+ * those rules locally can never hit a snapshot written under the old ones. In practice release
+ * builds are dirty too, because CI writes version bumps into the worktree before building; that
+ * costs nothing, since the stamp only has to be stable within an artifact.
+ *
+ * With no build at all (sources mode) the value is a constant. That deliberately trades away
+ * the dirty-worktree guarantee: it means someone running from sources exercises the restore
+ * path at all, rather than every snapshot being a guaranteed miss for the one audience most
+ * likely to find its defects. The exposure it reintroduces, editing pruning rules from sources
+ * and hitting a mirror written under the old ones, is the local-development gap the design
+ * already accepts, and `treeSnapshots: false` or deleting the directory clears it.
+ */
+export const ML_BUILD_STAMP: string = injectedStamp() ?? "sources";

lib/node/pl-middle-layer/src/middle_layer/index.ts

@@ -2,5 +2,6 @@ export { MiddleLayer } from "./middle_layer";
 export { Project } from "./project";
 export * from "./driver_kit";
 export * from "./ops";
+export type { TreeSnapshotMiss, TreeSnapshotStat } from "./tree_snapshot_store";
 export { ProjectsField } from "./project_list";
 export type { OutgoingShare, PendingShare } from "./sharing_list";

lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts

@@ -94,6 +94,13 @@ import type { Dispatcher } from "undici";
 import { RetryAgent } from "undici";
 import { getDebugFlags } from "../debug";
 import { ProjectHelper } from "../model/project_helper";
+import type { TreeSnapshotStat } from "./tree_snapshot_store";
+import { TreeSnapshotStore } from "./tree_snapshot_store";
+
+/** How long shutdown waits for close-boundary snapshot writes that are already running. Long
+ *  enough for a ten-megabyte encode and write on ordinary storage, short enough that a wedged
+ *  filesystem does not hold the quit open. */
+const SNAPSHOT_DRAIN_TIMEOUT_MS = 5_000;
 
 export interface MiddleLayerEnvironment {
   dispose(): Promise<void>;
@@ -112,6 +119,9 @@ export interface MiddleLayerEnvironment {
   readonly driverKit: MiddleLayerDriverKit;
   readonly serviceRegistry: ModelServiceRegistry;
   readonly projectHelper: ProjectHelper;
+  /** Persisted project tree mirrors. Undefined when snapshots are switched off, or when the
+   *  client is impersonating another user, in which case nothing is read or written. */
+  readonly treeSnapshots?: TreeSnapshotStore;
 }
 
 /**
@@ -941,6 +951,36 @@ export class MiddleLayer {
 
   private readonly openedProjects = new Map<ProjectId, Project>();
 
+  /** Snapshot writes started by {@link closeProject} and not yet finished. Held only so
+   *  {@link close} can give them a bounded chance to land. */
+  private readonly pendingSnapshotWrites = new Set<Promise<void>>();
+
+  private trackSnapshotWrite(write: Promise<void>): void {
+    this.pendingSnapshotWrites.add(write);
+    void write.finally(() => this.pendingSnapshotWrites.delete(write));
+  }
+
+  /** Waits for close-boundary snapshot writes that are already running, up to `timeoutMs`.
+   *
+   * This starts no work: quitting still performs no snapshot of its own. It only lets a write
+   * that a project close already began finish, so closing a project and immediately quitting
+   * does not routinely lose it. Bounded, because a wedged filesystem must not hang the quit,
+   * and losing the write costs one cold open rather than any correctness. */
+  private async drainSnapshotWrites(timeoutMs: number): Promise<void> {
+    if (this.pendingSnapshotWrites.size === 0) return;
+
+    let timer: NodeJS.Timeout | undefined;
+    const expiry = new Promise<void>((resolve) => {
+      timer = setTimeout(resolve, timeoutMs);
+      timer.unref?.();
+    });
+    try {
+      await Promise.race([Promise.allSettled(this.pendingSnapshotWrites), expiry]);
+    } finally {
+      if (timer !== undefined) clearTimeout(timer);
+    }
+  }
+
   /** Opens a project, and starts corresponding project maintenance loop. */
   public async openProject(id: ProjectId): Promise<void> {
     if (this.openedProjects.has(id)) throw new Error(`Project ${id} already opened`);
@@ -954,6 +994,16 @@ export class MiddleLayer {
     const prj = this.openedProjects.get(id);
     if (prj === undefined) throw new Error(`Project ${id} not found among opened projects`);
     this.openedProjects.delete(id);
+
+    // Snapshot before destroy, and here rather than inside destroy(): destroy() is also what
+    // application shutdown runs, and quitting should perform no snapshot work. Terminating the
+    // tree invalidates it, so the state has to be taken first either way.
+    //
+    // Started, not awaited. The capture happens synchronously inside, which is the part that
+    // needs the tree alive; the encode and write are up to ten megabytes of work that closing a
+    // project should not sit behind. Kept so shutdown can drain it.
+    this.trackSnapshotWrite(prj.snapshotOnClose());
+
     await prj.destroy();
     this.openedProjectsList.setValue([...this.openedProjects.keys()]);
   }
@@ -970,6 +1020,13 @@ export class MiddleLayer {
     return this.openedProjects.has(id);
   }
 
+  /** Counters for the persisted project tree mirrors, or undefined when they are switched off.
+   *  Reads and hits are what show whether a reopen was actually warm, and the miss breakdown
+   *  says why it was not. */
+  public get treeSnapshotStats(): Readonly<TreeSnapshotStat> | undefined {
+    return this.env.treeSnapshots?.stats;
+  }
+
   /**
    * Deallocates all runtime resources consumed by this object and awaits
    * actual termination of event loops and other processes associated with
@@ -985,6 +1042,7 @@ export class MiddleLayer {
       this.sharingStateTree.terminate(),
       this.pendingSharesTree.terminate(),
     ]);
+    await this.drainSnapshotWrites(SNAPSHOT_DRAIN_TIMEOUT_MS);
     await this.env.dispose();
     await this.pl.close();
   }
@@ -1092,6 +1150,24 @@ export class MiddleLayer {
 
     const serviceRegistry = createModelServiceRegistry({ logger });
 
+    const treeSnapshots = TreeSnapshotStore.create(pl, {
+      dir: ops.treeSnapshotPath,
+      maxSizeBytes: ops.treeSnapshotOps.maxSizeBytes,
+      enabled: ops.treeSnapshotOps.enabled,
+      logger,
+    });
+    if (ops.treeSnapshotOps.enabled) {
+      // Housekeeping before any project opens: drop snapshots from other builds, backends and
+      // users, then trim to the ceiling.
+      await treeSnapshots?.evict();
+    } else {
+      // Switched off, so reclaim what earlier sessions left on disk. The reason to reach for
+      // this switch is usually the disk itself, and leaving the files behind would answer the
+      // wrong half of that complaint. Keyed on the setting, not on the store being absent: it
+      // is also absent for an impersonated client, whose session must not delete anything.
+      await TreeSnapshotStore.purge(ops.treeSnapshotPath, logger);
+    }
+
     const env: MiddleLayerEnvironment = {
       pl,
       blockEventDispatcher: new BlockEventDispatcher(),
@@ -1112,6 +1188,7 @@ export class MiddleLayer {
       serviceRegistry,
       quickJs,
       projectHelper: new ProjectHelper(quickJs, logger),
+      treeSnapshots,
       dispose: async () => {
         await serviceRegistry.dispose();
         await retryHttpDispatcher.destroy();

lib/node/pl-middle-layer/src/middle_layer/ops.ts

@@ -222,6 +222,40 @@ export type DriverKitOpsConstructor = Omit<
 export type MiddleLayerOpsPaths = DriverKitOpsPaths & {
   /** Common root where to put frontend code. */
   readonly frontendDownloadPath: string;
+
+  /**
+   * Directory holding persisted project tree mirrors, one file per project. Like
+   * {@link DriverKitOpsPaths.parquetCachePath} and unlike the spill directories, it is NOT
+   * emptied on startup: surviving a restart is the entire point. It is pruned instead, see
+   * {@link TreeSnapshotOps.maxSizeBytes}.
+   */
+  readonly treeSnapshotPath: string;
+};
+
+/** Tuning for the persisted project tree mirrors. Their directory is
+ *  {@link MiddleLayerOpsPaths.treeSnapshotPath}; this carries the behaviour knobs. */
+export type TreeSnapshotOps = {
+  /**
+   * Whether project tree mirrors are persisted and restored at all.
+   *
+   * On by default. This is an operational kill switch, for a deployment where the cache
+   * directory turns out to be unwritable or otherwise troublesome, not a rollout gate: the
+   * floor of the feature is current behaviour, since a cache that never hits is a cold open.
+   */
+  readonly enabled: boolean;
+
+  /**
+   * Minimum wall-clock gap between periodic writes for one project.
+   *
+   * Can be generous, because a stale snapshot is less complete rather than wrong: final
+   * resources never change and are never refetched, so this only bounds how much recent work
+   * comes back from the non-final frontier on restore.
+   */
+  readonly writeInterval: number;
+
+  /** Total bytes the snapshot directory may occupy after startup eviction. Needed because a
+   *  heavy project runs to roughly ten megabytes. */
+  readonly maxSizeBytes: number;
 };
 
 /** Debug options for middle layer. */
@@ -253,6 +287,10 @@ export type MiddleLayerOpsSettings = DriverKitOpsSettings & {
    * `sharedAt + envelopeTtlMs`. Share-with-everybody envelopes never expire
    * (`expiresAt: null`) and ignore this. */
   readonly envelopeTtlMs: number;
+
+  /** Settings for persisting project tree mirrors to disk, so reopening a project transfers
+   * what changed rather than the tree again. */
+  readonly treeSnapshotOps: TreeSnapshotOps;
 };
 
 export type MiddleLayerOps = MiddleLayerOpsSettings & MiddleLayerOpsPaths;
@@ -266,6 +304,7 @@ export const DefaultMiddleLayerOpsSettings: Pick<
   | "devBlockUpdateRecheckInterval"
   | "debugOps"
   | "envelopeTtlMs"
+  | "treeSnapshotOps"
 > = {
   ...DefaultDriverKitOpsSettings,
   defaultTreeOptions: {
@@ -279,17 +318,23 @@ export const DefaultMiddleLayerOpsSettings: Pick<
   devBlockUpdateRecheckInterval: 1000,
   projectRefreshInterval: 2000,
   envelopeTtlMs: 14 * 24 * 3600 * 1000, // 14 days
+  treeSnapshotOps: {
+    enabled: true,
+    writeInterval: 5 * 60 * 1000, // 5 minutes
+    maxSizeBytes: 256 * 1024 * 1024, // 256 MB, roughly 25 heavy projects
+  },
 };
 
 export function DefaultMiddleLayerOpsPaths(
   workDir: string,
 ): Pick<
   MiddleLayerOpsPaths,
-  keyof ReturnType<typeof DefaultDriverKitOpsPaths> | "frontendDownloadPath"
+  keyof ReturnType<typeof DefaultDriverKitOpsPaths> | "frontendDownloadPath" | "treeSnapshotPath"
 > {
   return {
     ...DefaultDriverKitOpsPaths(workDir),
     frontendDownloadPath: path.join(workDir, "frontend"),
+    treeSnapshotPath: path.join(workDir, "treeSnapshots"),
   };
 }