diff --git a/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js b/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js
index b48257c6..5df43783 100644
--- a/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js
+++ b/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js
@@ -110,6 +110,10 @@ export async function run({ harness, expect }) {
report.totalCompacted,
(v) => typeof v === 'number' && v > 0
)
+ // @ref LLP 0220#tick-reports-degraded: the walk no longer aborts on a
+ // partition that throws, so a failure it swallowed would otherwise reach
+ // this smoke only as a missing compaction somewhere else in the cache.
+ expect.that('maintenance: no partition failed', report.totalFailed, (v) => v === 0)
// --- 4. Verify post-maintenance state ---
// Source-table layout: data lives under source=unknown (no client columns)
diff --git a/hypaware-core/smoke/flows/incremental_sink_compaction.js b/hypaware-core/smoke/flows/incremental_sink_compaction.js
index 989a7465..b4f1f4a0 100644
--- a/hypaware-core/smoke/flows/incremental_sink_compaction.js
+++ b/hypaware-core/smoke/flows/incremental_sink_compaction.js
@@ -156,6 +156,9 @@ export async function run({ harness, expect }) {
const before = readCursorSync(sourceDir).tableDir ?? 'table'
const maint = await maintainCache({ cacheRoot, force: true, compactOnly: true })
expect.that('compaction: at least one partition compacted', maint.totalCompacted, (v) => typeof v === 'number' && v > 0)
+ // @ref LLP 0220#tick-reports-degraded: a partition that throws no longer
+ // rejects the tick, so the failure has to be read off the report.
+ expect.that('compaction: no partition failed', maint.totalFailed, (v) => v === 0)
const after = readCursorSync(sourceDir).tableDir ?? 'table'
expect.that('compaction: generation directory swapped', [before, after], ([b, a]) => b !== a)
diff --git a/llp/0199-maintenance-compaction-convergence.decision.md b/llp/0199-maintenance-compaction-convergence.decision.md
index 7d576f79..0aa63ab7 100644
--- a/llp/0199-maintenance-compaction-convergence.decision.md
+++ b/llp/0199-maintenance-compaction-convergence.decision.md
@@ -6,7 +6,7 @@
**Author:** Kenny / Claude
**Date:** 2026-08-07
**Related:** LLP 0027
-**Extended-by:** [LLP 0207](./0207-foreign-sorted-replace-convergence.decision.md) (a baseline mismatch whose current snapshot is a sorted `replace` is a foreign rewrite, not growth: recognize it and re-baseline instead of compacting); [LLP 0209](./0209-compaction-file-size.decision.md) (the rewrite the baseline gate protects now sizes its output files by bytes written rather than by the in-memory batch estimate, so a compacted generation actually reaches `target_file_bytes`); [LLP 0217](./0217-compaction-effectiveness-verdict.decision.md) (the cursor records what a rewrite achieved, not only the count it produced, so a partition sitting on its baseline because the rewrite accomplished nothing is skipped explicitly and retried once when the writer changes)
+**Extended-by:** [LLP 0207](./0207-foreign-sorted-replace-convergence.decision.md) (a baseline mismatch whose current snapshot is a sorted `replace` is a foreign rewrite, not growth: recognize it and re-baseline instead of compacting); [LLP 0209](./0209-compaction-file-size.decision.md) (the rewrite the baseline gate protects now sizes its output files by bytes written rather than by the in-memory batch estimate, so a compacted generation actually reaches `target_file_bytes`); [LLP 0217](./0217-compaction-effectiveness-verdict.decision.md) (the cursor records what a rewrite achieved, not only the count it produced, so a partition sitting on its baseline because the rewrite accomplished nothing is skipped explicitly and retried once when the writer changes); [LLP 0220](./0220-maintenance-walk-survives-a-partition.decision.md) (the neediest-first order puts the likeliest partition to fail first, so the walk catches per partition and continues instead of losing every partition behind it)
> Maintenance stops re-flagging already-compacted partitions: a partition is
> only compaction-due when its live data-file count has moved off the count
diff --git a/llp/0218-compaction-failed-attempt-reported.decision.md b/llp/0218-compaction-failed-attempt-reported.decision.md
index 2ada3317..dfc67e02 100644
--- a/llp/0218-compaction-failed-attempt-reported.decision.md
+++ b/llp/0218-compaction-failed-attempt-reported.decision.md
@@ -6,6 +6,7 @@
**Author:** Kenny / Claude
**Date:** 2026-08-13
**Related:** LLP 0217 (the effectiveness verdict and the retry stamp this extends), LLP 0199 (the baseline gate both sit on), LLP 0207 (the recognition path that writes the same stamp)
+**Extended-by:** [LLP 0220](./0220-maintenance-walk-survives-a-partition.decision.md) (the tick in which the attempt fails is reported too: `failed` is this tick's error, beside the `compactionAttemptFailed` skip an earlier one recorded)
> The stamp a failed compaction retry writes records the moment the attempt
> failed, so the ticks that skip the partition afterwards can say why. A
diff --git a/llp/0220-maintenance-walk-survives-a-partition.decision.md b/llp/0220-maintenance-walk-survives-a-partition.decision.md
new file mode 100644
index 00000000..8cd745ad
--- /dev/null
+++ b/llp/0220-maintenance-walk-survives-a-partition.decision.md
@@ -0,0 +1,151 @@
+# LLP 0220: A maintenance walk survives the partition that throws, and says it lost one
+
+**Type:** Decision
+**Status:** Accepted
+**Systems:** Cache
+**Author:** Kenny / Claude
+**Date:** 2026-08-13
+**Related:** LLP 0199 (the neediest-first walk this makes survivable), LLP 0217 (the retry the attempt spends, which must still be spent), LLP 0218 (the skip-reason vocabulary this sits beside), LLP 0021 (the span helper that rethrows)
+
+> A partition whose maintenance throws ends that partition's work, not the
+> tick's. The walk records the failure on that partition's report and moves to
+> the next one. The tick then resolves with a report carrying the failures,
+> because the walk did complete; its callers read the failures off the report
+> and refuse to describe the run as clean.
+
+## Context {#context}
+
+`maintainCache` walks partitions neediest-first (LLP 0199#neediest-first):
+descending live data-file count, so the most fragmented partition is visited
+first. That is deliberate, and it is also why the walk is fragile. The
+neediest partition is the one most likely to fail a rewrite: in the #723 case
+318 MB, 230k rows and 1,521 files, doubling disk while it runs, and one torn
+data file in it is enough to throw.
+
+There was no per-partition catch, and `withSpan` rethrows after recording the
+exception (LLP 0021#span-helpers). So the first partition's error propagated
+out of the loop, out of `maintainCache`, and into
+`src/core/daemon/runtime.js`, which logged `daemon.maintenance_failed` and
+waited an hour to run into the same wall. Every partition behind the failure
+got neither compaction nor snapshot expiry, which is the guard against the
+unbounded metadata growth #723 was filed about in the first place.
+
+LLP 0217#retry-on-writer-change bounded how long this lasts: the failing
+attempt spends its writer generation's retry, so from the next tick the
+partition is skipped rather than re-attempted, and LLP 0218 made that skip a
+stated reason. Both leave the tick in which the failure happens fully lost,
+and that tick is the one that had the neediest partition's whole backlog
+queued behind it.
+
+Reported as #737, deliberately out of scope for the PR that settled the retry
+stamp, because continuing the walk is a behaviour change and not a fix to
+what that PR broke.
+
+## Decision {#decision}
+
+**One partition's error is one
+partition's outcome.** The maintenance loop catches around the
+`maintenance.partition` span, records the failure on that partition's report,
+and continues to the next partition. The catch is *outside* `withSpan` on
+purpose: the helper records the exception and an ERROR status on the partition
+span and then rethrows, so catching outside keeps the trace honest about which
+partition failed while the walk moves on. Catching inside the callback would
+publish an `ok` span for a partition that threw, which is the legibility
+defect one layer down from the one being fixed.
+
+The partition's report object is therefore built before the span opens rather
+than inside it, so a partition that threw part-way still reports what the run
+had established about it: its live data-file count, and any snapshots the
+expiry pass expired before compaction reached the error.
+
+Nothing about the failure path itself moves. LLP 0217's writer-generation
+stamp is written from inside `compactGeneration`'s own catch, on the way out,
+before the error reaches this one, so the attempt still spends the retry and
+a persistently failing partition is still attempted once per writer
+generation rather than once per tick.
+
+**The tick resolves, and reports what it
+lost.** `maintainCache` returns its report instead of rejecting when a
+partition threw, because the walk it was asked to perform completed. The
+report is the honest carrier: `MaintenancePartitionReport` gains `failed`,
+`errorKind` and `errorMessage`, and `MaintenanceReport` gains `totalFailed`.
+
+Rejecting after the walk instead would keep the daemon's existing wiring, but
+it throws the report away, so a tick that maintained forty partitions and lost
+one would print nothing about any of them. The report is the only thing that
+knows what the tick did.
+
+Every caller therefore reads the failures rather than a rejected promise. The
+daemon logs one `daemon.maintenance_failed` per failed partition, naming the
+dataset and partition (which the propagated exception never could), and marks
+the `maintenance.tick` span with `status: degraded` and `partitions_failed`
+as attributes *and* an `ERROR` span status code - set directly once the
+report is in hand, since `withSpan`'s status-attribute convention only reads
+that attribute's snapshot from before the callback runs and cannot see a
+verdict this late (`src/core/daemon/runtime.js`'s tick manages its span
+directly for this reason, rather than through `withSpan`). Its
+outer `.catch` stays, for the errors still outside the per-partition catch:
+partition discovery and the retired-generation sweep. `hyp query maintain`
+prints a `FAILED:` line for the partition and exits non-zero, so a script that
+gated on the old exit status does not start reading a degraded run as a clean
+one.
+
+**`failed` is not
+`compactionAttemptFailed`.** LLP 0218's field is read off the cursor and means
+*an earlier tick's attempt failed and nothing has been attempted since*; the
+partition was skipped, deliberately, for a stated reason. `failed` means *this
+tick attempted work here and it threw*. They never appear together, because a
+tick that attempted a rewrite is not a tick that skipped one, and an operator
+has to be able to tell a partition that just broke from one that has been
+quietly frozen for a week. Keeping one field for both would put the two
+readings back into the same absence the LLP 0218 report exists to remove.
+
+## Consequences {#consequences}
+
+- A cache holding one broken partition no longer has its whole walk starved
+ by it: the walk moves on to the next partition instead of aborting. The
+ per-tick budget guard (LLP 0199#neediest-first) still applies, and is keyed
+ off partitions actually *maintained*, not partitions merely visited - a
+ partition that failed did no work, so it cannot itself satisfy the guard's
+ "always work one partition before the budget can cut the tick short"
+ guarantee and stall everything behind it (`src/core/cache/maintenance.js`).
+ What this buys is "the walk keeps moving past a failure", not "every
+ healthy partition is guaranteed maintenance on the tick a partition
+ breaks": a tight budget can still cut a tick short once real work has
+ happened, exactly as it could before this document. The one case that
+ guard alone cannot bound - every partition in the cache failing - is capped
+ separately: the walk still breaks once `MAX_FAILURES_BEFORE_BUDGET_BREAK`
+ partitions have failed past the budget, so an all-failing cache cannot walk
+ unbounded either. The guard against unbounded metadata growth (snapshot
+ expiry, the orphan sweep) stops being hostage to the neediest partition's
+ health specifically; it is not a promise that one failure never costs any
+ other partition its turn in a budget-constrained tick.
+- `cleanRetiredEpochs` now runs on a tick that lost a partition, where the
+ abort used to skip it, so the half-written generation a failed rewrite left
+ behind is reclaimed by the orphan sweep on schedule instead of waiting for a
+ clean tick.
+- The budget still applies. A partition that fails fast costs its share of
+ `max_tick_ms` and no more, and the walk continues into the neediest healthy
+ partition rather than postponing the whole ranking by an hour.
+- `daemon.maintenance_failed` fires once per failed partition rather than once
+ per tick, and carries the partition identity. A cache with several broken
+ partitions logs several lines, which is the correct count of things that are
+ wrong.
+- Tests that asserted the whole tick rejected now assert the partition's
+ report instead. The thing they pin (the stamp, the committed cursor, the
+ recorded verdict) is unchanged; only how the failure is observed moved.
+- A failure that ought to abort the whole walk has nowhere to say so. Nothing
+ in the current maintenance path is such a failure (the shared state a
+ partition touches is its own directory and its own cursor), but a future step
+ that corrupts something cache-wide would need a way to stop the walk rather
+ than being reported as one partition's error.
+
+## Extends {#extends}
+
+LLP 0199 settled the walk order and the budget that cuts it short, and both
+stand: neediest-first is still the order, and a partition that fails is still
+visited in it. What this document adds is that the order's own consequence -
+the likeliest partition to fail goes first - no longer decides whether the
+rest of the walk happens. LLP 0218 settled that a partition maintenance leaves
+alone is left alone for a stated reason; this extends that vocabulary to the
+partition maintenance tried and could not finish.
diff --git a/src/core/cache/maintenance.js b/src/core/cache/maintenance.js
index 06c2e6b0..b8cfe0d3 100644
--- a/src/core/cache/maintenance.js
+++ b/src/core/cache/maintenance.js
@@ -57,6 +57,25 @@ const DEFAULTS = {
const GRACE_PERIOD_MS = 24 * 60 * 60 * 1000
+/**
+ * How many failed partitions the budget guard tolerates before it breaks
+ * the walk even though nothing has been maintained yet.
+ *
+ * The guard's job is "always work one partition before the budget can cut
+ * the tick short" (see below), and a partition that threw did no work, so
+ * it must not be the one that satisfies that guarantee - a first partition
+ * that fails slowly would otherwise starve every partition behind it, on
+ * every tick, which is #737 by another route. But gating the break on
+ * "maintained > 0" alone means a cache where every partition fails walks
+ * past `budgetMs` without bound, which trades one unbounded-tick risk for
+ * another. Four is deliberately small: it is bigger than the "one bad
+ * partition" case this whole PR is about, small enough that a tick spent
+ * entirely on failures is still bounded to roughly four partitions' worth
+ * of failure latency rather than the whole cache, and it does not require
+ * reading a config value that does not otherwise exist for this guard.
+ */
+export const MAX_FAILURES_BEFORE_BUDGET_BREAK = 4
+
/**
* How long an unreferenced (cursor-orphaned) table generation must sit
* untouched before the orphan sweep reclaims it. Long enough that an
@@ -143,58 +162,91 @@ export async function maintainCache(opts) {
let totalSnapshotsExpired = 0
let totalCompacted = 0
let totalRebaselined = 0
+ let totalFailed = 0
+ // A partition that threw did no work, so `reports.length > 0` cannot be
+ // what proves the budget guard's "always work one partition" guarantee -
+ // see MAX_FAILURES_BEFORE_BUDGET_BREAK.
+ let maintained = 0
for (const part of partitions) {
// Always work one partition before the budget can cut the tick short:
// the ranking pass above is itself unbudgeted, so on a large enough
// cache a bare cutoff here would break at iteration 0 every tick and
- // maintenance would never run at all.
- if (reports.length > 0 && Date.now() - startMs > budgetMs) break
-
- const report = await withSpan(
- 'maintenance.partition',
- {
- [Attr.COMPONENT]: 'cache',
- [Attr.OPERATION]: 'maintenance.partition',
- [Attr.DATASET]: part.dataset,
- partition: JSON.stringify(part.partition),
- status: 'ok',
- },
- async (span) => {
- /** @type {MaintenancePartitionReport} */
- const r = {
- dataset: part.dataset,
- partition: part.partition,
- path: part.path,
- snapshotsExpired: 0,
- compacted: false,
- rowCount: part.rowCount,
- dataFilesBefore: 0,
- dataFilesAfter: 0,
- }
+ // maintenance would never run at all. Gated on `maintained`, not on
+ // `reports.length`, so a first partition that fails (possibly slowly)
+ // cannot itself satisfy the guarantee and starve the walk behind it.
+ // The failure-count disjunct is the bounded escape hatch for the
+ // opposite case, a cache where every partition fails.
+ if (
+ Date.now() - startMs > budgetMs &&
+ (maintained > 0 || totalFailed >= MAX_FAILURES_BEFORE_BUDGET_BREAK)
+ ) break
+
+ // Built out here rather than inside the span callback so the catch
+ // below still has it: a partition that threw part-way keeps whatever
+ // the run had already established about it (its live file count, any
+ // snapshots expired before compaction reached the error) instead of
+ // being reported as a bare failure with zeroed counts.
+ /** @type {MaintenancePartitionReport} */
+ const report = {
+ dataset: part.dataset,
+ partition: part.partition,
+ path: part.path,
+ snapshotsExpired: 0,
+ compacted: false,
+ rowCount: part.rowCount,
+ dataFilesBefore: 0,
+ dataFilesAfter: 0,
+ }
- const cursor = readCursorSync(part.path)
- const settle = resolveSettleContext(opts, part.dataset)
-
- const done = await maintainGeneration(
- r, cursor, cfg, opts, settle, snapshotsExpiredCounter, compactionsCounter, rebaselinesCounter
- )
- // A compaction that "converged" is only healthy if it also shrank
- // the file count; publish both sides of that so a run that rewrites
- // a partition into the same fragmentation is visible in the trace
- // rather than only in a later disk audit.
- span.setAttribute('compacted', done.compacted)
- span.setAttribute('data_files_before', done.dataFilesBefore)
- span.setAttribute('data_files_after', done.dataFilesAfter)
- span.setAttribute('rows', done.rowCount)
- if (done.compactedBytesWritten !== undefined) {
- span.setAttribute('bytes_written', done.compactedBytesWritten)
- }
- return done
- },
- { component: 'cache' }
- )
+ try {
+ await withSpan(
+ 'maintenance.partition',
+ {
+ [Attr.COMPONENT]: 'cache',
+ [Attr.OPERATION]: 'maintenance.partition',
+ [Attr.DATASET]: part.dataset,
+ partition: JSON.stringify(part.partition),
+ status: 'ok',
+ },
+ async (span) => {
+ const cursor = readCursorSync(part.path)
+ const settle = resolveSettleContext(opts, part.dataset)
+
+ const done = await maintainGeneration(
+ report, cursor, cfg, opts, settle, snapshotsExpiredCounter, compactionsCounter, rebaselinesCounter
+ )
+ // A compaction that "converged" is only healthy if it also shrank
+ // the file count; publish both sides of that so a run that rewrites
+ // a partition into the same fragmentation is visible in the trace
+ // rather than only in a later disk audit.
+ span.setAttribute('compacted', done.compacted)
+ span.setAttribute('data_files_before', done.dataFilesBefore)
+ span.setAttribute('data_files_after', done.dataFilesAfter)
+ span.setAttribute('rows', done.rowCount)
+ if (done.compactedBytesWritten !== undefined) {
+ span.setAttribute('bytes_written', done.compactedBytesWritten)
+ }
+ },
+ { component: 'cache' }
+ )
+ } catch (err) {
+ // @ref LLP 0220#walk-survives-a-partition [implements]: one partition's
+ // error ends that partition's work, not the tick's. Outside `withSpan`
+ // on purpose: the helper rethrows after recording the exception and an
+ // ERROR status on `maintenance.partition`, so catching here keeps the
+ // span honest about the partition while the walk moves on. Catching
+ // inside the callback would hand every reader of the trace an `ok`
+ // span for a partition that failed. Everything the failure path itself
+ // has to do - LLP 0217's writer-generation stamp above all - already
+ // ran inside `maintainGeneration`, on the way out.
+ report.failed = true
+ report.errorKind = errorKindOf(err)
+ report.errorMessage = err instanceof Error ? err.message : String(err)
+ totalFailed++
+ }
reports.push(report)
+ if (!report.failed) maintained++
totalSnapshotsExpired += report.snapshotsExpired
if (report.compacted) totalCompacted++
if (report.rebaselined) totalRebaselined++
@@ -209,11 +261,29 @@ export async function maintainCache(opts) {
totalSnapshotsExpired,
totalCompacted,
totalRebaselined,
+ totalFailed,
dryRun: opts.dryRun ?? false,
elapsedMs: Date.now() - startMs,
}
}
+/**
+ * The error's own kind when it carries one (the convention sink
+ * materialization already uses for the same per-unit catch), else a kind
+ * naming where it was caught. Never the exception's class: what an
+ * operator needs off a maintenance report is which step of the tick gave
+ * up, and the message beside it carries the rest.
+ *
+ * @param {unknown} err
+ * @returns {string}
+ */
+function errorKindOf(err) {
+ if (err && typeof err === 'object' && 'errorKind' in err) {
+ return String(/** @type {{ errorKind: unknown }} */ (err).errorKind)
+ }
+ return 'maintenance_partition_failed'
+}
+
/**
* The five spots where the source-table and legacy epoch layouts differ.
* Everything else in maintenance, compaction, and status is
@@ -390,15 +460,25 @@ async function maintainGeneration(r, cursor, cfg, opts, settle, snapshotsExpired
// re-settle force: a leftover unmatchable fallback row must not undo
// the sorted layout every night. An explicit --force still rewrites.
if (!opts.force && foreignSortedReplace(tableInfo)) {
- r.rebaselined = true
// The counter proves a rebaseline happened at all, but it carries
// only the dataset; tagging the enclosing maintenance.partition span
// names the partition, so a trace query finds which day re-baselined
- // without cross-referencing the counter.
+ // without cross-referencing the counter. Left here rather than moved
+ // beside `r.rebaselined` below: the span carries ERROR anyway if the
+ // write that follows throws, so the intent is still worth recording.
getActiveSpan()?.setAttribute('rebaselined', true)
if (!opts.dryRun) {
await writeCursor(r.path, rebaselineCursor(cursor, dataFilesBefore))
+ // Set only once the cursor write that persists it has succeeded:
+ // a throw here must not leave the report (and `totalRebaselined`)
+ // claiming a rebaseline that never landed on disk.
+ r.rebaselined = true
rebaselinesCounter.add(1, { [Attr.DATASET]: r.dataset })
+ } else {
+ // No write happens in dry-run, so nothing to wait on: this is
+ // the preview of what a real run would do, same as `r.compacted`
+ // below for the ordinary rewrite path.
+ r.rebaselined = true
}
} else if (opts.dryRun) {
r.compacted = true
diff --git a/src/core/cache/types.d.ts b/src/core/cache/types.d.ts
index 8158846a..22842b0b 100644
--- a/src/core/cache/types.d.ts
+++ b/src/core/cache/types.d.ts
@@ -315,6 +315,17 @@ export interface MaintenancePartitionReport {
// When that attempt failed, as the cursor records it. Set whenever
// `compactionAttemptFailed` is.
compactionAttemptFailedAt?: string
+ // THIS tick's work on the partition ended in an error and the walk moved
+ // on (LLP 0220). Distinct from `compactionAttemptFailed`, which is read
+ // off the cursor and says an EARLIER tick's attempt failed and nothing
+ // has been attempted since: the two never appear together, because a
+ // tick that attempted a rewrite is not a tick that skipped one.
+ failed?: boolean
+ // The error's own `errorKind` when it carries one, else
+ // `maintenance_partition_failed`. Set whenever `failed` is.
+ errorKind?: string
+ // The error's message. Set whenever `failed` is.
+ errorMessage?: string
}
export interface MaintenanceReport {
@@ -322,6 +333,11 @@ export interface MaintenanceReport {
totalSnapshotsExpired: number
totalCompacted: number
totalRebaselined: number
+ // Partitions whose work threw during this tick. Non-zero means the walk
+ // completed but the tick is degraded: it did not maintain everything it
+ // set out to, and its callers report that rather than a clean run
+ // (LLP 0220#tick-reports-degraded).
+ totalFailed: number
dryRun: boolean
elapsedMs: number
}
diff --git a/src/core/commands/query.js b/src/core/commands/query.js
index 0f173eae..32c6fb4e 100644
--- a/src/core/commands/query.js
+++ b/src/core/commands/query.js
@@ -342,13 +342,33 @@ export async function runQueryMaintain(argv, ctx) {
if (p.compactionAttemptFailed) {
actions.push(`compaction skipped: the retry failed under this writer at ${p.compactionAttemptFailedAt}, --force to retry`)
}
+ // @ref LLP 0220#tick-reports-degraded: this tick's own failure, as
+ // opposed to the line above, which reports one an earlier tick already
+ // recorded on the cursor. The walk continued past this partition, so
+ // without the line the run reads as a clean one that happened to do
+ // less work.
+ if (p.failed) {
+ actions.push(`FAILED: ${p.errorMessage} (${p.errorKind}); the walk continued`)
+ }
if (actions.length > 0) {
ctx.stdout.write(` ${label}: ${actions.join(', ')}\n`)
}
}
const rebaselineNote = report.totalRebaselined > 0 ? `, ${report.totalRebaselined} rebaselined` : ''
- ctx.stdout.write(`maintenance: ${report.totalSnapshotsExpired} snapshots expired, ${report.totalCompacted} partitions compacted${rebaselineNote} (${report.elapsedMs}ms)\n`)
- return 0
+ const failedNote = report.totalFailed > 0 ? `, ${report.totalFailed} partitions failed` : ''
+ ctx.stdout.write(`maintenance: ${report.totalSnapshotsExpired} snapshots expired, ${report.totalCompacted} partitions compacted${rebaselineNote}${failedNote} (${report.elapsedMs}ms)\n`)
+ // Before this PR the exception propagated to bin/hypaware.js, which wrote
+ // `hyp: ` to stderr - so a caller that only captures stderr (a
+ // cron or systemd wrapper, `>/dev/null`) still needs a line there on a
+ // degraded tick, not just the stdout summary above.
+ if (report.totalFailed > 0) {
+ ctx.stderr.write(`hyp query maintain: ${report.totalFailed} partition(s) failed; the walk continued\n`)
+ }
+ // A partition that threw used to abort the walk and exit non-zero. The
+ // walk survives it now, but the tick did not do what it was asked, so the
+ // exit status still says so: a script that gated on this must not start
+ // reading a degraded run as a clean one.
+ return report.totalFailed > 0 ? 1 : 0
}
const QUERY_MAINTAIN_USAGE = 'usage: hyp query maintain [dataset] [--dry-run] [--force] [--compact-only] [--expire-only]'
diff --git a/src/core/daemon/runtime.js b/src/core/daemon/runtime.js
index 7ba0e756..f82aae1b 100644
--- a/src/core/daemon/runtime.js
+++ b/src/core/daemon/runtime.js
@@ -4,10 +4,13 @@ import process from 'node:process'
import {
Attr,
+ buildAttrs,
getKernelInstruments,
getLogger,
+ getTracer,
installObservability,
runRoot,
+ SpanStatusCode,
withSpan,
} from '../observability/index.js'
import { readObservabilityEnv } from '../observability/env.js'
@@ -659,17 +662,28 @@ export async function runDaemon(opts = {}) {
const { maintainCache, normalizeMaintenanceConfig } = await import('../cache/maintenance.js')
const mCfg = normalizeMaintenanceConfig(maintenanceCfg)
const intervalMs = mCfg.interval_minutes * 60 * 1000
+ // @ref LLP 0220#tick-reports-degraded [constrained-by]: not built on
+ // `withSpan`. `withSpan` (src/core/observability/span_helpers.js)
+ // derives the span's status code from the `status` attribute snapshot
+ // taken at span-creation time, and sets it from that snapshot after the
+ // callback resolves - so a status only known once the tick's report is
+ // in hand (clean vs. degraded) cannot be conveyed by setting the
+ // attribute inside the callback, the way every other `withSpan` caller
+ // does: the post-hoc `setStatus(OK)` clobbers it. Managed inline here
+ // with the tracer directly instead, so only this one call site's status
+ // handling changes and `withSpan` (and everything else that calls it)
+ // is untouched.
async function runMaintenance() {
- await withSpan(
- 'maintenance.tick',
- {
- [Attr.COMPONENT]: 'daemon',
- [Attr.OPERATION]: 'maintenance.tick',
- daemon_mode: mode,
- status: 'ok',
- },
- async () => {
- await maintainCache({
+ const tracer = getTracer('daemon')
+ const attrs = buildAttrs({
+ [Attr.COMPONENT]: 'daemon',
+ [Attr.OPERATION]: 'maintenance.tick',
+ daemon_mode: mode,
+ status: 'ok',
+ })
+ await tracer.startActiveSpan('maintenance.tick', { attributes: attrs }, async (span) => {
+ try {
+ const report = await maintainCache({
cacheRoot: boot.runtime.storage.cacheRoot,
budgetMs: mCfg.max_tick_ms,
config: mCfg,
@@ -680,9 +694,46 @@ export async function runDaemon(opts = {}) {
storage: boot.runtime.storage,
getSettleHook: (dataset) => boot.runtime.query.getDataset(dataset)?.resettleBatch,
})
- },
- { component: 'daemon' }
- ).catch((err) => {
+ // @ref LLP 0220#tick-reports-degraded [implements]: the walk now
+ // survives a partition that throws, so the rejected promise has
+ // stopped being how the daemon hears about one. Read the failures
+ // off the report instead, or a tick that lost its neediest
+ // partition would log exactly as a clean one does. The line names
+ // the partitions, which the propagated exception never could.
+ for (const p of report.partitions) {
+ if (!p.failed) continue
+ fileLog.error('daemon.maintenance_failed', {
+ [Attr.DATASET]: p.dataset,
+ partition: JSON.stringify(p.partition),
+ [Attr.ERROR_KIND]: p.errorKind,
+ message: p.errorMessage,
+ })
+ }
+ const degraded = report.totalFailed > 0
+ if (degraded) {
+ span.setAttribute('status', 'degraded')
+ span.setAttribute('partitions_failed', report.totalFailed)
+ }
+ span.setAttribute('partitions_maintained', report.partitions.length - report.totalFailed)
+ // The status code itself, not just the attribute: only knowable
+ // now that the report is in hand, so set directly rather than
+ // through the status-attribute snapshot `withSpan` would
+ // otherwise have read before the callback ran.
+ span.setStatus(degraded
+ ? { code: SpanStatusCode.ERROR, message: `${report.totalFailed} partition(s) failed` }
+ : { code: SpanStatusCode.OK })
+ } catch (error) {
+ const err = error instanceof Error ? error : new Error(String(error))
+ span.recordException(err)
+ span.setStatus({ code: SpanStatusCode.ERROR, message: err.message })
+ span.setAttribute('error_kind', attrs.error_kind ?? 'unhandled_exception')
+ throw err
+ } finally {
+ span.end()
+ }
+ }).catch((err) => {
+ // Still reachable: partition discovery, the retired-generation
+ // sweep, and anything else outside the per-partition catch.
const message = err instanceof Error ? err.message : String(err)
fileLog.error('daemon.maintenance_failed', { message })
})
diff --git a/test/core/cache-compaction-effectiveness.test.js b/test/core/cache-compaction-effectiveness.test.js
index 5a4b6b66..8ac07ea6 100644
--- a/test/core/cache-compaction-effectiveness.test.js
+++ b/test/core/cache-compaction-effectiveness.test.js
@@ -294,8 +294,12 @@ test('a retry whose rewrite throws still spends its writer generation', async ()
const [torn] = await liveDataFiles(dir)
await fs.truncate(torn, 4)
- await assert.rejects(
- maintainCache({ cacheRoot, compactOnly: true }),
+ // The walk survives the failure and reports it per partition
+ // (LLP 0220), so the fixture invariant is read off the report rather
+ // than off a rejected tick.
+ const attempt = await maintainCache({ cacheRoot, compactOnly: true })
+ assert.equal(
+ attempt.partitions[0].failed, true,
'fixture invariant: the retry must attempt a rewrite, and that rewrite must fail'
)
@@ -343,8 +347,9 @@ test('a partition frozen by a failed retry is reported as skipped on every later
const intact = await fs.readFile(torn)
await fs.truncate(torn, 4)
- await assert.rejects(
- maintainCache({ cacheRoot, compactOnly: true }),
+ const attempt = await maintainCache({ cacheRoot, compactOnly: true })
+ assert.equal(
+ attempt.partitions[0].failed, true,
'fixture invariant: the retry must attempt a rewrite, and that rewrite must fail'
)
const record = compactionRecord(dir)
@@ -409,7 +414,8 @@ test('a committed ineffective verdict outranks the failed attempt that followed
const retiringDir = path.join(dir, retiring)
await fs.chmod(retiringDir, 0o555)
try {
- await assert.rejects(maintainCache({ cacheRoot, compactOnly: true }))
+ const attempt = await maintainCache({ cacheRoot, compactOnly: true })
+ assert.equal(attempt.partitions[0].failed, true, 'fixture invariant: the tick must fail on this partition')
} finally {
await fs.chmod(retiringDir, 0o755)
}
@@ -461,8 +467,9 @@ test('a rewrite that throws after committing its cursor keeps the generation it
const retiringDir = path.join(dir, retiring)
await fs.chmod(retiringDir, 0o555)
try {
- await assert.rejects(
- maintainCache({ cacheRoot, compactOnly: true }),
+ const attempt = await maintainCache({ cacheRoot, compactOnly: true })
+ assert.equal(
+ attempt.partitions[0].failed, true,
'fixture invariant: the rewrite must commit its cursor and then fail'
)
diff --git a/test/core/cache-maintenance-walk-resilience.test.js b/test/core/cache-maintenance-walk-resilience.test.js
new file mode 100644
index 00000000..bf8f1d9a
--- /dev/null
+++ b/test/core/cache-maintenance-walk-resilience.test.js
@@ -0,0 +1,334 @@
+// @ts-check
+
+// The maintenance walk visits partitions neediest-first (LLP 0199), so the
+// most fragmented partition goes first - and that is exactly the partition
+// most likely to fail a rewrite. Without a per-partition catch one such
+// partition took the whole tick with it: every partition behind it got
+// neither compaction nor snapshot expiry, every hour, forever. These tests
+// pin the walk surviving one partition's error while still reporting it.
+
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+
+import { MAX_FAILURES_BEFORE_BUDGET_BREAK, maintainCache } from '../../src/core/cache/maintenance.js'
+import { appendRowsToSourceTable, readCursorSync, writeCursor } from '../../src/core/cache/partition.js'
+
+/**
+ * @import { ColumnSpec } from '../../hypaware-plugin-kernel-types.js'
+ * @import { MaintenancePartitionReport, PartitionCursor } from '../../src/core/cache/types.js'
+ */
+
+/** @type {ColumnSpec[]} */
+const SESSION_COLUMNS = [
+ { name: 'id', type: 'INT32', nullable: false },
+ { name: 'session_id', type: 'STRING', nullable: false },
+ { name: 'attributes', type: 'STRING', nullable: true },
+]
+
+/** @type {ColumnSpec[]} */
+const FLAT_COLUMNS = [
+ { name: 'id', type: 'INT32', nullable: false },
+ { name: 'attributes', type: 'STRING', nullable: true },
+]
+
+/**
+ * One Iceberg partition tuple per session, so ingest lands one data file
+ * per session and the partition is both the neediest in the cache and the
+ * one a torn file can break. Same fixture shape as the effectiveness
+ * tests: this is the production partition from #723.
+ */
+const SESSION_DECLARATION = {
+ source: { columns: ['source'] },
+ iceberg: { fields: [{ column: 'session_id', transform: 'identity' }] },
+}
+
+/**
+ * @param {string} cacheRoot
+ * @param {number} sessions
+ */
+async function seedNeediestPartition(cacheRoot, sessions) {
+ const rows = Array.from({ length: sessions }, (_, i) => ({
+ id: i,
+ session_id: `s-${i}`,
+ attributes: `{"gateway":{"session":"s-${i}"}}`,
+ }))
+ await appendRowsToSourceTable(
+ cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows,
+ { declaration: SESSION_DECLARATION }
+ )
+}
+
+/**
+ * A second, healthy partition holding strictly fewer live data files, so
+ * the neediest-first ranking always puts it behind the one that throws.
+ *
+ * @param {string} cacheRoot
+ * @param {number} waves
+ */
+async function seedHealthyPartition(cacheRoot, waves) {
+ for (let wave = 0; wave < waves; wave++) {
+ const rows = Array.from({ length: 5 }, (_, i) => ({
+ id: wave * 5 + i,
+ attributes: `wave ${wave} row ${i}`,
+ }))
+ await appendRowsToSourceTable(cacheRoot, 'logs', ['source=claude'], FLAT_COLUMNS, rows)
+ }
+}
+
+/** @param {string} cacheRoot @param {string} dataset @returns {string} */
+function partitionDir(cacheRoot, dataset) {
+ return path.join(cacheRoot, 'datasets', dataset, 'source=claude')
+}
+
+/**
+ * Plant the stamp-less compaction record an older HypAware wrote: its
+ * baseline sits on the live count, so the LLP 0199 gate skips the
+ * partition, and it names no writer generation, so LLP 0217 owes it
+ * exactly one retry under the writer running now. That retry is what the
+ * torn file below turns into a throw.
+ *
+ * @param {string} dir
+ * @param {number} baselineFiles
+ * @returns {Promise}
+ */
+async function plantStamplessRecord(dir, baselineFiles) {
+ const cursor = readCursorSync(dir)
+ /** @type {PartitionCursor} */
+ const next = {
+ ...cursor,
+ compaction: {
+ previousTableDir: 'table',
+ compactedAt: '2026-08-12T21:55:35.168Z',
+ resettleBaselineFiles: baselineFiles,
+ },
+ }
+ await writeCursor(dir, next)
+}
+
+/**
+ * Truncate one of the partition's live parquet files to a stub no reader
+ * can decode: the torn-write stand-in from the issue, which makes the
+ * rewrite's scan throw part-way through.
+ *
+ * @param {string} dir
+ * @returns {Promise}
+ */
+async function tearOneDataFile(dir) {
+ const cursor = readCursorSync(dir)
+ const dataDir = path.join(dir, cursor.tableDir ?? 'table', 'data')
+ const entries = await fs.readdir(dataDir, { withFileTypes: true })
+ const [torn] = entries
+ .filter((e) => e.isFile() && e.name.endsWith('.parquet'))
+ .map((e) => path.join(dataDir, e.name))
+ assert.ok(torn, 'fixture invariant: the partition must hold a live data file to tear')
+ await fs.truncate(torn, 4)
+}
+
+/** @param {string} dir @returns {Record} */
+function compactionRecord(dir) {
+ const { compaction } = readCursorSync(dir)
+ assert.ok(compaction && typeof compaction === 'object', 'expected a compaction record on the cursor')
+ return /** @type {Record} */ (compaction)
+}
+
+/**
+ * @param {{ partitions: MaintenancePartitionReport[] }} report
+ * @param {string} dataset
+ * @returns {MaintenancePartitionReport}
+ */
+function partitionReport(report, dataset) {
+ const found = report.partitions.find((p) => p.dataset === dataset)
+ assert.ok(found, `expected a report for ${dataset}; got ${report.partitions.map((p) => p.dataset).join(', ') || '(none)'}`)
+ return found
+}
+
+/**
+ * The cache both tests below run over: a neediest partition rigged to
+ * throw on its rewrite, and a healthy partition behind it in walk order.
+ *
+ * @returns {Promise}
+ */
+async function seedTornAndHealthy() {
+ const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-walk-'))
+ await seedNeediestPartition(cacheRoot, 8)
+ await seedHealthyPartition(cacheRoot, 3)
+ const torn = partitionDir(cacheRoot, 'ai_gateway_messages')
+ await plantStamplessRecord(torn, 8)
+ await tearOneDataFile(torn)
+ return cacheRoot
+}
+
+// @ref LLP 0220#walk-survives-a-partition [tests]: the whole point. The
+// neediest partition throws, and the partition behind it is still maintained
+// in the same tick rather than an hour later, or never.
+test('a partition whose compaction throws does not abort the rest of the walk', async () => {
+ const cacheRoot = await seedTornAndHealthy()
+ try {
+ const report = await maintainCache({ cacheRoot, compactOnly: true })
+
+ const torn = partitionReport(report, 'ai_gateway_messages')
+ assert.equal(torn.compacted, false, 'fixture invariant: the neediest partition must attempt a rewrite and fail it')
+ assert.equal(torn.failed, true, 'the partition that threw is reported as failed')
+
+ const healthy = partitionReport(report, 'logs')
+ assert.equal(
+ healthy.compacted, true,
+ 'the partition behind the failure must still be compacted in this tick'
+ )
+ assert.equal(healthy.failed, undefined, 'the healthy partition did not fail')
+ assert.equal(report.totalCompacted, 1)
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
+// @ref LLP 0217#retry-on-writer-change [tests]: the per-partition catch must
+// not swallow the failure before the stamp is written. The attempt spends the
+// writer generation's retry whether or not it succeeded, or the same partition
+// is re-attempted, and fails, on every tick forever.
+test('the partition that threw still spends its writer generation', async () => {
+ const cacheRoot = await seedTornAndHealthy()
+ try {
+ await maintainCache({ cacheRoot, compactOnly: true })
+
+ const dir = partitionDir(cacheRoot, 'ai_gateway_messages')
+ const record = compactionRecord(dir)
+ assert.equal(typeof record.writerGeneration, 'number', 'a spent attempt must stamp the cursor')
+ assert.equal(typeof record.attemptFailedAt, 'string', 'the stamp records when the attempt failed')
+ assert.equal(record.resettleBaselineFiles, 8, 'a failed rewrite must not move the baseline')
+ assert.equal(record.dataFilesBefore, undefined, 'a failed rewrite proves nothing about effectiveness')
+
+ // @ref LLP 0218#report-the-spent-attempt [tests]: and from the next tick
+ // on the partition is skipped for that stated reason, with the walk now
+ // reporting no failure at all because nothing was attempted.
+ const next = await maintainCache({ cacheRoot, compactOnly: true })
+ const torn = partitionReport(next, 'ai_gateway_messages')
+ assert.equal(torn.failed, undefined, 'nothing was attempted this tick, so nothing failed in it')
+ assert.equal(torn.compactionAttemptFailed, true, 'the skip states the spent attempt as its reason')
+ assert.equal(torn.compactionAttemptFailedAt, record.attemptFailedAt)
+ assert.equal(next.totalFailed, 0)
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
+// @ref LLP 0220#walk-survives-a-partition [tests]: compaction is not the only
+// thing a partition behind the failure was owed. Snapshot expiry is the
+// unbounded-growth guard (#723's metadata directories), and it runs before
+// compaction for every partition in the walk, so aborting the walk cost it too.
+test('snapshot expiry still runs for the partitions behind the one that threw', async () => {
+ const cacheRoot = await seedTornAndHealthy()
+ try {
+ // Expire everything but the current snapshot, so the healthy partition's
+ // three ingest waves leave something to expire.
+ const report = await maintainCache({
+ cacheRoot,
+ config: { min_snapshots_to_keep: 0, max_snapshot_age_hours: 0 },
+ })
+
+ const healthy = partitionReport(report, 'logs')
+ assert.ok(
+ healthy.snapshotsExpired > 0,
+ `the partition behind the failure must still have its snapshots expired; got ${healthy.snapshotsExpired}`
+ )
+ assert.ok(report.totalSnapshotsExpired > 0)
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
+// @ref LLP 0220#tick-reports-degraded [tests]: continuing the walk must not
+// cost the tick its failure signal. A tick that quietly returns a clean report
+// while a partition went unmaintained is the same defect class as #723: the
+// operator has nothing to read.
+test('a tick that lost a partition reports the loss rather than swallowing it', async () => {
+ const cacheRoot = await seedTornAndHealthy()
+ try {
+ const report = await maintainCache({ cacheRoot, compactOnly: true })
+
+ assert.equal(report.totalFailed, 1, 'the tick counts the partition it lost')
+ const torn = partitionReport(report, 'ai_gateway_messages')
+ assert.equal(torn.failed, true)
+ assert.equal(typeof torn.errorKind, 'string')
+ assert.ok(
+ typeof torn.errorMessage === 'string' && torn.errorMessage.length > 0,
+ 'the report carries the error that ended the partition, not just the fact of one'
+ )
+ // The failure is this tick's, and it is not the cursor-read skip reason
+ // LLP 0218 defined: an operator must be able to tell "this partition threw
+ // just now" from "this partition has been skipped since an earlier throw".
+ assert.equal(torn.compactionAttemptFailed, undefined)
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
+// --- the budget guard must not let a failed partition stand in for real work (round-1 review finding 1) ---
+
+// @ref LLP 0220#walk-survives-a-partition [tests]: a partition that failed
+// did no work, so it must not be the one that satisfies the budget guard's
+// "always work one partition before the budget can cut the tick short"
+// guarantee (src/core/cache/maintenance.js). Before this fix, `reports.length
+// > 0` was true the moment the torn partition's failure was recorded, so a
+// zero budget broke the walk right there and the healthy partition behind it
+// was never visited - #737 by another route, reached without any per-partition
+// catch missing at all.
+test('a zero budget does not let a failed first partition starve the healthy partition behind it', async () => {
+ const cacheRoot = await seedTornAndHealthy()
+ try {
+ const report = await maintainCache({ cacheRoot, compactOnly: true, budgetMs: 0 })
+
+ const torn = partitionReport(report, 'ai_gateway_messages')
+ assert.equal(torn.failed, true, 'fixture invariant: the neediest partition still throws')
+
+ const healthy = partitionReport(report, 'logs')
+ assert.equal(
+ healthy.compacted, true,
+ 'a zero budget must not let the failed first partition stand in for the "one partition maintained" guarantee'
+ )
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
+// @ref LLP 0220#walk-survives-a-partition [tests]: the other half of the
+// budget-guard fix. Gating the break on "a partition was actually
+// maintained" alone would let a cache where every partition fails walk past
+// its budget without bound - trading one starvation bug for an unbounded
+// one. `MAX_FAILURES_BEFORE_BUDGET_BREAK` caps it; pinned here with more
+// failing partitions seeded than the cap allows through.
+test('an all-failing cache does not walk past its budget unbounded: it stops at the failure cap', async () => {
+ const total = MAX_FAILURES_BEFORE_BUDGET_BREAK + 2
+ const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-all-failing-'))
+ try {
+ for (let i = 0; i < total; i++) {
+ const dataset = `walk-budget-bad-${i}`
+ const rows = Array.from({ length: 4 }, (_, j) => ({
+ id: j,
+ session_id: `s-${j}`,
+ attributes: `{"gateway":{"session":"s-${j}"}}`,
+ }))
+ await appendRowsToSourceTable(
+ cacheRoot, dataset, ['source=claude'], SESSION_COLUMNS, rows,
+ { declaration: SESSION_DECLARATION }
+ )
+ const dir = partitionDir(cacheRoot, dataset)
+ await plantStamplessRecord(dir, 4)
+ await tearOneDataFile(dir)
+ }
+
+ const report = await maintainCache({ cacheRoot, compactOnly: true, budgetMs: 0 })
+
+ assert.equal(
+ report.totalFailed, MAX_FAILURES_BEFORE_BUDGET_BREAK,
+ `the walk must stop once ${MAX_FAILURES_BEFORE_BUDGET_BREAK} partitions have failed, not after all ${total}`
+ )
+ assert.equal(report.partitions.length, MAX_FAILURES_BEFORE_BUDGET_BREAK)
+ assert.ok(report.partitions.every((p) => p.failed), 'every partition visited in this fixture fails')
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
diff --git a/test/core/cache-retention-maintenance.test.js b/test/core/cache-retention-maintenance.test.js
index 41aac363..5af6cb38 100644
--- a/test/core/cache-retention-maintenance.test.js
+++ b/test/core/cache-retention-maintenance.test.js
@@ -926,6 +926,58 @@ test('a foreign sorted replace re-baselines the cursor instead of being rewritte
}
})
+// @ref LLP 0220#walk-survives-a-partition [tests]: a rebaseline is a cursor
+// write with no rewrite behind it, and `writeCursor` can throw same as any
+// other write in this loop. `r.rebaselined` must not be set until that write
+// has actually landed, or a tick that lost this partition would still print
+// "rebaselined" and count it in `totalRebaselined` for a cursor that never
+// changed (round-1 review finding 2).
+test('a rebaseline that fails to persist is not reported as rebaselined, and is not counted', async () => {
+ const cacheRoot = await makeTmpDir('maint-foreign-replace-write-fail')
+ try {
+ const partDir = path.join(cacheRoot, 'datasets', 'ds1', 'date=2026-08-08')
+ const epoch0 = path.join(partDir, 'epoch=0')
+ for (let i = 0; i < 3; i++) {
+ await appendRowsToTable(epoch0, COLUMNS, [
+ { id: i, value: `v${i}`, timestamp: new Date().toISOString() },
+ ], { sortOrder: [{ column: 'id', direction: 'asc' }] })
+ }
+ await commitForeignReplace(epoch0)
+ // Same stale-baseline cursor as the passing recognition test: this is
+ // the fixture that would otherwise re-baseline cleanly.
+ await writeCursor(partDir, {
+ epoch: 0,
+ rowCount: 3,
+ layout: 'epoch',
+ compaction: { compactedAt: '2026-08-08T00:00:00.000Z', resettleBaselineFiles: 99 },
+ })
+
+ // `writeCursor` writes `cursor.json` directly into the partition dir,
+ // so making the dir read-only fails exactly that write and nothing
+ // upstream of it (the read of the epoch dir's own metadata is
+ // unaffected).
+ await fs.chmod(partDir, 0o555)
+ try {
+ const report = await maintainCache({ cacheRoot, compactOnly: true })
+ assert.equal(report.partitions[0].failed, true, 'fixture invariant: the cursor write must fail')
+ assert.notEqual(
+ report.partitions[0].rebaselined, true,
+ 'a rebaseline that never landed on disk must not be reported as one'
+ )
+ assert.equal(report.totalRebaselined, 0, 'an unpersisted rebaseline must not be counted')
+ } finally {
+ await fs.chmod(partDir, 0o755)
+ }
+
+ // The cursor on disk still carries the pre-rebaseline baseline: the
+ // write failure must not have partially landed.
+ const compaction = /** @type {{ resettleBaselineFiles: number }} */ (readCursorSync(partDir).compaction)
+ assert.equal(compaction.resettleBaselineFiles, 99)
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
test('foreign sorted replace recognition works on the source-table layout', async () => {
const cacheRoot = await makeTmpDir('maint-foreign-source')
try {
diff --git a/test/core/daemon-maintenance-tick-status.test.js b/test/core/daemon-maintenance-tick-status.test.js
new file mode 100644
index 00000000..b23e9b8c
--- /dev/null
+++ b/test/core/daemon-maintenance-tick-status.test.js
@@ -0,0 +1,158 @@
+// @ts-check
+
+// @ref LLP 0220#tick-reports-degraded [tests]: `withSpan` derives the span's
+// status code from a `status` attribute snapshot taken before the callback
+// runs, and always calls `setStatus` after the callback resolves - so a
+// `span.setAttribute('status', 'degraded')` made once the report is in hand
+// (this tick's own verdict) never reached the span's OTel status code; the
+// post-hoc `setStatus(OK)` clobbered it. `runtime.js`'s maintenance tick was
+// rewritten to manage its span directly instead of through `withSpan` so the
+// real verdict wins. This drives a real daemon through one degraded tick and
+// reads the exported span back off disk (round-1 review finding 4).
+
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+
+import { runDaemon } from '../../src/core/daemon/runtime.js'
+import { defaultConfigPath } from '../../src/core/config/schema.js'
+import { appendRowsToSourceTable, readCursorSync, writeCursor } from '../../src/core/cache/partition.js'
+
+/** @import { ColumnSpec } from '../../hypaware-plugin-kernel-types.js' */
+/** @import { PartitionCursor } from '../../src/core/cache/types.js' */
+
+/** @type {ColumnSpec[]} */
+const SESSION_COLUMNS = [
+ { name: 'id', type: 'INT32', nullable: false },
+ { name: 'session_id', type: 'STRING', nullable: false },
+ { name: 'attributes', type: 'STRING', nullable: true },
+]
+
+const SESSION_DECLARATION = {
+ source: { columns: ['source'] },
+ iceberg: { fields: [{ column: 'session_id', transform: 'identity' }] },
+}
+
+/** @param {string} dir @param {number} baselineFiles */
+async function plantStamplessRecord(dir, baselineFiles) {
+ const cursor = readCursorSync(dir)
+ /** @type {PartitionCursor} */
+ const next = {
+ ...cursor,
+ compaction: {
+ previousTableDir: 'table',
+ compactedAt: '2026-08-12T21:55:35.168Z',
+ resettleBaselineFiles: baselineFiles,
+ },
+ }
+ await writeCursor(dir, next)
+}
+
+/** @param {string} dir */
+async function tearOneDataFile(dir) {
+ const cursor = readCursorSync(dir)
+ const dataDir = path.join(dir, cursor.tableDir ?? 'table', 'data')
+ const entries = await fs.readdir(dataDir, { withFileTypes: true })
+ const [torn] = entries
+ .filter((e) => e.isFile() && e.name.endsWith('.parquet'))
+ .map((e) => path.join(dataDir, e.name))
+ assert.ok(torn, 'fixture invariant: the partition must hold a live data file to tear')
+ await fs.truncate(torn, 4)
+}
+
+/**
+ * Poll a JSONL file for a line matching `predicate`, since the tracer
+ * exporter writes to an `fs.WriteStream` with no flush hook this daemon
+ * ever calls: the write lands async relative to the tick's own promise
+ * resolving.
+ *
+ * @param {string} filePath
+ * @param {(record: any) => boolean} predicate
+ * @param {number} timeoutMs
+ * @returns {Promise}
+ */
+async function pollJsonlFor(filePath, predicate, timeoutMs) {
+ const deadline = Date.now() + timeoutMs
+ for (;;) {
+ try {
+ const raw = await fs.readFile(filePath, 'utf8')
+ for (const line of raw.split('\n')) {
+ if (!line) continue
+ const record = JSON.parse(line)
+ if (predicate(record)) return record
+ }
+ } catch (err) {
+ if (/** @type {NodeJS.ErrnoException} */ (err).code !== 'ENOENT') throw err
+ }
+ if (Date.now() > deadline) return undefined
+ await new Promise((resolve) => setTimeout(resolve, 25))
+ }
+}
+
+test('a degraded maintenance tick sets the span status code, not just the attribute', async () => {
+ const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hypaware-daemon-maint-status-'))
+ const savedHypHome = process.env.HYP_HOME
+ const savedDevTelemetry = process.env.HYP_DEV_TELEMETRY
+ let handle
+ try {
+ const cacheRoot = path.join(hypHome, 'hypaware', 'cache')
+ const rows = Array.from({ length: 4 }, (_, i) => ({
+ id: i,
+ session_id: `s-${i}`,
+ attributes: `{"gateway":{"session":"s-${i}"}}`,
+ }))
+ await appendRowsToSourceTable(
+ cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows,
+ { declaration: SESSION_DECLARATION }
+ )
+ const partDir = path.join(cacheRoot, 'datasets', 'ai_gateway_messages', 'source=claude')
+ await plantStamplessRecord(partDir, 4)
+ await tearOneDataFile(partDir)
+
+ const configPath = defaultConfigPath(hypHome)
+ await fs.mkdir(path.dirname(configPath), { recursive: true })
+ await fs.writeFile(configPath, JSON.stringify({
+ version: 2,
+ query: { cache: { maintenance: { interval_minutes: 0.001 } } },
+ }))
+
+ // `installObservability()` (called inside `runDaemon`) reads real
+ // `process.env`, not the `env` option below - the same mechanism the
+ // hermetic smokes use (CLAUDE.md's "temp HYP_HOME and
+ // HYP_DEV_TELEMETRY=1"), so the JSONL span exporter has to be armed here.
+ process.env.HYP_HOME = hypHome
+ process.env.HYP_DEV_TELEMETRY = '1'
+
+ handle = await runDaemon({
+ hypHome,
+ configPath,
+ env: { ...process.env, HYP_HOME: hypHome },
+ runId: 'maint-status-test',
+ tickIntervalMs: 0,
+ installSignalHandlers: false,
+ })
+
+ const tracesPath = path.join(hypHome, 'hypaware', 'dev-telemetry', `traces-${process.pid}.jsonl`)
+ const span = await pollJsonlFor(tracesPath, (r) => r.name === 'maintenance.tick', 5_000)
+
+ assert.ok(span, 'the maintenance.tick span must be exported within the poll window')
+ assert.equal(span.attributes.status, 'degraded', 'sanity: the attribute this tick set')
+ assert.equal(
+ span.status, 'failed',
+ 'the span status CODE (not just the attribute) must reflect the degraded tick - ' +
+ 'a bare setAttribute inside withSpan would leave this "ok"'
+ )
+ } finally {
+ if (handle) {
+ await handle.stop()
+ await handle.done
+ }
+ if (savedHypHome === undefined) delete process.env.HYP_HOME
+ else process.env.HYP_HOME = savedHypHome
+ if (savedDevTelemetry === undefined) delete process.env.HYP_DEV_TELEMETRY
+ else process.env.HYP_DEV_TELEMETRY = savedDevTelemetry
+ await fs.rm(hypHome, { recursive: true, force: true })
+ }
+})
diff --git a/test/core/query-maintain-cli.test.js b/test/core/query-maintain-cli.test.js
new file mode 100644
index 00000000..2bbcbaa9
--- /dev/null
+++ b/test/core/query-maintain-cli.test.js
@@ -0,0 +1,146 @@
+// @ts-check
+
+// @ref LLP 0220#tick-reports-degraded [tests]: `maintainCache` stopped
+// rejecting when a partition throws (it reports the failure instead), so a
+// caller that only captures stderr - a cron wrapper, `>/dev/null` - needs a
+// line there on a degraded tick, or it never learns the walk lost a
+// partition. These pin that the line appears exactly when it should
+// (round-1 review finding 3).
+
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+
+import { runQueryMaintain } from '../../src/core/commands/query.js'
+import { appendRowsToSourceTable, readCursorSync, writeCursor } from '../../src/core/cache/partition.js'
+
+/** @import { ColumnSpec } from '../../hypaware-plugin-kernel-types.js' */
+/** @import { PartitionCursor } from '../../src/core/cache/types.js' */
+
+/** @type {ColumnSpec[]} */
+const SESSION_COLUMNS = [
+ { name: 'id', type: 'INT32', nullable: false },
+ { name: 'session_id', type: 'STRING', nullable: false },
+ { name: 'attributes', type: 'STRING', nullable: true },
+]
+
+const SESSION_DECLARATION = {
+ source: { columns: ['source'] },
+ iceberg: { fields: [{ column: 'session_id', transform: 'identity' }] },
+}
+
+function makeBuf() {
+ let value = ''
+ return {
+ /** @param {string} chunk */
+ write(chunk) { value += String(chunk); return true },
+ text() { return value },
+ }
+}
+
+/**
+ * A minimal `CommandRunContext` stand-in: `runQueryMaintain` only reaches
+ * `ctx.storage.cacheRoot`, `ctx.config?.query?.cache?.maintenance`,
+ * `ctx.query.getDataset(...)?.resettleBatch`, and the two stream sinks.
+ *
+ * @param {string} cacheRoot
+ */
+function ctxFor(cacheRoot) {
+ const stdout = makeBuf()
+ const stderr = makeBuf()
+ return {
+ stdout,
+ stderr,
+ ctx: /** @type {any} */ ({
+ stdout,
+ stderr,
+ storage: { cacheRoot },
+ config: { version: 2 },
+ query: { getDataset: () => null },
+ env: {},
+ cwd: '/w/project',
+ }),
+ }
+}
+
+/** @param {string} dir @param {number} baselineFiles */
+async function plantStamplessRecord(dir, baselineFiles) {
+ const cursor = readCursorSync(dir)
+ /** @type {PartitionCursor} */
+ const next = {
+ ...cursor,
+ compaction: {
+ previousTableDir: 'table',
+ compactedAt: '2026-08-12T21:55:35.168Z',
+ resettleBaselineFiles: baselineFiles,
+ },
+ }
+ await writeCursor(dir, next)
+}
+
+/** @param {string} dir */
+async function tearOneDataFile(dir) {
+ const cursor = readCursorSync(dir)
+ const dataDir = path.join(dir, cursor.tableDir ?? 'table', 'data')
+ const entries = await fs.readdir(dataDir, { withFileTypes: true })
+ const [torn] = entries
+ .filter((e) => e.isFile() && e.name.endsWith('.parquet'))
+ .map((e) => path.join(dataDir, e.name))
+ assert.ok(torn, 'fixture invariant: the partition must hold a live data file to tear')
+ await fs.truncate(torn, 4)
+}
+
+test('hyp query maintain writes a stderr line when the walk loses a partition', async () => {
+ const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-cli-degraded-'))
+ try {
+ const rows = Array.from({ length: 4 }, (_, i) => ({
+ id: i,
+ session_id: `s-${i}`,
+ attributes: `{"gateway":{"session":"s-${i}"}}`,
+ }))
+ await appendRowsToSourceTable(
+ cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows,
+ { declaration: SESSION_DECLARATION }
+ )
+ const dir = path.join(cacheRoot, 'datasets', 'ai_gateway_messages', 'source=claude')
+ await plantStamplessRecord(dir, 4)
+ await tearOneDataFile(dir)
+
+ const { stderr, ctx } = ctxFor(cacheRoot)
+ const exitCode = await runQueryMaintain(['--compact-only'], ctx)
+
+ assert.equal(exitCode, 1, 'a degraded tick must still exit non-zero')
+ assert.match(
+ stderr.text(),
+ /^hyp query maintain: 1 partition\(s\) failed; the walk continued$/m,
+ 'a caller that only captures stderr must see the walk lost a partition'
+ )
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})
+
+test('hyp query maintain writes nothing to stderr on a clean tick', async () => {
+ const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-cli-clean-'))
+ try {
+ const rows = Array.from({ length: 4 }, (_, i) => ({
+ id: i,
+ session_id: `s-${i}`,
+ attributes: `{"gateway":{"session":"s-${i}"}}`,
+ }))
+ await appendRowsToSourceTable(
+ cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows,
+ { declaration: SESSION_DECLARATION }
+ )
+
+ const { stderr, ctx } = ctxFor(cacheRoot)
+ const exitCode = await runQueryMaintain(['--compact-only'], ctx)
+
+ assert.equal(exitCode, 0, 'a clean tick must exit zero')
+ assert.equal(stderr.text(), '', 'a clean tick must not write anything to stderr')
+ } finally {
+ await fs.rm(cacheRoot, { recursive: true, force: true })
+ }
+})