diff --git a/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js b/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js index b48257c6..5df43783 100644 --- a/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js +++ b/hypaware-core/smoke/flows/cache_lifecycle_maintenance.js @@ -110,6 +110,10 @@ export async function run({ harness, expect }) { report.totalCompacted, (v) => typeof v === 'number' && v > 0 ) + // @ref LLP 0220#tick-reports-degraded: the walk no longer aborts on a + // partition that throws, so a failure it swallowed would otherwise reach + // this smoke only as a missing compaction somewhere else in the cache. + expect.that('maintenance: no partition failed', report.totalFailed, (v) => v === 0) // --- 4. Verify post-maintenance state --- // Source-table layout: data lives under source=unknown (no client columns) diff --git a/hypaware-core/smoke/flows/incremental_sink_compaction.js b/hypaware-core/smoke/flows/incremental_sink_compaction.js index 989a7465..b4f1f4a0 100644 --- a/hypaware-core/smoke/flows/incremental_sink_compaction.js +++ b/hypaware-core/smoke/flows/incremental_sink_compaction.js @@ -156,6 +156,9 @@ export async function run({ harness, expect }) { const before = readCursorSync(sourceDir).tableDir ?? 'table' const maint = await maintainCache({ cacheRoot, force: true, compactOnly: true }) expect.that('compaction: at least one partition compacted', maint.totalCompacted, (v) => typeof v === 'number' && v > 0) + // @ref LLP 0220#tick-reports-degraded: a partition that throws no longer + // rejects the tick, so the failure has to be read off the report. + expect.that('compaction: no partition failed', maint.totalFailed, (v) => v === 0) const after = readCursorSync(sourceDir).tableDir ?? 'table' expect.that('compaction: generation directory swapped', [before, after], ([b, a]) => b !== a) diff --git a/llp/0199-maintenance-compaction-convergence.decision.md b/llp/0199-maintenance-compaction-convergence.decision.md index 7d576f79..0aa63ab7 100644 --- a/llp/0199-maintenance-compaction-convergence.decision.md +++ b/llp/0199-maintenance-compaction-convergence.decision.md @@ -6,7 +6,7 @@ **Author:** Kenny / Claude **Date:** 2026-08-07 **Related:** LLP 0027 -**Extended-by:** [LLP 0207](./0207-foreign-sorted-replace-convergence.decision.md) (a baseline mismatch whose current snapshot is a sorted `replace` is a foreign rewrite, not growth: recognize it and re-baseline instead of compacting); [LLP 0209](./0209-compaction-file-size.decision.md) (the rewrite the baseline gate protects now sizes its output files by bytes written rather than by the in-memory batch estimate, so a compacted generation actually reaches `target_file_bytes`); [LLP 0217](./0217-compaction-effectiveness-verdict.decision.md) (the cursor records what a rewrite achieved, not only the count it produced, so a partition sitting on its baseline because the rewrite accomplished nothing is skipped explicitly and retried once when the writer changes) +**Extended-by:** [LLP 0207](./0207-foreign-sorted-replace-convergence.decision.md) (a baseline mismatch whose current snapshot is a sorted `replace` is a foreign rewrite, not growth: recognize it and re-baseline instead of compacting); [LLP 0209](./0209-compaction-file-size.decision.md) (the rewrite the baseline gate protects now sizes its output files by bytes written rather than by the in-memory batch estimate, so a compacted generation actually reaches `target_file_bytes`); [LLP 0217](./0217-compaction-effectiveness-verdict.decision.md) (the cursor records what a rewrite achieved, not only the count it produced, so a partition sitting on its baseline because the rewrite accomplished nothing is skipped explicitly and retried once when the writer changes); [LLP 0220](./0220-maintenance-walk-survives-a-partition.decision.md) (the neediest-first order puts the likeliest partition to fail first, so the walk catches per partition and continues instead of losing every partition behind it) > Maintenance stops re-flagging already-compacted partitions: a partition is > only compaction-due when its live data-file count has moved off the count diff --git a/llp/0218-compaction-failed-attempt-reported.decision.md b/llp/0218-compaction-failed-attempt-reported.decision.md index 2ada3317..dfc67e02 100644 --- a/llp/0218-compaction-failed-attempt-reported.decision.md +++ b/llp/0218-compaction-failed-attempt-reported.decision.md @@ -6,6 +6,7 @@ **Author:** Kenny / Claude **Date:** 2026-08-13 **Related:** LLP 0217 (the effectiveness verdict and the retry stamp this extends), LLP 0199 (the baseline gate both sit on), LLP 0207 (the recognition path that writes the same stamp) +**Extended-by:** [LLP 0220](./0220-maintenance-walk-survives-a-partition.decision.md) (the tick in which the attempt fails is reported too: `failed` is this tick's error, beside the `compactionAttemptFailed` skip an earlier one recorded) > The stamp a failed compaction retry writes records the moment the attempt > failed, so the ticks that skip the partition afterwards can say why. A diff --git a/llp/0220-maintenance-walk-survives-a-partition.decision.md b/llp/0220-maintenance-walk-survives-a-partition.decision.md new file mode 100644 index 00000000..8cd745ad --- /dev/null +++ b/llp/0220-maintenance-walk-survives-a-partition.decision.md @@ -0,0 +1,151 @@ +# LLP 0220: A maintenance walk survives the partition that throws, and says it lost one + +**Type:** Decision +**Status:** Accepted +**Systems:** Cache +**Author:** Kenny / Claude +**Date:** 2026-08-13 +**Related:** LLP 0199 (the neediest-first walk this makes survivable), LLP 0217 (the retry the attempt spends, which must still be spent), LLP 0218 (the skip-reason vocabulary this sits beside), LLP 0021 (the span helper that rethrows) + +> A partition whose maintenance throws ends that partition's work, not the +> tick's. The walk records the failure on that partition's report and moves to +> the next one. The tick then resolves with a report carrying the failures, +> because the walk did complete; its callers read the failures off the report +> and refuse to describe the run as clean. + +## Context {#context} + +`maintainCache` walks partitions neediest-first (LLP 0199#neediest-first): +descending live data-file count, so the most fragmented partition is visited +first. That is deliberate, and it is also why the walk is fragile. The +neediest partition is the one most likely to fail a rewrite: in the #723 case +318 MB, 230k rows and 1,521 files, doubling disk while it runs, and one torn +data file in it is enough to throw. + +There was no per-partition catch, and `withSpan` rethrows after recording the +exception (LLP 0021#span-helpers). So the first partition's error propagated +out of the loop, out of `maintainCache`, and into +`src/core/daemon/runtime.js`, which logged `daemon.maintenance_failed` and +waited an hour to run into the same wall. Every partition behind the failure +got neither compaction nor snapshot expiry, which is the guard against the +unbounded metadata growth #723 was filed about in the first place. + +LLP 0217#retry-on-writer-change bounded how long this lasts: the failing +attempt spends its writer generation's retry, so from the next tick the +partition is skipped rather than re-attempted, and LLP 0218 made that skip a +stated reason. Both leave the tick in which the failure happens fully lost, +and that tick is the one that had the neediest partition's whole backlog +queued behind it. + +Reported as #737, deliberately out of scope for the PR that settled the retry +stamp, because continuing the walk is a behaviour change and not a fix to +what that PR broke. + +## Decision {#decision} + +**One partition's error is one +partition's outcome.** The maintenance loop catches around the +`maintenance.partition` span, records the failure on that partition's report, +and continues to the next partition. The catch is *outside* `withSpan` on +purpose: the helper records the exception and an ERROR status on the partition +span and then rethrows, so catching outside keeps the trace honest about which +partition failed while the walk moves on. Catching inside the callback would +publish an `ok` span for a partition that threw, which is the legibility +defect one layer down from the one being fixed. + +The partition's report object is therefore built before the span opens rather +than inside it, so a partition that threw part-way still reports what the run +had established about it: its live data-file count, and any snapshots the +expiry pass expired before compaction reached the error. + +Nothing about the failure path itself moves. LLP 0217's writer-generation +stamp is written from inside `compactGeneration`'s own catch, on the way out, +before the error reaches this one, so the attempt still spends the retry and +a persistently failing partition is still attempted once per writer +generation rather than once per tick. + +**The tick resolves, and reports what it +lost.** `maintainCache` returns its report instead of rejecting when a +partition threw, because the walk it was asked to perform completed. The +report is the honest carrier: `MaintenancePartitionReport` gains `failed`, +`errorKind` and `errorMessage`, and `MaintenanceReport` gains `totalFailed`. + +Rejecting after the walk instead would keep the daemon's existing wiring, but +it throws the report away, so a tick that maintained forty partitions and lost +one would print nothing about any of them. The report is the only thing that +knows what the tick did. + +Every caller therefore reads the failures rather than a rejected promise. The +daemon logs one `daemon.maintenance_failed` per failed partition, naming the +dataset and partition (which the propagated exception never could), and marks +the `maintenance.tick` span with `status: degraded` and `partitions_failed` +as attributes *and* an `ERROR` span status code - set directly once the +report is in hand, since `withSpan`'s status-attribute convention only reads +that attribute's snapshot from before the callback runs and cannot see a +verdict this late (`src/core/daemon/runtime.js`'s tick manages its span +directly for this reason, rather than through `withSpan`). Its +outer `.catch` stays, for the errors still outside the per-partition catch: +partition discovery and the retired-generation sweep. `hyp query maintain` +prints a `FAILED:` line for the partition and exits non-zero, so a script that +gated on the old exit status does not start reading a degraded run as a clean +one. + +**`failed` is not +`compactionAttemptFailed`.** LLP 0218's field is read off the cursor and means +*an earlier tick's attempt failed and nothing has been attempted since*; the +partition was skipped, deliberately, for a stated reason. `failed` means *this +tick attempted work here and it threw*. They never appear together, because a +tick that attempted a rewrite is not a tick that skipped one, and an operator +has to be able to tell a partition that just broke from one that has been +quietly frozen for a week. Keeping one field for both would put the two +readings back into the same absence the LLP 0218 report exists to remove. + +## Consequences {#consequences} + +- A cache holding one broken partition no longer has its whole walk starved + by it: the walk moves on to the next partition instead of aborting. The + per-tick budget guard (LLP 0199#neediest-first) still applies, and is keyed + off partitions actually *maintained*, not partitions merely visited - a + partition that failed did no work, so it cannot itself satisfy the guard's + "always work one partition before the budget can cut the tick short" + guarantee and stall everything behind it (`src/core/cache/maintenance.js`). + What this buys is "the walk keeps moving past a failure", not "every + healthy partition is guaranteed maintenance on the tick a partition + breaks": a tight budget can still cut a tick short once real work has + happened, exactly as it could before this document. The one case that + guard alone cannot bound - every partition in the cache failing - is capped + separately: the walk still breaks once `MAX_FAILURES_BEFORE_BUDGET_BREAK` + partitions have failed past the budget, so an all-failing cache cannot walk + unbounded either. The guard against unbounded metadata growth (snapshot + expiry, the orphan sweep) stops being hostage to the neediest partition's + health specifically; it is not a promise that one failure never costs any + other partition its turn in a budget-constrained tick. +- `cleanRetiredEpochs` now runs on a tick that lost a partition, where the + abort used to skip it, so the half-written generation a failed rewrite left + behind is reclaimed by the orphan sweep on schedule instead of waiting for a + clean tick. +- The budget still applies. A partition that fails fast costs its share of + `max_tick_ms` and no more, and the walk continues into the neediest healthy + partition rather than postponing the whole ranking by an hour. +- `daemon.maintenance_failed` fires once per failed partition rather than once + per tick, and carries the partition identity. A cache with several broken + partitions logs several lines, which is the correct count of things that are + wrong. +- Tests that asserted the whole tick rejected now assert the partition's + report instead. The thing they pin (the stamp, the committed cursor, the + recorded verdict) is unchanged; only how the failure is observed moved. +- A failure that ought to abort the whole walk has nowhere to say so. Nothing + in the current maintenance path is such a failure (the shared state a + partition touches is its own directory and its own cursor), but a future step + that corrupts something cache-wide would need a way to stop the walk rather + than being reported as one partition's error. + +## Extends {#extends} + +LLP 0199 settled the walk order and the budget that cuts it short, and both +stand: neediest-first is still the order, and a partition that fails is still +visited in it. What this document adds is that the order's own consequence - +the likeliest partition to fail goes first - no longer decides whether the +rest of the walk happens. LLP 0218 settled that a partition maintenance leaves +alone is left alone for a stated reason; this extends that vocabulary to the +partition maintenance tried and could not finish. diff --git a/src/core/cache/maintenance.js b/src/core/cache/maintenance.js index 06c2e6b0..b8cfe0d3 100644 --- a/src/core/cache/maintenance.js +++ b/src/core/cache/maintenance.js @@ -57,6 +57,25 @@ const DEFAULTS = { const GRACE_PERIOD_MS = 24 * 60 * 60 * 1000 +/** + * How many failed partitions the budget guard tolerates before it breaks + * the walk even though nothing has been maintained yet. + * + * The guard's job is "always work one partition before the budget can cut + * the tick short" (see below), and a partition that threw did no work, so + * it must not be the one that satisfies that guarantee - a first partition + * that fails slowly would otherwise starve every partition behind it, on + * every tick, which is #737 by another route. But gating the break on + * "maintained > 0" alone means a cache where every partition fails walks + * past `budgetMs` without bound, which trades one unbounded-tick risk for + * another. Four is deliberately small: it is bigger than the "one bad + * partition" case this whole PR is about, small enough that a tick spent + * entirely on failures is still bounded to roughly four partitions' worth + * of failure latency rather than the whole cache, and it does not require + * reading a config value that does not otherwise exist for this guard. + */ +export const MAX_FAILURES_BEFORE_BUDGET_BREAK = 4 + /** * How long an unreferenced (cursor-orphaned) table generation must sit * untouched before the orphan sweep reclaims it. Long enough that an @@ -143,58 +162,91 @@ export async function maintainCache(opts) { let totalSnapshotsExpired = 0 let totalCompacted = 0 let totalRebaselined = 0 + let totalFailed = 0 + // A partition that threw did no work, so `reports.length > 0` cannot be + // what proves the budget guard's "always work one partition" guarantee - + // see MAX_FAILURES_BEFORE_BUDGET_BREAK. + let maintained = 0 for (const part of partitions) { // Always work one partition before the budget can cut the tick short: // the ranking pass above is itself unbudgeted, so on a large enough // cache a bare cutoff here would break at iteration 0 every tick and - // maintenance would never run at all. - if (reports.length > 0 && Date.now() - startMs > budgetMs) break - - const report = await withSpan( - 'maintenance.partition', - { - [Attr.COMPONENT]: 'cache', - [Attr.OPERATION]: 'maintenance.partition', - [Attr.DATASET]: part.dataset, - partition: JSON.stringify(part.partition), - status: 'ok', - }, - async (span) => { - /** @type {MaintenancePartitionReport} */ - const r = { - dataset: part.dataset, - partition: part.partition, - path: part.path, - snapshotsExpired: 0, - compacted: false, - rowCount: part.rowCount, - dataFilesBefore: 0, - dataFilesAfter: 0, - } + // maintenance would never run at all. Gated on `maintained`, not on + // `reports.length`, so a first partition that fails (possibly slowly) + // cannot itself satisfy the guarantee and starve the walk behind it. + // The failure-count disjunct is the bounded escape hatch for the + // opposite case, a cache where every partition fails. + if ( + Date.now() - startMs > budgetMs && + (maintained > 0 || totalFailed >= MAX_FAILURES_BEFORE_BUDGET_BREAK) + ) break + + // Built out here rather than inside the span callback so the catch + // below still has it: a partition that threw part-way keeps whatever + // the run had already established about it (its live file count, any + // snapshots expired before compaction reached the error) instead of + // being reported as a bare failure with zeroed counts. + /** @type {MaintenancePartitionReport} */ + const report = { + dataset: part.dataset, + partition: part.partition, + path: part.path, + snapshotsExpired: 0, + compacted: false, + rowCount: part.rowCount, + dataFilesBefore: 0, + dataFilesAfter: 0, + } - const cursor = readCursorSync(part.path) - const settle = resolveSettleContext(opts, part.dataset) - - const done = await maintainGeneration( - r, cursor, cfg, opts, settle, snapshotsExpiredCounter, compactionsCounter, rebaselinesCounter - ) - // A compaction that "converged" is only healthy if it also shrank - // the file count; publish both sides of that so a run that rewrites - // a partition into the same fragmentation is visible in the trace - // rather than only in a later disk audit. - span.setAttribute('compacted', done.compacted) - span.setAttribute('data_files_before', done.dataFilesBefore) - span.setAttribute('data_files_after', done.dataFilesAfter) - span.setAttribute('rows', done.rowCount) - if (done.compactedBytesWritten !== undefined) { - span.setAttribute('bytes_written', done.compactedBytesWritten) - } - return done - }, - { component: 'cache' } - ) + try { + await withSpan( + 'maintenance.partition', + { + [Attr.COMPONENT]: 'cache', + [Attr.OPERATION]: 'maintenance.partition', + [Attr.DATASET]: part.dataset, + partition: JSON.stringify(part.partition), + status: 'ok', + }, + async (span) => { + const cursor = readCursorSync(part.path) + const settle = resolveSettleContext(opts, part.dataset) + + const done = await maintainGeneration( + report, cursor, cfg, opts, settle, snapshotsExpiredCounter, compactionsCounter, rebaselinesCounter + ) + // A compaction that "converged" is only healthy if it also shrank + // the file count; publish both sides of that so a run that rewrites + // a partition into the same fragmentation is visible in the trace + // rather than only in a later disk audit. + span.setAttribute('compacted', done.compacted) + span.setAttribute('data_files_before', done.dataFilesBefore) + span.setAttribute('data_files_after', done.dataFilesAfter) + span.setAttribute('rows', done.rowCount) + if (done.compactedBytesWritten !== undefined) { + span.setAttribute('bytes_written', done.compactedBytesWritten) + } + }, + { component: 'cache' } + ) + } catch (err) { + // @ref LLP 0220#walk-survives-a-partition [implements]: one partition's + // error ends that partition's work, not the tick's. Outside `withSpan` + // on purpose: the helper rethrows after recording the exception and an + // ERROR status on `maintenance.partition`, so catching here keeps the + // span honest about the partition while the walk moves on. Catching + // inside the callback would hand every reader of the trace an `ok` + // span for a partition that failed. Everything the failure path itself + // has to do - LLP 0217's writer-generation stamp above all - already + // ran inside `maintainGeneration`, on the way out. + report.failed = true + report.errorKind = errorKindOf(err) + report.errorMessage = err instanceof Error ? err.message : String(err) + totalFailed++ + } reports.push(report) + if (!report.failed) maintained++ totalSnapshotsExpired += report.snapshotsExpired if (report.compacted) totalCompacted++ if (report.rebaselined) totalRebaselined++ @@ -209,11 +261,29 @@ export async function maintainCache(opts) { totalSnapshotsExpired, totalCompacted, totalRebaselined, + totalFailed, dryRun: opts.dryRun ?? false, elapsedMs: Date.now() - startMs, } } +/** + * The error's own kind when it carries one (the convention sink + * materialization already uses for the same per-unit catch), else a kind + * naming where it was caught. Never the exception's class: what an + * operator needs off a maintenance report is which step of the tick gave + * up, and the message beside it carries the rest. + * + * @param {unknown} err + * @returns {string} + */ +function errorKindOf(err) { + if (err && typeof err === 'object' && 'errorKind' in err) { + return String(/** @type {{ errorKind: unknown }} */ (err).errorKind) + } + return 'maintenance_partition_failed' +} + /** * The five spots where the source-table and legacy epoch layouts differ. * Everything else in maintenance, compaction, and status is @@ -390,15 +460,25 @@ async function maintainGeneration(r, cursor, cfg, opts, settle, snapshotsExpired // re-settle force: a leftover unmatchable fallback row must not undo // the sorted layout every night. An explicit --force still rewrites. if (!opts.force && foreignSortedReplace(tableInfo)) { - r.rebaselined = true // The counter proves a rebaseline happened at all, but it carries // only the dataset; tagging the enclosing maintenance.partition span // names the partition, so a trace query finds which day re-baselined - // without cross-referencing the counter. + // without cross-referencing the counter. Left here rather than moved + // beside `r.rebaselined` below: the span carries ERROR anyway if the + // write that follows throws, so the intent is still worth recording. getActiveSpan()?.setAttribute('rebaselined', true) if (!opts.dryRun) { await writeCursor(r.path, rebaselineCursor(cursor, dataFilesBefore)) + // Set only once the cursor write that persists it has succeeded: + // a throw here must not leave the report (and `totalRebaselined`) + // claiming a rebaseline that never landed on disk. + r.rebaselined = true rebaselinesCounter.add(1, { [Attr.DATASET]: r.dataset }) + } else { + // No write happens in dry-run, so nothing to wait on: this is + // the preview of what a real run would do, same as `r.compacted` + // below for the ordinary rewrite path. + r.rebaselined = true } } else if (opts.dryRun) { r.compacted = true diff --git a/src/core/cache/types.d.ts b/src/core/cache/types.d.ts index 8158846a..22842b0b 100644 --- a/src/core/cache/types.d.ts +++ b/src/core/cache/types.d.ts @@ -315,6 +315,17 @@ export interface MaintenancePartitionReport { // When that attempt failed, as the cursor records it. Set whenever // `compactionAttemptFailed` is. compactionAttemptFailedAt?: string + // THIS tick's work on the partition ended in an error and the walk moved + // on (LLP 0220). Distinct from `compactionAttemptFailed`, which is read + // off the cursor and says an EARLIER tick's attempt failed and nothing + // has been attempted since: the two never appear together, because a + // tick that attempted a rewrite is not a tick that skipped one. + failed?: boolean + // The error's own `errorKind` when it carries one, else + // `maintenance_partition_failed`. Set whenever `failed` is. + errorKind?: string + // The error's message. Set whenever `failed` is. + errorMessage?: string } export interface MaintenanceReport { @@ -322,6 +333,11 @@ export interface MaintenanceReport { totalSnapshotsExpired: number totalCompacted: number totalRebaselined: number + // Partitions whose work threw during this tick. Non-zero means the walk + // completed but the tick is degraded: it did not maintain everything it + // set out to, and its callers report that rather than a clean run + // (LLP 0220#tick-reports-degraded). + totalFailed: number dryRun: boolean elapsedMs: number } diff --git a/src/core/commands/query.js b/src/core/commands/query.js index 0f173eae..32c6fb4e 100644 --- a/src/core/commands/query.js +++ b/src/core/commands/query.js @@ -342,13 +342,33 @@ export async function runQueryMaintain(argv, ctx) { if (p.compactionAttemptFailed) { actions.push(`compaction skipped: the retry failed under this writer at ${p.compactionAttemptFailedAt}, --force to retry`) } + // @ref LLP 0220#tick-reports-degraded: this tick's own failure, as + // opposed to the line above, which reports one an earlier tick already + // recorded on the cursor. The walk continued past this partition, so + // without the line the run reads as a clean one that happened to do + // less work. + if (p.failed) { + actions.push(`FAILED: ${p.errorMessage} (${p.errorKind}); the walk continued`) + } if (actions.length > 0) { ctx.stdout.write(` ${label}: ${actions.join(', ')}\n`) } } const rebaselineNote = report.totalRebaselined > 0 ? `, ${report.totalRebaselined} rebaselined` : '' - ctx.stdout.write(`maintenance: ${report.totalSnapshotsExpired} snapshots expired, ${report.totalCompacted} partitions compacted${rebaselineNote} (${report.elapsedMs}ms)\n`) - return 0 + const failedNote = report.totalFailed > 0 ? `, ${report.totalFailed} partitions failed` : '' + ctx.stdout.write(`maintenance: ${report.totalSnapshotsExpired} snapshots expired, ${report.totalCompacted} partitions compacted${rebaselineNote}${failedNote} (${report.elapsedMs}ms)\n`) + // Before this PR the exception propagated to bin/hypaware.js, which wrote + // `hyp: ` to stderr - so a caller that only captures stderr (a + // cron or systemd wrapper, `>/dev/null`) still needs a line there on a + // degraded tick, not just the stdout summary above. + if (report.totalFailed > 0) { + ctx.stderr.write(`hyp query maintain: ${report.totalFailed} partition(s) failed; the walk continued\n`) + } + // A partition that threw used to abort the walk and exit non-zero. The + // walk survives it now, but the tick did not do what it was asked, so the + // exit status still says so: a script that gated on this must not start + // reading a degraded run as a clean one. + return report.totalFailed > 0 ? 1 : 0 } const QUERY_MAINTAIN_USAGE = 'usage: hyp query maintain [dataset] [--dry-run] [--force] [--compact-only] [--expire-only]' diff --git a/src/core/daemon/runtime.js b/src/core/daemon/runtime.js index 7ba0e756..f82aae1b 100644 --- a/src/core/daemon/runtime.js +++ b/src/core/daemon/runtime.js @@ -4,10 +4,13 @@ import process from 'node:process' import { Attr, + buildAttrs, getKernelInstruments, getLogger, + getTracer, installObservability, runRoot, + SpanStatusCode, withSpan, } from '../observability/index.js' import { readObservabilityEnv } from '../observability/env.js' @@ -659,17 +662,28 @@ export async function runDaemon(opts = {}) { const { maintainCache, normalizeMaintenanceConfig } = await import('../cache/maintenance.js') const mCfg = normalizeMaintenanceConfig(maintenanceCfg) const intervalMs = mCfg.interval_minutes * 60 * 1000 + // @ref LLP 0220#tick-reports-degraded [constrained-by]: not built on + // `withSpan`. `withSpan` (src/core/observability/span_helpers.js) + // derives the span's status code from the `status` attribute snapshot + // taken at span-creation time, and sets it from that snapshot after the + // callback resolves - so a status only known once the tick's report is + // in hand (clean vs. degraded) cannot be conveyed by setting the + // attribute inside the callback, the way every other `withSpan` caller + // does: the post-hoc `setStatus(OK)` clobbers it. Managed inline here + // with the tracer directly instead, so only this one call site's status + // handling changes and `withSpan` (and everything else that calls it) + // is untouched. async function runMaintenance() { - await withSpan( - 'maintenance.tick', - { - [Attr.COMPONENT]: 'daemon', - [Attr.OPERATION]: 'maintenance.tick', - daemon_mode: mode, - status: 'ok', - }, - async () => { - await maintainCache({ + const tracer = getTracer('daemon') + const attrs = buildAttrs({ + [Attr.COMPONENT]: 'daemon', + [Attr.OPERATION]: 'maintenance.tick', + daemon_mode: mode, + status: 'ok', + }) + await tracer.startActiveSpan('maintenance.tick', { attributes: attrs }, async (span) => { + try { + const report = await maintainCache({ cacheRoot: boot.runtime.storage.cacheRoot, budgetMs: mCfg.max_tick_ms, config: mCfg, @@ -680,9 +694,46 @@ export async function runDaemon(opts = {}) { storage: boot.runtime.storage, getSettleHook: (dataset) => boot.runtime.query.getDataset(dataset)?.resettleBatch, }) - }, - { component: 'daemon' } - ).catch((err) => { + // @ref LLP 0220#tick-reports-degraded [implements]: the walk now + // survives a partition that throws, so the rejected promise has + // stopped being how the daemon hears about one. Read the failures + // off the report instead, or a tick that lost its neediest + // partition would log exactly as a clean one does. The line names + // the partitions, which the propagated exception never could. + for (const p of report.partitions) { + if (!p.failed) continue + fileLog.error('daemon.maintenance_failed', { + [Attr.DATASET]: p.dataset, + partition: JSON.stringify(p.partition), + [Attr.ERROR_KIND]: p.errorKind, + message: p.errorMessage, + }) + } + const degraded = report.totalFailed > 0 + if (degraded) { + span.setAttribute('status', 'degraded') + span.setAttribute('partitions_failed', report.totalFailed) + } + span.setAttribute('partitions_maintained', report.partitions.length - report.totalFailed) + // The status code itself, not just the attribute: only knowable + // now that the report is in hand, so set directly rather than + // through the status-attribute snapshot `withSpan` would + // otherwise have read before the callback ran. + span.setStatus(degraded + ? { code: SpanStatusCode.ERROR, message: `${report.totalFailed} partition(s) failed` } + : { code: SpanStatusCode.OK }) + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)) + span.recordException(err) + span.setStatus({ code: SpanStatusCode.ERROR, message: err.message }) + span.setAttribute('error_kind', attrs.error_kind ?? 'unhandled_exception') + throw err + } finally { + span.end() + } + }).catch((err) => { + // Still reachable: partition discovery, the retired-generation + // sweep, and anything else outside the per-partition catch. const message = err instanceof Error ? err.message : String(err) fileLog.error('daemon.maintenance_failed', { message }) }) diff --git a/test/core/cache-compaction-effectiveness.test.js b/test/core/cache-compaction-effectiveness.test.js index 5a4b6b66..8ac07ea6 100644 --- a/test/core/cache-compaction-effectiveness.test.js +++ b/test/core/cache-compaction-effectiveness.test.js @@ -294,8 +294,12 @@ test('a retry whose rewrite throws still spends its writer generation', async () const [torn] = await liveDataFiles(dir) await fs.truncate(torn, 4) - await assert.rejects( - maintainCache({ cacheRoot, compactOnly: true }), + // The walk survives the failure and reports it per partition + // (LLP 0220), so the fixture invariant is read off the report rather + // than off a rejected tick. + const attempt = await maintainCache({ cacheRoot, compactOnly: true }) + assert.equal( + attempt.partitions[0].failed, true, 'fixture invariant: the retry must attempt a rewrite, and that rewrite must fail' ) @@ -343,8 +347,9 @@ test('a partition frozen by a failed retry is reported as skipped on every later const intact = await fs.readFile(torn) await fs.truncate(torn, 4) - await assert.rejects( - maintainCache({ cacheRoot, compactOnly: true }), + const attempt = await maintainCache({ cacheRoot, compactOnly: true }) + assert.equal( + attempt.partitions[0].failed, true, 'fixture invariant: the retry must attempt a rewrite, and that rewrite must fail' ) const record = compactionRecord(dir) @@ -409,7 +414,8 @@ test('a committed ineffective verdict outranks the failed attempt that followed const retiringDir = path.join(dir, retiring) await fs.chmod(retiringDir, 0o555) try { - await assert.rejects(maintainCache({ cacheRoot, compactOnly: true })) + const attempt = await maintainCache({ cacheRoot, compactOnly: true }) + assert.equal(attempt.partitions[0].failed, true, 'fixture invariant: the tick must fail on this partition') } finally { await fs.chmod(retiringDir, 0o755) } @@ -461,8 +467,9 @@ test('a rewrite that throws after committing its cursor keeps the generation it const retiringDir = path.join(dir, retiring) await fs.chmod(retiringDir, 0o555) try { - await assert.rejects( - maintainCache({ cacheRoot, compactOnly: true }), + const attempt = await maintainCache({ cacheRoot, compactOnly: true }) + assert.equal( + attempt.partitions[0].failed, true, 'fixture invariant: the rewrite must commit its cursor and then fail' ) diff --git a/test/core/cache-maintenance-walk-resilience.test.js b/test/core/cache-maintenance-walk-resilience.test.js new file mode 100644 index 00000000..bf8f1d9a --- /dev/null +++ b/test/core/cache-maintenance-walk-resilience.test.js @@ -0,0 +1,334 @@ +// @ts-check + +// The maintenance walk visits partitions neediest-first (LLP 0199), so the +// most fragmented partition goes first - and that is exactly the partition +// most likely to fail a rewrite. Without a per-partition catch one such +// partition took the whole tick with it: every partition behind it got +// neither compaction nor snapshot expiry, every hour, forever. These tests +// pin the walk surviving one partition's error while still reporting it. + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { MAX_FAILURES_BEFORE_BUDGET_BREAK, maintainCache } from '../../src/core/cache/maintenance.js' +import { appendRowsToSourceTable, readCursorSync, writeCursor } from '../../src/core/cache/partition.js' + +/** + * @import { ColumnSpec } from '../../hypaware-plugin-kernel-types.js' + * @import { MaintenancePartitionReport, PartitionCursor } from '../../src/core/cache/types.js' + */ + +/** @type {ColumnSpec[]} */ +const SESSION_COLUMNS = [ + { name: 'id', type: 'INT32', nullable: false }, + { name: 'session_id', type: 'STRING', nullable: false }, + { name: 'attributes', type: 'STRING', nullable: true }, +] + +/** @type {ColumnSpec[]} */ +const FLAT_COLUMNS = [ + { name: 'id', type: 'INT32', nullable: false }, + { name: 'attributes', type: 'STRING', nullable: true }, +] + +/** + * One Iceberg partition tuple per session, so ingest lands one data file + * per session and the partition is both the neediest in the cache and the + * one a torn file can break. Same fixture shape as the effectiveness + * tests: this is the production partition from #723. + */ +const SESSION_DECLARATION = { + source: { columns: ['source'] }, + iceberg: { fields: [{ column: 'session_id', transform: 'identity' }] }, +} + +/** + * @param {string} cacheRoot + * @param {number} sessions + */ +async function seedNeediestPartition(cacheRoot, sessions) { + const rows = Array.from({ length: sessions }, (_, i) => ({ + id: i, + session_id: `s-${i}`, + attributes: `{"gateway":{"session":"s-${i}"}}`, + })) + await appendRowsToSourceTable( + cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows, + { declaration: SESSION_DECLARATION } + ) +} + +/** + * A second, healthy partition holding strictly fewer live data files, so + * the neediest-first ranking always puts it behind the one that throws. + * + * @param {string} cacheRoot + * @param {number} waves + */ +async function seedHealthyPartition(cacheRoot, waves) { + for (let wave = 0; wave < waves; wave++) { + const rows = Array.from({ length: 5 }, (_, i) => ({ + id: wave * 5 + i, + attributes: `wave ${wave} row ${i}`, + })) + await appendRowsToSourceTable(cacheRoot, 'logs', ['source=claude'], FLAT_COLUMNS, rows) + } +} + +/** @param {string} cacheRoot @param {string} dataset @returns {string} */ +function partitionDir(cacheRoot, dataset) { + return path.join(cacheRoot, 'datasets', dataset, 'source=claude') +} + +/** + * Plant the stamp-less compaction record an older HypAware wrote: its + * baseline sits on the live count, so the LLP 0199 gate skips the + * partition, and it names no writer generation, so LLP 0217 owes it + * exactly one retry under the writer running now. That retry is what the + * torn file below turns into a throw. + * + * @param {string} dir + * @param {number} baselineFiles + * @returns {Promise} + */ +async function plantStamplessRecord(dir, baselineFiles) { + const cursor = readCursorSync(dir) + /** @type {PartitionCursor} */ + const next = { + ...cursor, + compaction: { + previousTableDir: 'table', + compactedAt: '2026-08-12T21:55:35.168Z', + resettleBaselineFiles: baselineFiles, + }, + } + await writeCursor(dir, next) +} + +/** + * Truncate one of the partition's live parquet files to a stub no reader + * can decode: the torn-write stand-in from the issue, which makes the + * rewrite's scan throw part-way through. + * + * @param {string} dir + * @returns {Promise} + */ +async function tearOneDataFile(dir) { + const cursor = readCursorSync(dir) + const dataDir = path.join(dir, cursor.tableDir ?? 'table', 'data') + const entries = await fs.readdir(dataDir, { withFileTypes: true }) + const [torn] = entries + .filter((e) => e.isFile() && e.name.endsWith('.parquet')) + .map((e) => path.join(dataDir, e.name)) + assert.ok(torn, 'fixture invariant: the partition must hold a live data file to tear') + await fs.truncate(torn, 4) +} + +/** @param {string} dir @returns {Record} */ +function compactionRecord(dir) { + const { compaction } = readCursorSync(dir) + assert.ok(compaction && typeof compaction === 'object', 'expected a compaction record on the cursor') + return /** @type {Record} */ (compaction) +} + +/** + * @param {{ partitions: MaintenancePartitionReport[] }} report + * @param {string} dataset + * @returns {MaintenancePartitionReport} + */ +function partitionReport(report, dataset) { + const found = report.partitions.find((p) => p.dataset === dataset) + assert.ok(found, `expected a report for ${dataset}; got ${report.partitions.map((p) => p.dataset).join(', ') || '(none)'}`) + return found +} + +/** + * The cache both tests below run over: a neediest partition rigged to + * throw on its rewrite, and a healthy partition behind it in walk order. + * + * @returns {Promise} + */ +async function seedTornAndHealthy() { + const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-walk-')) + await seedNeediestPartition(cacheRoot, 8) + await seedHealthyPartition(cacheRoot, 3) + const torn = partitionDir(cacheRoot, 'ai_gateway_messages') + await plantStamplessRecord(torn, 8) + await tearOneDataFile(torn) + return cacheRoot +} + +// @ref LLP 0220#walk-survives-a-partition [tests]: the whole point. The +// neediest partition throws, and the partition behind it is still maintained +// in the same tick rather than an hour later, or never. +test('a partition whose compaction throws does not abort the rest of the walk', async () => { + const cacheRoot = await seedTornAndHealthy() + try { + const report = await maintainCache({ cacheRoot, compactOnly: true }) + + const torn = partitionReport(report, 'ai_gateway_messages') + assert.equal(torn.compacted, false, 'fixture invariant: the neediest partition must attempt a rewrite and fail it') + assert.equal(torn.failed, true, 'the partition that threw is reported as failed') + + const healthy = partitionReport(report, 'logs') + assert.equal( + healthy.compacted, true, + 'the partition behind the failure must still be compacted in this tick' + ) + assert.equal(healthy.failed, undefined, 'the healthy partition did not fail') + assert.equal(report.totalCompacted, 1) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + +// @ref LLP 0217#retry-on-writer-change [tests]: the per-partition catch must +// not swallow the failure before the stamp is written. The attempt spends the +// writer generation's retry whether or not it succeeded, or the same partition +// is re-attempted, and fails, on every tick forever. +test('the partition that threw still spends its writer generation', async () => { + const cacheRoot = await seedTornAndHealthy() + try { + await maintainCache({ cacheRoot, compactOnly: true }) + + const dir = partitionDir(cacheRoot, 'ai_gateway_messages') + const record = compactionRecord(dir) + assert.equal(typeof record.writerGeneration, 'number', 'a spent attempt must stamp the cursor') + assert.equal(typeof record.attemptFailedAt, 'string', 'the stamp records when the attempt failed') + assert.equal(record.resettleBaselineFiles, 8, 'a failed rewrite must not move the baseline') + assert.equal(record.dataFilesBefore, undefined, 'a failed rewrite proves nothing about effectiveness') + + // @ref LLP 0218#report-the-spent-attempt [tests]: and from the next tick + // on the partition is skipped for that stated reason, with the walk now + // reporting no failure at all because nothing was attempted. + const next = await maintainCache({ cacheRoot, compactOnly: true }) + const torn = partitionReport(next, 'ai_gateway_messages') + assert.equal(torn.failed, undefined, 'nothing was attempted this tick, so nothing failed in it') + assert.equal(torn.compactionAttemptFailed, true, 'the skip states the spent attempt as its reason') + assert.equal(torn.compactionAttemptFailedAt, record.attemptFailedAt) + assert.equal(next.totalFailed, 0) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + +// @ref LLP 0220#walk-survives-a-partition [tests]: compaction is not the only +// thing a partition behind the failure was owed. Snapshot expiry is the +// unbounded-growth guard (#723's metadata directories), and it runs before +// compaction for every partition in the walk, so aborting the walk cost it too. +test('snapshot expiry still runs for the partitions behind the one that threw', async () => { + const cacheRoot = await seedTornAndHealthy() + try { + // Expire everything but the current snapshot, so the healthy partition's + // three ingest waves leave something to expire. + const report = await maintainCache({ + cacheRoot, + config: { min_snapshots_to_keep: 0, max_snapshot_age_hours: 0 }, + }) + + const healthy = partitionReport(report, 'logs') + assert.ok( + healthy.snapshotsExpired > 0, + `the partition behind the failure must still have its snapshots expired; got ${healthy.snapshotsExpired}` + ) + assert.ok(report.totalSnapshotsExpired > 0) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + +// @ref LLP 0220#tick-reports-degraded [tests]: continuing the walk must not +// cost the tick its failure signal. A tick that quietly returns a clean report +// while a partition went unmaintained is the same defect class as #723: the +// operator has nothing to read. +test('a tick that lost a partition reports the loss rather than swallowing it', async () => { + const cacheRoot = await seedTornAndHealthy() + try { + const report = await maintainCache({ cacheRoot, compactOnly: true }) + + assert.equal(report.totalFailed, 1, 'the tick counts the partition it lost') + const torn = partitionReport(report, 'ai_gateway_messages') + assert.equal(torn.failed, true) + assert.equal(typeof torn.errorKind, 'string') + assert.ok( + typeof torn.errorMessage === 'string' && torn.errorMessage.length > 0, + 'the report carries the error that ended the partition, not just the fact of one' + ) + // The failure is this tick's, and it is not the cursor-read skip reason + // LLP 0218 defined: an operator must be able to tell "this partition threw + // just now" from "this partition has been skipped since an earlier throw". + assert.equal(torn.compactionAttemptFailed, undefined) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + +// --- the budget guard must not let a failed partition stand in for real work (round-1 review finding 1) --- + +// @ref LLP 0220#walk-survives-a-partition [tests]: a partition that failed +// did no work, so it must not be the one that satisfies the budget guard's +// "always work one partition before the budget can cut the tick short" +// guarantee (src/core/cache/maintenance.js). Before this fix, `reports.length +// > 0` was true the moment the torn partition's failure was recorded, so a +// zero budget broke the walk right there and the healthy partition behind it +// was never visited - #737 by another route, reached without any per-partition +// catch missing at all. +test('a zero budget does not let a failed first partition starve the healthy partition behind it', async () => { + const cacheRoot = await seedTornAndHealthy() + try { + const report = await maintainCache({ cacheRoot, compactOnly: true, budgetMs: 0 }) + + const torn = partitionReport(report, 'ai_gateway_messages') + assert.equal(torn.failed, true, 'fixture invariant: the neediest partition still throws') + + const healthy = partitionReport(report, 'logs') + assert.equal( + healthy.compacted, true, + 'a zero budget must not let the failed first partition stand in for the "one partition maintained" guarantee' + ) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + +// @ref LLP 0220#walk-survives-a-partition [tests]: the other half of the +// budget-guard fix. Gating the break on "a partition was actually +// maintained" alone would let a cache where every partition fails walk past +// its budget without bound - trading one starvation bug for an unbounded +// one. `MAX_FAILURES_BEFORE_BUDGET_BREAK` caps it; pinned here with more +// failing partitions seeded than the cap allows through. +test('an all-failing cache does not walk past its budget unbounded: it stops at the failure cap', async () => { + const total = MAX_FAILURES_BEFORE_BUDGET_BREAK + 2 + const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-all-failing-')) + try { + for (let i = 0; i < total; i++) { + const dataset = `walk-budget-bad-${i}` + const rows = Array.from({ length: 4 }, (_, j) => ({ + id: j, + session_id: `s-${j}`, + attributes: `{"gateway":{"session":"s-${j}"}}`, + })) + await appendRowsToSourceTable( + cacheRoot, dataset, ['source=claude'], SESSION_COLUMNS, rows, + { declaration: SESSION_DECLARATION } + ) + const dir = partitionDir(cacheRoot, dataset) + await plantStamplessRecord(dir, 4) + await tearOneDataFile(dir) + } + + const report = await maintainCache({ cacheRoot, compactOnly: true, budgetMs: 0 }) + + assert.equal( + report.totalFailed, MAX_FAILURES_BEFORE_BUDGET_BREAK, + `the walk must stop once ${MAX_FAILURES_BEFORE_BUDGET_BREAK} partitions have failed, not after all ${total}` + ) + assert.equal(report.partitions.length, MAX_FAILURES_BEFORE_BUDGET_BREAK) + assert.ok(report.partitions.every((p) => p.failed), 'every partition visited in this fixture fails') + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) diff --git a/test/core/cache-retention-maintenance.test.js b/test/core/cache-retention-maintenance.test.js index 41aac363..5af6cb38 100644 --- a/test/core/cache-retention-maintenance.test.js +++ b/test/core/cache-retention-maintenance.test.js @@ -926,6 +926,58 @@ test('a foreign sorted replace re-baselines the cursor instead of being rewritte } }) +// @ref LLP 0220#walk-survives-a-partition [tests]: a rebaseline is a cursor +// write with no rewrite behind it, and `writeCursor` can throw same as any +// other write in this loop. `r.rebaselined` must not be set until that write +// has actually landed, or a tick that lost this partition would still print +// "rebaselined" and count it in `totalRebaselined` for a cursor that never +// changed (round-1 review finding 2). +test('a rebaseline that fails to persist is not reported as rebaselined, and is not counted', async () => { + const cacheRoot = await makeTmpDir('maint-foreign-replace-write-fail') + try { + const partDir = path.join(cacheRoot, 'datasets', 'ds1', 'date=2026-08-08') + const epoch0 = path.join(partDir, 'epoch=0') + for (let i = 0; i < 3; i++) { + await appendRowsToTable(epoch0, COLUMNS, [ + { id: i, value: `v${i}`, timestamp: new Date().toISOString() }, + ], { sortOrder: [{ column: 'id', direction: 'asc' }] }) + } + await commitForeignReplace(epoch0) + // Same stale-baseline cursor as the passing recognition test: this is + // the fixture that would otherwise re-baseline cleanly. + await writeCursor(partDir, { + epoch: 0, + rowCount: 3, + layout: 'epoch', + compaction: { compactedAt: '2026-08-08T00:00:00.000Z', resettleBaselineFiles: 99 }, + }) + + // `writeCursor` writes `cursor.json` directly into the partition dir, + // so making the dir read-only fails exactly that write and nothing + // upstream of it (the read of the epoch dir's own metadata is + // unaffected). + await fs.chmod(partDir, 0o555) + try { + const report = await maintainCache({ cacheRoot, compactOnly: true }) + assert.equal(report.partitions[0].failed, true, 'fixture invariant: the cursor write must fail') + assert.notEqual( + report.partitions[0].rebaselined, true, + 'a rebaseline that never landed on disk must not be reported as one' + ) + assert.equal(report.totalRebaselined, 0, 'an unpersisted rebaseline must not be counted') + } finally { + await fs.chmod(partDir, 0o755) + } + + // The cursor on disk still carries the pre-rebaseline baseline: the + // write failure must not have partially landed. + const compaction = /** @type {{ resettleBaselineFiles: number }} */ (readCursorSync(partDir).compaction) + assert.equal(compaction.resettleBaselineFiles, 99) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + test('foreign sorted replace recognition works on the source-table layout', async () => { const cacheRoot = await makeTmpDir('maint-foreign-source') try { diff --git a/test/core/daemon-maintenance-tick-status.test.js b/test/core/daemon-maintenance-tick-status.test.js new file mode 100644 index 00000000..b23e9b8c --- /dev/null +++ b/test/core/daemon-maintenance-tick-status.test.js @@ -0,0 +1,158 @@ +// @ts-check + +// @ref LLP 0220#tick-reports-degraded [tests]: `withSpan` derives the span's +// status code from a `status` attribute snapshot taken before the callback +// runs, and always calls `setStatus` after the callback resolves - so a +// `span.setAttribute('status', 'degraded')` made once the report is in hand +// (this tick's own verdict) never reached the span's OTel status code; the +// post-hoc `setStatus(OK)` clobbered it. `runtime.js`'s maintenance tick was +// rewritten to manage its span directly instead of through `withSpan` so the +// real verdict wins. This drives a real daemon through one degraded tick and +// reads the exported span back off disk (round-1 review finding 4). + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { runDaemon } from '../../src/core/daemon/runtime.js' +import { defaultConfigPath } from '../../src/core/config/schema.js' +import { appendRowsToSourceTable, readCursorSync, writeCursor } from '../../src/core/cache/partition.js' + +/** @import { ColumnSpec } from '../../hypaware-plugin-kernel-types.js' */ +/** @import { PartitionCursor } from '../../src/core/cache/types.js' */ + +/** @type {ColumnSpec[]} */ +const SESSION_COLUMNS = [ + { name: 'id', type: 'INT32', nullable: false }, + { name: 'session_id', type: 'STRING', nullable: false }, + { name: 'attributes', type: 'STRING', nullable: true }, +] + +const SESSION_DECLARATION = { + source: { columns: ['source'] }, + iceberg: { fields: [{ column: 'session_id', transform: 'identity' }] }, +} + +/** @param {string} dir @param {number} baselineFiles */ +async function plantStamplessRecord(dir, baselineFiles) { + const cursor = readCursorSync(dir) + /** @type {PartitionCursor} */ + const next = { + ...cursor, + compaction: { + previousTableDir: 'table', + compactedAt: '2026-08-12T21:55:35.168Z', + resettleBaselineFiles: baselineFiles, + }, + } + await writeCursor(dir, next) +} + +/** @param {string} dir */ +async function tearOneDataFile(dir) { + const cursor = readCursorSync(dir) + const dataDir = path.join(dir, cursor.tableDir ?? 'table', 'data') + const entries = await fs.readdir(dataDir, { withFileTypes: true }) + const [torn] = entries + .filter((e) => e.isFile() && e.name.endsWith('.parquet')) + .map((e) => path.join(dataDir, e.name)) + assert.ok(torn, 'fixture invariant: the partition must hold a live data file to tear') + await fs.truncate(torn, 4) +} + +/** + * Poll a JSONL file for a line matching `predicate`, since the tracer + * exporter writes to an `fs.WriteStream` with no flush hook this daemon + * ever calls: the write lands async relative to the tick's own promise + * resolving. + * + * @param {string} filePath + * @param {(record: any) => boolean} predicate + * @param {number} timeoutMs + * @returns {Promise} + */ +async function pollJsonlFor(filePath, predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs + for (;;) { + try { + const raw = await fs.readFile(filePath, 'utf8') + for (const line of raw.split('\n')) { + if (!line) continue + const record = JSON.parse(line) + if (predicate(record)) return record + } + } catch (err) { + if (/** @type {NodeJS.ErrnoException} */ (err).code !== 'ENOENT') throw err + } + if (Date.now() > deadline) return undefined + await new Promise((resolve) => setTimeout(resolve, 25)) + } +} + +test('a degraded maintenance tick sets the span status code, not just the attribute', async () => { + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hypaware-daemon-maint-status-')) + const savedHypHome = process.env.HYP_HOME + const savedDevTelemetry = process.env.HYP_DEV_TELEMETRY + let handle + try { + const cacheRoot = path.join(hypHome, 'hypaware', 'cache') + const rows = Array.from({ length: 4 }, (_, i) => ({ + id: i, + session_id: `s-${i}`, + attributes: `{"gateway":{"session":"s-${i}"}}`, + })) + await appendRowsToSourceTable( + cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows, + { declaration: SESSION_DECLARATION } + ) + const partDir = path.join(cacheRoot, 'datasets', 'ai_gateway_messages', 'source=claude') + await plantStamplessRecord(partDir, 4) + await tearOneDataFile(partDir) + + const configPath = defaultConfigPath(hypHome) + await fs.mkdir(path.dirname(configPath), { recursive: true }) + await fs.writeFile(configPath, JSON.stringify({ + version: 2, + query: { cache: { maintenance: { interval_minutes: 0.001 } } }, + })) + + // `installObservability()` (called inside `runDaemon`) reads real + // `process.env`, not the `env` option below - the same mechanism the + // hermetic smokes use (CLAUDE.md's "temp HYP_HOME and + // HYP_DEV_TELEMETRY=1"), so the JSONL span exporter has to be armed here. + process.env.HYP_HOME = hypHome + process.env.HYP_DEV_TELEMETRY = '1' + + handle = await runDaemon({ + hypHome, + configPath, + env: { ...process.env, HYP_HOME: hypHome }, + runId: 'maint-status-test', + tickIntervalMs: 0, + installSignalHandlers: false, + }) + + const tracesPath = path.join(hypHome, 'hypaware', 'dev-telemetry', `traces-${process.pid}.jsonl`) + const span = await pollJsonlFor(tracesPath, (r) => r.name === 'maintenance.tick', 5_000) + + assert.ok(span, 'the maintenance.tick span must be exported within the poll window') + assert.equal(span.attributes.status, 'degraded', 'sanity: the attribute this tick set') + assert.equal( + span.status, 'failed', + 'the span status CODE (not just the attribute) must reflect the degraded tick - ' + + 'a bare setAttribute inside withSpan would leave this "ok"' + ) + } finally { + if (handle) { + await handle.stop() + await handle.done + } + if (savedHypHome === undefined) delete process.env.HYP_HOME + else process.env.HYP_HOME = savedHypHome + if (savedDevTelemetry === undefined) delete process.env.HYP_DEV_TELEMETRY + else process.env.HYP_DEV_TELEMETRY = savedDevTelemetry + await fs.rm(hypHome, { recursive: true, force: true }) + } +}) diff --git a/test/core/query-maintain-cli.test.js b/test/core/query-maintain-cli.test.js new file mode 100644 index 00000000..2bbcbaa9 --- /dev/null +++ b/test/core/query-maintain-cli.test.js @@ -0,0 +1,146 @@ +// @ts-check + +// @ref LLP 0220#tick-reports-degraded [tests]: `maintainCache` stopped +// rejecting when a partition throws (it reports the failure instead), so a +// caller that only captures stderr - a cron wrapper, `>/dev/null` - needs a +// line there on a degraded tick, or it never learns the walk lost a +// partition. These pin that the line appears exactly when it should +// (round-1 review finding 3). + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { runQueryMaintain } from '../../src/core/commands/query.js' +import { appendRowsToSourceTable, readCursorSync, writeCursor } from '../../src/core/cache/partition.js' + +/** @import { ColumnSpec } from '../../hypaware-plugin-kernel-types.js' */ +/** @import { PartitionCursor } from '../../src/core/cache/types.js' */ + +/** @type {ColumnSpec[]} */ +const SESSION_COLUMNS = [ + { name: 'id', type: 'INT32', nullable: false }, + { name: 'session_id', type: 'STRING', nullable: false }, + { name: 'attributes', type: 'STRING', nullable: true }, +] + +const SESSION_DECLARATION = { + source: { columns: ['source'] }, + iceberg: { fields: [{ column: 'session_id', transform: 'identity' }] }, +} + +function makeBuf() { + let value = '' + return { + /** @param {string} chunk */ + write(chunk) { value += String(chunk); return true }, + text() { return value }, + } +} + +/** + * A minimal `CommandRunContext` stand-in: `runQueryMaintain` only reaches + * `ctx.storage.cacheRoot`, `ctx.config?.query?.cache?.maintenance`, + * `ctx.query.getDataset(...)?.resettleBatch`, and the two stream sinks. + * + * @param {string} cacheRoot + */ +function ctxFor(cacheRoot) { + const stdout = makeBuf() + const stderr = makeBuf() + return { + stdout, + stderr, + ctx: /** @type {any} */ ({ + stdout, + stderr, + storage: { cacheRoot }, + config: { version: 2 }, + query: { getDataset: () => null }, + env: {}, + cwd: '/w/project', + }), + } +} + +/** @param {string} dir @param {number} baselineFiles */ +async function plantStamplessRecord(dir, baselineFiles) { + const cursor = readCursorSync(dir) + /** @type {PartitionCursor} */ + const next = { + ...cursor, + compaction: { + previousTableDir: 'table', + compactedAt: '2026-08-12T21:55:35.168Z', + resettleBaselineFiles: baselineFiles, + }, + } + await writeCursor(dir, next) +} + +/** @param {string} dir */ +async function tearOneDataFile(dir) { + const cursor = readCursorSync(dir) + const dataDir = path.join(dir, cursor.tableDir ?? 'table', 'data') + const entries = await fs.readdir(dataDir, { withFileTypes: true }) + const [torn] = entries + .filter((e) => e.isFile() && e.name.endsWith('.parquet')) + .map((e) => path.join(dataDir, e.name)) + assert.ok(torn, 'fixture invariant: the partition must hold a live data file to tear') + await fs.truncate(torn, 4) +} + +test('hyp query maintain writes a stderr line when the walk loses a partition', async () => { + const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-cli-degraded-')) + try { + const rows = Array.from({ length: 4 }, (_, i) => ({ + id: i, + session_id: `s-${i}`, + attributes: `{"gateway":{"session":"s-${i}"}}`, + })) + await appendRowsToSourceTable( + cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows, + { declaration: SESSION_DECLARATION } + ) + const dir = path.join(cacheRoot, 'datasets', 'ai_gateway_messages', 'source=claude') + await plantStamplessRecord(dir, 4) + await tearOneDataFile(dir) + + const { stderr, ctx } = ctxFor(cacheRoot) + const exitCode = await runQueryMaintain(['--compact-only'], ctx) + + assert.equal(exitCode, 1, 'a degraded tick must still exit non-zero') + assert.match( + stderr.text(), + /^hyp query maintain: 1 partition\(s\) failed; the walk continued$/m, + 'a caller that only captures stderr must see the walk lost a partition' + ) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +}) + +test('hyp query maintain writes nothing to stderr on a clean tick', async () => { + const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-maintain-cli-clean-')) + try { + const rows = Array.from({ length: 4 }, (_, i) => ({ + id: i, + session_id: `s-${i}`, + attributes: `{"gateway":{"session":"s-${i}"}}`, + })) + await appendRowsToSourceTable( + cacheRoot, 'ai_gateway_messages', ['source=claude'], SESSION_COLUMNS, rows, + { declaration: SESSION_DECLARATION } + ) + + const { stderr, ctx } = ctxFor(cacheRoot) + const exitCode = await runQueryMaintain(['--compact-only'], ctx) + + assert.equal(exitCode, 0, 'a clean tick must exit zero') + assert.equal(stderr.text(), '', 'a clean tick must not write anything to stderr') + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +})