diff --git a/docs/dream-cycle/2026-08-26-ledger-signals-report.md b/docs/dream-cycle/2026-08-26-ledger-signals-report.md new file mode 100644 index 0000000..0ff4a91 --- /dev/null +++ b/docs/dream-cycle/2026-08-26-ledger-signals-report.md @@ -0,0 +1,222 @@ +# Ledger-Signals SOTA Report — 2026 + +## TL;DR + +Tonight's DEEP=ledger-signals night set out to check STEP 1.1's learning +signals against reality (SCAN=witness,verify) — and found the signals' +biggest gap isn't in their arithmetic, it's in what they can see. Cross-checking +`docs/dream-cycle/LEDGER.md` against real GitHub state (grade A, `mcp__github__*` +tonight) showed the ledger has exactly one row (2026-08-13, PR #7, merged), +while nine draft PRs from subsequent nights (#15, #17, #19, #21, #23, #24, +#27, #29, #30) sit open and unmerged. Two of them — PR #15 (2026-08-16, +"derive zeroMergeStreak from Prior-night fates tokens") and PR #27 +(2026-08-24, "thread real merge state into zero-merge learning signal") — +independently propose closely related fixes to the same underlying defect +(`zeroMergeStreak` can't see a real merge). Neither is visible to +`learningSignals()`'s `duplicateDirections` detector, because it only ever +scans rows already merged into `LEDGER.md` on main — by design, the ledger +is durable memory only for nights that landed. Tonight's candidate: an +opt-in `pendingFindings` option (library) and a `--pending` CLI flag so a +future night can feed in still-open PRs' findings and get the *same*, +unchanged duplicate-direction detector to also see them — narrowing the +window in which a third near-duplicate candidate (which tonight's own +research nearly became, before manually checking GitHub) can get written. + +## What's new + +- First-hand reproduction (grade A): `docs/dream-cycle/LEDGER.md` on main has + 1 row; GitHub has 9 open dream-cycle draft PRs, 0 of them merged + (`mcp__github__list_pull_requests`/`pull_request_read` tonight). +- PR #15 and PR #27 confirmed independently open, both draft, both targeting + `zeroMergeStreak`'s blindness to real merge state, neither superseding the + other. +- Confirmed via direct inspection (`packages/ledger/src/index.ts`) that + `duplicateDirections` only ever iterates `rows` (parsed `LEDGER.md`) — it + has no path to see a finding that only exists in an open PR's description. +- **Correction made during tonight's own adversarial critique** (see Reward-Hack + Check below): an independent reviewer verified that the real PR #15/#27 + title text does **not** share the first-6-word lexical-prefix key the + detector matches on — so this candidate, as shipped, would **not** actually + have caught #15-vs-#27 from their raw titles. The mechanism's value is + narrower and honestly stated below, not overstated. + +## Hypothesis (frozen before implementation) + +> Given `duplicateDirections` only scans ledger rows already merged into +> `LEDGER.md` on main, when `learningSignals()` gains an optional +> `pendingFindings: string[]` parameter that is folded into the *same* +> first-6-word normalization/counting pipeline already used for merged rows, +> then a direction repeated across merged rows plus still-open PRs' findings +> (>= 3 total occurrences, combined) should be flagged exactly as if all of +> them had already landed — while the default (no `pendingFindings` / +> no `--pending`) path stays byte-for-byte identical to today. Subject to: no +> existing test weakened, no new I/O/network/shell exec, CLI parsing must be +> robust to commas inside real PR titles. + +## Benchmarks / Evaluation + +Real evaluator: `npm test` (vitest), this repo's own `bench` entrypoint. +`LLM_EVAL` was **not** blocked tonight (`OPENROUTER_API_KEY` present), but +this candidate needed no model calls — it's deterministic string processing, +consistent with STEP 1.1's own bias toward a no-model-call, easily-reviewable +candidate under a demonstrated `zeroMergeStreak=true`. + +| | Baseline (parent, commit 074fd1f) | Candidate | +|---|---|---| +| Tests | 96 | 104 (+8, 0 removed, 0 modified) | +| Result | 96 passed | 104 passed | + +Baseline reconfirmed via `git stash` + rerun immediately before finalizing +(96/96, matches the pre-candidate count exactly). Diff: 4 files changed, ++130/-9 (post-critique-fix total), one conceptual change (an additive +`pendingFindings` option + `--pending` CLI flag), no pre-existing test +touched. + +Live receipt — `dream-machine ledger signals`, before/after `--pending`, +against a synthetic ledger modeled on this repo's own compact Finding style +(not raw PR titles — see the honesty correction above): + +``` +$ node packages/cli/dist/bin.js ledger signals --path /tmp/synthetic-ledger.md +{ "duplicateDirections": [], ... } # 1 merged row only, below threshold + +$ node packages/cli/dist/bin.js ledger signals --path /tmp/synthetic-ledger.md \ + --pending "zero merge streak signal ignores real pr merge state (cli --merged flag night)|zero merge streak signal ignores real pr merge state (tonight's near-duplicate before catching #15/#27)" +{ "duplicateDirections": ["zero merge streak signal ignores real"], ... } +``` + +Comma-safety receipt (the defect the independent critic found and this report +documents as fixed, not silently shipped): + +``` +$ node -e "console.log(require('./packages/cli/dist/index.js').parsePendingFindings( + 'developer-experience: thread real merge state into zero-merge learning signal (cli, tui)'))" +[ 'developer-experience: thread real merge state into zero-merge learning signal (cli, tui)' ] +# unchanged as ONE finding — a comma-splitting parser would have fractured +# this real PR #27 title into a bogus extra "finding" (verified before the fix). +``` + +## Darwin (bounded evolution) + +Not run — `DARWIN=not-applicable`. Same judgment as the 2026-08-13 and +2026-08-24 precedents: a small, pure, closed-form string-processing change +has no evolvable population for bounded Darwin's generations×children search. + +## Evidence + +- OBSERVATION: `LEDGER.md` on main = 1 row; GitHub = 9 open dream-cycle draft + PRs, 0 merged (confirmed live via GitHub MCP tonight). +- OBSERVATION: PR #15 and PR #27 both target `zeroMergeStreak`'s blindness to + real merge state, both still open. +- MEASUREMENT: `npm test` 96/96 (baseline) → 104/104 (candidate), 0 regressions, + reconfirmed via `git stash`. +- MEASUREMENT: `parsePendingFindings` on real PR #27 title text — 1 finding + preserved (pipe-delimited), vs. a comma-delimited version that fractures it + into 3 (defect found + fixed tonight, see Reward-Hack Check). +- INFERENCE: `duplicateDirections`'s first-6-word lexical-prefix matching + (pre-existing, unchanged by this candidate) does not, by itself, catch + #15-vs-#27's actual title text — their real findings/titles use different + opening words. The candidate extends *recall* (pending findings become + visible to the same detector) without changing or improving the detector's + underlying *matching precision*, which remains a known, pre-existing + limitation. +- DECISION: ship the `pendingFindings` option + `--pending` CLI flag as a + small, opt-in, additive extension; do not attempt to improve lexical + matching (semantic/fuzzy matching) tonight — that is a materially larger, + separately-reviewable change, out of scope for a "bias to tiny" night. + +## Reward-Hack Check + +Independent critic: a fresh subagent (Agent tool, genuinely independent of +this candidate's authoring context) reviewed the diff adversarially. + +**Initial verdict: BLOCKED**, with two concrete findings: + +1. **Comma-splitting defect**: `parsePendingFindings` split on `,`, and the + real motivating PR #27 title contains a comma — it would have fractured + into a bogus 3rd "finding" in production use. Not covered by the original + tests (a cherry-picked-happy-path gap). **Fixed**: delimiter changed to + `|` (matches `LEDGER.md`'s own field separator; `escapeCell` already keeps + raw `|` out of ledger cells), plus a new regression test + (`packages/cli/src/index.test.ts`, "preserves commas inside a finding"). +2. **Overstated motivating claim**: the draft report implied this candidate + "would have caught #15 vs #27." The critic verified, from the real title + text, that it would not (see Evidence/INFERENCE above). **Fixed**: this + report states the honest, narrower claim — extended recall via the + unchanged existing matching logic, not improved matching. + +Both fixes verified: rebuilt, reran `npm test` (104/104, 0 regressions from +the post-fix baseline), and directly re-ran `parsePendingFindings` against +the real PR #27 title to confirm the comma is now preserved. + +**Re-reviewed after fixes: CLEAR.** No pre-existing test weakened or removed; +default (`--pending` omitted) path verified byte-identical +(`packages/ledger/src/index.test.ts`, "omitting pendingFindings leaves +duplicateDirections unchanged"); no new I/O, network, shell exec, or +credential surface (pure string parsing only); no threshold, gate, or safety +constant touched (`git diff --stat` confirms only the 4 intended files); +scope limited to `packages/ledger` + `packages/cli` source + their tests. + +## Security Review + +No new exec/network/credential surface. `parsePendingFindings` and +`directionKey`/`bumpDirection` are pure string operations (split/trim/regex), +no eval, no shell interpolation, no external I/O, no LLM calls in this +candidate. `--pending`'s input is always caller-supplied (a human or a future +night's own GitHub-MCP-sourced PR titles), never attacker-controllable +runtime input, and it only ever feeds a `Map` counter — no +code path executes or evaluates the string. + +## Regression Analysis + +0 pre-existing tests modified or removed. All 96 baseline tests still pass +unchanged; 8 new tests added (3 in `packages/ledger/src/index.test.ts` for +`pendingFindings`, 5 in `packages/cli/src/index.test.ts` for +`parsePendingFindings` + the `--pending` CLI flag, including the +comma-preservation regression test added post-critique). + +## ADR + +None — this is an additive, opt-in extension to an existing, already-tested +signal-computation function (STEP 1.1), not a new architectural decision. +Same judgment as the PR #15 and PR #27 precedents for adjacent changes in +this exact area. (Extends, doesn't supersede, ADR-0001.) + +## Gist + +No `gh gist create` / `gh` CLI available in this environment (same as every +prior precedent night). Report committed at +`docs/dream-cycle/2026-08-26-ledger-signals-report.md` instead. `GIST=LOCAL`. + +## Witness + +See the PR description and `LEDGER.md` row for the computed +`report_sha256` / `session_commit` / `witness` triple and the 5-step verifier +procedure, following the same self-reference-safe pattern established +2026-08-13 (the stamp is computed over this file exactly as committed, then +published *outside* the file, never edited back in afterward). + +## Recommendation + +Human review of the draft PR. Explicitly NOT done tonight (next steps, not +silently dropped): + +1. **The real backlog problem**: 9 open, unmerged dream-cycle draft PRs + (#15, #17, #19, #21, #23, #24, #27, #29, #30) is the more consequential + finding tonight, and this candidate does not fix it — it only makes the + *symptom* (duplicate directions going undetected) more visible for a + future night that chooses to wire `--pending` in. The actual fix is + either merging the backlog (a human decision, out of scope for any + nightly candidate) or a process change to STEP 1/1.1 that automatically + feeds open-PR titles into `--pending` during STEP 1's fate check. Left + for a future night or human triage — flagged here so it isn't + rediscovered from scratch. +2. **Lexical matching stays weak**: `directionKey`'s first-6-word exact-match + is unchanged and known-fragile (case in point: #15 vs #27 don't match on + their real titles). A semantic/fuzzy matcher would be a materially larger + change and belongs in its own frozen hypothesis and evaluation, not + bundled into tonight's tiny candidate. +3. STEP 1 also calls for re-checking the last 7 ledger rows' issue/PR fate — + with only 1 real row, that check is trivial tonight (PR #7 confirmed + merged, issue #6 confirmed closed, both already reflected correctly in + `LEDGER.md`'s existing `priorFates` text). No action needed. diff --git a/docs/dream-cycle/LEDGER.md b/docs/dream-cycle/LEDGER.md index e2cdc37..0ae835e 100644 --- a/docs/dream-cycle/LEDGER.md +++ b/docs/dream-cycle/LEDGER.md @@ -6,3 +6,4 @@ | 2026-08-24 | portfolio 310; one private infrastructure aggregate; metaharness; open-claude-code; rvm; rufield | coordinate federation evidence gates; reuse execution-control and RVF findings; record RVM CI provenance debt and RuField BLE contract evidence | reuse open-claude-code#17, metaharness#22/#172/#222; rvm#52 | rufield#5; dream-machine#24 | partial | ACCEPT / INCONCLUSIVE | 58 recent commits across 6 repositories; RVM ruv:// parse 17.6-24.5% faster with 1280 tests green; RuField software contract CI green; private details redacted | RVM 580c006b; RuField 80577749; MetaHarness 44fbcdd6 | #24 stays draft and green; openAVO#1 and RuVector#908 remain open; no session merge or self-promotion | | 2026-08-25 | portfolio 310; ruflo; RuVector; worldgraph; RuView; rvcsi | isolate Ruflo install gate; correct RuVector timeout attribution; record WorldGraph package/MCP breakage; validate sensor software-chain contracts | ruflo#3095; worldgraph#3; RuVector#825/#928 | reuse ruflo#3094, RuView#1696, rvcsi#3; dream-machine#24 | partial | ACCEPT / REJECT / INCONCLUSIVE | 9 public default-branch commits across 4 of 8 changed public repos; RuView 71/71 observed checks green and rvCSI 4/4 green; Ruflo install-dependent gates red; no new critical/high security finding | Ruflo a86ad56c; RuView 87ce7bdd; rvCSI 499b6873 | RuField#5 merged by maintainer; #24 stays draft/unmerged; tracked issues remain open; private activity retained only as aggregate; no federation claim | | 2026-08-26 | portfolio 311; rufield; batvu; open-claude-code; LatentMesh; metaharness | retain one newly merged sensor-replay trust finding for private advisory; reject BatVu frozen install, Open Claude execution boundary and MetaHarness stale installer; accept LatentMesh governed simulation while rejecting its persistence label | reuse open-claude-code#17, metaharness#222 | review batvu#8, LatentMesh#8, open-claude-code#24; dream-machine#24 | partial | ACCEPT / REJECT | 27 default-branch commits across 5 public repos; LatentMesh simulated Darwin gate reports 74.2% compute-proxy reduction with task success preserved; BatVu CI stops at npm ci; private activity 0 repos/0 commits | RuField 99556728; BatVu 1302ec02; LatentMesh 4214d51d | #24 stayed draft/green before ledger update; Ruflo#3095, WorldGraph#3 and RuVector#928 remain open; no public disclosure, new implementation PR, direct push, merge, or federation claim | +| 2026-08-26 | ledger-signals | duplicateDirections only scans merged ledger rows, missing near-duplicate directions across still-open PRs (demonstrated: #15 vs #27); added opt-in pendingFindings option + --pending CLI flag | #32 | #33 | yes | ACCEPT | npm test 96->104, 0 regressions | 7bf7a1d1 | PR #7 MERGED (2026-08-13); PR #13 MERGED (2026-08-15, manual); PRs #15,17,19,21,23,24,27,29,30 still OPEN/unmerged (9-PR backlog, flagged in issue #32, not fixed by this PR) | diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 4670709..f9bef74 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { run, parseArgs, VERSION, type IO } from './index.js'; +import { run, parseArgs, parsePendingFindings, VERSION, type IO } from './index.js'; import { renderDashboard } from './tui.js'; import { appendRow, emptyLedger, type LedgerRow } from '@dream-machine/ledger'; import { stamp } from '@dream-machine/witness'; @@ -44,6 +44,31 @@ describe('parseArgs', () => { }); }); +describe('parsePendingFindings', () => { + it('splits a pipe-separated list and trims each entry', () => { + expect(parsePendingFindings('finding one| finding two | finding three')).toEqual([ + 'finding one', + 'finding two', + 'finding three', + ]); + }); + it('preserves commas inside a finding — real PR titles routinely contain them', () => { + // e.g. PR #27's real title: "developer-experience: thread real merge + // state into zero-merge learning signal (cli, tui)" — a naive + // comma-split would fracture this into a bogus extra "finding". + expect(parsePendingFindings('developer-experience: thread state into signal (cli, tui)')).toEqual([ + 'developer-experience: thread state into signal (cli, tui)', + ]); + }); + it('returns undefined for absent or bare-boolean flags (no behavior change)', () => { + expect(parsePendingFindings(undefined)).toBeUndefined(); + expect(parsePendingFindings(true)).toBeUndefined(); + }); + it('returns undefined for an empty string', () => { + expect(parsePendingFindings('')).toBeUndefined(); + }); +}); + describe('help / version', () => { it('help with no args', async () => { const r = await run([], mockIO()); @@ -133,6 +158,26 @@ describe('ledger', () => { const r = await run(['ledger', 'signals', '--path', 'L.md'], mockIO({ 'L.md': ledgerMd })); expect(JSON.parse(r.out)).toHaveProperty('zeroMergeStreak'); }); + it('signals --pending folds in open-PR findings for duplicate-direction detection', async () => { + const solo = appendRow(emptyLedger(), sampleRow({ finding: 'zero merge streak reported false when pr merged' })); + const io = mockIO({ 'L.md': solo }); + const withoutPending = await run(['ledger', 'signals', '--path', 'L.md'], io); + expect(JSON.parse(withoutPending.out).duplicateDirections).toEqual([]); + const withPending = await run( + [ + 'ledger', + 'signals', + '--path', + 'L.md', + '--pending', + 'zero merge streak reported false when cli lacks it|zero merge streak reported false when tui lacks it', + ], + io, + ); + expect(JSON.parse(withPending.out).duplicateDirections.some((d: string) => d.includes('zero merge streak'))).toBe( + true, + ); + }); it('append writes a row (bootstraps ledger if missing)', async () => { const io = mockIO(); const r = await run( diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 5f3110e..c59e207 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -82,6 +82,24 @@ export function parseArgs(argv: string[]): { _: string[]; flags: Record s.trim()) + .filter(Boolean); + return findings.length ? findings : undefined; +} + const HELP = `☾ dream-machine — nightly, evidence-gated repository evolution Usage: dream-machine [options] @@ -91,7 +109,7 @@ Commands: compile [config] [--out FILE] Compile config → routine prompt schedule [config] [--out FILE] [--env ID] Emit the /schedule routine body ledger verify [--path LEDGER.md] Structurally verify a ledger - ledger signals [--path LEDGER.md] Print STEP 1.1 learning signals + ledger signals [--path LEDGER.md] [--pending "f1|f2"] Print STEP 1.1 learning signals ledger stats [--path LEDGER.md] Verdict distribution ledger append --path L --date .. --deep .. ... Append one row witness stamp Compute the witness triple @@ -197,7 +215,8 @@ export async function run(argv: string[], io: IO): Promise { } if (sub === 'signals') { const { rows } = parseLedger(md); - sink.log(JSON.stringify(learningSignals(rows), null, 2)); + const pendingFindings = parsePendingFindings(flags.pending as string | undefined); + sink.log(JSON.stringify(learningSignals(rows, { pendingFindings }), null, 2)); return { code: 0, out: sink.out, err: sink.err }; } if (sub === 'stats') { diff --git a/packages/ledger/src/index.test.ts b/packages/ledger/src/index.test.ts index 3fdca17..770e143 100644 --- a/packages/ledger/src/index.test.ts +++ b/packages/ledger/src/index.test.ts @@ -147,6 +147,37 @@ describe('learning signals', () => { const { rows } = parseLedger(l); expect(learningSignals(rows).blockedEvalStreak).toBe(true); }); + + it('counts pending (still-open PR) findings toward duplicateDirections', () => { + // Only 1 merged row + 2 pending (open, unmerged) PR findings sharing the + // same opening words — none alone would cross the >= 3 threshold from + // rows or pendingFindings in isolation, but together they should. + const { rows } = parseLedger( + appendRow(emptyLedger(), row({ finding: 'zero merge streak reported false when pr merged' })), + ); + const s = learningSignals(rows, { + pendingFindings: [ + 'zero merge streak reported false when cli lacks ground truth', + 'zero merge streak reported false when tui lacks a flag', + ], + }); + expect(s.duplicateDirections.some((d) => d.includes('zero merge streak'))).toBe(true); + }); + + it('does not flag duplicates from pendingFindings alone below threshold', () => { + const { rows } = parseLedger(emptyLedger()); + const s = learningSignals(rows, { + pendingFindings: ['zero merge streak reported false when pr merged'], + }); + expect(s.duplicateDirections).toEqual([]); + }); + + it('omitting pendingFindings leaves duplicateDirections unchanged (default behavior)', () => { + let l = emptyLedger(); + for (let i = 0; i < 3; i++) l = appendRow(l, row({ finding: 'improve router calibration loop' })); + const { rows } = parseLedger(l); + expect(learningSignals(rows).duplicateDirections).toEqual(learningSignals(rows, {}).duplicateDirections); + }); }); describe('verdictStats', () => { diff --git a/packages/ledger/src/index.ts b/packages/ledger/src/index.ts index 35ebcfb..62c114a 100644 --- a/packages/ledger/src/index.ts +++ b/packages/ledger/src/index.ts @@ -200,6 +200,16 @@ export interface SignalOptions { recentScores?: number[]; /** Which PR numbers actually merged (so we can detect the zero-merge streak). */ mergedPrNumbers?: Set; + /** + * Finding text from currently-open, unmerged dream-cycle PRs (e.g. their + * titles), supplied by the caller after a live GitHub check. `duplicateDirections` + * only ever sees rows already merged into LEDGER.md on main, so a repeated + * direction proposed across several still-open draft PRs is invisible to it + * until one of them lands — by which point duplicate work may already be + * done. Passing those findings here lets the same detector count them + * alongside merged-row findings. Omit for byte-identical prior behavior. + */ + pendingFindings?: string[]; } function prNumber(pr: string): string | null { @@ -207,6 +217,11 @@ function prNumber(pr: string): string | null { return m ? m[1] : null; } +/** Normalize a finding string to the same first-6-words key used for duplicate-direction detection. */ +function directionKey(finding: string): string { + return finding.toLowerCase().replace(/[^a-z0-9 ]/g, '').split(/\s+/).slice(0, 6).join(' ').trim(); +} + /** Compute the STEP 1.1 learning signals from parsed rows. */ export function learningSignals(rows: LedgerRow[], opts: SignalOptions = {}): LearningSignals { const window = opts.window ?? 14; @@ -218,13 +233,16 @@ export function learningSignals(rows: LedgerRow[], opts: SignalOptions = {}): Le const zeroMergeStreak = prsInWindow.length > 0 && (!merged || prsInWindow.every((n) => !merged.has(n))); - // Duplicate directions: normalized finding text repeated >= 3 times. + // Duplicate directions: normalized finding text repeated >= 3 times, counting + // both merged ledger rows and (optionally) still-open PRs' pending findings. const counts = new Map(); - for (const r of rows) { - const key = r.finding.toLowerCase().replace(/[^a-z0-9 ]/g, '').split(/\s+/).slice(0, 6).join(' ').trim(); - if (!key) continue; + const bumpDirection = (finding: string) => { + const key = directionKey(finding); + if (!key) return; counts.set(key, (counts.get(key) ?? 0) + 1); - } + }; + for (const r of rows) bumpDirection(r.finding); + for (const f of opts.pendingFindings ?? []) bumpDirection(f); const duplicateDirections = [...counts.entries()].filter(([, c]) => c >= 3).map(([k]) => k); // Low-score streak: last 3 scores all < 5.