From db76a3b4d6c990978804708b21e5809fafcace3f Mon Sep 17 00:00:00 2001 From: test Date: Sat, 15 Aug 2026 06:08:53 +0000 Subject: [PATCH 1/2] Pin the icebird absent-column contract at the SQL surface (#778) A read of a column a partition physically lacks was documented nowhere for the icebird-backed cache, and five successive written descriptions of it were measured false during PR #740's review. This measures it instead, at the pair `hyp query sql` actually runs (executeSql + collect) over a staged icebird cache, and records only what the runs showed. The contract turns out to be path-dependent and not single-valued: nothing throws, a scan whose hint set is exactly one column reads null (the scanColumn fast path, which withSchemaColumns null-normalizes), and anything widening it to two reads undefined. A literal sibling does not widen it; a WHERE on an unrelated column does, so the same projection flips value when a date filter is added. Both are live readings of the same absent cell. Measuring it exposed a correctness hole. LLP 0098#wrapper-duties already requires withSchemaColumns to strip a predicate naming a declared-but-absent column, but only scanColumn implemented it; scan forwarded it verbatim. icebird converts it to a hyparquet filter over a column its schema never had, filters nothing, and still reports appliedWhere: true, so the engine trusts the unfiltered stream: on a single-partition cache, WHERE git_remote = 'zzz' returned every row. The union's own gate hid it, so only a fresh install with one client was affected. The row path now applies the same gate. LLP 0240 records the measured table and the gate. LLP 0015 and LLP 0098 are Accepted/Active and gain only forward-refs. Co-Authored-By: Claude --- .../ai-gateway/src/dataset.js | 30 +- .../ai-gateway/src/message_projector.js | 9 +- llp/0015-query-and-datasets.spec.md | 8 + ...0098-scancolumn-where-pushdown.decision.md | 7 + ...icebird-absent-column-contract.decision.md | 134 ++++++++ .../core/ai-gateway-absent-column-sql.test.js | 299 ++++++++++++++++++ 6 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 llp/0240-icebird-absent-column-contract.decision.md create mode 100644 test/core/ai-gateway-absent-column-sql.test.js diff --git a/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js b/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js index dd8cf192..156454be 100644 --- a/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js +++ b/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js @@ -163,11 +163,13 @@ const SCHEMA_COLUMN_NAMES = AI_GATEWAY_SCHEMA_COLUMNS.map((c) => c.name) * (e.g. `git_remote`/`head_sha`/`repo_root` in v7, LLP 0032). Squirreling's * `validateScan` rejects a SELECT that names a column absent from the source's * `columns`, so without this a contract or query that reads a freshly-added - * column would throw `ColumnNotFoundError` over any pre-bump partition. The scan - * itself is unchanged: a row object that lacks the key simply reads as null, - * which is the correct value for "this partition predates the column". + * column would throw `ColumnNotFoundError` over any pre-bump partition. Over + * the icebird-backed cache the value such a read yields is `null` on the + * single-column `scanColumn` path and `undefined` on the row path, never a + * throw; LLP 0240 records the measured contract and the tests that pin it. * * @ref LLP 0032#capture [implements]: additive columns stay queryable over old partitions; no partition-label bump / cache wipe needed + * @ref LLP 0240#contract [implements]: the wrapper is what makes an absent column addressable; its read values are pinned at the SQL surface * @param {AsyncDataSource} source * @returns {AsyncDataSource} */ @@ -177,8 +179,28 @@ function withSchemaColumns(source) { const wrapped = { columns, numRows: source.numRows, + // The row path owes the same predicate gate as `scanColumn` below. A + // `where` naming a declared-but-physically-absent column must not reach + // the source: an icebird partition builds a hyparquet filter on a column + // its schema never had, matches nothing away, and still reports + // `appliedWhere: true`, so the engine trusts the unfiltered stream and + // `WHERE git_remote = 'x'` returns every row. Forwarding it verbatim was + // wrong in exactly the shape LLP 0098 already forbids; the union hides it + // (its own gate fires first) so only a single-partition cache was hit. + // @ref LLP 0098#wrapper-duties [implements]: a predicate naming a declared-but-absent column is stripped on the row path too, not only on scanColumn + // @ref LLP 0240#where-gate [implements]: an ungated row-path where made a single icebird partition answer predicates on an absent column wrongly scan(options) { - return source.scan(options) + const pushable = !options?.where || canPushWhere(source, whereColumns(options.where)) + if (pushable) return source.scan(options) + // Stripping the predicate also strips limit/offset: they are only + // meaningful after the filter, and a source that ignored the predicate + // but honored a slice would silently drop matching rows. + const inner = source.scan({ ...options, where: undefined, limit: undefined, offset: undefined }) + return { + appliedWhere: false, + appliedLimitOffset: false, + rows: () => inner.rows(), + } }, } // Forward the column-stream hook so single-column aggregates stay on the diff --git a/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js b/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js index 8f285259..d95b0baf 100644 --- a/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js +++ b/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js @@ -22,8 +22,13 @@ const DATASET_NAME = 'ai_gateway_messages' * of which adapter projector produced the messages (projector-defined * fields map onto these named columns directly). `schema_version` 7 added * the `git_remote` / `head_sha` / `repo_root` capture columns (LLP 0032); - * the additions are nullable, so old partitions read them as null and no - * partition-label bump is needed. + * the additions are nullable and no partition-label bump is needed, so old + * partitions carry no such column at all. A read of one over the + * icebird-backed cache never throws, but it is `null` on the single-column + * scan path and `undefined` on the row path: LLP 0240 has the measured table. + * Treat both as absent; do not branch on which one you got. + * + * @ref LLP 0240#contract [constrained-by]: an additive nullable column reads null or undefined depending on the scan path, never one canonical value * * @type {ReadonlyArray} */ diff --git a/llp/0015-query-and-datasets.spec.md b/llp/0015-query-and-datasets.spec.md index 67ef1b29..e3f5f6cf 100644 --- a/llp/0015-query-and-datasets.spec.md +++ b/llp/0015-query-and-datasets.spec.md @@ -22,6 +22,14 @@ > OOM the host by buffering an unbounded scan > ([hyparam/hypaware-server#9](https://github.com/hyparam/hypaware-server/issues/9)). +> **Extended by [LLP 0240](./0240-icebird-absent-column-contract.decision.md).** +> The union section below describes the parquet-backed sources it was written +> against. Over the **icebird**-backed cache (which is what +> `ai_gateway_messages` reads from) a declared-but-physically-absent column +> never throws, and reads as `null` or `undefined` depending on which scan +> path the engine takes. LLP 0240 records the measured values and the +> SQL-surface tests that pin them. + ## Query is intrinsic Query and Iceberg storage are intrinsic services. Plugins register datasets; diff --git a/llp/0098-scancolumn-where-pushdown.decision.md b/llp/0098-scancolumn-where-pushdown.decision.md index a6b68e70..cd633db0 100644 --- a/llp/0098-scancolumn-where-pushdown.decision.md +++ b/llp/0098-scancolumn-where-pushdown.decision.md @@ -12,6 +12,13 @@ > flags, so a filtered `COUNT` keeps the streaming fast path instead of > falling back to per-row materialization. +> **Extended by [LLP 0240](./0240-icebird-absent-column-contract.decision.md).** +> `#wrapper-duties` below states the `withSchemaColumns` predicate gate as a +> duty of the wrapper; it was implemented only on `scanColumn`, and the row +> `scan` forwarded the predicate verbatim. LLP 0240 extends the same gate to +> the row path and records what the gap cost on an icebird-backed partition, +> which reports the filter applied instead of throwing. + ## Context LLP 0055 lit the engine's streaming-aggregate fast path by implementing diff --git a/llp/0240-icebird-absent-column-contract.decision.md b/llp/0240-icebird-absent-column-contract.decision.md new file mode 100644 index 00000000..c7ecd015 --- /dev/null +++ b/llp/0240-icebird-absent-column-contract.decision.md @@ -0,0 +1,134 @@ +# LLP 0240: What an icebird-backed read of an absent column actually yields + +**Type:** Decision +**Status:** Accepted +**Systems:** Query, Cache +**Author:** Claude +**Date:** 2026-08-15 +**Related:** LLP 0015 (#multi-partition-union: the union contract this +completes for the icebird backing), LLP 0032 (#capture: the additive v7 +columns that create the drift), LLP 0098 (#wrapper-duties: the predicate gate +this extends from `scanColumn` to the row path), LLP 0055 + +> Extends [LLP 0015](./0015-query-and-datasets.spec.md). LLP 0015 settles what +> the union does with a column a partition physically lacks, in terms of the +> parquet-backed sources it was written against. It does not settle what the +> **icebird**-backed cache yields for such a read, and the flagship dataset +> `ai_gateway_messages` is icebird-backed. This decision records that, measured +> at the SQL surface rather than derived from the code, and closes a +> correctness hole the measurement exposed. + +## Context + +`ai_gateway_messages` declares more columns than any given partition +physically has. Schema v7 added `git_remote` / `head_sha` / `repo_root` as +nullable (LLP 0032) with no partition-label bump, so every partition written +before the bump lacks them, and `withSchemaColumns` in the ai-gateway plugin +advertises the declared set on top of whatever the storage source reports so +that a SELECT naming one of them plans at all. + +What such a read then *yields* had never been pinned. Only the raw `scan()` +rows and the `scanColumn()` chunks were tested; no test ran a full SELECT +through `executeSql` + `collect`, which is the pair `hyp query sql` uses. In +the absence of a test, five successive written descriptions of this mechanism +were each measured false during the review of #731 / PR #740, and the +maintainer descoped the icebird half rather than ship a sixth guess. + +Everything below was obtained by running the query and recording the answer. + +## Decision + +### The contract + +Over an icebird-backed partition that physically lacks a declared column, +**nothing throws**, and the value read depends on which path the engine takes: + +| query shape | value | rendering | +| --- | --- | --- | +| `SELECT git_remote FROM t` | `null` | `{"git_remote":null}` | +| `SELECT git_remote AS gr FROM t` | `null` | `{"gr":null}` | +| `SELECT git_remote, 1 AS n FROM t` | `null` | `{"git_remote":null,"n":1}` | +| `SELECT id, git_remote FROM t` | `undefined` | `{"id":1}` (key dropped) | +| `SELECT git_remote FROM t WHERE date >= '...'` | `undefined` | `{}` | +| `SELECT * FROM t` | key absent from the row | `{"id":1,"date":"..."}` | + +The discriminator is **the size of the scan's hint column set, not the shape +of the SELECT list.** Squirreling routes a scan whose hints name exactly one +column through `scanColumn` (`execute.js`, gated on +`plan.hints.columns?.length === 1`, with no aggregate required), and +`withSchemaColumns` normalizes the hole to `null` on exactly that path. +Anything that widens the hint set to two columns takes the row path instead +and reads `undefined`. A literal or expression sibling reads no column, so it +does **not** widen it; a `WHERE` on an unrelated column does, which is why +adding a date filter silently flips the same projection from `null` to +`undefined`. + +On the row path the value is `undefined` rather than a throw because icebird +builds each row with squirreling's `asyncRow(obj, requestedColumns)` over the +**requested** column list: the cell exists as a thunk that resolves to +`obj[name]`, which is `undefined`, and the pre-materialized `resolved` map +that `collect()` reads simply has no entry for it. This is the whole +difference from a parquet-backed partition, whose `asyncRow` is built over +`Object.keys(data[0])`, the row's **physical** keys, so the cell does not +exist and anything evaluating it throws `ColumnNotFoundError`. Hence, on +icebird, `ORDER BY`, `GROUP BY`, `DISTINCT`, an expression, and an aggregate +over the absent column all answer (with `null`, or a count that skips it) +where the parquet union throws. + +Consequently **`null` and `undefined` are both live readings of the same +absent cell, and neither is "the" value.** A consumer that must distinguish +"no value" from "column predates this partition" cannot do it from the read; +callers should treat both as absent, and no code should branch on which one it +got. + +### The row path owes the same predicate gate as `scanColumn` + +LLP 0098 (#wrapper-duties) already requires `withSchemaColumns` to strip a +predicate naming a declared-but-physically-absent column before it reaches the +source. Only `scanColumn` implemented it; `scan` forwarded `options` verbatim. +Measuring the contract exposed what that costs on icebird, which does not +throw where parquet does: + +- icebird converts the predicate to a hyparquet filter over a column its + schema never had, +- filters nothing away, +- and still reports `appliedWhere: true`, + +so the engine trusts the stream and does not re-filter. On a cache with a +**single** partition lacking the column, `SELECT id FROM t WHERE git_remote = +'zzz'` returned every row, and so did `WHERE git_remote IS NOT NULL`. Two or +more partitions hid it, because `createDataSource` then wraps `unionSources`, +whose own per-partition gate (LLP 0015#multi-partition-union) fires first. The +exposed shape is therefore the ordinary one: a fresh install with one client. + +`withSchemaColumns.scan` now applies the same gate as its `scanColumn`: when +the predicate names a column the wrapped source does not advertise, drop the +predicate along with `limit`/`offset` (only meaningful post-filter) and report +`appliedWhere: false` / `appliedLimitOffset: false`, handing the filter back +to the engine. A predicate the source can satisfy is still pushed and still +claimed, so the ordinary filtered read keeps its pushdown. + +### Pinned at the SQL surface + +The contract is pinned by +[`test/core/ai-gateway-absent-column-sql.test.js`](../test/core/ai-gateway-absent-column-sql.test.js), +which runs `executeSql` + `collect` over a staged icebird cache in two shapes: +one partition lacking the column (no union in the way), and a drifted pair. +Every value is asserted exactly, as `null` versus `undefined` versus key +absence, never through a tolerant `?? null`. That form is deliberate: a +tolerant assertion is what let the mechanism be described wrongly five times +while the suite stayed green. + +## Consequences + +- Documentation of this contract belongs here, not in LLP 0015. LLP 0015 is + Active and its union section is being corrected separately for the parquet + backing (#731 / PR #740); this doc carries the icebird half and 0015 gains + only a forward-ref. +- The `null`-versus-`undefined` split is a property of the engine's fast-path + gate, not of the cache. If squirreling ever widens or narrows that gate, the + values in the table above move, and the pinning tests are what will say so. +- `SELECT *, git_remote FROM t` over a drifted union was observed to + mis-assign a value into a neighbouring declared column. That is a star + expansion defect above this layer, is not part of this contract, and is left + unaddressed here; the tests deliberately do not cover it. diff --git a/test/core/ai-gateway-absent-column-sql.test.js b/test/core/ai-gateway-absent-column-sql.test.js new file mode 100644 index 00000000..6c4218d7 --- /dev/null +++ b/test/core/ai-gateway-absent-column-sql.test.js @@ -0,0 +1,299 @@ +// @ts-check + +// SQL-surface pins for the icebird-backed absent-column contract of +// `ai_gateway_messages` (LLP 0240). Every assertion here was derived by +// running the query and recording what came back, not by reading the code: +// five successive from-the-code descriptions of this mechanism were each +// measured wrong during PR #740's review. The values are asserted exactly +// (`strictEqual` against `null` / `undefined`, key presence, the JSON +// rendering) rather than through a tolerant `?? null`, because the whole +// point is which of those it is. +// +// `test/core/ai-gateway-dataset.test.js` pins the raw `scan()` rows and the +// `scanColumn()` chunks. This file pins what a user actually types: a full +// SELECT through `executeSql` + `collect`, the pair `hyp query sql` runs. + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { collect, executeSql } from 'squirreling' + +import { appendRowsToSourceTable } from '../../src/core/cache/partition.js' +import { createQueryStorageService } from '../../src/core/cache/storage.js' +import { + createDataSource, + DATASET_NAME, + discoverParts, +} from '../../hypaware-core/plugins-workspace/ai-gateway/src/dataset.js' + +/** + * @import { ColumnSpec, QueryScope } from '../../hypaware-plugin-kernel-types.js' + * @import { AsyncDataSource, ExprNode } from 'squirreling/src/types.js' + */ + +/** Two columns every fixture partition carries. */ +/** @type {ColumnSpec[]} */ +const NARROW_COLUMNS = [ + { name: 'id', type: 'INT32', nullable: false }, + { name: 'date', type: 'STRING', nullable: false }, +] + +/** The same plus `git_remote`, a real v7 capture column (LLP 0032). */ +/** @type {ColumnSpec[]} */ +const WIDE_COLUMNS = [...NARROW_COLUMNS, { name: 'git_remote', type: 'STRING', nullable: true }] + +const REMOTE = 'git@example.com:acme/app.git' + +/** + * Stage an icebird-backed `ai_gateway_messages` cache and return the dataset + * source a query would run against. + * + * `shape` picks the drift: + * - `lone`: ONE partition whose iceberg schema never had `git_remote`. This + * is the shape that skips `unionSources` entirely (`createDataSource` + * returns `withSchemaColumns(sources[0])`), so nothing but the wrapper + * stands between the query and icebird. + * - `drifted`: TWO partitions, one with `git_remote` and one without, joined + * by `unionSources`. + * + * @param {'lone' | 'drifted'} shape + * @returns {Promise<{ cacheRoot: string, source: AsyncDataSource }>} + */ +async function stageFixture(shape) { + const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), `hyp-absent-col-${shape}-`)) + await appendRowsToSourceTable( + cacheRoot, DATASET_NAME, ['source=claude'], + NARROW_COLUMNS, [{ id: 1, date: '2026-05-26' }] + ) + if (shape === 'drifted') { + await appendRowsToSourceTable( + cacheRoot, DATASET_NAME, ['source=codex'], + WIDE_COLUMNS, [{ id: 2, date: '2026-05-27', git_remote: REMOTE }] + ) + } + const storage = createQueryStorageService({ cacheRoot }) + /** @type {QueryScope} */ + const scope = { limit: 1000 } + const partitions = await discoverParts({ cacheDir: cacheRoot, scope, config: { version: 2 } }) + const source = await createDataSource(partitions, { scope, storage }) + return { cacheRoot, source } +} + +/** + * Run a SELECT the way `hyp query sql` does and return the collected rows, + * ordered by `id` so partition scan order can't make an assertion flap. + * + * @param {AsyncDataSource} source + * @param {string} query + * @returns {Promise[]>} + */ +async function runSql(source, query) { + const rows = await collect(executeSql({ tables: { t: source }, query })) + return /** @type {Record[]} */ (rows) +} + +/** + * @param {'lone' | 'drifted'} shape + * @param {(source: AsyncDataSource) => Promise} body + */ +async function withFixture(shape, body) { + const { cacheRoot, source } = await stageFixture(shape) + try { + await body(source) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +} + +// --- what the wrapper makes addressable at all --- + +test('absent column: an icebird source advertises the declared column it physically lacks', async () => { + await withFixture('lone', async (source) => { + assert.ok(source.columns.includes('git_remote'), 'declared v7 column is addressable') + assert.ok(source.columns.includes('id'), 'physical column is addressable') + // Without this the SELECTs below would fail in validateScan, not in the + // scan: the contract only becomes interesting because planning succeeds. + assert.equal(typeof source.scanColumn, 'function', 'the column-stream hook survives the wrapper') + }) +}) + +// --- the two read paths yield DIFFERENT values --- + +test('absent column: a single-column bare projection reads null (scanColumn fast path)', async () => { + // The engine routes a scan whose hint set is exactly one column through + // `scanColumn` (squirreling execute.js gates on `columns?.length === 1`, + // with no aggregate required), and `withSchemaColumns` normalizes the + // hole to null on that path. So this yields null, and RENDERS as null. + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT git_remote FROM t') + assert.equal(rows.length, 1) + assert.ok('git_remote' in rows[0]) + assert.strictEqual(rows[0].git_remote, null, 'single-column projection reads null, not undefined') + assert.equal(JSON.stringify(rows[0]), '{"git_remote":null}', 'null survives JSON rendering') + }) +}) + +test('absent column: an aliased single-column projection is still the null path', async () => { + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT git_remote AS gr FROM t') + assert.deepEqual(rows.map((r) => r.gr), [null]) + }) +}) + +test('absent column: a non-identifier sibling does NOT change the value or throw', async () => { + // Recorded because the natural prediction is the opposite. On the parquet + // union a literal sibling collapses the resolveable fast path and the + // drifted column's thunk throws. Here it does not: the hint set is still + // the single column `git_remote` (a literal reads no column), so the query + // stays on the `scanColumn` path and still reads null. + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT git_remote, 1 AS n FROM t') + assert.equal(rows.length, 1) + assert.strictEqual(rows[0].git_remote, null) + assert.equal(rows[0].n, 1) + }) +}) + +test('absent column: a multi-column bare projection reads undefined and JSON drops the key', async () => { + // Two hint columns take the row path instead. icebird builds the row with + // `asyncRow(obj, requestedColumns)`, so the cell EXISTS but resolves to + // undefined, and the pre-materialized `resolved` map `collect()` reads + // simply has no entry. The key is present on the output row with the value + // undefined, which `JSON.stringify` omits. + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT id, git_remote FROM t') + assert.equal(rows.length, 1) + assert.equal(rows[0].id, 1) + assert.ok('git_remote' in rows[0], 'the key is present on the row') + assert.strictEqual(rows[0].git_remote, undefined, 'multi-column projection reads undefined, not null') + assert.equal(JSON.stringify(rows[0]), '{"id":1}', 'JSON.stringify drops the undefined key') + }) +}) + +test('absent column: a WHERE on a present column pushes the projection onto the row path', async () => { + // The trap this pins: the query LOOKS like the single-column null case, + // but the predicate's column joins the hint set, making it two, so the + // same SELECT list reads undefined instead of null. Nothing about the + // projection changed. + await withFixture('lone', async (source) => { + const rows = await runSql(source, "SELECT git_remote FROM t WHERE date >= '2026-01-01'") + assert.equal(rows.length, 1) + assert.strictEqual(rows[0].git_remote, undefined) + assert.equal(JSON.stringify(rows[0]), '{}') + }) +}) + +test('absent column: SELECT * omits the key entirely for the partition that lacks it', async () => { + await withFixture('drifted', async (source) => { + const rows = (await runSql(source, 'SELECT * FROM t')).sort((a, b) => Number(a.id) - Number(b.id)) + assert.equal(rows.length, 2) + assert.equal('git_remote' in rows[0], false, 'star keeps each partition\'s own row shape') + assert.equal(rows[1].git_remote, REMOTE) + }) +}) + +// --- nothing on this path throws --- + +test('absent column: evaluating, ordering, grouping and aggregating never throw on icebird', async () => { + // The parquet-backed union throws `ColumnNotFoundError` for these, because + // its rows carry no cell for the column at all. icebird's do (a thunk that + // resolves to undefined), so every one of these answers instead. This is + // the single sharpest difference between the two backings. + await withFixture('lone', async (source) => { + assert.deepEqual(await runSql(source, "SELECT git_remote || 'x' AS e FROM t"), [{ e: null }]) + assert.deepEqual(await runSql(source, 'SELECT id FROM t ORDER BY git_remote'), [{ id: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT DISTINCT git_remote FROM t'), [{ git_remote: null }]) + assert.deepEqual( + await runSql(source, 'SELECT git_remote, COUNT(*) AS n FROM t GROUP BY git_remote'), + [{ git_remote: null, n: 1 }] + ) + assert.deepEqual(await runSql(source, 'SELECT COUNT(git_remote) AS n FROM t'), [{ n: 0 }]) + assert.deepEqual(await runSql(source, 'SELECT MAX(git_remote) AS m FROM t'), [{ m: null }]) + }) +}) + +// --- predicates on the absent column answer correctly --- + +test('absent column: a lone icebird partition answers predicates on the column it lacks', async () => { + // Regression pin. `withSchemaColumns.scan` used to forward the predicate + // verbatim; icebird converted it to a hyparquet filter over a column its + // schema never had, filtered nothing, and still reported + // `appliedWhere: true`, so the engine trusted the unfiltered stream and + // BOTH of these returned the row. The union's own gate hid it, so only a + // single-partition cache (a fresh install with one client) was affected. + // @ref LLP 0240#where-gate [tests]: an ungated row-path where made a lone icebird partition answer predicates on an absent column wrongly + await withFixture('lone', async (source) => { + assert.deepEqual(await runSql(source, "SELECT id FROM t WHERE git_remote = 'zzz'"), []) + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NOT NULL'), []) + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NULL'), [{ id: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT COUNT(*) AS n FROM t WHERE git_remote IS NULL'), [{ n: 1 }]) + assert.deepEqual(await runSql(source, "SELECT COUNT(*) AS n FROM t WHERE git_remote = 'zzz'"), [{ n: 0 }]) + }) +}) + +test('absent column: the wrapper reports appliedWhere false for a predicate it had to strip', async () => { + // The flag is what the engine trusts; assert it directly so a regression + // shows up as a flag, not only as a wrong row count. + await withFixture('lone', async (source) => { + /** @type {ExprNode} */ + const where = { + type: 'binary', + op: '=', + left: { type: 'identifier', name: 'git_remote', positionStart: 0, positionEnd: 0 }, + right: { type: 'literal', value: 'zzz', positionStart: 0, positionEnd: 0 }, + positionStart: 0, + positionEnd: 0, + } + const stripped = source.scan({ columns: ['id', 'git_remote'], where, limit: 1 }) + assert.equal(stripped.appliedWhere, false, 'a predicate on a physically absent column is not claimed') + assert.equal(stripped.appliedLimitOffset, false, 'the slice is handed back with the filter') + + // A predicate the partition CAN satisfy is still pushed and claimed, so + // the gate does not cost the ordinary filtered read its pushdown. + /** @type {ExprNode} */ + const pushable = { + type: 'binary', + op: '=', + left: { type: 'identifier', name: 'id', positionStart: 0, positionEnd: 0 }, + right: { type: 'literal', value: 1, positionStart: 0, positionEnd: 0 }, + positionStart: 0, + positionEnd: 0, + } + assert.equal(source.scan({ columns: ['id'], where: pushable }).appliedWhere, true) + }) +}) + +// --- the drifted union: present values are untouched --- + +test('absent column: a drifted union reads real values alongside the holes', async () => { + await withFixture('drifted', async (source) => { + const bare = (await runSql(source, 'SELECT git_remote FROM t')) + .map((r) => r.git_remote).sort() + assert.deepEqual(bare, [REMOTE, null], 'the hole is null on the single-column path, the value is intact') + + const pair = (await runSql(source, 'SELECT id, git_remote FROM t')) + .sort((a, b) => Number(a.id) - Number(b.id)) + assert.strictEqual(pair[0].git_remote, undefined, 'the hole is undefined on the row path') + assert.equal(pair[1].git_remote, REMOTE) + + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NOT NULL'), [{ id: 2 }]) + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NULL'), [{ id: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT COUNT(git_remote) AS n FROM t'), [{ n: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT MAX(git_remote) AS m FROM t'), [{ m: REMOTE }]) + }) +}) + +test('absent column: a present column is unaffected on either path', async () => { + // The control. If these ever drift, the assertions above are measuring the + // fixture rather than the contract. + await withFixture('drifted', async (source) => { + const ids = (await runSql(source, 'SELECT id FROM t')).map((r) => r.id).sort() + assert.deepEqual(ids, [1, 2]) + const pairs = (await runSql(source, 'SELECT id, date FROM t')) + .sort((a, b) => Number(a.id) - Number(b.id)) + assert.deepEqual(pairs, [{ id: 1, date: '2026-05-26' }, { id: 2, date: '2026-05-27' }]) + }) +}) From bca70d2e75ffb8fdc7970a3a7455050c71e43a4c Mon Sep 17 00:00:00 2001 From: neutral Date: Sat, 15 Aug 2026 07:06:53 +0000 Subject: [PATCH 2/2] LLP 0240 links the star-expansion defect it defers to issue #788 Review follow-up: the Consequences bullet records an observed star-expansion defect and says it is left unaddressed, but pointed at no tracking issue. #788 was filed for it; cite it so a reader of the doc can find where it went. Editorial link only; nothing the doc settled changes. Co-Authored-By: Claude --- llp/0240-icebird-absent-column-contract.decision.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llp/0240-icebird-absent-column-contract.decision.md b/llp/0240-icebird-absent-column-contract.decision.md index c7ecd015..6c7c8f29 100644 --- a/llp/0240-icebird-absent-column-contract.decision.md +++ b/llp/0240-icebird-absent-column-contract.decision.md @@ -131,4 +131,5 @@ while the suite stayed green. - `SELECT *, git_remote FROM t` over a drifted union was observed to mis-assign a value into a neighbouring declared column. That is a star expansion defect above this layer, is not part of this contract, and is left - unaddressed here; the tests deliberately do not cover it. + unaddressed here; the tests deliberately do not cover it. It is tracked as + [hyparam/hypaware#788](https://github.com/hyparam/hypaware/issues/788).