diff --git a/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js b/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js index e0611694..ac068b4f 100644 --- a/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js +++ b/hypaware-core/plugins-workspace/ai-gateway/src/dataset.js @@ -163,12 +163,15 @@ const SCHEMA_COLUMN_NAMES = AI_GATEWAY_SCHEMA_COLUMNS.map((c) => c.name) * (e.g. `git_remote`/`head_sha`/`repo_root` in v7, LLP 0032). Squirreling's * `validateScan` rejects a SELECT that names a column absent from the source's * `columns`, so without this a contract or query that reads a freshly-added - * column would throw `ColumnNotFoundError` over any pre-bump partition. The scan - * itself is unchanged: a column an old partition physically lacks stays - * addressable, and the exact value a read of it yields depends on the read path - * (LLP 0015#multi-partition-union). + * column would throw `ColumnNotFoundError` over any pre-bump partition. Over + * the icebird-backed cache the value such a read yields is `null` on the + * single-column `scanColumn` path and `undefined` on the row path, never a + * throw; LLP 0240 records the measured contract and the tests that pin it. + * LLP 0015#multi-partition-union states the parquet-backed contract, which is + * a different one (undefined-or-throws) and does not govern this dataset. * * @ref LLP 0032#capture [implements]: additive columns stay queryable over old partitions; no partition-label bump / cache wipe needed + * @ref LLP 0240#contract [implements]: the wrapper is what makes an absent column addressable; its read values are pinned at the SQL surface * @param {AsyncDataSource} source * @returns {AsyncDataSource} */ @@ -178,6 +181,16 @@ function withSchemaColumns(source) { const wrapped = { columns, numRows: source.numRows, + // The row path owes the same predicate gate as `scanColumn` below. A + // `where` naming a declared-but-physically-absent column must not reach + // the source: an icebird partition builds a hyparquet filter on a column + // its schema never had, matches nothing away, and still reports + // `appliedWhere: true`, so the engine trusts the unfiltered stream and + // `WHERE git_remote = 'x'` returns every row. Forwarding it verbatim was + // wrong in exactly the shape LLP 0098 already forbids; the union hides it + // (its own gate fires first) so only a single-partition cache was hit. + // @ref LLP 0098#wrapper-duties [implements]: a predicate naming a declared-but-absent column is stripped on the row path too, not only on scanColumn + // @ref LLP 0240#where-gate [implements]: an ungated row-path where made a single icebird partition answer predicates on an absent column wrongly scan(options) { // The engine names this scan's output columns from the list advertised // here, but fills them from each row's own `columns`. A partition that @@ -185,14 +198,25 @@ function withSchemaColumns(source) { // output name past the gap onto its neighbour's value: over a drifted // union `SELECT *, git_remote` answered with git_remote's value under // the name of the column that happened to follow the star's short - // width. Pad each row back out to the advertised list. + // width. Pad each row back out to the advertised list. Stripping the + // predicate below does not narrow it: the gate only drops `where`. // @ref LLP 0241#alignment [implements]: a declared-but-absent column becomes a padded cell, not a missing slot the star can slide through const scanColumns = options?.columns ?? columns - const result = source.scan(options) + const pushable = !options?.where || canPushWhere(source, whereColumns(options.where)) + // Stripping the predicate also strips limit/offset: they are only + // meaningful after the filter, and a source that ignored the predicate + // but honored a slice would silently drop matching rows. + const inner = pushable + ? source.scan(options) + : source.scan({ ...options, where: undefined, limit: undefined, offset: undefined }) return { - appliedWhere: result.appliedWhere, - appliedLimitOffset: result.appliedLimitOffset, - rows: () => alignRows(result.rows(), scanColumns), + appliedWhere: pushable && inner.appliedWhere, + appliedLimitOffset: pushable && inner.appliedLimitOffset, + // The two duties compose in one direction: the gate hands an + // unfiltered stream back to the engine, and the padding is what lets + // the engine's own re-filter read the absent column as `undefined` + // instead of missing it on a short row (LLP 0241#alignment). + rows: () => alignRows(inner.rows(), scanColumns), } }, } diff --git a/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js b/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js index 9182470e..6f229640 100644 --- a/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js +++ b/hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js @@ -22,10 +22,14 @@ const DATASET_NAME = 'ai_gateway_messages' * of which adapter projector produced the messages (projector-defined * fields map onto these named columns directly). `schema_version` 7 added * the `git_remote` / `head_sha` / `repo_root` capture columns (LLP 0032); - * the additions are nullable and no partition-label bump is needed. An old - * partition physically lacks them; `withSchemaColumns` in `dataset.js` is the - * only reason they stay addressable at all, and the exact value a read of one - * yields depends on the read path (LLP 0015#multi-partition-union). + * the additions are nullable and no partition-label bump is needed, so old + * partitions carry no such column at all. `withSchemaColumns` in `dataset.js` + * is the only reason they stay addressable. A read of one over the + * icebird-backed cache never throws, but it is `null` on the single-column + * scan path and `undefined` on the row path: LLP 0240 has the measured table. + * Treat both as absent; do not branch on which one you got. + * + * @ref LLP 0240#contract [constrained-by]: an additive nullable column reads null or undefined depending on the scan path, never one canonical value * * @type {ReadonlyArray} */ diff --git a/llp/0015-query-and-datasets.spec.md b/llp/0015-query-and-datasets.spec.md index 739ea6d9..e388b794 100644 --- a/llp/0015-query-and-datasets.spec.md +++ b/llp/0015-query-and-datasets.spec.md @@ -22,6 +22,14 @@ > OOM the host by buffering an unbounded scan > ([hyparam/hypaware-server#9](https://github.com/hyparam/hypaware-server/issues/9)). +> **Extended by [LLP 0240](./0240-icebird-absent-column-contract.decision.md).** +> The union section below describes the parquet-backed sources it was written +> against. Over the **icebird**-backed cache (which is what +> `ai_gateway_messages` reads from) a declared-but-physically-absent column +> never throws, and reads as `null` or `undefined` depending on which scan +> path the engine takes. LLP 0240 records the measured values and the +> SQL-surface tests that pin them. + ## Query is intrinsic Query and Iceberg storage are intrinsic services. Plugins register datasets; diff --git a/llp/0098-scancolumn-where-pushdown.decision.md b/llp/0098-scancolumn-where-pushdown.decision.md index a6b68e70..cd633db0 100644 --- a/llp/0098-scancolumn-where-pushdown.decision.md +++ b/llp/0098-scancolumn-where-pushdown.decision.md @@ -12,6 +12,13 @@ > flags, so a filtered `COUNT` keeps the streaming fast path instead of > falling back to per-row materialization. +> **Extended by [LLP 0240](./0240-icebird-absent-column-contract.decision.md).** +> `#wrapper-duties` below states the `withSchemaColumns` predicate gate as a +> duty of the wrapper; it was implemented only on `scanColumn`, and the row +> `scan` forwarded the predicate verbatim. LLP 0240 extends the same gate to +> the row path and records what the gap cost on an icebird-backed partition, +> which reports the filter applied instead of throwing. + ## Context LLP 0055 lit the engine's streaming-aggregate fast path by implementing diff --git a/llp/0240-icebird-absent-column-contract.decision.md b/llp/0240-icebird-absent-column-contract.decision.md new file mode 100644 index 00000000..1fdfd544 --- /dev/null +++ b/llp/0240-icebird-absent-column-contract.decision.md @@ -0,0 +1,154 @@ +# LLP 0240: What an icebird-backed read of an absent column actually yields + +**Type:** Decision +**Status:** Accepted +**Systems:** Query, Cache +**Author:** Claude +**Date:** 2026-08-15 +**Related:** LLP 0015 (#multi-partition-union: the union contract this +completes for the icebird backing), LLP 0032 (#capture: the additive v7 +columns that create the drift), LLP 0098 (#wrapper-duties: the predicate gate +this extends from `scanColumn` to the row path), LLP 0055 + +> Extends [LLP 0015](./0015-query-and-datasets.spec.md). LLP 0015 settles what +> the union does with a column a partition physically lacks, in terms of the +> parquet-backed sources it was written against. It does not settle what the +> **icebird**-backed cache yields for such a read, and the flagship dataset +> `ai_gateway_messages` is icebird-backed. This decision records that, measured +> at the SQL surface rather than derived from the code, and closes a +> correctness hole the measurement exposed. + +> **Amended by [LLP 0241 §alignment](./0241-scan-rows-carry-advertised-columns.decision.md#alignment), +> which landed first.** 0241 pads every scanned row out to the column list the +> scan advertised, which moves exactly one cell of the table below: under +> `SELECT *` the absent column's key now **exists** and holds `undefined`, +> where it was previously not on the row at all. The rendering is unchanged +> and no other row of the table moves; re-measured on the merged tree, and +> pinned by the same test file. 0241 also fixes the star-expansion defect the +> Consequences below deferred to issue #788. + +## Context + +`ai_gateway_messages` declares more columns than any given partition +physically has. Schema v7 added `git_remote` / `head_sha` / `repo_root` as +nullable (LLP 0032) with no partition-label bump, so every partition written +before the bump lacks them, and `withSchemaColumns` in the ai-gateway plugin +advertises the declared set on top of whatever the storage source reports so +that a SELECT naming one of them plans at all. + +What such a read then *yields* had never been pinned. Only the raw `scan()` +rows and the `scanColumn()` chunks were tested; no test ran a full SELECT +through `executeSql` + `collect`, which is the pair `hyp query sql` uses. In +the absence of a test, five successive written descriptions of this mechanism +were each measured false during the review of #731 / PR #740, and the +maintainer descoped the icebird half rather than ship a sixth guess. + +Everything below was obtained by running the query and recording the answer. + +## Decision + +### The contract + +Over an icebird-backed partition that physically lacks a declared column, +**nothing throws**, and the value read depends on which path the engine takes: + +| query shape | value | rendering | +| --- | --- | --- | +| `SELECT git_remote FROM t` | `null` | `{"git_remote":null}` | +| `SELECT git_remote AS gr FROM t` | `null` | `{"gr":null}` | +| `SELECT git_remote, 1 AS n FROM t` | `null` | `{"git_remote":null,"n":1}` | +| `SELECT id, git_remote FROM t` | `undefined` | `{"id":1}` (key dropped) | +| `SELECT git_remote FROM t WHERE date >= '...'` | `undefined` | `{}` | +| `SELECT * FROM t` | `undefined`, under a key that exists (LLP 0241) | `{"id":1,"date":"..."}` | + +The discriminator is **the size of the scan's hint column set, not the shape +of the SELECT list.** Squirreling routes a scan whose hints name exactly one +column through `scanColumn` (`execute.js`, gated on +`plan.hints.columns?.length === 1`, with no aggregate required), and +`withSchemaColumns` normalizes the hole to `null` on exactly that path. +Anything that widens the hint set to two columns takes the row path instead +and reads `undefined`. A literal or expression sibling reads no column, so it +does **not** widen it; a `WHERE` on an unrelated column does, which is why +adding a date filter silently flips the same projection from `null` to +`undefined`. + +On the row path the value is `undefined` rather than a throw because icebird +builds each row with squirreling's `asyncRow(obj, requestedColumns)` over the +**requested** column list: the cell exists as a thunk that resolves to +`obj[name]`, which is `undefined`, and the pre-materialized `resolved` map +that `collect()` reads simply has no entry for it. This is the whole +difference from a parquet-backed partition, whose `asyncRow` is built over +`Object.keys(data[0])`, the row's **physical** keys, so the cell does not +exist and anything evaluating it throws `ColumnNotFoundError`. Hence, on +icebird, `ORDER BY`, `GROUP BY`, `DISTINCT`, an expression, and an aggregate +over the absent column all answer (with `null`, or a count that skips it) +where the parquet union throws. + +Consequently **`null` and `undefined` are both live readings of the same +absent cell, and neither is "the" value.** A consumer that must distinguish +"no value" from "column predates this partition" cannot do it from the read; +callers should treat both as absent, and no code should branch on which one it +got. + +### The row path owes the same predicate gate as `scanColumn` + +LLP 0098 (#wrapper-duties) already requires `withSchemaColumns` to strip a +predicate naming a declared-but-physically-absent column before it reaches the +source. Only `scanColumn` implemented it; `scan` forwarded `options` verbatim. +Measuring the contract exposed what that costs on icebird, which does not +throw where parquet does: + +- icebird converts the predicate to a hyparquet filter over a column its + schema never had, +- filters nothing away, +- and still reports `appliedWhere: true`, + +so the engine trusts the stream and does not re-filter. On a cache with a +**single** partition lacking the column, `SELECT id FROM t WHERE git_remote = +'zzz'` returned every row, and so did `WHERE git_remote IS NOT NULL`. Two or +more partitions hid it, because `createDataSource` then wraps `unionSources`, +whose own per-partition gate (LLP 0015#multi-partition-union) fires first. The +exposed shape is therefore the ordinary one: a fresh install with one client. + +`withSchemaColumns.scan` now applies the same gate as its `scanColumn`: when +the predicate names a column the wrapped source does not advertise, drop the +predicate along with `limit`/`offset` (only meaningful post-filter) and report +`appliedWhere: false` / `appliedLimitOffset: false`, handing the filter back +to the engine. A predicate the source can satisfy is still pushed and still +claimed, so the ordinary filtered read keeps its pushdown. + +### Pinned at the SQL surface + +The contract is pinned by +[`test/core/ai-gateway-absent-column-sql.test.js`](../test/core/ai-gateway-absent-column-sql.test.js), +which runs `executeSql` + `collect` over a staged icebird cache in two shapes: +one partition lacking the column (no union in the way), and a drifted pair. +Every value is asserted exactly, as `null` versus `undefined` versus key +absence, never through a tolerant `?? null`. That form is deliberate: a +tolerant assertion is what let the mechanism be described wrongly five times +while the suite stayed green. + +## Consequences + +- Documentation of this contract belongs here, not in LLP 0015. LLP 0015 is + Active and its union section was corrected separately for the parquet + backing (#731 / PR #740, now on master); this doc carries the icebird half + and 0015 gains only a forward-ref. +- The `null`-versus-`undefined` split is a property of the engine's fast-path + gate, not of the cache. If squirreling ever widens or narrows that gate, the + values in the table above move, and the pinning tests are what will say so. +- `SELECT *, git_remote FROM t` over a drifted union was observed to + mis-assign a value into a neighbouring declared column. That is a star + expansion defect above this layer, is not part of this contract, and is left + unaddressed here; the tests deliberately do not cover it. It is tracked as + [hyparam/hypaware#788](https://github.com/hyparam/hypaware/issues/788), and + is **now fixed** by [LLP 0241](./0241-scan-rows-carry-advertised-columns.decision.md), + which landed on master first. +- The #where-gate and 0241's padding are not independent at the star. A star + carries no `columns` hint, so its rows are only as wide as the partition + physically is; the gate then hands the predicate back to the engine, which + reads the absent column off `row.cells`. Measured with 0241's two alignment + call sites reverted, `SELECT * FROM t WHERE git_remote IS NULL` raises + `ColumnNotFoundError` from `filterRows` instead of answering. The padding is + what makes the handed-back filter evaluable, so the two must ship together; + the composition has its own pin in the test file. diff --git a/test/core/ai-gateway-absent-column-sql.test.js b/test/core/ai-gateway-absent-column-sql.test.js new file mode 100644 index 00000000..b47de001 --- /dev/null +++ b/test/core/ai-gateway-absent-column-sql.test.js @@ -0,0 +1,335 @@ +// @ts-check + +// SQL-surface pins for the icebird-backed absent-column contract of +// `ai_gateway_messages` (LLP 0240). Every assertion here was derived by +// running the query and recording what came back, not by reading the code: +// five successive from-the-code descriptions of this mechanism were each +// measured wrong during PR #740's review. The values are asserted exactly +// (`strictEqual` against `null` / `undefined`, key presence, the JSON +// rendering) rather than through a tolerant `?? null`, because the whole +// point is which of those it is. +// +// `test/core/ai-gateway-dataset.test.js` pins the raw `scan()` rows and the +// `scanColumn()` chunks. This file pins what a user actually types: a full +// SELECT through `executeSql` + `collect`, the pair `hyp query sql` runs. + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { collect, executeSql } from 'squirreling' + +import { appendRowsToSourceTable } from '../../src/core/cache/partition.js' +import { createQueryStorageService } from '../../src/core/cache/storage.js' +import { + createDataSource, + DATASET_NAME, + discoverParts, +} from '../../hypaware-core/plugins-workspace/ai-gateway/src/dataset.js' + +/** + * @import { ColumnSpec, QueryScope } from '../../hypaware-plugin-kernel-types.js' + * @import { AsyncDataSource, ExprNode } from 'squirreling/src/types.js' + */ + +/** Two columns every fixture partition carries. */ +/** @type {ColumnSpec[]} */ +const NARROW_COLUMNS = [ + { name: 'id', type: 'INT32', nullable: false }, + { name: 'date', type: 'STRING', nullable: false }, +] + +/** The same plus `git_remote`, a real v7 capture column (LLP 0032). */ +/** @type {ColumnSpec[]} */ +const WIDE_COLUMNS = [...NARROW_COLUMNS, { name: 'git_remote', type: 'STRING', nullable: true }] + +const REMOTE = 'git@example.com:acme/app.git' + +/** + * Stage an icebird-backed `ai_gateway_messages` cache and return the dataset + * source a query would run against. + * + * `shape` picks the drift: + * - `lone`: ONE partition whose iceberg schema never had `git_remote`. This + * is the shape that skips `unionSources` entirely (`createDataSource` + * returns `withSchemaColumns(sources[0])`), so nothing but the wrapper + * stands between the query and icebird. + * - `drifted`: TWO partitions, one with `git_remote` and one without, joined + * by `unionSources`. + * + * @param {'lone' | 'drifted'} shape + * @returns {Promise<{ cacheRoot: string, source: AsyncDataSource }>} + */ +async function stageFixture(shape) { + const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), `hyp-absent-col-${shape}-`)) + await appendRowsToSourceTable( + cacheRoot, DATASET_NAME, ['source=claude'], + NARROW_COLUMNS, [{ id: 1, date: '2026-05-26' }] + ) + if (shape === 'drifted') { + await appendRowsToSourceTable( + cacheRoot, DATASET_NAME, ['source=codex'], + WIDE_COLUMNS, [{ id: 2, date: '2026-05-27', git_remote: REMOTE }] + ) + } + const storage = createQueryStorageService({ cacheRoot }) + /** @type {QueryScope} */ + const scope = { limit: 1000 } + const partitions = await discoverParts({ cacheDir: cacheRoot, scope, config: { version: 2 } }) + const source = await createDataSource(partitions, { scope, storage }) + return { cacheRoot, source } +} + +/** + * Run a SELECT the way `hyp query sql` does and return the collected rows, + * ordered by `id` so partition scan order can't make an assertion flap. + * + * @param {AsyncDataSource} source + * @param {string} query + * @returns {Promise[]>} + */ +async function runSql(source, query) { + const rows = await collect(executeSql({ tables: { t: source }, query })) + return /** @type {Record[]} */ (rows) +} + +/** + * @param {'lone' | 'drifted'} shape + * @param {(source: AsyncDataSource) => Promise} body + */ +async function withFixture(shape, body) { + const { cacheRoot, source } = await stageFixture(shape) + try { + await body(source) + } finally { + await fs.rm(cacheRoot, { recursive: true, force: true }) + } +} + +// --- what the wrapper makes addressable at all --- + +test('absent column: an icebird source advertises the declared column it physically lacks', async () => { + await withFixture('lone', async (source) => { + assert.ok(source.columns.includes('git_remote'), 'declared v7 column is addressable') + assert.ok(source.columns.includes('id'), 'physical column is addressable') + // Without this the SELECTs below would fail in validateScan, not in the + // scan: the contract only becomes interesting because planning succeeds. + assert.equal(typeof source.scanColumn, 'function', 'the column-stream hook survives the wrapper') + }) +}) + +// --- the two read paths yield DIFFERENT values --- + +test('absent column: a single-column bare projection reads null (scanColumn fast path)', async () => { + // The engine routes a scan whose hint set is exactly one column through + // `scanColumn` (squirreling execute.js gates on `columns?.length === 1`, + // with no aggregate required), and `withSchemaColumns` normalizes the + // hole to null on that path. So this yields null, and RENDERS as null. + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT git_remote FROM t') + assert.equal(rows.length, 1) + assert.ok('git_remote' in rows[0]) + assert.strictEqual(rows[0].git_remote, null, 'single-column projection reads null, not undefined') + assert.equal(JSON.stringify(rows[0]), '{"git_remote":null}', 'null survives JSON rendering') + }) +}) + +test('absent column: an aliased single-column projection is still the null path', async () => { + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT git_remote AS gr FROM t') + assert.deepEqual(rows.map((r) => r.gr), [null]) + }) +}) + +test('absent column: a non-identifier sibling does NOT change the value or throw', async () => { + // Recorded because the natural prediction is the opposite. On the parquet + // union a literal sibling collapses the resolveable fast path and the + // drifted column's thunk throws. Here it does not: the hint set is still + // the single column `git_remote` (a literal reads no column), so the query + // stays on the `scanColumn` path and still reads null. + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT git_remote, 1 AS n FROM t') + assert.equal(rows.length, 1) + assert.strictEqual(rows[0].git_remote, null) + assert.equal(rows[0].n, 1) + }) +}) + +test('absent column: a multi-column bare projection reads undefined and JSON drops the key', async () => { + // Two hint columns take the row path instead. icebird builds the row with + // `asyncRow(obj, requestedColumns)`, so the cell EXISTS but resolves to + // undefined, and the pre-materialized `resolved` map `collect()` reads + // simply has no entry. The key is present on the output row with the value + // undefined, which `JSON.stringify` omits. + await withFixture('lone', async (source) => { + const rows = await runSql(source, 'SELECT id, git_remote FROM t') + assert.equal(rows.length, 1) + assert.equal(rows[0].id, 1) + assert.ok('git_remote' in rows[0], 'the key is present on the row') + assert.strictEqual(rows[0].git_remote, undefined, 'multi-column projection reads undefined, not null') + assert.equal(JSON.stringify(rows[0]), '{"id":1}', 'JSON.stringify drops the undefined key') + }) +}) + +test('absent column: a WHERE on a present column pushes the projection onto the row path', async () => { + // The trap this pins: the query LOOKS like the single-column null case, + // but the predicate's column joins the hint set, making it two, so the + // same SELECT list reads undefined instead of null. Nothing about the + // projection changed. + await withFixture('lone', async (source) => { + const rows = await runSql(source, "SELECT git_remote FROM t WHERE date >= '2026-01-01'") + assert.equal(rows.length, 1) + assert.strictEqual(rows[0].git_remote, undefined) + assert.equal(JSON.stringify(rows[0]), '{}') + }) +}) + +test('absent column: SELECT * carries the key but renders nothing for the partition that lacks it', async () => { + // Measured before LLP 0241, the star left the narrow partition's own row + // shape alone and the key was simply not there. LLP 0241 pads every row out + // to the advertised list, so the key now EXISTS and holds `undefined`. Only + // key presence moved: the value is the same `undefined` the row path always + // read, and the JSON rendering is byte-identical either way. Reverting + // either alignment call site flips `in` back to false and nothing else. + // @ref LLP 0241#alignment [tests]: a star's rows carry the advertised column list, so an absent column is a present key holding undefined + await withFixture('drifted', async (source) => { + const rows = (await runSql(source, 'SELECT * FROM t')).sort((a, b) => Number(a.id) - Number(b.id)) + assert.equal(rows.length, 2) + assert.equal('git_remote' in rows[0], true, 'star pads the narrow row out to the advertised list') + assert.strictEqual(rows[0].git_remote, undefined, 'the padded cell reads undefined, not null') + assert.equal( + JSON.stringify(rows[0]), '{"id":1,"date":"2026-05-26"}', + 'the padded key is absent from the rendering, exactly as before LLP 0241' + ) + assert.equal(rows[1].git_remote, REMOTE) + }) +}) + +// --- nothing on this path throws --- + +test('absent column: evaluating, ordering, grouping and aggregating never throw on icebird', async () => { + // The parquet-backed union throws `ColumnNotFoundError` for these, because + // its rows carry no cell for the column at all. icebird's do (a thunk that + // resolves to undefined), so every one of these answers instead. This is + // the single sharpest difference between the two backings. + await withFixture('lone', async (source) => { + assert.deepEqual(await runSql(source, "SELECT git_remote || 'x' AS e FROM t"), [{ e: null }]) + assert.deepEqual(await runSql(source, 'SELECT id FROM t ORDER BY git_remote'), [{ id: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT DISTINCT git_remote FROM t'), [{ git_remote: null }]) + assert.deepEqual( + await runSql(source, 'SELECT git_remote, COUNT(*) AS n FROM t GROUP BY git_remote'), + [{ git_remote: null, n: 1 }] + ) + assert.deepEqual(await runSql(source, 'SELECT COUNT(git_remote) AS n FROM t'), [{ n: 0 }]) + assert.deepEqual(await runSql(source, 'SELECT MAX(git_remote) AS m FROM t'), [{ m: null }]) + }) +}) + +// --- predicates on the absent column answer correctly --- + +test('absent column: a lone icebird partition answers predicates on the column it lacks', async () => { + // Regression pin. `withSchemaColumns.scan` used to forward the predicate + // verbatim; icebird converted it to a hyparquet filter over a column its + // schema never had, filtered nothing, and still reported + // `appliedWhere: true`, so the engine trusted the unfiltered stream and + // BOTH of these returned the row. The union's own gate hid it, so only a + // single-partition cache (a fresh install with one client) was affected. + // @ref LLP 0240#where-gate [tests]: an ungated row-path where made a lone icebird partition answer predicates on an absent column wrongly + await withFixture('lone', async (source) => { + assert.deepEqual(await runSql(source, "SELECT id FROM t WHERE git_remote = 'zzz'"), []) + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NOT NULL'), []) + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NULL'), [{ id: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT COUNT(*) AS n FROM t WHERE git_remote IS NULL'), [{ n: 1 }]) + assert.deepEqual(await runSql(source, "SELECT COUNT(*) AS n FROM t WHERE git_remote = 'zzz'"), [{ n: 0 }]) + }) +}) + +test('absent column: the wrapper reports appliedWhere false for a predicate it had to strip', async () => { + // The flag is what the engine trusts; assert it directly so a regression + // shows up as a flag, not only as a wrong row count. + await withFixture('lone', async (source) => { + /** @type {ExprNode} */ + const where = { + type: 'binary', + op: '=', + left: { type: 'identifier', name: 'git_remote', positionStart: 0, positionEnd: 0 }, + right: { type: 'literal', value: 'zzz', positionStart: 0, positionEnd: 0 }, + positionStart: 0, + positionEnd: 0, + } + const stripped = source.scan({ columns: ['id', 'git_remote'], where, limit: 1 }) + assert.equal(stripped.appliedWhere, false, 'a predicate on a physically absent column is not claimed') + assert.equal(stripped.appliedLimitOffset, false, 'the slice is handed back with the filter') + + // A predicate the partition CAN satisfy is still pushed and claimed, so + // the gate does not cost the ordinary filtered read its pushdown. + /** @type {ExprNode} */ + const pushable = { + type: 'binary', + op: '=', + left: { type: 'identifier', name: 'id', positionStart: 0, positionEnd: 0 }, + right: { type: 'literal', value: 1, positionStart: 0, positionEnd: 0 }, + positionStart: 0, + positionEnd: 0, + } + assert.equal(source.scan({ columns: ['id'], where: pushable }).appliedWhere, true) + }) +}) + +test('absent column: a star filtered on the absent column needs BOTH the gate and the padding', async () => { + // The composition of LLP 0240#where-gate and LLP 0241#alignment, which + // neither pinned on its own: the gate's tests all project a named column + // (which puts it in the scan's `columns` hint, so icebird builds a cell for + // it either way), and the alignment's tests never strip a predicate. + // + // A star gets NO `columns` hint, so its rows are as wide as the partition + // physically is. The gate drops the predicate and hands filtering back to + // the engine, and the engine reads `git_remote` off `row.cells`. Measured + // with the two alignment call sites reverted, that lookup raises + // `ColumnNotFoundError: Column "git_remote" not found. Available columns: + // id, date` from `filterRows`. Padding is what gives it a cell to read. + // Revert the gate instead and the `= 'zzz'` case comes back with the row. + // @ref LLP 0241#alignment [tests]: padding is what lets the engine re-filter a predicate the wrapper's gate handed back + await withFixture('lone', async (source) => { + assert.deepEqual(await runSql(source, "SELECT * FROM t WHERE git_remote = 'zzz'"), []) + const kept = await runSql(source, 'SELECT * FROM t WHERE git_remote IS NULL') + assert.equal(kept.length, 1) + assert.equal(JSON.stringify(kept[0]), '{"id":1,"date":"2026-05-26"}') + assert.equal(JSON.stringify(await runSql(source, 'SELECT * FROM t ORDER BY git_remote')), + '[{"id":1,"date":"2026-05-26"}]') + }) +}) + +// --- the drifted union: present values are untouched --- + +test('absent column: a drifted union reads real values alongside the holes', async () => { + await withFixture('drifted', async (source) => { + const bare = (await runSql(source, 'SELECT git_remote FROM t')) + .map((r) => r.git_remote).sort() + assert.deepEqual(bare, [REMOTE, null], 'the hole is null on the single-column path, the value is intact') + + const pair = (await runSql(source, 'SELECT id, git_remote FROM t')) + .sort((a, b) => Number(a.id) - Number(b.id)) + assert.strictEqual(pair[0].git_remote, undefined, 'the hole is undefined on the row path') + assert.equal(pair[1].git_remote, REMOTE) + + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NOT NULL'), [{ id: 2 }]) + assert.deepEqual(await runSql(source, 'SELECT id FROM t WHERE git_remote IS NULL'), [{ id: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT COUNT(git_remote) AS n FROM t'), [{ n: 1 }]) + assert.deepEqual(await runSql(source, 'SELECT MAX(git_remote) AS m FROM t'), [{ m: REMOTE }]) + }) +}) + +test('absent column: a present column is unaffected on either path', async () => { + // The control. If these ever drift, the assertions above are measuring the + // fixture rather than the contract. + await withFixture('drifted', async (source) => { + const ids = (await runSql(source, 'SELECT id FROM t')).map((r) => r.id).sort() + assert.deepEqual(ids, [1, 2]) + const pairs = (await runSql(source, 'SELECT id, date FROM t')) + .sort((a, b) => Number(a.id) - Number(b.id)) + assert.deepEqual(pairs, [{ id: 1, date: '2026-05-26' }, { id: 2, date: '2026-05-27' }]) + }) +})