Pin the icebird absent-column contract at the SQL surface (#778)

hypaware-core/plugins-workspace/ai-gateway/src/dataset.js

@@ -163,12 +163,15 @@ const SCHEMA_COLUMN_NAMES = AI_GATEWAY_SCHEMA_COLUMNS.map((c) => c.name)
  * (e.g. `git_remote`/`head_sha`/`repo_root` in v7, LLP 0032). Squirreling's
  * `validateScan` rejects a SELECT that names a column absent from the source's
  * `columns`, so without this a contract or query that reads a freshly-added
- * column would throw `ColumnNotFoundError` over any pre-bump partition. The scan
- * itself is unchanged: a column an old partition physically lacks stays
- * addressable, and the exact value a read of it yields depends on the read path
- * (LLP 0015#multi-partition-union).
+ * column would throw `ColumnNotFoundError` over any pre-bump partition. Over
+ * the icebird-backed cache the value such a read yields is `null` on the
+ * single-column `scanColumn` path and `undefined` on the row path, never a
+ * throw; LLP 0240 records the measured contract and the tests that pin it.
+ * LLP 0015#multi-partition-union states the parquet-backed contract, which is
+ * a different one (undefined-or-throws) and does not govern this dataset.
  *
  * @ref LLP 0032#capture [implements]: additive columns stay queryable over old partitions; no partition-label bump / cache wipe needed
+ * @ref LLP 0240#contract [implements]: the wrapper is what makes an absent column addressable; its read values are pinned at the SQL surface
  * @param {AsyncDataSource} source
  * @returns {AsyncDataSource}
  */
@@ -178,21 +181,42 @@ function withSchemaColumns(source) {
   const wrapped = {
     columns,
     numRows: source.numRows,
+    // The row path owes the same predicate gate as `scanColumn` below. A
+    // `where` naming a declared-but-physically-absent column must not reach
+    // the source: an icebird partition builds a hyparquet filter on a column
+    // its schema never had, matches nothing away, and still reports
+    // `appliedWhere: true`, so the engine trusts the unfiltered stream and
+    // `WHERE git_remote = 'x'` returns every row. Forwarding it verbatim was
+    // wrong in exactly the shape LLP 0098 already forbids; the union hides it
+    // (its own gate fires first) so only a single-partition cache was hit.
+    // @ref LLP 0098#wrapper-duties [implements]: a predicate naming a declared-but-absent column is stripped on the row path too, not only on scanColumn
+    // @ref LLP 0240#where-gate [implements]: an ungated row-path where made a single icebird partition answer predicates on an absent column wrongly
     scan(options) {
       // The engine names this scan's output columns from the list advertised
       // here, but fills them from each row's own `columns`. A partition that
       // predates a declared column yields a SHORTER row, which slides every
       // output name past the gap onto its neighbour's value: over a drifted
       // union `SELECT *, git_remote` answered with git_remote's value under
       // the name of the column that happened to follow the star's short
-      // width. Pad each row back out to the advertised list.
+      // width. Pad each row back out to the advertised list. Stripping the
+      // predicate below does not narrow it: the gate only drops `where`.
       // @ref LLP 0241#alignment [implements]: a declared-but-absent column becomes a padded cell, not a missing slot the star can slide through
       const scanColumns = options?.columns ?? columns
-      const result = source.scan(options)
+      const pushable = !options?.where || canPushWhere(source, whereColumns(options.where))
+      // Stripping the predicate also strips limit/offset: they are only
+      // meaningful after the filter, and a source that ignored the predicate
+      // but honored a slice would silently drop matching rows.
+      const inner = pushable
+        ? source.scan(options)
+        : source.scan({ ...options, where: undefined, limit: undefined, offset: undefined })
       return {
-        appliedWhere: result.appliedWhere,
-        appliedLimitOffset: result.appliedLimitOffset,
-        rows: () => alignRows(result.rows(), scanColumns),
+        appliedWhere: pushable && inner.appliedWhere,
+        appliedLimitOffset: pushable && inner.appliedLimitOffset,
+        // The two duties compose in one direction: the gate hands an
+        // unfiltered stream back to the engine, and the padding is what lets
+        // the engine's own re-filter read the absent column as `undefined`
+        // instead of missing it on a short row (LLP 0241#alignment).
+        rows: () => alignRows(inner.rows(), scanColumns),
       }
     },
   }

hypaware-core/plugins-workspace/ai-gateway/src/message_projector.js

@@ -22,10 +22,14 @@ const DATASET_NAME = 'ai_gateway_messages'
  * of which adapter projector produced the messages (projector-defined
  * fields map onto these named columns directly). `schema_version` 7 added
  * the `git_remote` / `head_sha` / `repo_root` capture columns (LLP 0032);
- * the additions are nullable and no partition-label bump is needed. An old
- * partition physically lacks them; `withSchemaColumns` in `dataset.js` is the
- * only reason they stay addressable at all, and the exact value a read of one
- * yields depends on the read path (LLP 0015#multi-partition-union).
+ * the additions are nullable and no partition-label bump is needed, so old
+ * partitions carry no such column at all. `withSchemaColumns` in `dataset.js`
+ * is the only reason they stay addressable. A read of one over the
+ * icebird-backed cache never throws, but it is `null` on the single-column
+ * scan path and `undefined` on the row path: LLP 0240 has the measured table.
+ * Treat both as absent; do not branch on which one you got.
+ *
+ * @ref LLP 0240#contract [constrained-by]: an additive nullable column reads null or undefined depending on the scan path, never one canonical value
  *
  * @type {ReadonlyArray<ColumnSpec>}
  */

llp/0015-query-and-datasets.spec.md

@@ -22,6 +22,14 @@
 > OOM the host by buffering an unbounded scan
 > ([hyparam/hypaware-server#9](https://github.com/hyparam/hypaware-server/issues/9)).
 
+> **Extended by [LLP 0240](./0240-icebird-absent-column-contract.decision.md).**
+> The union section below describes the parquet-backed sources it was written
+> against. Over the **icebird**-backed cache (which is what
+> `ai_gateway_messages` reads from) a declared-but-physically-absent column
+> never throws, and reads as `null` or `undefined` depending on which scan
+> path the engine takes. LLP 0240 records the measured values and the
+> SQL-surface tests that pin them.
+
 ## Query is intrinsic
 
 Query and Iceberg storage are intrinsic services. Plugins register datasets;

llp/0098-scancolumn-where-pushdown.decision.md

@@ -12,6 +12,13 @@
 > flags, so a filtered `COUNT` keeps the streaming fast path instead of
 > falling back to per-row materialization.
 
+> **Extended by [LLP 0240](./0240-icebird-absent-column-contract.decision.md).**
+> `#wrapper-duties` below states the `withSchemaColumns` predicate gate as a
+> duty of the wrapper; it was implemented only on `scanColumn`, and the row
+> `scan` forwarded the predicate verbatim. LLP 0240 extends the same gate to
+> the row path and records what the gap cost on an icebird-backed partition,
+> which reports the filter applied instead of throwing.
+
 ## Context
 
 LLP 0055 lit the engine's streaming-aggregate fast path by implementing

llp/0240-icebird-absent-column-contract.decision.md

@@ -0,0 +1,154 @@
+# LLP 0240: What an icebird-backed read of an absent column actually yields
+
+**Type:** Decision
+**Status:** Accepted
+**Systems:** Query, Cache
+**Author:** Claude
+**Date:** 2026-08-15
+**Related:** LLP 0015 (#multi-partition-union: the union contract this
+completes for the icebird backing), LLP 0032 (#capture: the additive v7
+columns that create the drift), LLP 0098 (#wrapper-duties: the predicate gate
+this extends from `scanColumn` to the row path), LLP 0055
+
+> Extends [LLP 0015](./0015-query-and-datasets.spec.md). LLP 0015 settles what
+> the union does with a column a partition physically lacks, in terms of the
+> parquet-backed sources it was written against. It does not settle what the
+> **icebird**-backed cache yields for such a read, and the flagship dataset
+> `ai_gateway_messages` is icebird-backed. This decision records that, measured
+> at the SQL surface rather than derived from the code, and closes a
+> correctness hole the measurement exposed.
+
+> **Amended by [LLP 0241 §alignment](./0241-scan-rows-carry-advertised-columns.decision.md#alignment),
+> which landed first.** 0241 pads every scanned row out to the column list the
+> scan advertised, which moves exactly one cell of the table below: under
+> `SELECT *` the absent column's key now **exists** and holds `undefined`,
+> where it was previously not on the row at all. The rendering is unchanged
+> and no other row of the table moves; re-measured on the merged tree, and
+> pinned by the same test file. 0241 also fixes the star-expansion defect the
+> Consequences below deferred to issue #788.
+
+## Context
+
+`ai_gateway_messages` declares more columns than any given partition
+physically has. Schema v7 added `git_remote` / `head_sha` / `repo_root` as
+nullable (LLP 0032) with no partition-label bump, so every partition written
+before the bump lacks them, and `withSchemaColumns` in the ai-gateway plugin
+advertises the declared set on top of whatever the storage source reports so
+that a SELECT naming one of them plans at all.
+
+What such a read then *yields* had never been pinned. Only the raw `scan()`
+rows and the `scanColumn()` chunks were tested; no test ran a full SELECT
+through `executeSql` + `collect`, which is the pair `hyp query sql` uses. In
+the absence of a test, five successive written descriptions of this mechanism
+were each measured false during the review of #731 / PR #740, and the
+maintainer descoped the icebird half rather than ship a sixth guess.
+
+Everything below was obtained by running the query and recording the answer.
+
+## Decision
+
+### <a id="contract"></a>The contract
+
+Over an icebird-backed partition that physically lacks a declared column,
+**nothing throws**, and the value read depends on which path the engine takes:
+
+| query shape | value | rendering |
+| --- | --- | --- |
+| `SELECT git_remote FROM t` | `null` | `{"git_remote":null}` |
+| `SELECT git_remote AS gr FROM t` | `null` | `{"gr":null}` |
+| `SELECT git_remote, 1 AS n FROM t` | `null` | `{"git_remote":null,"n":1}` |
+| `SELECT id, git_remote FROM t` | `undefined` | `{"id":1}` (key dropped) |
+| `SELECT git_remote FROM t WHERE date >= '...'` | `undefined` | `{}` |
+| `SELECT * FROM t` | `undefined`, under a key that exists (LLP 0241) | `{"id":1,"date":"..."}` |
+
+The discriminator is **the size of the scan's hint column set, not the shape
+of the SELECT list.** Squirreling routes a scan whose hints name exactly one
+column through `scanColumn` (`execute.js`, gated on
+`plan.hints.columns?.length === 1`, with no aggregate required), and
+`withSchemaColumns` normalizes the hole to `null` on exactly that path.
+Anything that widens the hint set to two columns takes the row path instead
+and reads `undefined`. A literal or expression sibling reads no column, so it
+does **not** widen it; a `WHERE` on an unrelated column does, which is why
+adding a date filter silently flips the same projection from `null` to
+`undefined`.
+
+On the row path the value is `undefined` rather than a throw because icebird
+builds each row with squirreling's `asyncRow(obj, requestedColumns)` over the
+**requested** column list: the cell exists as a thunk that resolves to
+`obj[name]`, which is `undefined`, and the pre-materialized `resolved` map
+that `collect()` reads simply has no entry for it. This is the whole
+difference from a parquet-backed partition, whose `asyncRow` is built over
+`Object.keys(data[0])`, the row's **physical** keys, so the cell does not
+exist and anything evaluating it throws `ColumnNotFoundError`. Hence, on
+icebird, `ORDER BY`, `GROUP BY`, `DISTINCT`, an expression, and an aggregate
+over the absent column all answer (with `null`, or a count that skips it)
+where the parquet union throws.
+
+Consequently **`null` and `undefined` are both live readings of the same
+absent cell, and neither is "the" value.** A consumer that must distinguish
+"no value" from "column predates this partition" cannot do it from the read;
+callers should treat both as absent, and no code should branch on which one it
+got.
+
+### <a id="where-gate"></a>The row path owes the same predicate gate as `scanColumn`
+
+LLP 0098 (#wrapper-duties) already requires `withSchemaColumns` to strip a
+predicate naming a declared-but-physically-absent column before it reaches the
+source. Only `scanColumn` implemented it; `scan` forwarded `options` verbatim.
+Measuring the contract exposed what that costs on icebird, which does not
+throw where parquet does:
+
+- icebird converts the predicate to a hyparquet filter over a column its
+  schema never had,
+- filters nothing away,
+- and still reports `appliedWhere: true`,
+
+so the engine trusts the stream and does not re-filter. On a cache with a
+**single** partition lacking the column, `SELECT id FROM t WHERE git_remote =
+'zzz'` returned every row, and so did `WHERE git_remote IS NOT NULL`. Two or
+more partitions hid it, because `createDataSource` then wraps `unionSources`,
+whose own per-partition gate (LLP 0015#multi-partition-union) fires first. The
+exposed shape is therefore the ordinary one: a fresh install with one client.
+
+`withSchemaColumns.scan` now applies the same gate as its `scanColumn`: when
+the predicate names a column the wrapped source does not advertise, drop the
+predicate along with `limit`/`offset` (only meaningful post-filter) and report
+`appliedWhere: false` / `appliedLimitOffset: false`, handing the filter back
+to the engine. A predicate the source can satisfy is still pushed and still
+claimed, so the ordinary filtered read keeps its pushdown.
+
+### <a id="pinned"></a>Pinned at the SQL surface
+
+The contract is pinned by
+[`test/core/ai-gateway-absent-column-sql.test.js`](../test/core/ai-gateway-absent-column-sql.test.js),
+which runs `executeSql` + `collect` over a staged icebird cache in two shapes:
+one partition lacking the column (no union in the way), and a drifted pair.
+Every value is asserted exactly, as `null` versus `undefined` versus key
+absence, never through a tolerant `?? null`. That form is deliberate: a
+tolerant assertion is what let the mechanism be described wrongly five times
+while the suite stayed green.
+
+## Consequences
+
+- Documentation of this contract belongs here, not in LLP 0015. LLP 0015 is
+  Active and its union section was corrected separately for the parquet
+  backing (#731 / PR #740, now on master); this doc carries the icebird half
+  and 0015 gains only a forward-ref.
+- The `null`-versus-`undefined` split is a property of the engine's fast-path
+  gate, not of the cache. If squirreling ever widens or narrows that gate, the
+  values in the table above move, and the pinning tests are what will say so.
+- `SELECT *, git_remote FROM t` over a drifted union was observed to
+  mis-assign a value into a neighbouring declared column. That is a star
+  expansion defect above this layer, is not part of this contract, and is left
+  unaddressed here; the tests deliberately do not cover it. It is tracked as
+  [hyparam/hypaware#788](https://github.com/hyparam/hypaware/issues/788), and
+  is **now fixed** by [LLP 0241](./0241-scan-rows-carry-advertised-columns.decision.md),
+  which landed on master first.
+- The #where-gate and 0241's padding are not independent at the star. A star
+  carries no `columns` hint, so its rows are only as wide as the partition
+  physically is; the gate then hands the predicate back to the engine, which
+  reads the absent column off `row.cells`. Measured with 0241's two alignment
+  call sites reverted, `SELECT * FROM t WHERE git_remote IS NULL` raises
+  `ColumnNotFoundError` from `filterRows` instead of answering. The padding is
+  what makes the handed-back filter evaluable, so the two must ship together;
+  the composition has its own pin in the test file.