diff --git a/CLAUDE.md b/CLAUDE.md index 2bb13c8..9df3bf2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ - Release tags use 3-part SemVer (`vMAJOR.MINOR.PATCH`, e.g. `v1.0.0`). This is required, not stylistic: ColdFront is a Go module (`github.com/pgedge/coldfront`), and the toolchain only treats full `vX.Y.Z` tags as releases — a 2-part `v1.0` tag yields pseudo-versions. Git tags, GitHub releases, container image tags, and the changelog all use this form. The patch field keeps a bugfix-only release (`v1.0.1`) distinct from a feature release (`v1.1.0`). - The PostgreSQL extension keeps the conventional 2-part version in `extension/coldfront/coldfront.control` (`default_version`) and the `coldfront--X.Y.sql` / `coldfront--X.Y--X.Z.sql` upgrade-script filenames, per PG convention. Extension `1.0` ships inside release `v1.0.0`; a patch release may carry the same extension version or bump it with an upgrade script when the SQL changes. - Build 4 static binaries per release: linux-amd64, linux-arm64, darwin-amd64, darwin-arm64 -- Build command: `CGO_ENABLED=0 go build -ldflags="-s -w"` +- Build command: `CGO_ENABLED=0 go build -ldflags="-s -w -X github.com/pgedge/coldfront/internal/version.Version=$TAG -X github.com/pgedge/coldfront/internal/version.BuildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)"` (the compactor stamps `-X main.Version` / `-X main.BuildTime`); `make build` derives the stamp from `git describe` - Release notes format: `## Added` / `## Changed` / `## Fixed` - Only list user-facing changes — no internal test additions, no same-cycle fix churn - `## Fixed` is for bugs that existed in the previous release, not things broken and fixed in the same cycle diff --git a/Makefile b/Makefile index 78ff414..c85f09d 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,12 @@ # golangci-lint path: PATH first, else the default go install location. ci/matrix.sh # passes GOLANGCI= so the compactor gate uses the same linter. GOLANGCI ?= $(shell command -v golangci-lint 2>/dev/null || echo $(HOME)/go/bin/golangci-lint) +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo unknown) +BUILD_TIME := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") build: - CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/archiver ./cmd/archiver - CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/partitioner ./cmd/partitioner + CGO_ENABLED=0 go build -ldflags="-s -w -X github.com/pgedge/coldfront/internal/version.Version=$(VERSION) -X github.com/pgedge/coldfront/internal/version.BuildTime=$(BUILD_TIME)" -o bin/archiver ./cmd/archiver + CGO_ENABLED=0 go build -ldflags="-s -w -X github.com/pgedge/coldfront/internal/version.Version=$(VERSION) -X github.com/pgedge/coldfront/internal/version.BuildTime=$(BUILD_TIME)" -o bin/partitioner ./cmd/partitioner # compactor: a SEPARATE Go module (cmd/compactor/go.mod) so iceberg-go's heavy # dependency tree never links into the lean archiver. Its full gate — vet, lint, @@ -18,7 +20,7 @@ compactor: cd cmd/compactor && go vet ./... cd cmd/compactor && "$(GOLANGCI)" run --timeout=5m cd cmd/compactor && go test ./... - cd cmd/compactor && CGO_ENABLED=0 go build -ldflags="-s -w" -o $(CURDIR)/bin/compactor . + cd cmd/compactor && CGO_ENABLED=0 go build -ldflags="-s -w -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME)" -o $(CURDIR)/bin/compactor . test: go test -race -v ./... diff --git a/README.md b/README.md index 357317a..38f968b 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ The following table lists the ColdFront guides and what each one covers: | Doc | Contents | |---|---| +| **[Embeddings](docs/usage_vectors.md)** | Storing and searching embeddings with the pgvector interface | | **[Usage](docs/usage.md)** | Day-to-day use - both modes plus the standalone partition manager, one-time setup, reading/writing, supported types, the partition CLI, storage backends, distributed (mesh) setup, tuning | | **[Installation](docs/installation.md)** | Build from source (Docker or bare-metal); Testing & CI | | **[Object store setup](docs/object_store.md)** | Get ColdFront running on cloud S3 (virtual-hosted), end-to-end | @@ -128,6 +129,7 @@ The following table lists the ColdFront guides and what each one covers: | **[Architecture](docs/architecture.md)** | Shared architecture and core mechanics | | **[Architecture: tiered](docs/architecture_tiered.md)** | Tiered (hot PG + cold Iceberg) deep dive | | **[Architecture: decoupled](docs/architecture_decoupled.md)** | Decoupled (iceberg-only) deep dive | +| **[Architecture: vectors](docs/architecture_vectors.md)** | Vector storage internals - type mapping, routing state, cluster assignment, layout | ## Least-privilege application roles @@ -180,7 +182,7 @@ pgedge-coldfront/ │ ├── partcfg/ ← in-DB, Spock-replicated per-table lifecycle config │ ├── partition/ ← partition create/find/detach/drop (time + id modes) │ ├── sqlutil/ ← shared SQL helpers -│ ├── view/ ← unified view + trigger generation +│ ├── view/ ← unified view generation │ └── watermark/ ← archive_watermark table CRUD ├── extension/coldfront/ ← PGXS C extension (DML hooks, bakery, registry, SQL) ├── ci/ @@ -201,7 +203,8 @@ pgedge-coldfront/ │ └── seaweedfs-s3.json ← SeaweedFS S3 auth config (example) ├── docs/ ← MkDocs site (user docs; mkdocs.yml at repo root) │ ├── index.md · installation.md · object_store.md · usage.md · compaction.md -│ ├── architecture.md · architecture_tiered.md · architecture_decoupled.md · changelog.md +│ ├── architecture.md · architecture_tiered.md · architecture_decoupled.md +│ ├── architecture_vectors.md · usage_vectors.md · changelog.md │ └── formal/ ← TLA+ model of the bakery protocol (Bakery_v2.tla) ├── docker-compose.yml ← END-USER single-node stack (ports published) ├── docker-compose.matrix.yml ← CI only: single-node vanilla matrix diff --git a/ci/journey.sh b/ci/journey.sh index a51462b..74e9054 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -587,6 +587,597 @@ EOSQL assert_contains "TC-099: oid rejected at provisioning (use bigint instead)" "oid values as bigint" "$OE" } +# ─────────────────────────────────────────────────────────────────────────── +# Story 5b — Embeddings round-trip. A pgvector column tiers as an Iceberg +# list and comes back element-for-element through the view, over all three +# cold-write paths: the archiver's bulk export, the INSTEAD OF trigger's cold +# INSERT, and the decoupled hook. The view exposes the column as real[], and PG +# spells an array {1,2,3} where DuckDB's list cast takes only [1,2,3], so each +# path has to rewrite the delimiters; comparing the stored value to a literal is +# what catches a path that does not. +# ─────────────────────────────────────────────────────────────────────────── +story_vector() { + step "5b. Embeddings round-trip (vector → list, tiered + decoupled)" + qf "$HOST" <<'EOSQL' >/dev/null +SET search_path = public; +CREATE EXTENSION IF NOT EXISTS vector; +CREATE TABLE IF NOT EXISTS chunks ( + id bigint GENERATED ALWAYS AS IDENTITY, + ts timestamptz NOT NULL, + body text, + embedding vector(3), + PRIMARY KEY (id, ts) +) PARTITION BY RANGE (ts); +-- m4 (cold after archiving) and m1 (hot), named from now()-relative months. +DO $do$ +DECLARE m date; +BEGIN + FOREACH m IN ARRAY ARRAY[(date_trunc('month',now()) - interval '4 months')::date, + (date_trunc('month',now()) - interval '1 month')::date] LOOP + EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF chunks FOR VALUES FROM (%L) TO (%L)', + 'chunks_p_' || to_char(m, 'YYYY_MM'), m, (m + interval '1 month')); + END LOOP; +END $do$; +-- The hot column is a real vector, so the unadorned pgvector literal is what a +-- user writes. Both rows carry the same value: one archives, one stays hot. +INSERT INTO chunks (ts, body, embedding) +VALUES (date_trunc('month',now()) - interval '4 months' + interval '9 days', 'cold', '[1.5,-2,3]'), + (date_trunc('month',now()) - interval '1 month' + interval '9 days', 'hot', '[1.5,-2,3]'); +EOSQL + local ret_days; ret_days=$(hot_days) + cat > /tmp/journey-chunks.yaml </tmp/journey-chunks.log 2>&1; then + fail "import chunks into partition_config — see /tmp/journey-chunks.log"; tail -5 /tmp/journey-chunks.log; return + fi + if "$ARCHIVER" --config /tmp/journey-chunks.yaml >>/tmp/journey-chunks.log 2>&1; then + pass "vector table archived (m4 → cold)" + else + fail "vector archive — see /tmp/journey-chunks.log"; tail -5 /tmp/journey-chunks.log; return + fi + + # Verify what landed in Iceberg, read straight from the cold table. Equality + # against a real[] literal is the whole assertion: a stringified write could + # not have landed at all, since a FLOAT[] column rejects PG's {…} array text. + # + # Not through the tiered view: that plan carries the hot table's pgvector + # column, which pg_duckdb cannot scan, so a view read of this column depends + # on PostgreSQL happening to prune the hot branch. + local O; O=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'COLD_VEC:' || (r['embedding']::real[] = ARRAY[1.5,-2,3]::real[])::text + FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] = 'cold'; +EOSQL +) + assert_eq "cold vector round-trip (bulk export)" "true" "$(extract COLD_VEC "$O")" + + # Through the view, with the predicate a caller actually writes. The watermark + # classifies this as cold-only, so the read must reach the Iceberg side alone: + # the hot branch scans a pgvector column, which fails the whole plan even when + # no hot row can match. + # Through the view, with a now()-relative bound. The watermark sits at the + # archived partition's upper bound, which date_trunc lands on exactly, so the + # read is cold-only however long ago the fixture was built. + local V; V=$(qf "$HOST" <<'EOSQL' +SELECT 'VIEW_VEC:' || (embedding = ARRAY[1.5,-2,3]::real[])::text FROM chunks + WHERE body = 'cold' AND ts < date_trunc('month', now()) - interval '3 months'; +SELECT 'VIEW_X:' || count(*)::text FROM chunks WHERE embedding IS NOT NULL; +SELECT 'VIEW_XB:' || string_agg(body, ',' ORDER BY body) FROM chunks WHERE embedding IS NOT NULL; +EOSQL +) + assert_eq "cold vector read through the view" "true" "$(extract VIEW_VEC "$V")" + # No tier bound: one hot row and one archived row, both projecting the vector, + # which is the read that needs the hot side to be scannable at all. + assert_eq "cross-tier read projecting the vector" "2" "$(extract VIEW_X "$V")" + assert_eq "cross-tier read spans both tiers" "cold,hot" "$(extract VIEW_XB "$V")" + assert_eq "cold vector read through the view (classified, now()-relative)" "true" "$(extract VIEW_VEC "$V")" + + # Cold INSERT through the view trigger (ts < cutoff): a second write path, + # and the one that renders NEW.embedding::text. The vector literal exercises + # pgvector's implicit vector -> real[] cast on the way into the view column. + local CI; CI=$(qf "$HOST" <<'EOSQL' +INSERT INTO chunks (ts, body, embedding) +VALUES (date_trunc('month',now()) - interval '4 months' + interval '19 days', 'coldins', '[4,5.5,-6]'::vector); +SELECT coldfront.ensure_attached(); +SELECT 'COLDINS_VEC:' || (r['embedding']::real[] = ARRAY[4,5.5,-6]::real[])::text + FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] = 'coldins'; +EOSQL +) + assert_eq "cold-INSERT-via-trigger vector round-trip" "true" "$(extract COLDINS_VEC "$CI")" + + # The search itself: pgvector-shaped syntax over the view, ranking hot and cold + # rows together in one statement. The query vector is one of the stored rows, so + # the nearest is exact rather than a tie. + local S; S=$(qf "$HOST" <<'EOSQL' +SELECT 'NEAREST:' || body FROM chunks ORDER BY embedding <=> ARRAY[4,5.5,-6]::real[] LIMIT 1; +SELECT 'RANKED:' || string_agg(body, ',' ORDER BY d) FROM ( + SELECT body, embedding <=> ARRAY[4,5.5,-6]::real[] AS d FROM chunks) t; +EOSQL +) + assert_eq "top-k over both tiers finds the nearest row" "coldins" "$(extract NEAREST "$S")" + assert_contains "top-k ranks the exact match first" "coldins," "$(extract RANKED "$S")" + + # Decoupled: no hot tier, its own generated trigger, its own type map. Shares + # none of the archiver's plumbing, so it is a distinct path. + local i + for i in 1 2 3 4 5; do + q_may "$HOST" "SELECT coldfront.create_iceberg_table('public','icevec','[{\"name\":\"id\",\"type\":\"bigint\"},{\"name\":\"ts\",\"type\":\"timestamptz\"},{\"name\":\"embedding\",\"type\":\"vector(3)\"}]'::jsonb);" >/dev/null 2>&1 + [ "$(q "$HOST" "SELECT count(*) FROM pg_class WHERE relname='icevec' AND relkind='v' AND relnamespace='public'::regnamespace;")" = "1" ] && break + sleep 2 + done + local D; D=$(qf "$HOST" <<'EOSQL' +INSERT INTO icevec VALUES (1, date_trunc('month',now()) + interval '11 hours', '[7,-8.25,9]'::vector); +SELECT 'DEC_VEC:' || (embedding = ARRAY[7,-8.25,9]::real[])::text FROM icevec WHERE id = 1; +EOSQL +) + assert_eq "decoupled vector round-trip" "true" "$(extract DEC_VEC "$D")" + + # Decoupled declares its own schema, so the append has a second implementation. + local DL; DL=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'DEC_ICE_LIST:' || count(*) FROM duckdb.query($$SELECT column_name FROM (DESCRIBE ice.public.icevec)$$) AS t(r) + WHERE r['column_name']::text = '_cf_vec_list_embedding'; +SELECT 'DEC_VIEW_LIST:' || count(*) FROM pg_attribute + WHERE attrelid = 'public.icevec'::regclass AND attname = '_cf_vec_list_embedding' AND attnum > 0; +EOSQL +) + assert_eq "decoupled Iceberg schema carries the cluster column" "1" "$(extract DEC_ICE_LIST "$DL")" + assert_eq "decoupled view does not project it" "0" "$(extract DEC_VIEW_LIST "$DL")" + + # A decoupled INSERT is rewritten in C, a different generator again, and it + # must stamp the cluster in the same statement once a generation is live. + q "$HOST" "INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) VALUES ('public','icevec','embedding',2,1);" >/dev/null + q "$HOST" "INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) VALUES ('public','icevec','embedding',1,0,ARRAY[7,-8.25,9]::real[]),('public','icevec','embedding',1,1,ARRAY[-1,-1,-1]::real[]);" >/dev/null + q "$HOST" "UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'icevec';" >/dev/null + local DA; DA=$(qf "$HOST" <<'EOSQL' +INSERT INTO icevec VALUES (2, date_trunc('month',now()) + interval '12 hours', '[7,-8.25,9]'::vector); +SELECT coldfront.ensure_attached(); +SELECT 'DEC_ASG:' || r['_cf_vec_list_embedding'] FROM iceberg_scan('ice.public.icevec') r WHERE r['id'] = 2; +EOSQL +) + assert_eq "a decoupled INSERT stamps the nearest cluster" "0" "$(extract DEC_ASG "$DA")" + + # Two vector columns on one table. Each gets its own cluster column and its own + # assignment; only the first gets the file sort order, so only its probe prunes. + # Its own table, so nothing above depends on the second column existing. + local i2 + for i2 in 1 2 3 4 5; do + q_may "$HOST" "SELECT coldfront.create_iceberg_table('public','icetwo','[{\"name\":\"id\",\"type\":\"bigint\"},{\"name\":\"ts\",\"type\":\"timestamptz\"},{\"name\":\"embedding\",\"type\":\"vector(3)\"},{\"name\":\"summary\",\"type\":\"vector(2)\"}]'::jsonb);" >/dev/null 2>&1 + [ "$(q "$HOST" "SELECT count(*) FROM pg_class WHERE relname='icetwo' AND relkind='v';")" = "1" ] && break + sleep 2 + done + q "$HOST" "INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) VALUES ('public','icetwo','embedding',2,1),('public','icetwo','summary',2,1);" >/dev/null + q "$HOST" "INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) VALUES ('public','icetwo','embedding',1,0,ARRAY[1,0,0]::real[]),('public','icetwo','embedding',1,1,ARRAY[0,0,1]::real[]),('public','icetwo','summary',1,0,ARRAY[1,0]::real[]),('public','icetwo','summary',1,1,ARRAY[0,1]::real[]);" >/dev/null + q "$HOST" "UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'icetwo';" >/dev/null + local T2; T2=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'TWO_REG:' || array_to_string(vec_columns, ',') FROM coldfront.tiered_views WHERE relname = 'icetwo'; +SELECT 'TWO_COLS:' || string_agg(r['column_name']::text, ',' ORDER BY r['column_name']::text) + FROM duckdb.query($$SELECT column_name FROM (DESCRIBE ice.public.icetwo)$$) AS t(r) + WHERE starts_with(r['column_name']::text, '_cf_vec_list_'); +INSERT INTO icetwo VALUES (1, date_trunc('month',now()) + interval '13 hours', '[0,0,1]'::vector, '[0,1]'::vector); +SELECT coldfront.ensure_attached(); +SELECT 'TWO_ASG:' || r['_cf_vec_list_embedding'] || '/' || r['_cf_vec_list_summary'] + FROM iceberg_scan('ice.public.icetwo') r WHERE r['id'] = 1; +EOSQL +) + assert_eq "the registry records both vector columns" "embedding,summary" "$(extract TWO_REG "$T2")" + assert_eq "the Iceberg schema carries a cluster column each" "_cf_vec_list_embedding,_cf_vec_list_summary" "$(extract TWO_COLS "$T2")" + # embedding [0,0,1] is nearest its centroid 1; summary [0,1] is nearest its 1. + assert_eq "one INSERT assigns every vector column" "1/1" "$(extract TWO_ASG "$T2")" + # Only the first column is in the sort key, which is what makes its probe the + # only one that prunes. + local SK; SK=$(q "$HOST" "SELECT coldfront._vec_sort_key((SELECT vec_columns[1] FROM coldfront.tiered_views WHERE relname='icetwo'), NULL);") + assert_eq "the sort key names only the first vector column" "_cf_vec_list_embedding" "$SK" + + # Multi-row INSERT ... SELECT of cold rows: the extension rewrites this in C, + # a different generator from the single-row trigger above, and it builds its + # projection from the hot table's own column list. + local B; B=$(qf "$HOST" <<'EOSQL' +INSERT INTO chunks (ts, body, embedding) +SELECT date_trunc('month',now()) - interval '4 months' + interval '20 days' + (i || ' hours')::interval, + 'bulk' || i, ARRAY[i, i + 1, i + 2]::real[]::vector + FROM generate_series(1, 3) i; +SELECT 'BULK_N:' || count(*) FROM chunks WHERE body LIKE 'bulk%'; +SELECT 'BULK_VEC:' || (embedding = ARRAY[2,3,4]::real[])::text FROM chunks WHERE body = 'bulk2'; +EOSQL +) + assert_eq "bulk cold INSERT of vectors lands" "3" "$(extract BULK_N "$B")" + assert_eq "bulk cold INSERT round-trips the vector" "true" "$(extract BULK_VEC "$B")" + + # The cluster column belongs to the Iceberg schema and to nothing else: no + # branch of the view projects it, so no query written against the view can + # name it and SELECT * stays the user's own column list. + local H; H=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'ICE_LIST:' || count(*) FROM duckdb.query($$SELECT column_name FROM (DESCRIBE ice.public.chunks)$$) AS t(r) + WHERE r['column_name']::text = '_cf_vec_list_embedding'; +SELECT 'VIEW_LIST:' || count(*) FROM pg_attribute + WHERE attrelid = 'public.chunks'::regclass AND attname = '_cf_vec_list_embedding' AND attnum > 0; +SELECT 'HOT_LIST:' || count(*) FROM pg_attribute + WHERE attrelid = 'public._chunks'::regclass AND attname = '_cf_vec_list_embedding' AND attnum > 0; +EOSQL +) + assert_eq "the cluster column is in the Iceberg schema" "1" "$(extract ICE_LIST "$H")" + assert_eq "the view does not project it" "0" "$(extract VIEW_LIST "$H")" + assert_eq "the hot table does not carry it" "0" "$(extract HOT_LIST "$H")" + + # Training reads the cold corpus through DuckDB and writes a centroid set back + # into PostgreSQL. Two clusters over the archived rows is enough to prove the + # loop runs, the means come back as real[], and the generation pointer moves. + # CALL, not SELECT: reading a DuckDB result is only possible outside a function. + q "$HOST" "INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) VALUES ('public','chunks','embedding',2,1);" >/dev/null + local T; T=$(qf "$HOST" <<'EOSQL' +CALL coldfront.vector_train('public','chunks','embedding'); +SELECT 'GEN:' || generation FROM coldfront.vector_config WHERE table_name = 'chunks'; +SELECT 'NCENT:' || count(*) FROM coldfront.vector_centroids + WHERE table_name = 'chunks' AND generation = (SELECT generation FROM coldfront.vector_config WHERE table_name = 'chunks'); +SELECT 'DIMS:' || string_agg(DISTINCT array_length(centroid,1)::text, ',') FROM coldfront.vector_centroids WHERE table_name = 'chunks'; +SELECT 'FINITE:' || bool_and(c > '-Infinity'::real AND c < 'Infinity'::real)::text + FROM coldfront.vector_centroids, unnest(centroid) c WHERE table_name = 'chunks'; +-- k-means++ excludes a point it has already chosen (its distance to the seed set is +-- zero), so no two centroids may start from the same row. Distinct after the Lloyd +-- means as well, which a collapsed pair would not be. +SELECT 'DISTINCT:' || count(DISTINCT centroid::text) FROM coldfront.vector_centroids + WHERE table_name = 'chunks' + AND generation = (SELECT generation FROM coldfront.vector_config WHERE table_name = 'chunks'); +EOSQL +) + assert_eq "training writes a new generation" "1" "$(extract GEN "$T")" + assert_eq "training stores the trained centroids" "2" "$(extract NCENT "$T")" + assert_eq "centroids keep the column's dimension" "3" "$(extract DIMS "$T")" + assert_eq "centroid means are finite" "true" "$(extract FINITE "$T")" + assert_eq "seeding picked distinct starting points" "2" "$(extract DISTINCT "$T")" + + # A retrain is a new generation, never an edit of the one queries are resolving. + local T2; T2=$(qf "$HOST" <<'EOSQL' +CALL coldfront.vector_train('public','chunks','embedding'); +SELECT 'GEN2:' || generation FROM coldfront.vector_config WHERE table_name = 'chunks'; +SELECT 'GENS:' || string_agg(DISTINCT generation::text, ',' ORDER BY generation::text) FROM coldfront.vector_centroids WHERE table_name = 'chunks'; +EOSQL +) + assert_eq "a retrain moves the pointer forward" "2" "$(extract GEN2 "$T2")" + assert_eq "the previous generation is left intact" "1,2" "$(extract GENS "$T2")" + + # Training fewer centroids than the configured nprobe clamps nprobe in the + # same write: k-means returns at most one cluster per distinct point, and an + # unclamped row would fail vc_nprobe_fit and abort the transaction that did + # the training work. + local CL; CL=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.create_iceberg_table('public','vecclamp','[ + {"name":"id","type":"bigint"},{"name":"ts","type":"timestamptz"},{"name":"embedding","type":"vector(2)"} +]'::jsonb); +INSERT INTO vecclamp VALUES (1, now(), '[1,0]'::vector), (2, now(), '[0,1]'::vector); +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) +VALUES ('public','vecclamp','embedding',8,4); +CALL coldfront.vector_train('public','vecclamp','embedding'); +SELECT 'CLAMP:' || nlist || '/' || nprobe FROM coldfront.vector_config WHERE table_name = 'vecclamp'; +EOSQL +) + assert_eq "training clamps nprobe to the trained count" "2/2" "$(extract CLAMP "$CL")" + + # With a generation live, every path that writes a vector into the cold tier + # stamps the cluster in the same statement. Two paths, one vector, read back + # from Iceberg: the view cannot project the column, by design. + local A; A=$(qf "$HOST" <<'EOSQL' +INSERT INTO chunks (ts, body, embedding) +VALUES (date_trunc('month',now()) - interval '4 months' + interval '21 days', 'asg_trig', '[1.5,-2,3]'::vector); +INSERT INTO chunks (ts, body, embedding) +SELECT date_trunc('month',now()) - interval '4 months' + interval '22 days', 'asg_bulk' || i, ARRAY[1.5,-2,3]::real[]::vector + FROM generate_series(1, 1) i; +SELECT coldfront.ensure_attached(); +SELECT 'ASG:' || r['body'] || '=' || r['_cf_vec_list_embedding'] + FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] IN ('asg_trig', 'asg_bulk1') ORDER BY r['body']; +SELECT 'ASG_AGREE:' || count(DISTINCT r['_cf_vec_list_embedding']::int) + FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] IN ('asg_trig', 'asg_bulk1'); +SELECT 'ASG_NOTNULL:' || bool_and(r['_cf_vec_list_embedding'] IS NOT NULL)::text + FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] IN ('asg_trig', 'asg_bulk1'); +EOSQL +) + assert_eq "every write path agrees on the cluster" "1" "$(extract ASG_AGREE "$A")" + assert_eq "a written vector is never left unassigned" "true" "$(extract ASG_NOTNULL "$A")" + + # The assignment is the nearest centroid, not just any value: computed here + # independently of the generator the write paths used. + local E; E=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'EXPECT:' || (SELECT centroid_id FROM coldfront.vector_centroids + WHERE table_name = 'chunks' AND generation = (SELECT generation FROM coldfront.vector_config WHERE table_name = 'chunks') + ORDER BY centroid <=> ARRAY[1.5,-2,3]::real[] LIMIT 1); +SELECT 'ACTUAL:' || min(r['_cf_vec_list_embedding']::int) FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] = 'asg_trig'; +EOSQL +) + assert_eq "the stamped cluster is the nearest centroid" "$(extract EXPECT "$E")" "$(extract ACTUAL "$E")" + + # Rows archived before any training stay unassigned, which a probe reads + # through the null arm of its predicate rather than missing. + local U; U=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'PRE:' || count(*) FROM iceberg_scan('ice.public.chunks') r + WHERE r['body'] = 'cold' AND r['_cf_vec_list_embedding'] IS NULL; +EOSQL +) + assert_eq "a row written before training stays unassigned" "1" "$(extract PRE "$U")" + + # A cold UPDATE that sets a new embedding must re-derive the cluster in the + # same statement. The row is found by its new cluster and no longer by its + # old one, which is the silent-invisibility bug stated as a test. + # + # The new vector is the centroid farthest from the one this row holds, read from + # the trained set rather than written here: that makes the move a real change of + # cluster whatever training produced, and it leaves the two assigned rows in + # different clusters, which is what the probe assertions below need. + local FARCENT + FARCENT=$(q "$HOST" "SELECT array_to_string(centroid, ',') FROM coldfront.vector_centroids + WHERE table_name = 'chunks' + AND generation = (SELECT generation FROM coldfront.vector_config WHERE table_name = 'chunks') + ORDER BY centroid <=> ARRAY[1.5,-2,3]::real[] DESC LIMIT 1;") + local OA; OA=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'OLD_ASG:' || r['_cf_vec_list_embedding'] FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] = 'asg_trig'; +EOSQL +) + local OLD_ASG; OLD_ASG=$(extract OLD_ASG "$OA") + local UP; UP=$(qf "$HOST" < ARRAY[$FARCENT]::real[] LIMIT 1); +EOSQL +) + assert_eq "a cold UPDATE writes the new embedding" "true" "$(extract NEW_VEC "$UP")" + assert_eq "a cold UPDATE re-derives the cluster" "$(extract NEAREST_TO_NEW "$UP")" "$(extract NEW_ASG "$UP")" + assert_ne "the row left its old cluster" "$OLD_ASG" "$(extract NEW_ASG "$UP")" + + # An untrained column is a clear error, not a silent empty probe set. + local TE; TE=$(q_may "$HOST" "CALL coldfront.vector_train('public','chunks','nosuchcol',2);") + assert_err "training an unconfigured column is refused" "no configuration" "$TE" + + # The probe. A recognised top-k reads only the clusters nearest the query + # vector, and the counts are what prove it: the predicate cannot be observed + # from the caller's side, since the column it tests is in no branch of the view. + # + # The query vector is the cluster asg_trig was just moved into, so a + # single-cluster probe reads that one and skips asg_bulk1's. nlist is 2, so + # nprobe=2 is exhaustive by construction and is the reference every other count + # is compared against. Each query is its own session, so a SET cannot leak into + # the next; psql prints the SET's own tag, which is not a row. + local topk="SELECT body FROM chunks ORDER BY embedding <=> ARRAY[$FARCENT]::real[]" + local ALL OFF ONE NOLIMIT + ALL=$(q "$HOST" "SET coldfront.vector_nprobe = 2; $topk LIMIT 100;" | grep -vx SET | grep -c . || true) + OFF=$(q "$HOST" "SET coldfront.vector_probe = off; $topk LIMIT 100;" | grep -vx SET | grep -c . || true) + ONE=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $topk LIMIT 100;" | grep -vx SET | grep -c . || true) + NOLIMIT=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $topk;" | grep -vx SET | grep -c . || true) + assert_eq "an exhaustive probe returns the exact answer" "$OFF" "$ALL" + assert_gt "a narrower probe reads fewer rows" "$ONE" "$ALL" + assert_eq "an unrecognised shape keeps its exact answer" "$ALL" "$NOLIMIT" + + # Which rows it drops, and which it must never drop. + local ONE_ROWS + ONE_ROWS=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $topk LIMIT 100;" | grep -vx SET) + assert_eq "a probe reads the cluster it aimed at" "1" "$(printf '%s\n' "$ONE_ROWS" | grep -c '^asg_trig$' || true)" + assert_eq "a probe skips the cluster it did not look in" "0" "$(printf '%s\n' "$ONE_ROWS" | grep -c '^asg_bulk1$' || true)" + assert_eq "a probe keeps every unassigned row" "1" "$(printf '%s\n' "$ONE_ROWS" | grep -c '^coldins$' || true)" + assert_eq "a probe keeps every hot row" "1" "$(printf '%s\n' "$ONE_ROWS" | grep -c '^hot$' || true)" + + # The predicate reaches the Iceberg scan itself rather than filtering above it, + # and a shape the rewrite declines carries none at all. + local PLAN_TOPK PLAN_PLAIN + PLAN_TOPK=$(q "$HOST" "SET coldfront.vector_nprobe = 1; EXPLAIN (COSTS OFF, VERBOSE) $topk LIMIT 100;" 2>/dev/null) + PLAN_PLAIN=$(q "$HOST" "SET coldfront.vector_nprobe = 1; EXPLAIN (COSTS OFF, VERBOSE) $topk;" 2>/dev/null) + assert_contains "the probe predicate reaches the scan" "_cf_vec_list_embedding" "$PLAN_TOPK" + assert_eq "a declined shape carries no predicate" "0" "$(printf '%s\n' "$PLAN_PLAIN" | grep -c '_cf_vec_list_embedding' || true)" + + # The null arm carrying its weight, stated as an answer rather than a count: + # coldins holds this exact vector and was written before there was a generation, + # so a probe returns it only through that disjunct. Asserted as membership in the + # top few rather than as the single winner: asg_trig carries a centroid, and a + # centroid of a cluster with one member IS that member's vector, so the two can + # tie at distance zero and either is then a correct answer to LIMIT 1. + local NEAR; NEAR=$(q "$HOST" "SET coldfront.vector_nprobe = 1; SELECT body FROM chunks ORDER BY embedding <=> ARRAY[4,5.5,-6]::real[] LIMIT 3;" | grep -vx SET | grep -c '^coldins$' || true) + assert_eq "a probed search still finds an unassigned row" "1" "$NEAR" + + # Grouping, aggregates, windows and DISTINCT ride the same narrowed scan: the + # probe rewrites the statement they sit in, and everything in it computes over + # the rows the probe reads. At an exhaustive probe each answers exactly what it + # answers with the probe off, which also proves the rewrite round-trips these + # shapes through deparse and reparse. + local shapes=( + "SELECT body FROM chunks GROUP BY body, embedding ORDER BY embedding <=> ARRAY[$FARCENT]::real[] LIMIT 100" + "SELECT count(*)::text FROM chunks GROUP BY embedding ORDER BY embedding <=> ARRAY[$FARCENT]::real[] LIMIT 100" + "SELECT body || '/' || (rank() OVER (ORDER BY embedding <=> ARRAY[$FARCENT]::real[]))::text FROM chunks ORDER BY embedding <=> ARRAY[$FARCENT]::real[] LIMIT 100" + "SELECT DISTINCT body, embedding <=> ARRAY[$FARCENT]::real[] AS d FROM chunks ORDER BY d LIMIT 100" + ) + local names=("a grouped search" "an aggregated search" "a windowed search" "a DISTINCT search") + local si exact probed np + for si in 0 1 2 3; do + exact=$(q "$HOST" "SET coldfront.vector_probe = off; ${shapes[$si]};" | grep -vx SET | sort | tr '\n' ' ') + probed=$(q "$HOST" "SET coldfront.vector_nprobe = 2; ${shapes[$si]};" | grep -vx SET | sort | tr '\n' ' ') + assert_eq "${names[$si]} answers exactly at an exhaustive probe" "$exact" "$probed" + # Without this, the equality above passes vacuously when the shape is + # declined: both arms would be the same exact scan. + np=$(q "$HOST" "SET coldfront.vector_nprobe = 1; EXPLAIN (COSTS OFF, VERBOSE) ${shapes[$si]};" | grep -c "cf_vec_list" || true) + assert_gt "${names[$si]} carries the probe predicate" "0" "$np" + done + # And the probe is on the scan, not on the grouping: narrowed, the grouped + # search reads the same rows the plain one does. + local GN PN + GN=$(q "$HOST" "SET coldfront.vector_nprobe = 1; ${shapes[0]};" | grep -vx SET | sort | tr '\n' ' ') + PN=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $topk LIMIT 100;" | grep -vx SET | sort | tr '\n' ' ') + assert_eq "a narrowed grouped search reads the plain search's rows" "$PN" "$GN" + + # Decoupled: no hot arm, so the predicate becomes the whole of the cold arm's + # WHERE rather than an addition to a cutoff qual — the other branch of + # coldfront._vec_probed_viewdef. Row 1 predates the generation and row 2 sits in + # the cluster this probe does not look in. + local dtopk="SELECT id FROM icevec ORDER BY embedding <=> ARRAY[-1,-1,-1]::real[]" + local DONE_ DALL + DONE_=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $dtopk LIMIT 100;" | grep -vx SET | grep -c . || true) + DALL=$(q "$HOST" "SET coldfront.vector_nprobe = 2; $dtopk LIMIT 100;" | grep -vx SET | grep -c . || true) + assert_eq "a decoupled probe reads one cluster and the unassigned" "1" "$DONE_" + assert_eq "an exhaustive decoupled probe reads everything" "2" "$DALL" + + # vector_status reports what a retrain decision needs. Every count is compared + # against the same number computed independently from the cold table, so the + # assertions hold whatever the fixture grows into. + local S; S=$(qf "$HOST" <<'EOSQL' +CALL coldfront.vector_status('public', 'chunks'); +SELECT coldfront.ensure_attached(); +SELECT 'REPORTED:' || count(*) FROM cf_vector_status; +SELECT 'GEN:' || generation FROM cf_vector_status; +SELECT 'TRAINED:' || clusters_trained FROM cf_vector_status; +SELECT 'REPORTS:' || rows_total || '/' || rows_unassigned || '/' || clusters_occupied + FROM cf_vector_status; +SELECT 'ACTUAL:' || count(*) || '/' || count(*) FILTER (WHERE r['_cf_vec_list_embedding'] IS NULL) + || '/' || count(DISTINCT r['_cf_vec_list_embedding']::int) + FROM iceberg_scan('ice.public.chunks') r; +SELECT 'FRACTION_SANE:' || (probe_fraction > 0 AND probe_fraction <= 1)::text FROM cf_vector_status; +SELECT 'FLOOR:' || (clusters_below_row_group = clusters_occupied)::text FROM cf_vector_status; +SELECT 'ADVICE:' || (advice IS NOT NULL)::text FROM cf_vector_status; +EOSQL +) + assert_eq "vector_status reports the clustered table" "1" "$(extract REPORTED "$S")" + assert_eq "it reports the live generation" "2" "$(extract GEN "$S")" + assert_eq "it reports that generation's centroid count" "2" "$(extract TRAINED "$S")" + assert_eq "the distribution matches the cold table" "$(extract ACTUAL "$S")" "$(extract REPORTS "$S")" + assert_eq "the probe fraction is a fraction" "true" "$(extract FRACTION_SANE "$S")" + # Every cluster here holds one row, far under a row group, which is exactly the + # condition the floor exists to name. + assert_eq "clusters under one row group are counted" "true" "$(extract FLOOR "$S")" + assert_eq "a fixture this small draws advice" "true" "$(extract ADVICE "$S")" + + # Assigning the rows that predate training. These are the rows vector_status + # just counted as unassigned, and they are read by every probe whatever clusters + # it looks in, so on a corpus tiered before training they are the entire cost of + # a search. One claimed UPDATE, reusing the SET item a cold UPDATE emits. + local unasg_before; unasg_before=$(extract REPORTS "$S" | cut -d/ -f2) + local A; A=$(qf "$HOST" <<'EOSQL' +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); +SELECT coldfront.ensure_attached(); +SELECT 'LEFT:' || count(*) FROM iceberg_scan('ice.public.chunks') r + WHERE r['_cf_vec_list_embedding'] IS NULL; +SELECT 'ROWS:' || count(*) FROM iceberg_scan('ice.public.chunks') r; +SELECT 'MATCHES:' || count(*) FROM iceberg_scan('ice.public.chunks') r + WHERE r['_cf_vec_list_embedding']::int = ( + SELECT centroid_id FROM coldfront.vector_centroids c + WHERE c.table_name = 'chunks' + AND c.generation = (SELECT generation FROM coldfront.vector_config WHERE table_name = 'chunks') + ORDER BY c.centroid <=> r['embedding']::real[] LIMIT 1); +EOSQL +) + assert_gt "there were rows to assign" "0" "$unasg_before" + assert_eq "no cold row is left without a cluster" "0" "$(extract LEFT "$A")" + assert_eq "assigning rewrote rows, it did not add any" "7" "$(extract ROWS "$A")" + assert_eq "every row sits in its nearest cluster" "$(extract ROWS "$A")" "$(extract MATCHES "$A")" + + # Which is the point: with nothing unassigned, a narrow probe stops reading the + # whole table. + local narrowed + narrowed=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $topk LIMIT 100;" | grep -vx SET | grep -c . || true) + assert_gt "a narrow probe now reads less than everything" "$narrowed" "$ALL" + + # Idempotent: nothing to do the second time, and no rewrite to pay for. + local A2; A2=$(q_may "$HOST" "CALL coldfront.vector_assign('public','chunks','embedding');") + assert_contains "a second assign has nothing to do" "already assigned" "$A2" + + # A NULL embedding through the identity-omitted slow path. The Iceberg schema + # declares one cluster column per vector column unconditionally, so the row's + # prefix slot must survive a NULL value: the insert succeeds, lands unassigned, + # and the view returns it through the null arm. A cold DELETE then restores the + # fixture's shape for the compaction story. + local NV; NV=$(qf "$HOST" <<'EOSQL' +INSERT INTO chunks (ts, body, embedding) +VALUES (date_trunc('month',now()) - interval '4 months' + interval '23 days', 'nullvec', NULL); +SELECT coldfront.ensure_attached(); +SELECT 'NV_LAND:' || count(*) FROM iceberg_scan('ice.public.chunks') r + WHERE r['body'] = 'nullvec' AND r['embedding'] IS NULL AND r['_cf_vec_list_embedding'] IS NULL; +SELECT 'NV_VIEW:' || count(*) FROM chunks WHERE body = 'nullvec'; +EOSQL +) + assert_eq "a NULL embedding rides the slow path unassigned" "1" "$(extract NV_LAND "$NV")" + assert_eq "the view returns the NULL-embedding row" "1" "$(extract NV_VIEW "$NV")" + local ND; ND=$(qf "$HOST" <<'EOSQL' +DELETE FROM chunks WHERE body = 'nullvec'; +SELECT coldfront.ensure_attached(); +SELECT 'NV_GONE:' || count(*) FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] = 'nullvec'; +EOSQL +) + assert_eq "the NULL-embedding row deletes cleanly" "0" "$(extract NV_GONE "$ND")" + + story_vector_compaction +} + +# ─────────────────────────────────────────────────────────────────────────── +# Story 5c — Compaction on a clustered table. Each cold write leaves a file +# sorted within itself, so what accumulates is a set of files with overlapping +# cluster ranges, and the rewrite has to merge them on the sort column rather +# than append them. That the merge orders rows is asserted exactly in +# cmd/compactor's unit tests, which read back through iceberg-go; what only a +# real deployment can show is that the pass runs against a live catalog with the +# claim held, keeps every row, converges, and leaves a probe answering the same +# question. +# ─────────────────────────────────────────────────────────────────────────── +story_vector_compaction() { + step "5c. Compaction of a clustered vector table" + require_compactor || return + + # The answer a probed search gives, captured before the merge so it can be + # compared with the answer after it. Sorted and unbounded by the probe, because + # what must not change is the set of rows, not their order among equal distances. + local probe_q="SELECT body FROM chunks ORDER BY embedding <=> ARRAY[4,5.5,-6]::real[] LIMIT 100" + local answer_before; answer_before=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $probe_q;" | grep -vx SET | sort | tr '\n' ' ') + + local rows_before; rows_before=$(q "$HOST" "SELECT count(*) FROM chunks;") + local before; before=$("$COMPACTOR" --config /tmp/journey-chunks.yaml --table chunks --dry-run 2>&1) + if echo "$before" | grep -q "group(s)"; then + pass "a clustered table with small files reports work to do" + else + fail "nothing to compact on the clustered table: $before"; return + fi + + if "$COMPACTOR" --config /tmp/journey-chunks.yaml --table chunks >/tmp/journey-compact-vec.log 2>&1; then + pass "the sort-key merge committed against a live catalog" + else + fail "compaction of chunks failed — see /tmp/journey-compact-vec.log" + tail -8 /tmp/journey-compact-vec.log; return + fi + assert_eq "the merge preserved every row" "$rows_before" "$(q "$HOST" "SELECT count(*) FROM chunks;")" + + # A merge that did not converge would rewrite the same table forever. + local again; again=$("$COMPACTOR" --config /tmp/journey-chunks.yaml --table chunks --dry-run 2>&1) + if echo "$again" | grep -q "nothing to compact"; then + pass "a second pass over the merged table is a no-op" + else + fail "clustered table still reports work after compaction: $again" + fi + + # The rows the cold UPDATE had marked deleted must not come back: the merge + # reads through the scan, which applies the position deletes it found. + local D; D=$(qf "$HOST" <<'EOSQL' +SELECT coldfront.ensure_attached(); +SELECT 'ASG_ROWS:' || count(*) FROM iceberg_scan('ice.public.chunks') r WHERE r['body'] = 'asg_trig'; +SELECT 'ASG_VEC:' || count(*) FROM iceberg_scan('ice.public.chunks') r + WHERE r['body'] = 'asg_trig' AND r['_cf_vec_list_embedding'] IS NOT NULL; +EOSQL +) + assert_eq "the merge applied the deletes it read through" "1" "$(extract ASG_ROWS "$D")" + assert_eq "the updated row kept its cluster" "1" "$(extract ASG_VEC "$D")" + + # A merge rewrites where every row lives, so the one thing it must not change is + # which rows a probe finds. Same query, same probe count, same answer. + local answer_after; answer_after=$(q "$HOST" "SET coldfront.vector_nprobe = 1; $probe_q;" | grep -vx SET | sort | tr '\n' ' ') + assert_eq "a probed search answers the same after compaction" "$answer_before" "$answer_after" +} + # ─────────────────────────────────────────────────────────────────────────── # Story 6 — Writes via the view (proven assertions from run-ci-local). # ─────────────────────────────────────────────────────────────────────────── @@ -4148,8 +4739,9 @@ if [ "$MODE" = "tiered" ]; then [ "$MESH" = 1 ] && story_mesh_tiered # cross-node tiered, while hot+cold coexist story_reads story_types + story_vector # embeddings: vector → list over all three cold-write paths story_writes - story_compaction # iceberg-go RewriteDataFiles, now that the manifest-list + story_compaction # iceberg-go RewriteDataFiles; the manifest-list # format-version interop patch makes the cold tier's # manifests iceberg-go-readable story_maintenance # iceberg-go ExpireSnapshots + DeleteOrphanFiles — reclaim the diff --git a/ci/lib.sh b/ci/lib.sh index 4408189..01e57f6 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -22,6 +22,8 @@ note() { echo -e "${YELLOW} NOTE: $1${NC}"; } assert_eq() { if [ "$2" = "$3" ]; then pass "$1"; else fail "$1 — expected '$2', got '$3'"; fi; } assert_gt() { if [ "$3" -gt "$2" ] 2>/dev/null; then pass "$1"; else fail "$1 — expected > $2, got '$3'"; fi; } assert_contains() { case "$3" in *"$2"*) pass "$1";; *) fail "$1 — '$3' does not contain '$2'";; esac; } +# Assert two values differ (for "this must have changed" checks). +assert_ne() { if [ "$2" != "$3" ]; then pass "$1"; else fail "$1 — both sides are '$2'"; fi; } # Assert a command/SQL produced a specific error fragment (for blocked-op / read-only stories). assert_err() { case "$3" in *"$2"*) pass "$1";; *) fail "$1 — error did not contain '$2'; got: $3";; esac; } diff --git a/ci/matrix.sh b/ci/matrix.sh index 8340bca..4c89c99 100755 --- a/ci/matrix.sh +++ b/ci/matrix.sh @@ -64,10 +64,15 @@ preflight() { step "preflight 5: build" if ! make build 2>&1; then fail "build"; exit 1; fi pass "build ($(ls -lh bin/archiver 2>/dev/null | awk '{print $5}'))" + if ! bin/archiver --version | grep -q . || ! bin/partitioner --version | grep -q .; then + fail "--version"; exit 1 + fi + pass "--version ($(bin/archiver --version))" step "preflight 6: compactor module (separate go.mod — iceberg-go)" if ! make compactor GOLANGCI="$linter" 2>&1; then fail "compactor module"; exit 1; fi pass "compactor (vet, lint, test, build $(ls -lh bin/compactor 2>/dev/null | awk '{print $5}'))" + if ! bin/compactor --version | grep -q .; then fail "compactor --version"; exit 1; fi step "preflight 7: docs (mkdocs build --strict)" if ! command -v mkdocs >/dev/null 2>&1; then diff --git a/cmd/archiver/main.go b/cmd/archiver/main.go index c18b3b1..3f4be45 100644 --- a/cmd/archiver/main.go +++ b/cmd/archiver/main.go @@ -10,6 +10,7 @@ import ( "log" "os" "os/signal" + "path/filepath" "regexp" "sort" "strings" @@ -23,6 +24,7 @@ import ( "github.com/pgedge/coldfront/internal/partcfg" "github.com/pgedge/coldfront/internal/partition" "github.com/pgedge/coldfront/internal/sqlutil" + "github.com/pgedge/coldfront/internal/version" "github.com/pgedge/coldfront/internal/view" "github.com/pgedge/coldfront/internal/watermark" ) @@ -51,8 +53,14 @@ func main() { "sleep this long after Phase 2 (capture+bulk-export) and before Phase 3 "+ "(replay+cutover). Test-only knob to widen the window so concurrent "+ "writes deterministically race into the capture trigger.") + showVersion := flag.Bool("version", false, "print the version and exit") flag.Parse() + if *showVersion { + fmt.Printf("%s %s (built %s)\n", filepath.Base(os.Args[0]), version.Version, version.BuildTime) + return + } + cfg, err := config.Load(*configPath) if err != nil { log.Fatalf("load config: %v", err) @@ -646,9 +654,17 @@ func (ac *archiveCycle) bootstrapTieredView(ctx context.Context, columns []view. } hotTable := pgx.Identifier{t.SourceSchema, "_" + t.SourceTable}.Sanitize() if err := registerTieredView(ctx, ac.conn, t.SourceSchema, t.SourceTable, - hotTable, iceTable, t.PartitionColumn); err != nil { + hotTable, iceTable, t.PartitionColumn, vectorColumns(columns)); err != nil { return fmt.Errorf("register tiered view: %w", err) } + // The INSTEAD OF INSERT trigger, from the extension's one builder. After the + // registration, because the builder reads the registry for the hot table, the + // Iceberg ref and the vector columns. + if _, err := ac.conn.Exec(ctx, + "SELECT coldfront._rebuild_write_trigger($1, $2)", + t.SourceSchema, t.SourceTable); err != nil { + return fmt.Errorf("build write trigger: %w", err) + } return nil } @@ -1195,41 +1211,129 @@ func wipeIcebergRange(ctx context.Context, conn *pgx.Conn, iceTable, partCol str // in S" → replayed. The replay is idempotent (DELETE+INSERT keyed on PK), so // the duplicate work is correct, just wasted. For typical workloads this // window is sub-millisecond. -// stageSelectList builds the SELECT projection for the bulk export, casting -// only the VARCHAR-backed rich types (jsonb/json/interval — Type=="VARCHAR" -// with a surface ViewCastType) to ::text so they land as VARCHAR in Iceberg -// and the transparent view casts them back on read. +// vecCompanion names the generated real[] column that carries a vector column's +// scannable form on the hot table. coldfront._vec_companion derives the same name +// for the SQL-side view rebuild; the two have to agree. +func vecCompanion(col string) string { return "_cf_vec_" + col } + +// vecLayoutProps asks the extension for the CREATE TABLE properties a clustered +// table needs, or "" when the table has no vector. The values live in the +// extension so both modes create the same layout; the primary key rides along as +// the sort key's tiebreak, which only this side knows. +func vecLayoutProps(ctx context.Context, conn *pgx.Conn, columns []view.Column) (string, error) { + if len(vectorColumns(columns)) == 0 { + return "", nil + } + var pk []string + for _, c := range columns { + if c.IsPK { + pk = append(pk, c.Name) + } + } + var props string + if err := conn.QueryRow(ctx, + "SELECT coldfront._vec_layout_props(coldfront._vec_sort_key($1, $2))", + vectorColumns(columns)[0], pk).Scan(&props); err != nil { + return "", fmt.Errorf("layout properties: %w", err) + } + return props, nil +} + +// pkOrder spells the primary key as trailing ORDER BY terms, or "" when the table +// has none. A tiebreak for determinism, not a pruning aid: sorting by cluster +// scatters a cluster's rows through key space. +func pkOrder(columns []view.Column) string { + var terms []string + for _, c := range columns { + if c.IsPK { + terms = append(terms, "s."+pgx.Identifier{c.Name}.Sanitize()) + } + } + if len(terms) == 0 { + return "" + } + return ", " + strings.Join(terms, ", ") +} + +// vectorColumns names the table's vector columns in column order, empty when it has +// none. The order is the contract: the Iceberg schema declares one cluster column +// per entry in this order, a positional cold write fills them in it, and the first +// entry is the one that owns the file sort order. +func vectorColumns(columns []view.Column) []string { + var out []string + for _, c := range columns { + if c.IsVector() { + out = append(out, c.Name) + } + } + return out +} + +// vecListPrefix asks the extension for the expression that assigns a cluster to +// each staged row, leading the Iceberg INSERT's projection, or "" when the table +// has no vector. // -// A ViewCastType alone does NOT mean text-backed: bytea (storage BLOB) and -// double precision (storage DOUBLE) carry a ViewCastType only to give the view -// a PG-parseable hot-side cast (`::bytea`/`::double precision` instead of the -// non-PG `::BLOB`/`::DOUBLE`). Iceberg stores those natively (binary / double), -// so they must be exported AS-IS — ::text-casting bytea would stringify the -// bytes ('\xdeadbeef') and corrupt the BLOB column. Gating on Type=="VARCHAR" -// selects exactly the text-backed types and leaves native ones untouched. +// The extension owns the expression rather than the archiver spelling its own: +// a row whose cluster disagrees with its vector is invisible to its own search +// and reports no error, so every write path derives from one definition. It also +// reads the centroids over pglocal, which is why the caller attaches it. +func vecListPrefix(ctx context.Context, conn *pgx.Conn, t *config.TableConfig, columns []view.Column) (string, error) { + vecCols := vectorColumns(columns) + if len(vecCols) == 0 { + return "", nil + } + if _, err := conn.Exec(ctx, "SELECT coldfront.ensure_pg_attached()"); err != nil { + return "", fmt.Errorf("attach pglocal: %w", err) + } + exprs := make([]string, len(vecCols)) + for i, c := range vecCols { + exprs[i] = "s." + pgx.Identifier{c}.Sanitize() + } + var prefix string + if err := conn.QueryRow(ctx, + "SELECT coldfront._vec_list_prefix($1, $2, $3, $4)", + t.SourceSchema, t.SourceTable, vecCols, exprs, + ).Scan(&prefix); err != nil { + return "", fmt.Errorf("cluster expression: %w", err) + } + return prefix, nil +} + +// stageSelectList builds the SELECT projection for the bulk export. Which +// columns need a cast, and to what, is view.Column.ExportCast's decision, and +// which hot-side column is read is view.Column.HotRef's: this only spells the +// projection. func stageSelectList(columns []view.Column) string { if len(columns) == 0 { return "*" } - parts := make([]string, len(columns)) - for i, c := range columns { + parts := make([]string, 0, len(columns)) + for _, c := range columns { id := pgx.Identifier{c.Name}.Sanitize() - if c.Type == "VARCHAR" && c.ViewCastType != "" { - parts[i] = id + "::text AS " + id - } else { - parts[i] = id + ref, aliased := c.HotRef() + switch { + case aliased: + // The companion is already real[]; reading it needs no cast. + parts = append(parts, ref+" AS "+id) + case c.ExportCast() != "": + parts = append(parts, ref+"::"+c.ExportCast()+" AS "+id) + default: + parts = append(parts, ref) } } return strings.Join(parts, ", ") } -// needsPGTextStage reports whether the column set contains a type pg_duckdb's -// PG reader cannot scan, requiring a PostgreSQL-side text-cast staging table -// before the DuckDB stage. jsonb (ViewCastType "json") scans fine, so it does -// NOT trigger the detour. interval is included defensively (Iceberg-VARCHAR- -// backed, and not worth a separate scan probe). inet/cidr were the original -// offenders but are no longer supported (pg_duckdb rejects inet outright). -func needsPGTextStage(columns []view.Column) bool { +// needsPGStage reports whether the column set contains a type pg_duckdb's PG +// reader cannot scan, so PostgreSQL has to materialise a cast copy in a plain +// temp table before the DuckDB stage reads it. +// +// interval is included defensively (Iceberg-VARCHAR-backed, and not worth a +// separate scan probe). jsonb, bytea and double scan fine and must not trigger +// it: the detour copies every partition twice. A vector does not trigger it +// either, because the export reads its generated real[] companion rather than the +// pgvector column (see view.Column.HotRef). +func needsPGStage(columns []view.Column) bool { for _, c := range columns { if c.ViewCastType == "interval" { return true @@ -1252,19 +1356,25 @@ func bulkExportWithSnapshot(ctx context.Context, conn *pgx.Conn, t *config.Table // (inet/cidr were the original Oid-869 offenders that motivated this detour // but are no longer supported — see pgFormatTypeToDuckDB.) jsonb-only / // plain tables skip the detour (single copy, fast path). + // The projection belongs to whichever statement reads the real partition: it is + // what casts the VARCHAR-backed types and what reads a vector through its + // generated companion instead of the pgvector column pg_duckdb cannot scan. + // After the detour, cf_pgstage already holds those columns under their own + // names, so the DuckDB stage takes them as they are. src := pgx.Identifier{t.SourceSchema, partName}.Sanitize() - if needsPGTextStage(columns) { + proj := stageSelectList(columns) + if needsPGStage(columns) { pgStageSQL := fmt.Sprintf( - "CREATE TEMP TABLE cf_pgstage AS SELECT %s FROM %s", - stageSelectList(columns), src) + "CREATE TEMP TABLE cf_pgstage AS SELECT %s FROM %s", proj, src) if _, err := conn.Exec(ctx, pgStageSQL); err != nil { // nosemgrep return "", fmt.Errorf("pg text-stage: %w", err) } defer func() { _, _ = conn.Exec(ctx, "DROP TABLE IF EXISTS cf_pgstage") }() // nosemgrep src = "cf_pgstage" + proj = "*" } stageSQL := fmt.Sprintf( - "CREATE TEMP TABLE duck_stage USING duckdb AS SELECT * FROM %s", src) + "CREATE TEMP TABLE duck_stage USING duckdb AS SELECT %s FROM %s", proj, src) if _, err := conn.Exec(ctx, stageSQL); err != nil { // nosemgrep return "", fmt.Errorf("stage: %w", err) } @@ -1275,7 +1385,19 @@ func bulkExportWithSnapshot(ctx context.Context, conn *pgx.Conn, t *config.Table // statement on this dedicated conn, so the claim/lock is released at this // statement's commit. duck_stage was created on the same conn in the prior // statement, so only one DuckDB-database write happens inside this tx. - insertSQL, err := dollarQuote(fmt.Sprintf("INSERT INTO %s SELECT * FROM pg_temp.duck_stage", iceTable)) + vecPrefix, err := vecListPrefix(ctx, conn, t, columns) + if err != nil { + return "", err + } + // Ordered by cluster, so this file's own row groups each hold roughly one + // cluster and a probe skips the rest of it. Nothing outside this file is + // touched: the sorted regions accumulate and compaction folds them together. + order := "" + if vecPrefix != "" { + order = " ORDER BY 1" + pkOrder(columns) + } + insertSQL, err := dollarQuote(fmt.Sprintf( + "INSERT INTO %s SELECT %ss.* FROM pg_temp.duck_stage s%s", iceTable, vecPrefix, order)) if err != nil { return "", fmt.Errorf("iceberg insert: %w", err) } @@ -1311,15 +1433,23 @@ func ensureIcebergTable(ctx context.Context, conn *pgx.Conn, t *config.TableConf if err != nil { return fmt.Errorf("get columns: %w", err) } - var colDefs string - for i, c := range columns { - if i > 0 { - colDefs += ", " - } - colDefs += fmt.Sprintf("%s %s", pgx.Identifier{c.Name}.Sanitize(), c.Type) + defs := make([]string, 0, len(columns)+1) + // One cluster column per vector column, leading the schema in column order, which + // is the order a positional cold write fills them in. + for _, c := range vectorColumns(columns) { + defs = append(defs, pgx.Identifier{view.VecListColumn(c)}.Sanitize()+" INTEGER") } + for _, c := range columns { + defs = append(defs, fmt.Sprintf("%s %s", pgx.Identifier{c.Name}.Sanitize(), c.Type)) + } + colDefs := strings.Join(defs, ", ") - if err := execDuckDB(ctx, conn, fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", iceTable, colDefs)); err != nil { + props, err := vecLayoutProps(ctx, conn, columns) + if err != nil { + return err + } + if err := execDuckDB(ctx, conn, fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)%s", + iceTable, colDefs, props)); err != nil { return fmt.Errorf("create iceberg table: %w", err) } return nil @@ -1328,15 +1458,16 @@ func ensureIcebergTable(ctx context.Context, conn *pgx.Conn, t *config.TableConf // registerTieredView upserts a row in coldfront.tiered_views so the // coldfront C extension can identify this view as a tiered target and // rewrite UPDATE/DELETE into dual-tier CTEs. Called after every view recreate. -func registerTieredView(ctx context.Context, conn *pgx.Conn, schema, table, hotTable, icebergTable, partitionCol string) error { +func registerTieredView(ctx context.Context, conn *pgx.Conn, schema, table, hotTable, icebergTable, partitionCol string, vecColumns []string) error { _, err := conn.Exec(ctx /* nosemgrep */, ` - INSERT INTO coldfront.tiered_views (schema_name, relname, hot_table, iceberg_table, partition_col) - VALUES ($1, $2, $3, $4, $5) + INSERT INTO coldfront.tiered_views (schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) + VALUES ($1, $2, $3, $4, $5, NULLIF($6, '{}'::text[])) ON CONFLICT (schema_name, relname) DO UPDATE SET hot_table = EXCLUDED.hot_table, iceberg_table = EXCLUDED.iceberg_table, - partition_col = EXCLUDED.partition_col`, - schema, table, hotTable, icebergTable, partitionCol) + partition_col = EXCLUDED.partition_col, + vec_columns = EXCLUDED.vec_columns`, + schema, table, hotTable, icebergTable, partitionCol, vecColumns) return err } @@ -1385,6 +1516,20 @@ func pgFormatTypeToDuckDB(s string) (storage, viewCastType string, err error) { return "VARCHAR", "", nil } + // pgvector. format_type emits the dimension as a typmod (vector(1536)) + // unless the column was declared without one. Both types widen losslessly + // to float4 and store as an Iceberg list. The view cast is real[] + // rather than FLOAT[]: PG reads FLOAT as an alias for double precision, so + // ::FLOAT[] on the hot branch would widen to double precision[] and the two + // UNION branches would disagree on the column type. coldfront's SQL twin + // (_iceberg_storage_type / _iceberg_view_cast_type) returns the same pair + // for the decoupled path. sparsevec is absent deliberately: densifying it + // is a 100x storage blowup, so it stays hot-only and errors below. + if s == "vector" || strings.HasPrefix(s, "vector(") || + s == "halfvec" || strings.HasPrefix(s, "halfvec(") { + return "FLOAT[]", "real[]", nil + } + // numeric(P,S) → DECIMAL(P,S). Iceberg supports decimal up to P=38. if m := numericTypeRe.FindStringSubmatch(s); m != nil { return "DECIMAL(" + m[1] + "," + m[2] + ")", "", nil @@ -1400,9 +1545,11 @@ func pgFormatTypeToDuckDB(s string) (storage, viewCastType string, err error) { "PG type %q has no Iceberg-compatible mapping. Supported: bigint, integer, "+ "smallint, real, double precision, boolean, timestamp with/without time "+ "zone, date, time without time zone, uuid, text, character varying(N), "+ - "character(N), bytea, numeric(P,S) with P<=38, json, jsonb, interval. "+ + "character(N), bytea, numeric(P,S) with P<=38, json, jsonb, interval, "+ + "vector(N), halfvec(N). "+ "inet/cidr/oid are not supported (pg_duckdb cannot process them in "+ - "Iceberg-backed queries): store IP data as text and oid values as bigint", s) + "Iceberg-backed queries): store IP data as text and oid values as bigint. "+ + "sparsevec is not supported: keep it in the hot tier", s) } // pgTypeMap holds the 1:1 PG-format_type → (storage, viewCast) mappings used by @@ -1500,7 +1647,8 @@ func scanColumns(ctx context.Context, db querier, schema, actualName string) ([] rows, err := db.Query(ctx /* nosemgrep */, ` SELECT a.attname, format_type(a.atttypid, a.atttypmod), - a.attidentity::text + a.attidentity::text, + a.attgenerated::text FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace @@ -1514,20 +1662,31 @@ func scanColumns(ctx context.Context, db querier, schema, actualName string) ([] var cols []view.Column for rows.Next() { - var name, pgFormatType, attidentity string - if err := rows.Scan(&name, &pgFormatType, &attidentity); err != nil { + var name, pgFormatType, attidentity, attgenerated string + if err := rows.Scan(&name, &pgFormatType, &attidentity, &attgenerated); err != nil { return nil, err } + // A companion is ColdFront's own generated column, not a column of the + // table as far as the Iceberg schema and the view are concerned. Skipping + // it keeps exactly one column per user column. A generated column the user + // wrote is left alone and treated like any other. + if attgenerated != "" && strings.HasPrefix(name, vecCompanion("")) { + continue + } storage, viewCastType, err := pgFormatTypeToDuckDB(pgFormatType) if err != nil { return nil, fmt.Errorf("column %s.%s.%s: %w", schema, actualName, name, err) } - cols = append(cols, view.Column{ + col := view.Column{ Name: name, Type: storage, ViewCastType: viewCastType, IsIdentity: attidentity == "a", - }) + } + if col.IsVector() { + col.HotSource = vecCompanion(name) + } + cols = append(cols, col) } return cols, rows.Err() } diff --git a/cmd/archiver/main_test.go b/cmd/archiver/main_test.go index aad3c78..35b6fd4 100644 --- a/cmd/archiver/main_test.go +++ b/cmd/archiver/main_test.go @@ -177,6 +177,20 @@ func TestPgFormatTypeToDuckDB(t *testing.T) { {pg: "json", wantStorage: "VARCHAR", wantViewCastTyp: "json"}, {pg: "interval", wantStorage: "VARCHAR", wantViewCastTyp: "interval"}, + // pgvector. Both widen losslessly to float4 and store as list. + // The view cast is real[], not FLOAT[]: PG parses FLOAT as an alias for + // double precision, so ::FLOAT[] on the hot branch would widen to + // double precision[] and the two branches would disagree. + // coldfront._iceberg_storage_type / _iceberg_view_cast_type must return + // this same pair for the decoupled path; test/sql/vector_type_map.sql + // asserts these literals on that side. + {pg: "vector(1536)", wantStorage: "FLOAT[]", wantViewCastTyp: "real[]"}, + {pg: "vector", wantStorage: "FLOAT[]", wantViewCastTyp: "real[]"}, + {pg: "halfvec(768)", wantStorage: "FLOAT[]", wantViewCastTyp: "real[]"}, + {pg: "halfvec", wantStorage: "FLOAT[]", wantViewCastTyp: "real[]"}, + // sparsevec stays hot-only: densifying it is a 100x storage blowup. + {pg: "sparsevec(65536)", wantErr: true}, + // Errors. inet/cidr/oid are rejected: pg_duckdb cannot process them in // an Iceberg-backed query, and every tiered read is planned by pg_duckdb, // so there is no cast that makes them readable (oid would archive fine but @@ -424,21 +438,54 @@ func TestStageSelectList(t *testing.T) { assert.Equal(t, `"i", "j"::text AS "j", "iv"::text AS "iv", "b", "d"`, got) } +// The bulk export reads a vector through its generated real[] companion, so +// pg_duckdb never scans the pgvector type. Already real[], so no cast is added. +func TestStageSelectList_Vector(t *testing.T) { + cols := []view.Column{ + {Name: "id", Type: "BIGINT"}, + {Name: "embedding", Type: "FLOAT[]", ViewCastType: "real[]", HotSource: "_cf_vec_embedding"}, + } + assert.Equal(t, `"id", "_cf_vec_embedding" AS "embedding"`, stageSelectList(cols)) +} + +// The staging table holds the user's own columns. The cluster is derived by the +// statement that writes Iceberg, which is the one DuckDB executes and so the only +// one that can read the centroids. +func TestStageSelectList_NoClusterColumnStaged(t *testing.T) { + cols := []view.Column{ + {Name: "id", Type: "BIGINT"}, + {Name: "embedding", Type: "FLOAT[]", ViewCastType: "real[]", HotSource: "_cf_vec_embedding"}, + } + assert.NotContains(t, stageSelectList(cols), "_cf_vec_list") +} + +// The companion is what makes a vector scannable, so its name has to be derived +// the same way here and in coldfront._vec_companion, which the SQL side uses. +func TestVecCompanion(t *testing.T) { + assert.Equal(t, "_cf_vec_embedding", vecCompanion("embedding")) + assert.Equal(t, "_cf_vec_My Col", vecCompanion("My Col")) +} + func TestStageSelectList_Empty(t *testing.T) { assert.Equal(t, "*", stageSelectList(nil)) } -// The PG text-stage detour is only needed for types pg_duckdb's reader cannot -// scan natively — now just interval (kept defensively). bytea, jsonb and -// double scan fine and must not trigger it. -func TestNeedsPGTextStage(t *testing.T) { - assert.True(t, needsPGTextStage([]view.Column{{Name: "iv", Type: "VARCHAR", ViewCastType: "interval"}})) - assert.False(t, needsPGTextStage([]view.Column{ +// The PG staging detour is for source types pg_duckdb's reader cannot scan, which +// is interval, kept defensively. bytea, jsonb and double scan fine and must not +// trigger it, since the detour costs a full extra copy of every partition. +func TestNeedsPGStage(t *testing.T) { + assert.True(t, needsPGStage([]view.Column{{Name: "iv", Type: "VARCHAR", ViewCastType: "interval"}})) + // A vector exports through its generated real[] companion, so the detour + // would copy every partition twice for nothing. + assert.False(t, needsPGStage([]view.Column{ + {Name: "embedding", Type: "FLOAT[]", ViewCastType: "real[]", HotSource: "_cf_vec_embedding"}, + })) + assert.False(t, needsPGStage([]view.Column{ {Name: "j", Type: "VARCHAR", ViewCastType: "json"}, {Name: "b", Type: "BLOB", ViewCastType: "bytea"}, {Name: "d", Type: "DOUBLE", ViewCastType: "double precision"}, })) - assert.False(t, needsPGTextStage(nil)) + assert.False(t, needsPGStage(nil)) } // TestDollarQuote proves the dollar-quote wrapper cannot be broken out of by any diff --git a/cmd/compactor/compact.go b/cmd/compactor/compact.go index 619ce66..68c8ecf 100644 --- a/cmd/compactor/compact.go +++ b/cmd/compactor/compact.go @@ -4,8 +4,13 @@ import ( "context" "errors" "fmt" + "iter" "strings" + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/catalog" "github.com/apache/iceberg-go/catalog/rest" iceio "github.com/apache/iceberg-go/io" @@ -107,8 +112,44 @@ func excludeFromSigning(headers ...string) func(*middleware.Stack) error { // planResult bundles the rewrite groups with the planner's summary (for logging // and the no-op decision). type planResult struct { - groups []table.CompactionTaskGroup - plan compaction.Plan + groups []table.CompactionTaskGroup + plan compaction.Plan + sorted bool // the table declares a sort key: merge, do not concatenate + sortKey iceberg.NestedField // the column to merge on, when sorted + skipped string // non-empty: why the rewrite must leave this table alone +} + +// sortKeyProp names the column a table's data files are ordered by. Set it on +// tables whose query speed depends on row-group pruning; a table without it is +// rewritten as it arrives, with no ordering step. +// +// With it, a rewrite merges each group on that column rather than appending its +// files, so every output row group spans adjacent key values and its statistics +// stay useful. Row group *size* is a separate property: iceberg-go cuts groups +// only by write.parquet.row-group-limit (a row count, default 1,048,576) and +// never reads write.parquet.row-group-size-bytes, so a table that wants small +// groups sets the row-count property too. +const sortKeyProp = "coldfront.sort-key" + +// sortField returns the schema field named by sortKeyProp. The bool is false when +// the property is absent, which is not an error: an unsorted table has no order +// to preserve. A property naming a column that is not in the schema IS an error, +// because rewriting anyway would scramble the layout it was meant to protect. +// +// Only the leading column of a compound key matters here. Files are ordered by +// it; within one of its values a secondary key never straddles two files. +func sortField(props iceberg.Properties, sc *iceberg.Schema) (iceberg.NestedField, bool, error) { + name, _, _ := strings.Cut(props[sortKeyProp], ",") + name = strings.TrimSpace(name) + if name == "" { + return iceberg.NestedField{}, false, nil + } + field, ok := sc.FindFieldByName(name) + if !ok { + return iceberg.NestedField{}, false, fmt.Errorf( + "%s names column %q, which the table schema does not have", sortKeyProp, name) + } + return field, true, nil } // loadTableErr turns a LoadTable failure into the message the operator sees. @@ -131,6 +172,10 @@ func planCompaction(ctx context.Context, cat *rest.Catalog, ns, name string, tar if err != nil { return nil, nil, loadTableErr(ns, name, err) } + sortKey, sorted, err := sortField(tbl.Properties(), tbl.Schema()) + if err != nil { + return tbl, &planResult{skipped: err.Error()}, nil + } tasks, err := tbl.Scan().PlanFiles(ctx) if err != nil { return nil, nil, fmt.Errorf("plan files for %s.%s: %w", ns, name, err) @@ -158,7 +203,7 @@ func planCompaction(ctx context.Context, cat *rest.Catalog, ns, name string, tar TotalSizeBytes: g.TotalSizeBytes, } } - return tbl, &planResult{groups: groups, plan: plan}, nil + return tbl, &planResult{groups: groups, plan: plan, sorted: sorted, sortKey: sortKey}, nil } // rewrite executes the planned compaction as a single atomic rewrite snapshot @@ -170,13 +215,16 @@ func planCompaction(ctx context.Context, cat *rest.Catalog, ns, name string, tar // iceberg-go has no bakery-aware re-stamp patch, the claim is held across the // WHOLE read->rewrite->commit so the CAS parent is captured under the claim — // the stock-ordering discipline proved safe in docs/formal (Bakery_v2.cfg). -func rewrite(ctx context.Context, tbl *table.Table, groups []table.CompactionTaskGroup, targetSize int64) (*table.RewriteResult, error) { +func rewrite(ctx context.Context, tbl *table.Table, p *planResult, targetSize int64) (*table.RewriteResult, error) { + if p.sorted { + return rewriteSorted(ctx, tbl, p.groups, p.sortKey, targetSize) + } txn := tbl.NewTransaction() opts := table.RewriteDataFilesOptions{} if targetSize > 0 { opts.GroupOptions = []table.CompactionGroupOption{table.WithCompactionTargetFileSize(targetSize)} } - res, err := txn.RewriteDataFiles(ctx, groups, opts) + res, err := txn.RewriteDataFiles(ctx, p.groups, opts) if err != nil { return nil, fmt.Errorf("rewrite data files: %w", err) } @@ -185,3 +233,190 @@ func rewrite(ctx context.Context, tbl *table.Table, groups []table.CompactionTas } return res, nil } + +// rewriteSorted compacts a sorted table by merging each group on its sort column +// instead of concatenating it, and stages every group on one rewrite snapshot. +// +// Concatenation preserves order only while a group's input ranges are disjoint, +// which is true of a table built by one sorted pass and false as soon as writes +// land clustered: a batch cold write orders its own rows, so each new file spans +// the whole of key space and appending two of them interleaves two sorted runs. +// The cost of that is a run count, and a probe reads at least one row group per +// run, so bounding file count without merging the runs bounds the wrong thing. +// +// This drives the same two halves iceberg-go's own group executor drives, with a +// sort between them, which is the seam its documentation points distributed +// coordinators at. Reading through Scan.ReadTasks applies the position deletes a +// cold UPDATE or DELETE left behind, and writing through WriteRecords produces +// files with field ids, statistics and the table's row-group limit. Neither is +// true of touching the Parquet directly. +func rewriteSorted(ctx context.Context, tbl *table.Table, groups []table.CompactionTaskGroup, + field iceberg.NestedField, targetSize int64) (*table.RewriteResult, error) { + txn := tbl.NewTransaction() + rw := txn.NewRewrite(nil) + res := &table.RewriteResult{} + + for _, g := range groups { + gr, err := mergeGroup(ctx, tbl, g, field, targetSize) + if err != nil { + return nil, err + } + rw.ApplyResult(gr) + res.RewrittenGroups++ + res.AddedDataFiles += len(gr.NewDataFiles) + res.RemovedDataFiles += len(gr.OldDataFiles) + res.RemovedPositionDeleteFiles += len(gr.SafePosDeletes) + res.BytesBefore += gr.BytesBefore + res.BytesAfter += gr.BytesAfter + } + + if err := rw.Commit(ctx); err != nil { + return nil, fmt.Errorf("stage sorted rewrite: %w", err) + } + if _, err := txn.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit rewrite: %w", err) + } + return res, nil +} + +// mergeGroup reads one group with its deletes applied, sorts it by field, and +// writes the result back as new data files. A group is bin-packed to the +// file-size target, so holding one is bounded by that target rather than by the +// table. +func mergeGroup(ctx context.Context, tbl *table.Table, group table.CompactionTaskGroup, + field iceberg.NestedField, targetSize int64) (table.CompactionGroupResult, error) { + var zero table.CompactionGroupResult + + unsorted, err := readGroupTable(ctx, tbl, group) + if err != nil { + return zero, err + } + if unsorted == nil { + return table.CompactionGroupResult{PartitionKey: group.PartitionKey}, nil + } + defer unsorted.Release() + + sorted, err := sortedByField(ctx, unsorted, field) + if err != nil { + return zero, fmt.Errorf("group %q: %w", group.PartitionKey, err) + } + defer sorted.Release() + + newFiles, bytesAfter, err := writeSorted(ctx, tbl, sorted, targetSize) + if err != nil { + return zero, fmt.Errorf("write merged files for group %q: %w", group.PartitionKey, err) + } + + oldFiles := make([]iceberg.DataFile, 0, len(group.Tasks)) + for _, t := range group.Tasks { + oldFiles = append(oldFiles, t.File) + } + return table.CompactionGroupResult{ + PartitionKey: group.PartitionKey, + OldDataFiles: oldFiles, + NewDataFiles: newFiles, + SafePosDeletes: table.CollectSafePositionDeletes(group.Tasks), + BytesBefore: group.TotalSizeBytes, + BytesAfter: bytesAfter, + }, nil +} + +// readGroupTable reads a group's tasks with their deletes applied into one Arrow +// table, released by the caller, or nil when the group holds no rows. +func readGroupTable(ctx context.Context, tbl *table.Table, group table.CompactionTaskGroup) (arrow.Table, error) { + if len(group.Tasks) == 0 { + return nil, nil + } + // One reader: the sort decides the order, so a concurrent read would only + // shuffle its input to no effect. + schema, records, err := tbl.Scan(table.WitMaxConcurrency(1)).ReadTasks(ctx, group.Tasks) + if err != nil { + return nil, fmt.Errorf("read group %q: %w", group.PartitionKey, err) + } + var batches []arrow.RecordBatch + defer func() { + for _, b := range batches { + b.Release() + } + }() + for rec, err := range records { + if err != nil { + return nil, fmt.Errorf("read group %q: %w", group.PartitionKey, err) + } + rec.Retain() + batches = append(batches, rec) + } + if len(batches) == 0 { + return nil, nil + } + return array.NewTableFromRecords(schema, batches), nil +} + +// sortedByField returns the table's rows reordered ascending by field, in a new +// table released by the caller. Nulls sort last and therefore land together: a +// row another engine appended carries no assignment, and keeping those rows +// contiguous is what lets a probe read them in proportion to their own size +// instead of scattering them through every row group. +func sortedByField(ctx context.Context, unsorted arrow.Table, field iceberg.NestedField) (arrow.Table, error) { + idx := unsorted.Schema().FieldIndices(field.Name) + if len(idx) != 1 { + return nil, fmt.Errorf("sort column %q is not a single column of the read schema", field.Name) + } + order, err := compute.SortIndicesTable(ctx, unsorted, []compute.SortKey{{ + ColumnIndex: idx[0], + Order: compute.SortOrderAscending, + NullPlacement: compute.SortNullsAtEnd, + }}) + if err != nil { + return nil, fmt.Errorf("sort by %q: %w", field.Name, err) + } + defer order.Release() + + taken, err := compute.Take(ctx, *compute.DefaultTakeOptions(), + compute.NewDatumWithoutOwning(unsorted), compute.NewDatumWithoutOwning(order)) + if err != nil { + return nil, fmt.Errorf("reorder by %q: %w", field.Name, err) + } + sorted := taken.(*compute.TableDatum).Value + sorted.Retain() + taken.Release() + return sorted, nil +} + +// writeSorted writes the table back through WriteRecords, clustered and +// bin-packed to targetSize, returning the new data files and their total size. +func writeSorted(ctx context.Context, tbl *table.Table, sorted arrow.Table, targetSize int64) ([]iceberg.DataFile, int64, error) { + writeOpts := []table.WriteRecordOption{table.WithClusteredWrite()} + if targetSize > 0 { + writeOpts = append(writeOpts, table.WithTargetFileSize(targetSize)) + } + rdr := array.NewTableReader(sorted, 0) + defer rdr.Release() + + var ( + files []iceberg.DataFile + size int64 + ) + for df, err := range table.WriteRecords(ctx, tbl, sorted.Schema(), recordSeq(rdr), writeOpts...) { + if err != nil { + return nil, 0, err + } + files = append(files, df) + size += df.FileSizeBytes() + } + return files, size, nil +} + +// recordSeq adapts an Arrow table reader to the iterator WriteRecords consumes. +func recordSeq(rdr array.RecordReader) iter.Seq2[arrow.RecordBatch, error] { + return func(yield func(arrow.RecordBatch, error) bool) { + for rdr.Next() { + if !yield(rdr.RecordBatch(), nil) { + return + } + } + if err := rdr.Err(); err != nil { + yield(nil, err) + } + } +} diff --git a/cmd/compactor/compact_test.go b/cmd/compactor/compact_test.go index 52ca745..93789f0 100644 --- a/cmd/compactor/compact_test.go +++ b/cmd/compactor/compact_test.go @@ -2,23 +2,52 @@ package main import ( "errors" - "fmt" "testing" - "github.com/apache/iceberg-go/catalog" + "github.com/apache/iceberg-go" ) -func TestLoadTableErr_NotFound(t *testing.T) { - // The REST catalog maps a 404 to catalog.ErrNoSuchTable, but its Error() - // renders the server's wording ("NoSuchTableException: Error getting - // tabular from catalog"), which says nothing useful to whoever ran the - // command. Only the identifier they typed matters. - raw := fmt.Errorf("NoSuchTableException: Error getting tabular from catalog: %w", - catalog.ErrNoSuchTable) - got := loadTableErr("public", "no_such_table", raw) - want := `table "public.no_such_table" not found in catalog` - if got.Error() != want { - t.Errorf("got %q, want %q", got.Error(), want) +func testSchema() *iceberg.Schema { + return iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "list", Type: iceberg.PrimitiveTypes.Int32}, + iceberg.NestedField{ID: 3, Name: "label", Type: iceberg.PrimitiveTypes.String}, + ) +} + +func TestSortField(t *testing.T) { + sc := testSchema() + tests := []struct { + name string + props iceberg.Properties + wantID int + wantOK bool + wantErr bool + }{ + {name: "absent leaves compaction alone", props: iceberg.Properties{}}, + {name: "nil props", props: nil}, + {name: "blank is absent", props: iceberg.Properties{sortKeyProp: " "}}, + {name: "single column", props: iceberg.Properties{sortKeyProp: "list"}, wantID: 2, wantOK: true}, + // Only the leading column orders files: within one list value the + // secondary key never straddles two files. + {name: "compound key uses the first", props: iceberg.Properties{sortKeyProp: "list, id"}, wantID: 2, wantOK: true}, + // A key naming a column that is not there is a misconfiguration, and + // rewriting anyway would scramble the layout it was meant to protect. + {name: "unknown column errors", props: iceberg.Properties{sortKeyProp: "nope"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + field, ok, err := sortField(tt.props, sc) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr = %v", err, tt.wantErr) + } + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if ok && field.ID != tt.wantID { + t.Errorf("field.ID = %d, want %d", field.ID, tt.wantID) + } + }) } } diff --git a/cmd/compactor/go.mod b/cmd/compactor/go.mod index 702ed42..16b0d53 100644 --- a/cmd/compactor/go.mod +++ b/cmd/compactor/go.mod @@ -3,6 +3,7 @@ module github.com/pgedge/coldfront/cmd/compactor go 1.26.5 require ( + github.com/apache/arrow-go/v18 v18.6.0 github.com/apache/iceberg-go v0.6.0 github.com/aws/aws-sdk-go-v2 v1.41.7 github.com/aws/smithy-go v1.25.1 @@ -35,7 +36,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect - github.com/apache/arrow-go/v18 v18.6.0 // indirect github.com/apache/thrift v0.23.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect diff --git a/cmd/compactor/main.go b/cmd/compactor/main.go index 53e6414..66dd70b 100644 --- a/cmd/compactor/main.go +++ b/cmd/compactor/main.go @@ -16,6 +16,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "time" "github.com/jackc/pgx/v5" @@ -52,8 +53,14 @@ func main() { keepFiles := flag.Bool("expire-keep-files", false, "with --expire-snapshots: expire metadata only, leave freed files for an --orphans pass (iceberg-go WithPostCommit(false))") orphans := flag.Bool("orphans", false, "also delete orphan files (under the table location, referenced by no retained snapshot)") orphanAge := flag.Duration("orphan-age", 72*time.Hour, "with --orphans: only delete files older than this (in-flight-write safety; never 0 in production)") + showVersion := flag.Bool("version", false, "print the version and exit") flag.Parse() + if *showVersion { + fmt.Printf("%s %s (built %s)\n", filepath.Base(os.Args[0]), Version, BuildTime) + return + } + if *cfgPath == "" || *tableName == "" { fmt.Fprintln(os.Stderr, "usage: compactor --config --table [--target-size-mb N] [--dry-run]"+ " [--expire-snapshots [--expire-retain-last N]] [--orphans [--orphan-age D]]") @@ -155,6 +162,15 @@ func doCompaction(ctx context.Context, cat *rest.Catalog, ns, tableName string, if err != nil { return err } + if plan.skipped != "" { + // Refusing beats rewriting: a rewrite that cannot keep the file order + // scrambles the layout its queries prune on, and says nothing about it. + // File count stops being bounded until the cause is fixed, so this is + // loud and names the cause. + fmt.Fprintf(os.Stderr, "compactor: %s.%s NOT compacted: %s. Fix %s or unset it; "+ + "file count is unbounded until then\n", ns, tableName, plan.skipped, sortKeyProp) + return nil + } if len(plan.groups) == 0 { fmt.Fprintf(os.Stderr, "compactor: %s.%s — nothing to compact (%d files scanned, none below target)\n", ns, tableName, plan.plan.TotalInputFiles) @@ -168,7 +184,7 @@ func doCompaction(ctx context.Context, cat *rest.Catalog, ns, tableName string, var res *table.RewriteResult if err := claim(func() error { var rerr error - res, rerr = rewrite(ctx, tbl, plan.groups, o.targetSize) + res, rerr = rewrite(ctx, tbl, plan, o.targetSize) return rerr }); err != nil { return err diff --git a/cmd/compactor/merge_test.go b/cmd/compactor/merge_test.go new file mode 100644 index 0000000..22aa449 --- /dev/null +++ b/cmd/compactor/merge_test.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + "github.com/apache/iceberg-go/catalog/hadoop" + "github.com/apache/iceberg-go/table" +) + +// mergeSchema is the smallest table that can show the difference between a merge +// and an append: a sort column, and a payload column to prove rows travel with it. +func mergeSchema() *iceberg.Schema { + return iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "list", Type: iceberg.PrimitiveTypes.Int32}, + ) +} + +// localTable creates a real Iceberg table on disk through the filesystem catalog, +// so the merge under test runs against the same read, write and commit paths it +// uses in production rather than against a stub. Filesystem rather than a SQL +// catalog because the sqlite driver would add nineteen modules, one of them CGO, +// to a module whose whole point is to quarantine heavy dependencies. +func localTable(t *testing.T, props iceberg.Properties) (context.Context, *table.Table, catalog.Catalog) { + t.Helper() + ctx := context.Background() + cat, err := hadoop.NewCatalog("test", "file://"+t.TempDir(), nil) + if err != nil { + t.Fatalf("load catalog: %v", err) + } + if err := cat.CreateNamespace(ctx, catalog.ToIdentifier("ns"), nil); err != nil { + t.Fatalf("create namespace: %v", err) + } + tbl, err := cat.CreateTable(ctx, catalog.ToIdentifier("ns", "t"), mergeSchema(), + catalog.WithProperties(props)) + if err != nil { + t.Fatalf("create table: %v", err) + } + return ctx, tbl, cat +} + +// appendRun commits one data file holding the given rows, in the order given. A +// negative list value stands for NULL, which is what an unassigned row carries. +func appendRun(t *testing.T, ctx context.Context, tbl *table.Table, ids []int64, lists []int32) *table.Table { + t.Helper() + arrowSchema, err := table.SchemaToArrowSchema(tbl.Schema(), nil, true, false) + if err != nil { + t.Fatalf("arrow schema: %v", err) + } + bld := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) + defer bld.Release() + for i := range ids { + bld.Field(0).(*array.Int64Builder).Append(ids[i]) + if lists[i] < 0 { + bld.Field(1).(*array.Int32Builder).AppendNull() + } else { + bld.Field(1).(*array.Int32Builder).Append(lists[i]) + } + } + rec := bld.NewRecordBatch() + defer rec.Release() + + at := array.NewTableFromRecords(arrowSchema, []arrow.RecordBatch{rec}) + defer at.Release() + + txn := tbl.NewTransaction() + if err := txn.AppendTable(ctx, at, int64(len(ids)), nil); err != nil { + t.Fatalf("append run: %v", err) + } + out, err := txn.Commit(ctx) + if err != nil { + t.Fatalf("commit run: %v", err) + } + return out +} + +// readColumn returns the table's list column in the order the scan yields it, +// with NULL rendered as -1 so one slice can express both. +func readColumn(t *testing.T, ctx context.Context, tbl *table.Table) []int32 { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + if err != nil { + t.Fatalf("scan: %v", err) + } + defer at.Release() + + var got []int32 + col := at.Column(1).Data() + for _, chunk := range col.Chunks() { + vals := chunk.(*array.Int32) + for i := 0; i < vals.Len(); i++ { + if vals.IsNull(i) { + got = append(got, -1) + continue + } + got = append(got, vals.Value(i)) + } + } + return got +} + +// groupsFor bin-packs every file in the current snapshot into one group, which is +// what the planner produces for a table of small files. +func groupsFor(t *testing.T, ctx context.Context, tbl *table.Table) []table.CompactionTaskGroup { + t.Helper() + tasks, err := tbl.Scan().PlanFiles(ctx) + if err != nil { + t.Fatalf("plan files: %v", err) + } + var total int64 + for _, task := range tasks { + total += task.File.FileSizeBytes() + } + return []table.CompactionTaskGroup{{Tasks: tasks, TotalSizeBytes: total}} +} + +func TestRewriteSorted_MergesOverlappingRuns(t *testing.T) { + // Two files, each already sorted, whose ranges interleave. This is what a + // clustered table accumulates: every batch cold write orders its own rows, so + // each new file spans the whole of key space. Appending them in file order + // would yield 0,5,10,1,6,11 and leave a probe reading both halves; only a merge + // on the column produces one ordered run. + ctx, tbl, cat := localTable(t, iceberg.Properties{sortKeyProp: "list"}) + tbl = appendRun(t, ctx, tbl, []int64{1, 2, 3}, []int32{0, 5, 10}) + tbl = appendRun(t, ctx, tbl, []int64{4, 5, 6}, []int32{1, 6, 11}) + + field, ok := tbl.Schema().FindFieldByName("list") + if !ok { + t.Fatal("sort column missing from schema") + } + if _, err := rewriteSorted(ctx, tbl, groupsFor(t, ctx, tbl), field, 0); err != nil { + t.Fatalf("rewriteSorted: %v", err) + } + + merged, err := cat.LoadTable(ctx, catalog.ToIdentifier("ns", "t")) + if err != nil { + t.Fatalf("reload: %v", err) + } + want := []int32{0, 1, 5, 6, 10, 11} + got := readColumn(t, ctx, merged) + if len(got) != len(want) { + t.Fatalf("row count = %d, want %d (%v)", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("merged order = %v, want %v", got, want) + } + } +} + +func TestRewriteSorted_UnassignedRowsSortLast(t *testing.T) { + // A row another engine appended carries no assignment. Keeping those rows + // together at the end is what lets a probe read them in proportion to their own + // size rather than meeting them in every row group. + ctx, tbl, cat := localTable(t, iceberg.Properties{sortKeyProp: "list"}) + tbl = appendRun(t, ctx, tbl, []int64{1, 2}, []int32{-1, 7}) + tbl = appendRun(t, ctx, tbl, []int64{3, 4}, []int32{2, -1}) + + field, _ := tbl.Schema().FindFieldByName("list") + if _, err := rewriteSorted(ctx, tbl, groupsFor(t, ctx, tbl), field, 0); err != nil { + t.Fatalf("rewriteSorted: %v", err) + } + merged, err := cat.LoadTable(ctx, catalog.ToIdentifier("ns", "t")) + if err != nil { + t.Fatalf("reload: %v", err) + } + got := readColumn(t, ctx, merged) + want := []int32{2, 7, -1, -1} + if len(got) != len(want) { + t.Fatalf("row count = %d, want %d (%v)", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("merged order = %v, want %v (-1 is NULL)", got, want) + } + } +} + +func TestRewriteSorted_PreservesRowsWithNoSortableOrder(t *testing.T) { + // One file, already ordered: the merge must be a no-op on content. This is the + // idempotence a second compaction pass depends on. + ctx, tbl, cat := localTable(t, iceberg.Properties{sortKeyProp: "list"}) + tbl = appendRun(t, ctx, tbl, []int64{1, 2, 3}, []int32{4, 8, 12}) + + field, _ := tbl.Schema().FindFieldByName("list") + if _, err := rewriteSorted(ctx, tbl, groupsFor(t, ctx, tbl), field, 0); err != nil { + t.Fatalf("rewriteSorted: %v", err) + } + merged, err := cat.LoadTable(ctx, catalog.ToIdentifier("ns", "t")) + if err != nil { + t.Fatalf("reload: %v", err) + } + got := readColumn(t, ctx, merged) + want := []int32{4, 8, 12} + if len(got) != len(want) { + t.Fatalf("row count = %d, want %d (%v)", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("order = %v, want %v", got, want) + } + } +} + +func TestRewriteSorted_RowsStayIntact(t *testing.T) { + // The sort permutes every column or none: a reorder that moved the sort column + // alone would leave each row carrying another row's payload, which no assertion + // on the sort column's order can see. + ctx, tbl, cat := localTable(t, iceberg.Properties{sortKeyProp: "list"}) + tbl = appendRun(t, ctx, tbl, []int64{10, 20, 30}, []int32{0, 5, 10}) + tbl = appendRun(t, ctx, tbl, []int64{40, 50, 60}, []int32{1, 6, 11}) + + field, _ := tbl.Schema().FindFieldByName("list") + if _, err := rewriteSorted(ctx, tbl, groupsFor(t, ctx, tbl), field, 0); err != nil { + t.Fatalf("rewriteSorted: %v", err) + } + merged, err := cat.LoadTable(ctx, catalog.ToIdentifier("ns", "t")) + if err != nil { + t.Fatalf("reload: %v", err) + } + at, err := merged.Scan().ToArrowTable(ctx) + if err != nil { + t.Fatalf("scan: %v", err) + } + defer at.Release() + + // Each id is paired with exactly one list value, so the pairing itself is + // asserted, not a formula over the ids. + want := map[int32]int64{0: 10, 5: 20, 10: 30, 1: 40, 6: 50, 11: 60} + ids := at.Column(0).Data() + lists := at.Column(1).Data() + var n int + for c := range ids.Chunks() { + idv := ids.Chunk(c).(*array.Int64) + lsv := lists.Chunk(c).(*array.Int32) + for i := 0; i < idv.Len(); i++ { + if got := idv.Value(i); got != want[lsv.Value(i)] { + t.Errorf("list %d carries id %d, want %d", lsv.Value(i), got, want[lsv.Value(i)]) + } + n++ + } + } + if n != len(want) { + t.Fatalf("read %d rows, want %d", n, len(want)) + } +} diff --git a/cmd/compactor/version.go b/cmd/compactor/version.go new file mode 100644 index 0000000..bc85c69 --- /dev/null +++ b/cmd/compactor/version.go @@ -0,0 +1,10 @@ +package main + +// Version and BuildTime are set via ldflags at build time. A copy of +// internal/version, carried here because this module quarantines its heavy +// dependencies from the main module and takes no edge back to it for two +// variables. +var ( + Version = "unknown" + BuildTime = "unknown" +) diff --git a/cmd/partitioner/main.go b/cmd/partitioner/main.go index f180a9b..721382c 100644 --- a/cmd/partitioner/main.go +++ b/cmd/partitioner/main.go @@ -14,6 +14,7 @@ import ( "log" "os" "os/signal" + "path/filepath" "strings" "syscall" "time" @@ -23,6 +24,7 @@ import ( "github.com/pgedge/coldfront/internal/config" "github.com/pgedge/coldfront/internal/partcfg" "github.com/pgedge/coldfront/internal/partition" + "github.com/pgedge/coldfront/internal/version" ) // reconcileFailed logs a reconcile error and reports whether it should fail the @@ -48,7 +50,12 @@ func main() { } cfgPath := flag.String("config", "", "path to the YAML config file") + showVersion := flag.Bool("version", false, "print the version and exit") flag.Parse() + if *showVersion { + fmt.Printf("%s %s (built %s)\n", filepath.Base(os.Args[0]), version.Version, version.BuildTime) + return + } if *cfgPath == "" { log.Fatal("--config is required") } diff --git a/docker/Dockerfile.duckdb15 b/docker/Dockerfile.duckdb15 index 9d092a7..cf81b66 100644 --- a/docker/Dockerfile.duckdb15 +++ b/docker/Dockerfile.duckdb15 @@ -53,6 +53,14 @@ COPY --from=cf-build /out/usr/pgsql-${PG_MAJOR}/lib/ /usr/pgsql-${PG COPY --from=cf-build /out/usr/pgsql-${PG_MAJOR}/share/extension/ /usr/pgsql-${PG_MAJOR}/share/extension/ COPY docker/entrypoint.sh /usr/local/bin/coldfront-entrypoint.sh RUN chmod +x /usr/local/bin/coldfront-entrypoint.sh +# pgvector, from the same pgEdge repo the base is built from. A tiered vector +# table carries the type on its hot side, and both the cold read and cold write +# paths rely on pgvector's implicit vector -> real[] cast. It sits in the app +# layer for the same reason as entrypoint.sh: baked only into the separately +# published base, a stale base would silently lack it and local CI would diverge +# from GitHub CI. +RUN dnf install -y --setopt=install_weak_deps=False pgedge-pgvector_${PG_MAJOR} \ + && dnf clean all # License + third-party notices travel with the binaries we redistribute (DuckDB # family is MIT — the notice must accompany the distributed Software). COPY THIRD_PARTY_NOTICES.md LICENSE.md /usr/share/doc/coldfront/ diff --git a/docs/architecture_vectors.md b/docs/architecture_vectors.md new file mode 100644 index 0000000..7a41515 --- /dev/null +++ b/docs/architecture_vectors.md @@ -0,0 +1,469 @@ +# Vector storage architecture + +Reference for maintainers. Embeddings live in the cold (Iceberg) tier as +`list`; PostgreSQL holds the hot rows, the routing centroids, and nothing +proportional to the corpus. + +## Type mapping and the two column forms + +A `vector(n)` or `halfvec(n)` column maps to Iceberg `FLOAT[]`, and the tiered +view exposes it as `real[]`, because DuckDB has no `vector` type and both arms of +a `UNION ALL` view must agree on one type. `real[]` and `FLOAT[]` are the same +type under two spellings. + +pg_duckdb maps PostgreSQL types by OID and has no entry for an extension type. +The refusal lands on the column reference while the plan is built, so no cast in +a projection can rescue it. Any plan carrying a pgvector column fails regardless +of how few rows can match. + +The hot table therefore carries a companion: + +```sql +_cf_vec_ real[] GENERATED ALWAYS AS (::real[]) STORED +``` + +Every hot-side read goes through it: the view's hot branch, the bulk export's +projection, and the SQL-side view rebuild. `view.Column.HotRef` decides this on +the Go side and `coldfront._vec_companion` derives the same name on the SQL side; +the two must agree. `coldfront._is_vec_companion` keeps the companion out of +every column list that describes the user's table, so the Iceberg schema, the +view, the INSERT lists and the cross-tier move each carry exactly one column per +user column. Teardown drops it. + +`STORED` is required: PostgreSQL rejects `VIRTUAL` for a user-defined function's +expression. A hot row therefore stores its embedding twice. Cold rows, which are +all of the data by design, do not. + +## The search operators + +`coldfront.install_vector_ops()` creates three functions and three operators on +`(real[], real[])` in pgvector's own schema, resolved from the catalog rather +than hardcoded: + +| Operator | Function | Metric | +|---|---|---| +| `<=>` | `list_cosine_distance` | cosine | +| `<->` | `list_distance` | Euclidean | +| `<#>` | `list_negative_inner_product` | negative inner product | + +pg_duckdb resolves an operator by its implementing function's *name* in DuckDB's +catalog, so the names above are the mechanism, not a convention. The PostgreSQL +bodies are real implementations that delegate to pgvector, because a hot-only +(pre-cutover) view has no Iceberg scan to pull the query into DuckDB and +PostgreSQL executes them itself. + +The function runs at onboarding rather than at `CREATE EXTENSION`, and installs +pgvector if it is absent. `CREATE EXTENSION coldfront` never requires pgvector. + +A caller's own `vector` literal resolves without substitution: `vector -> real[]` +is an implicit cast, only implicit coercions count during operator resolution, +and pgvector's cast function is immutable so a constant folds. The same cast +makes an `INSERT` of a `vector` value coerce to the column. + +## Routing state + +Two tables, both name-keyed so a Spock mesh replicates them by value and every +node resolves a vector to the same cluster id without sharing OIDs: + +| Table | Holds | +|---|---| +| `coldfront.vector_config` | `nlist`, `nprobe`, the live `generation`, `addition_cap`, per (schema, table, column) | +| `coldfront.vector_centroids` | the centroids themselves, keyed additionally by `(generation, centroid_id)`, with `parent_id` for an adaptive addition | + +Both are registered in Spock's `default` replication set by +`coldfront._ensure_vector_state_replicated()`, gated on the spock extension so +vanilla is a no-op, and both are `pg_extension_config_dump`-marked: losing them +makes every stored cluster id uninterpretable and forces a retrain. + +`vector_centroids.centroid` is `real[]`, not a pgvector value. These tables are +created with the extension, which must install on a database that has no vectors +and may never have any. Scoring against them goes through the same `<=>` shim a +caller uses. + +A generation is immutable. A retrain writes a new one and moves the pointer, so +an assignment already stored keeps meaning what it meant, and the primary key +rejects a repeated `centroid_id` within one generation. + +`coldfront.tiered_views.vec_columns` records which columns are clustered, in order. +It cannot be derived afterwards in either mode: the view exposes `real[]` rather than +the pgvector type, and a decoupled table names its types without holding them. + +## The cluster columns, and which one owns the sort order + +A table may carry several vector columns. The Iceberg schema gets one cluster column +per vector column, `_cf_vec_list_`, leading the schema in column order, and +every write path assigns all of them. The registry records the ordered list in +`tiered_views.vec_columns`, and that order is a contract: a cold INSERT is +positional, so the prefix must fill the cluster columns in exactly the order the +schema declares them. `_vec_list_prefix` raises rather than emitting a short prefix, +because a short one would land every following value in the wrong column. + +**Only `vec_columns[1]` appears in `coldfront.sort-key`.** A Parquet file has one +physical row order, and pruning depends on a cluster's rows being adjacent so the +reader can skip row groups on their statistics. Ordering by a second cluster column +after the first would scatter its values inside every band of the first, leaving its +statistics bounding the whole file. So a later column's probe filters the rows +scored and prunes nothing read. Column read and decode is ~95% of query cost, so that +is worth single-digit percent rather than a multiple. +`cf_vector_status.prunes` reports which column is which. + +The read path needs no registry lookup to pick between them: it takes the column +name off the `ORDER BY` expression's Var and derives that column's cluster column +from it. A column with no configuration resolves to no probe set and the rewrite +declines. + +`_cf_vec_list_ integer` exists in the Iceberg schema and nowhere else. It is not a +column of the hot table and neither branch of the view projects it, so no query +written against the view can name it. + +It **leads** the Iceberg schema. Iceberg schema evolution appends, and a cold +INSERT is positional, so a column added later has to land after everything both +sides already agree on. Trailing the cluster column would put a user's +`ADD COLUMN` on the far side of an internal column and silently misalign every +positional write. + +`coldfront._vec_list_col(column)` and `view.VecListColumn(column)` are the two +spellings of the name. + +## Assignment + +`coldfront._vec_list_expr(schema, table, column, vec_expr)` is the only place a +cluster assignment is defined. Given the text of an expression that yields the +vector as DuckDB sees it, it returns: + +```sql +(SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, )) + FROM pglocal.coldfront.vector_centroids c + WHERE c.schema_name = … AND c.table_name = … AND c.column_name = … + AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc + WHERE …)) +``` + +A row whose cluster disagrees with its vector is invisible to its own search and +reports no error, which is why every path emits this and none derives its own. + +**Centroids are read over `pglocal`.** Inside `duckdb.raw_query` DuckDB has no +PostgreSQL catalog at all: `duckdb_tables()` is empty and neither +`pgduckdb.public.` nor `public.` resolves. pg_duckdb's in-process reads of +PostgreSQL tables exist only for statements PostgreSQL plans, where the planner +binds the relation and hands the scan down as part of the converted plan. A cold +write is a `raw_query` string that DuckDB binds itself, so an attachment is the +only route in. `coldfront.ensure_pg_attached()` loads DuckDB's `postgres` +extension and attaches the local instance as `pglocal`, with the DSN from the +`coldfront.local_pg_dsn` GUC. Consequences: the PostgreSQL table stays the only +copy of the centroids, no path inlines a centroid set, and no path keeps a +session copy it has no way to check. + +**The generation is resolved by the emitted SQL, not baked into it.** A statement +generated once, such as a trigger body, keeps assigning against the live +generation after a retrain instead of filtering on one that no longer exists. + +Before any training the config carries no generation, the inner query matches +nothing, and the expression yields `NULL`. Unassigned is a legitimate value: rows +a foreign engine appended straight to Iceberg carry none either, and the read +path handles them explicitly. + +A retrain cannot interleave with a cold write, because an operation that rewrites +the table holds the table's claim and every cold write serialises on that same +claim. + +### The seven paths + +| Path | Where | Shape | +|---|---|---| +| bulk archive | the Iceberg INSERT in `cmd/archiver`, not the staging SELECT | set-based | +| tiered trigger INSERT | `internal/view` (Go) and `coldfront._rebuild_tiered_view` (SQL twin) | per statement | +| slow per-row INSERT | `coldfront._tiered_insert_cold` | per row, cursor loop | +| cross-tier move | `coldfront._move_row_literal` | per row | +| replay drain | `coldfront.replay_archive_delta` | set-based | +| decoupled INSERT | the C rewrite | per statement | +| cold UPDATE that sets the vector | the C rewrite | expression text | + +The archiver derives in the statement that writes Iceberg rather than in the +staging SELECT, because only the DuckDB statement can reach the centroids. The +staging table holds the user's own columns. + +The decoupled INSERT is targeted, so it is re-emitted over a derived table: + +```sql +INSERT INTO (_cf_vec_list_, ) +SELECT , FROM () AS coldfront_src() +``` + +The cold UPDATE adds one SET item before the statement's own WHERE, or at the end +when it has none. `find_toplevel_where` locates it by tracking quotes before +parens, because a literal can contain the word, a sublink carries its own one +level down, and a literal can hold an unbalanced paren. This lives in +`build_cold_dml` rather than in a caller: the cold path and the dual path both +build their cold half through it, and an ambiguous predicate takes the dual path. + +The replay drain casts a vector to `real[]` in its scratch projection, because +DuckDB reads that scratch over libpq and cannot scan the pgvector type. + +`pglocal` is attached only where a lookup will run: `_exec_iceberg_with_claim` +attaches when the statement names it, the generated triggers emit the attach only +for a clustered table, and the per-row paths guard on +`coldfront._types_have_vector`. + +The generated trigger's placeholder list is apostrophe-escaped, because the +INSERT template is itself a single-quoted string and the assignment expression +carries the literals that name its configuration row. + +## Training + +`CALL coldfront.vector_train(schema, table, column, nlist, sample, iterations)`. + +A `PROCEDURE`, not a function, and this is a hard requirement. pg_duckdb refuses +to execute a DuckDB query inside a function (`DuckDB execution is not supported +inside functions`); a procedure and a `DO` block are exempt. `raw_query` does run +inside a function but is a bare DuckDB channel with no PostgreSQL catalog, so a +function can move data in neither direction. + +Lloyd iterations run as DuckDB statements over a reservoir sample, with fixed +`REPEATABLE` seeds so a retrain on unchanged data reproduces the same centroids. +The mean recompute unnests the vector against a matching `range` so the two lists +advance together, because DuckDB has no `WITH ORDINALITY`. The sample and the +working tables live in `memory.main`, which is session-scoped, so the whole loop +must run in one call. + +The centroids return through a temporary heap table. A single +`INSERT … SELECT FROM duckdb.query(…)` fails with `DuckDB does not support +modifying Postgres tables`, because a DuckDB source makes the whole statement +DuckDB's, so the read and the write are separate statements. + +Empty clusters do not come back from the mean, so the stored count can be below +`nlist`; the actual count is recorded rather than padded. A `vector_config` row +must exist first, since it holds the generation pointer the procedure writes. + +Seeds come from k-means++: one sample row at random, then each next drawn with +probability proportional to its squared distance from the nearest seed already +chosen, by an exponential race (the minimum of `-ln(u)/w` is a weighted draw, in one +pass and with no cumulative sum). The running distance folds in only the seed just +added, keyed on an insertion sequence rather than a row id, so a round is one pass +over the sample. + +Seeding costs about 46 ms per seed on a 20,000-row sample, so `nlist` 1000 adds +roughly 45 seconds and `nlist` 10,000 about eight minutes, on top of Lloyd's +iterations. Training is a one-time operation and nothing a query pays. + +What the spread start defends against is seeds clumping in a dense region, which +Lloyd cannot repair because it only moves centroids locally. That matters most at the +lower dimensionalities many embedding models produce; above about 1000 dimensions +distances concentrate and the starting spread makes little difference either way. + +## Layout + +Three properties, set at `CREATE TABLE`: + +| Property | Value | Read by | +|---|---|---| +| `write.parquet.row-group-limit` | `2048` | iceberg-go | +| `write.target-file-size-bytes` | `536870912` | both | +| `coldfront.sort-key` | the cluster column, then the key | the compactor | + +Row groups are the pruning granularity: the Parquet reader skips a row group +whose statistics cannot match the filter, and at 2048 rows a group holds a median +of one cluster. The two writers each read one row-group property and ignore the +other. DuckDB reads only `write.parquet.row-group-size-bytes`, and a table +carrying that property refuses every DuckDB write to it (`ROW_GROUP_SIZE_BYTES +does not work while preserving insertion order`), so it is not set. A DuckDB +write therefore emits one row group per file and compaction is what cuts them. + +The file target is large because on object storage every file a query touches is +a billed round trip. + +The sort key's leading column is what a compaction orders its inputs by. The key +after it is a tiebreak for determinism, not a pruning aid: sorting by cluster +scatters a cluster's rows through key space. + +Properties cannot be altered after creation on this build, so a table that +predates its vector column keeps the defaults. + +**Batch cold writes order by cluster.** The archiver's Iceberg INSERT and the C +bulk INSERT append `ORDER BY 1` (the cluster leads the projection) plus the key, +so each new file is internally sorted and its own row groups prune. No existing +file is touched: sorted regions accumulate, and a probe reads the matching row +group in each of them. + +**Compaction merges those regions rather than appending them.** A table carrying +`coldfront.sort-key` is rewritten group by group through `rewriteSorted` +(`cmd/compactor/compact.go`), which reads the group with `Scan.ReadTasks`, sorts +it on the sort column, and writes it back with `WriteRecords`. Appending the files +in key order, which is what the compactor did while the only clustered files came +from a single sorted pass, preserves order only while their ranges are disjoint, +and an incremental write's file spans the whole of cluster space by construction. +What that would cost is a run count: a probe reads at least one row group per +sorted run, so bounding file count without merging the runs bounds the wrong +thing. + +The two halves are iceberg-go's own, which is what makes the merge safe rather +than merely correct on a good day. Reading through the scan applies the position +deletes a cold UPDATE or DELETE left behind; writing through `WriteRecords` +produces files with field ids, column statistics and the table's row-group limit. +Touching the Parquet directly would have none of that, and would reinstate every +deleted row. Nulls sort last, so rows another engine appended without an +assignment stay contiguous instead of appearing in every row group. + +Each group is bin-packed to the file-size target, so a merge holds one group in +memory rather than one table. + +## Reading: the probe + +`cf_maybe_inject_probe` runs on the read path, alongside the hot-tier reroute and +the jsonb normalisation, and it is what makes the layout worth maintaining. It +recognises one shape and rewrites it: + +- a single-relation `SELECT` on a registered view with a clustered vector column, +- ordered by exactly one cosine distance between that column and a constant, +- with a `LIMIT`, +- and at the top level of the statement: the hook sees one `Query`, so a top-k + nested in a subquery or a CTE is not the query it is looking at. Wrapping a + search to aggregate over it therefore makes it exact. + +Grouping, aggregation, window functions and `DISTINCT` above that `ORDER BY` are +accepted, and they compute over the narrowed scan: the probe restricts rows, and +the statement's own semantics apply to what was read. The structural declines are +joins, CTEs, set operations, sublinks and row-marks. On PostgreSQL 18 a grouped +query carries an `RTE_GROUP` entry and its sort expression references grouping +expressions as Vars of that RTE; the hook counts that entry as no second relation +and resolves such Vars through `groupexprs` before matching the shape. + +Everything else is left byte-identical. That is an exact scan over both tiers, +which is correct, and it is what the product did before there was a layout. + +Three of those conditions are load-bearing. **The `LIMIT`** is part of the shape +because a probe trades recall for reads: that is the bargain a top-k asks for, and +not one to impose on a query that asked for every row in order. **Cosine only**, +because the centroids were trained under cosine, and ordering by `<->` or `<#>` +would route to clusters chosen under a different metric and quietly return the +wrong rows. **A constant query vector**, because pg_duckdb converts neither a +`vector` nor a `real[]` bound parameter, so a search that could only be resolved +from a parameter could not have run at all. + +The rewrite resolves the nearest `nprobe` centroid ids +(`coldfront._vec_probe_ids`), turns them into a predicate +(`coldfront._vec_probe_qual`), and substitutes the view reference for the view's +own definition carrying that predicate on its cold arm +(`coldfront._vec_probed_viewdef`): + +```sql +… WHERE r['ts'] < + AND (r['_cf_vec_list_embedding']::integer IN (3, 17) + OR r['_cf_vec_list_embedding']::integer IS NULL) +``` + +The substitution exists because the predicate has nowhere else to go: the cluster +column is in no branch of the view, so no query written against the view can name +it. Adding the test inside the definition puts it where the column exists and +leaves the user's column list alone. It is a range-table entry swapped for a +subquery, not text surgery on the caller's SQL, and PostgreSQL deparses the +result. + +The hot arm is untouched: hot rows carry no assignment and every one of them is +returned. + +**The null disjunct is not optional.** Rows another engine appended straight to +Iceberg carry no assignment, and a bare `IN` drops them silently. It is also not +expensive, because the reader prunes on each row group's null count: unassigned +rows are read in proportion to their own size rather than the table's. + +**Declining is total and silent.** No centroid generation, an empty probe set, a +view with no cold arm: each keeps today's query. This is the one place in the +vector path that fails open, and deliberately so. A read that loses its predicate +is slower, never wrong. A *write* that cannot resolve a generation fails loudly +instead, because a wrong cluster id makes a row invisible to its own probe. + +Two session knobs, both `PGC_USERSET`: + +| GUC | Default | Effect | +|---|---|---| +| `coldfront.vector_probe` | `on` | `off` gives the exact scan a recall measurement compares against | +| `coldfront.vector_nprobe` | `0` | `0` uses the column's configured `nprobe`; a value at or above `nlist` is exhaustive | + +## Assigning what predates training + +`CALL coldfront.vector_assign(schema, table, column)` gives a cluster to the cold +rows that have none, which are exactly those written before the generation existed. +It is one claimed UPDATE whose SET item is the same generator a cold UPDATE uses +when a caller changes an embedding, applied to the embedding already stored, so it +serialises through the bakery like every other cold write and adds no new way to +compute an assignment. + +It refuses without a live generation rather than reporting success. The lookup would +resolve to NULL for every row, so the table would be rewritten in full and left +exactly as it was. + +Two constraints shape the body, and both bite anything else written here: + +- **A DuckDB transaction may write one attached database.** Staging the row count in + a `memory.main` table would spend this transaction's one database on `memory` and + leave the UPDATE unable to write `ice` at all. +- **`duckdb.query` needs a constant at plan time, not a literal in the source.** + Built through `EXECUTE format(...)` the argument is a constant again, which is how + a dynamic table name is read without staging anything. `vector_status` reads its + distribution the same way. + +What it leaves behind is a merge-on-read delete per rewritten row, and rows in +update order rather than cluster order. Compaction resolves both. The sequence is +train, assign, compact. + +## Reporting + +`CALL coldfront.vector_status([schema, table])` fills a session-lifetime temporary +table `cf_vector_status`, one row per registered clustered column. A procedure +writing a table rather than a function returning rows, for the reason +`vector_train` is one: pg_duckdb refuses to execute a DuckDB query inside a +function, and a single `INSERT … SELECT` over a DuckDB scan is planned as DuckDB's, +which cannot write a PostgreSQL table. Session lifetime rather than `ON COMMIT DROP` +because a bare `CALL` is its own transaction. + +Nothing is staged on the way. DuckDB groups the table by cluster and aggregates +that grouping in one query, so a row of scalars crosses back per table, read through +the same `EXECUTE format(...)` form `vector_assign` uses. The work list is a pair of +key arrays walked by index, because a `FOR` over a query would hold a portal open +for its body and pg_duckdb refuses a DuckDB read while one is. `vector_train`'s +`memory.main` tables are its algorithm's own state, not a way to move a result +across, and nothing else here needs one. + +The headline is `probe_fraction`: the share of the corpus a probe reads on average, +which is the cost the layout exists to lower. Everything beside it explains that +number when it disappoints. `rows_unassigned` is the part no probe can skip, since +a row with no cluster is read by every one of them. `clusters_below_row_group` +counts occupied clusters holding fewer rows than a row group, which is the measured +floor on `nlist`. `advice` names the first condition that holds, or is NULL. + +File count, bytes and row-group structure are deliberately absent. Reaching them +means resolving a table's metadata location, which is a Lakekeeper HTTP call, and +the SQL layer makes no HTTP calls. The compactor reports files and bytes on every +pass, and file count is the wrong health signal regardless: a merge bounds it +without changing what a probe reads. + +## Current gaps + +These are properties of the code as it stands, not plans. + +- **`coldfront._tiered_insert_cold` writes unsorted.** Its cursor loop appends in + cursor order and would need buffering to sort. It is the fallback path for a + tiered INSERT that omits an IDENTITY column. + +## Constraints that are correctness + +- **The query vector is a literal, never a bound parameter.** A parameter typed + `vector` fails to convert, and so does one typed `real[]`, on a custom plan as + much as a generic one. A scalar parameter elsewhere in the same query is fine. +- **`'{…}'::real[]` does not work as the query vector.** It reaches DuckDB as a + VARCHAR and fails to cast. The spelling is `ARRAY[…]::real[]`. +- **`embedding::vector <=> …` fails** with `Type with name vector does not + exist!`, and materialising the read does not help. The unadorned form resolves, + so nothing needs the cast. +- **There is no PostgreSQL-side fallback.** Once a view embeds `iceberg_scan`, + DuckDB owns the whole query and a function it lacks is a hard error rather than + a slow path. Every expression the product wants users to write has to resolve + in DuckDB. +- **Cosine everywhere.** Assignment and search both use cosine. `list_distance` + is Euclidean and would be silently wrong against cosine centroids. +- **No `EXCEPTION` block in any of this plpgsql.** pg_duckdb rejects + subtransactions outright. +- **`coldfront.local_pg_dsn` must be set** for a clustered table, or the + assignment lookup fails with `Catalog 'pglocal' does not exist`. The shipped + container configuration sets it. diff --git a/docs/compaction.md b/docs/compaction.md index 70e69ad..e42b840 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -37,6 +37,7 @@ each reclaims: | `--expire-keep-files` | off | expire metadata only; leave the freed files for an `--orphans` pass | | `--orphans` | off | delete files under the table location that no retained snapshot references | | `--orphan-age D` | 72h | with `--orphans`: only delete files older than D - protects in-flight writes; never set 0 in production | +| `--version` | | print the version and exit | | `--dry-run` | off | report what each step would do; change nothing | Compaction always runs (a no-op when nothing is below target); diff --git a/docs/usage.md b/docs/usage.md index c655470..d08edba 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -124,6 +124,9 @@ Run the archiver (typically via cron): ./bin/archiver --config config.yaml ``` +`--version` prints the build's version (the release tag, or the commit it was +built from) and exits; the partitioner and compactor accept the same flag. + The first run renames `events` → `_events`, creates the unified view `events`, and registers it. From then on every cycle (1) tiers partitions older than `hot_period` from hot PG to cold Iceberg and @@ -847,6 +850,13 @@ The following GUCs adjust write behaviour and execution; tune them as needed: target one tier. `on` emits a dual-tier CTE; `off` rejects with an error and a hint. Not relevant in decoupled mode (every write is single-tier by definition). +- `coldfront.vector_probe` (bool, default `on`) - whether a recognised + similarity search reads only the clusters nearest its query vector. `off` + gives an exact scan of the whole corpus. Only affects a table with a + trained vector column ([usage_vectors.md](usage_vectors.md)). +- `coldfront.vector_nprobe` (int, default `0`) - clusters such a search + reads, overriding the column's own `nprobe`. `0` uses the configured + value; at or above the column's `nlist` the search is exhaustive. - `duckdb.force_execution` - bench it before flipping: on a mixed workload it helps `count(distinct)` and similar but regresses index lookups, top-K with PK ordering, and JSON access. **Default off.** diff --git a/docs/usage_vectors.md b/docs/usage_vectors.md new file mode 100644 index 0000000..6c678ef --- /dev/null +++ b/docs/usage_vectors.md @@ -0,0 +1,314 @@ +# Working with embeddings + +ColdFront stores embeddings in the cold tier as Iceberg `list` and keeps +the pgvector interface you already write. A vector column works in both modes: a +tiered table whose recent rows stay in PostgreSQL, and a decoupled table that +lives entirely in Iceberg. + +## Creating a table + +Tiered, through the archiver's normal configuration: + +```sql +CREATE TABLE chunks ( + id bigserial, + ts timestamptz NOT NULL, + body text, + embedding vector(1536) +) PARTITION BY RANGE (ts); +``` + +Decoupled, declared in one call: + +```sql +SELECT coldfront.create_iceberg_table('public', 'chunks', '[ + {"name": "id", "type": "bigint"}, + {"name": "ts", "type": "timestamptz"}, + {"name": "embedding", "type": "vector(1536)"} +]'::jsonb); +``` + +pgvector is installed for you the first time a table declares a vector column. +You do not need it in a database that has none. + +## Writing + +Ordinary pgvector syntax, on either tier: + +```sql +INSERT INTO chunks (ts, body, embedding) +VALUES (now(), 'hello', '[0.1, 0.2, 0.3, …]'::vector); +``` + +The value coerces to the column whether the row lands hot or cold, and an +`UPDATE` that sets a new embedding works the same way. Nothing about writes +changes when a table is clustered (below): assignments are maintained for you in +the same statement as the write. + +## Reading and searching + +The view exposes the column as `real[]`, which is the same type Iceberg stores. +Read it like any column: + +```sql +SELECT id, body, embedding FROM chunks WHERE id = 42; +``` + +Search with the pgvector operator you would use anyway, and the query spans both +tiers in one statement: + +```sql +SELECT id, body + FROM chunks + ORDER BY embedding <=> ARRAY[0.1, 0.2, 0.3, …]::real[] + LIMIT 10; +``` + +All three pgvector operators are available: `<=>` cosine, `<->` Euclidean, `<#>` +negative inner product. Use `<=>` unless you have a reason not to; clustering is +built on cosine. + +### Three rules for the query vector + +These are requirements, not style. Each fails clearly if broken. + +**Build the vector into the statement text.** A bound parameter does not work, +whether you type it `vector` or `real[]`. Other parameters in the same query are +fine, so only the vector itself has to be inlined: + +```sql +-- works +… ORDER BY embedding <=> ARRAY[0.1, 0.2]::real[] LIMIT $1; +-- fails +… ORDER BY embedding <=> $1 LIMIT 10; +``` + +**Spell it `ARRAY[…]::real[]`.** The `'{0.1,0.2}'::real[]` form reaches the cold +tier as text and fails to cast. + +**Do not cast the column.** `embedding <=> ARRAY[…]::real[]` resolves; +`embedding::vector <=> …` fails with `Type with name vector does not exist!`. Your +own `'[…]'::vector` literal on the right-hand side is fine. + +## Clustering + +Clustering is optional. Without it, a search is an exact scan of both tiers, +which is correct and needs no configuration. + +To prepare a column for clustering, record its settings and train a centroid set: + +```sql +INSERT INTO coldfront.vector_config + (schema_name, table_name, column_name, nlist, nprobe) +VALUES ('public', 'chunks', 'embedding', 500, 20); + +CALL coldfront.vector_train('public', 'chunks', 'embedding'); +``` + +`CALL`, not `SELECT`: this is a procedure. It samples the cold tier, trains +`nlist` centroids, and stores them as a new generation. Every cold write after it +stamps the row's nearest cluster in the same statement, on every write path, and a +retrain does not require regenerating anything. + +**Rows already in the cold tier are not clustered by training.** Training only +writes the centroids; the rows that predate it carry no cluster, and every search +reads all of them. If you are clustering a column that already has cold data, give +those rows a cluster and then compact: + +```sql +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); +``` + +That rewrites the unassigned rows in one operation, serialised like any other cold +write. Compacting afterwards is what puts them in cluster order and clears the +delete markers the rewrite leaves; without it the rows are assigned but a search +still reads more of the table than it needs to. On a column clustered before any +data arrives there is nothing to assign and nothing to run. + +Choosing `nlist` is a floor rather than a formula: aim for at least one row +group's worth of rows per cluster, roughly 2048. Below that, extra clusters stop +reducing the data read. Above it there is a wide plateau. + +### Choosing `nprobe` + +`nprobe` is how many clusters a search reads, and it is the dial between recall and +speed. Measured on a 10 million row corpus of 1024-dimension embeddings, `nlist` +1000, against the exact answer for 100 queries: + +| `nprobe` | clusters read | recall | median query | +|---|---|---|---| +| 1 | 0.1% | 57% | 49 ms | +| 10 | 1% | 87% | 307 ms | +| 20 | 2% | 93% | 592 ms | +| 50 | 5% | 97% | 1.9 s | +| 100 | 10% | 98.5% | 5.1 s | + +The same queries scanned exactly take 32 s each, so `nprobe` 20 is roughly 54 times +faster for 93% of the exact answer. + +**Start at 2% of `nlist`.** Recall gets rapidly more expensive as you buy more of +it: on that corpus the first 23 points of recall cost 111 ms, and the last 1.8 +points cost 3.2 s. The rate worsens from 5 ms per recall point to 1776, and the +knee is at 2% of the clusters. Above it you are paying several hundred milliseconds +per point. + +That also fixes `nlist`, given the row-group floor above. **Aim for `nlist` ≈ rows +/ 10,000**, which leaves a few row groups per cluster: 1000 clusters for 10 million +rows. Dividing more finely than one row group per cluster costs latency without +buying anything, because a cluster read pulls a whole row group either way. + +Two things the averages hide. **Recall is an average over queries, and the tail is +worse**: at `nprobe` 20 the worst of those 100 queries returned 4 of its true 10. +If every query matters, measure the worst case rather than the mean. **A more finely +divided index is not automatically better**: `nlist` 4999 returned more recall per +cluster read, but cost more time for it, and only came out ahead above about 98% +recall. Below that, fewer and larger clusters were faster at the same recall. + +Your corpus is not this one. Take these as the shape of the curve, and measure your +own against a sample of queries you have exact answers for. + +### What a clustered search does + +Once a column has a trained generation, a search that ends in `ORDER BY +<=> LIMIT n` reads only the `nprobe` clusters nearest your query vector. +The query you write does not change. The result becomes approximate, in the same +way it does with any vector index: raise `nprobe` for recall, lower it for speed. + +A search that also groups, aggregates, windows or applies `DISTINCT` is narrowed +the same way: the probe restricts which rows are scanned, and everything in the +statement is computed over those rows. A grouped count, for example, counts probed +rows only. There is no approximate-inside, exact-outside form of one statement: +wrapping the search in a subquery makes the whole thing exact. + +Anything that is not that shape stays an exact scan of both tiers, which is +correct. Two cases worth knowing: a search with no `LIMIT` is answered exactly, +because asking for every row in order is not a request to approximate; and so is +one wrapped in a subquery or CTE, because the clustering is applied to the +statement you write rather than to a nested part of it. + +```sql +-- narrowed to the nearest clusters +SELECT id, body FROM chunks ORDER BY embedding <=> ARRAY[…]::real[] LIMIT 10; +-- exact: the search is not the statement +SELECT string_agg(body, ',') FROM ( + SELECT body FROM chunks ORDER BY embedding <=> ARRAY[…]::real[] LIMIT 10) t; +``` + +Rows written before the column was trained have no cluster, and they are returned +by every search regardless of which clusters it reads. So is every hot-tier row. +Nothing goes missing because it predates the clustering. + +Two settings, per session: + +```sql +-- read every cluster: exact, and the reference to compare recall against +SET coldfront.vector_nprobe = 500; -- at or above nlist +-- or turn the narrowing off entirely +SET coldfront.vector_probe = off; +``` + +Leave `coldfront.vector_nprobe` at its default of `0` to use the `nprobe` you +recorded for the column. + +### Checking whether the clustering is earning its keep + +```sql +CALL coldfront.vector_status(); +SELECT table_name, rows_total, rows_unassigned, clusters_occupied, + probe_fraction, advice + FROM cf_vector_status; +``` + +`probe_fraction` is the number to watch: the share of the cold rows a search reads +on average. Lower is faster. `advice` is filled in only when something specific is +holding that number up, and says which: + +| What it says | What to do | +|---|---| +| no trained generation | `CALL coldfront.vector_train(...)` | +| over half the rows predate training | `CALL coldfront.vector_assign(...)`, then compact | +| over half the clusters hold less than one row group | retrain with a smaller `nlist` | +| the largest clusters hold over 4x the median | expect some queries to be slower than `probe_fraction` suggests; uneven clusters are mostly a property of the embeddings and a retrain rarely changes it | + +Pass a schema and table to report on one column: `CALL +coldfront.vector_status('public', 'chunks')`. The results land in a temporary table +that lasts for your session, and each call replaces the last. + +## More than one vector column + +A table may carry as many vector columns as you like. Each gets its own +configuration, its own centroids and its own generation, and each is assigned on +every write path: + +```sql +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) +VALUES ('public', 'docs', 'body_embedding', 1000, 20), + ('public', 'docs', 'title_embedding', 1000, 20); +CALL coldfront.vector_train('public', 'docs', 'body_embedding'); +CALL coldfront.vector_train('public', 'docs', 'title_embedding'); +``` + +**Only the first vector column's searches get the full speedup.** Not a policy +choice: a Parquet file has one physical row order, and the clustering works by +putting a cluster's rows next to each other so the reader can skip whole row groups. +The first column in table order gets that order. A second column's clusters are +scattered through it, so its row-group statistics cover most of the file and nothing +gets skipped. + +What a later column still gets is the filter. Its predicate cuts the rows that have +to be *scored*, just not the rows that have to be *read*, and reading is about 95% of +the cost. Expect single-digit percent rather than the 54x the first column gets. + +`vector_status` reports which is which: + +```sql +SELECT table_name, column_name, prunes, probe_fraction FROM cf_vector_status; +``` + +`prunes` is true for the one column that owns the sort order. For the others, +`probe_fraction` describes the rows scored rather than the rows read. + +If a second vector column needs to be fast, the honest answer is a second table +holding that column and a key, ordered by its own clustering. That is what a +secondary index is, and Iceberg gives us no way to have two orders in one file. + +## What a clustered table needs from the deployment + +The assignment lookup reads the centroid tables through a local connection, so +`coldfront.local_pg_dsn` must be set. The shipped container image sets it. On +bare metal, add it to `postgresql.conf`: + +``` +coldfront.local_pg_dsn = 'host=/var/run/postgresql dbname= user=' +``` + +Without it, a cold write to a clustered table fails with `Catalog 'pglocal' does +not exist` rather than writing an unassigned row. + +## Compaction + +Compaction stays mandatory on these tables, as it is on any ColdFront table: +every small write makes a file and query cost grows with file count. Run the +compactor as you already do. + +On a clustered table it does more than consolidate. Each write leaves a file +sorted within itself, and a search has to look in every one of them; compaction +merges them on the sort column, one ordered run per size-bounded merge group, so +the number of places a search looks is set by data volume rather than by write +count. Nothing to configure: the table records its own sort column at creation +and the compactor reads it. + +## Limits worth knowing + +- A hot pgvector index is optional and capped by pgvector itself: HNSW refuses a + `vector` column beyond 2,000 dimensions and a `halfvec` beyond 4,000, while + storage tops out at 16,000 for both. A 3,072-dimension model gets no hot HNSW + as a plain `vector`, and searches do not assume one exists. +- One vector column per table owns the physical sort order. Every vector column + gets centroids, assignments and a probe filter, but only the sorted column's + probes skip row groups; the others cut the rows scored, not the rows read. +- `SELECT *` returns your own columns. The cluster assignment is internal and no + branch of the view projects it, so no query can reference it. +- A hot row stores its embedding twice, once as `vector` and once in a generated + `real[]` column that the cold reader can scan. Cold rows, which are the bulk of + the data, store it once. diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 318a887..3f7b12d 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -24,7 +24,7 @@ REGRESS = load_order update_unregistered_view update_heap_table \ param_cold_via_plpgsql async_requires_patch \ storage_secret_azure storage_secret_vended privilege_model \ partition_config_interval self_join_rejected returning_cold_rejected \ - schema_collision drop_iceberg_table + schema_collision drop_iceberg_table vector_centroids vector_type_map vector_cold_render vector_param_render vector_ops vector_probe vector_status vector_assign vector_multicolumn REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index dccd183..6a3d0bf 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -24,6 +24,15 @@ CREATE TABLE coldfront.tiered_views ( iceberg_table text NOT NULL, -- DuckDB ref, e.g. 'ice.myapp.events' partition_col text, -- 'ts' (tiered) or NULL (iceberg-only) is_iceberg_only boolean NOT NULL DEFAULT false, + -- The table's vector columns, in the order the Iceberg schema declares their + -- cluster columns, or NULL when it has none. Recorded at registration because it + -- cannot be derived afterwards in either mode: the view exposes real[] rather + -- than the pgvector type, and a decoupled table names its types without holding + -- them. Every cold write path asks here which columns its assignments describe. + -- + -- The first element owns the file sort order and is the only one whose probe + -- prunes row groups; see _vec_sort_key. + vec_columns text[], PRIMARY KEY (schema_name, relname) ); @@ -161,6 +170,41 @@ CREATE TABLE IF NOT EXISTS coldfront.partition_config ( CONSTRAINT pc_strategy_part CHECK (expiration_strategy = 'drop' OR hot_period IS NULL) -- 'detach' is partition-only ); +-- Routing state for a vector column. Name-keyed like partition_config, so a mesh +-- replicates it by value and every node assigns identical cluster ids. +-- +-- A generation is immutable: a retrain writes a new one and moves the pointer, so +-- an assignment already stored keeps meaning what it meant. Centroids are real[], +-- not pgvector values: these tables are created with the extension, which must +-- install on a database that has no vectors and may never have any. Scoring a query +-- against them goes through the same `<=>` shim a caller uses, which delegates to +-- pgvector, and by then a vector column exists. +CREATE TABLE IF NOT EXISTS coldfront.vector_config ( + schema_name text NOT NULL DEFAULT 'public', + table_name text NOT NULL, + column_name text NOT NULL, + nlist int NOT NULL, + nprobe int NOT NULL, + generation int NOT NULL DEFAULT 0, -- 0 ⇒ nothing trained yet + addition_cap int NOT NULL DEFAULT 0, -- post-generation centroids allowed + PRIMARY KEY (schema_name, table_name, column_name), + CONSTRAINT vc_nlist_pos CHECK (nlist >= 1), + CONSTRAINT vc_nprobe_pos CHECK (nprobe >= 1), + CONSTRAINT vc_nprobe_fit CHECK (nprobe <= nlist) +); + +CREATE TABLE IF NOT EXISTS coldfront.vector_centroids ( + schema_name text NOT NULL DEFAULT 'public', + table_name text NOT NULL, + column_name text NOT NULL, + generation int NOT NULL, + centroid_id int NOT NULL, + parent_id int, -- set on an adaptive addition + centroid real[] NOT NULL, + PRIMARY KEY (schema_name, table_name, column_name, generation, centroid_id), + CONSTRAINT vcent_gen_pos CHECK (generation >= 1) +); + -- Carry the durable tiering metadata across pg_dump/restore so a restored node -- re-attaches to the same Iceberg cold tier with no re-provisioning. These are -- extension-member tables, whose data pg_dump would otherwise omit; @@ -171,6 +215,10 @@ CREATE TABLE IF NOT EXISTS coldfront.partition_config ( SELECT pg_extension_config_dump('coldfront.tiered_views', ''); SELECT pg_extension_config_dump('coldfront.archive_watermark', ''); SELECT pg_extension_config_dump('coldfront.partition_config', ''); +-- Losing these means every stored cluster id is uninterpretable and the table has +-- to be retrained from scratch, so they travel with a dump like the rest. +SELECT pg_extension_config_dump('coldfront.vector_config', ''); +SELECT pg_extension_config_dump('coldfront.vector_centroids', ''); -- ensure_attached() issues ATTACH IF NOT EXISTS for the Lakekeeper catalog -- using the coldfront.warehouse and coldfront.lakekeeper_endpoint GUCs. Called @@ -257,9 +305,9 @@ $$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog; -- needs it will fail with a clear "Catalog 'pglocal' does not exist" rather -- than silently doing the wrong thing. -- --- READ_ONLY on the ATTACH is deliberate: this connection is for *reading* --- PG tables to feed Iceberg writes; coldfront never wants writes flowing --- back through pglocal into PG. +-- Every use is a read: the PG tables that feed an Iceberg write, and the centroid +-- lookup a cold write on a clustered table carries. The ATTACH is a plain one, so +-- that is a property of what coldfront emits rather than one it enforces. CREATE OR REPLACE FUNCTION coldfront.ensure_pg_attached() RETURNS void AS $$ DECLARE dsn text := current_setting('coldfront.local_pg_dsn', true); @@ -657,7 +705,8 @@ BEGIN JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = p_schema AND c.relname = p_part - AND a.attnum > 0 AND NOT a.attisdropped; + AND a.attnum > 0 AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); -- Idempotent reset. CASCADE on DROP FUNCTION removes the AFTER-row / -- BEFORE-TRUNCATE triggers on the partition that reference these @@ -758,6 +807,8 @@ DECLARE v_hot_relname text; v_identity_col text; v_identity_seq text; + vec_cols text[]; + vec_exprs text[]; full_cols text[]; full_types text[]; full_defaults text[]; @@ -847,7 +898,8 @@ BEGIN JOIN pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum WHERE n.nspname = v_hot_schema AND c.relname = v_hot_relname - AND a.attnum > 0 AND NOT a.attisdropped; + AND a.attnum > 0 AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); target_csv := array_to_string( ARRAY(SELECT quote_ident(c) FROM unnest(p_target_cols) c), ', '); @@ -875,6 +927,12 @@ BEGIN END IF; END LOOP; + -- Only the cluster lookup reads pglocal, so a table without a vector must not + -- pay for the attach. + IF coldfront._types_have_vector(full_types) THEN + PERFORM coldfront.ensure_pg_attached(); + END IF; + OPEN cur FOR EXECUTE format( 'SELECT %s FROM (%s) AS coldfront_src(%s) WHERE %I < %L', cursor_proj, p_source_sql, target_csv, v_partcol, v_cutoff); @@ -884,7 +942,9 @@ BEGIN EXIT WHEN NOT FOUND; payload := to_jsonb(rec); - row_lit := ''; + row_lit := ''; + vec_cols := '{}'; + vec_exprs := '{}'; -- The cursor already projected every underlying column with the -- right value (user-supplied / DEFAULT / NULL stub for IDENTITY), @@ -902,21 +962,29 @@ BEGIN ELSIF payload ? col AND (payload->col) IS NOT NULL AND jsonb_typeof(payload->col) <> 'null' THEN val_text := payload->>col; - IF full_types[i] = 'bytea' THEN - -- val_text is PG bytea text '\xHEX' (bytea_output pinned to - -- hex above). DuckDB stores a BLOB, so rebuild the exact - -- bytes from the hex digits via from_hex(). Passing the - -- '\xHEX' string straight to a BLOB column would make DuckDB - -- mis-parse the \x escapes and corrupt the value. - row_lit := row_lit || format('from_hex(%L)', substr(val_text, 3)); + IF coldfront._is_vector_type(full_types[i]) THEN + vec_cols := vec_cols || col; + vec_exprs := vec_exprs || coldfront._render_cold_value(val_text, full_types[i]); + row_lit := row_lit || vec_exprs[cardinality(vec_exprs)]; ELSE - row_lit := row_lit || quote_literal(val_text); + row_lit := row_lit || coldfront._render_cold_value(val_text, full_types[i]); END IF; ELSE + -- A NULL vector still owns its prefix slot: the Iceberg schema + -- declares one cluster column per vector column unconditionally, + -- so a shorter prefix would misalign the positional tuple. + IF coldfront._is_vector_type(full_types[i]) THEN + vec_cols := vec_cols || col; + vec_exprs := vec_exprs || 'NULL'::text; + END IF; row_lit := row_lit || 'NULL'; END IF; END LOOP; + -- The cluster leads the tuple, derived from this row's own vector literal. + row_lit := coldfront._vec_list_prefix(p_view_schema, p_view_name, vec_cols, vec_exprs) + || row_lit; + cold_buf := cold_buf || (CASE WHEN cold_count = 0 THEN '' ELSE ', ' END) || '(' || row_lit || ')'; @@ -1042,7 +1110,8 @@ BEGIN FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace nn ON nn.oid = c.relnamespace WHERE nn.nspname = v_hot_schema AND c.relname = v_hot_relname - AND a.attnum > 0 AND NOT a.attisdropped; + AND a.attnum > 0 AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); SELECT string_agg(quote_ident(col), ', ' ORDER BY ord), string_agg(format('r[%L]::%s AS %I', col, COALESCE(NULLIF(coldfront._iceberg_view_cast_type(typ), ''), @@ -1067,6 +1136,9 @@ BEGIN v_inner := format('SELECT %s FROM iceberg_scan(%L) r WHERE r[%L] < %s', v_cold_read, v_iceberg, v_partcol, v_cut_lit); PERFORM coldfront.ensure_attached(); + IF coldfront._types_have_vector(full_types) THEN + PERFORM coldfront.ensure_pg_attached(); + END IF; -- Reject a cold→hot target with no covering hot partition, naming the VIEW -- (never the internal heap name): the heap is RANGE-partitioned with no default @@ -1128,7 +1200,8 @@ BEGIN IF (payload->>'cf_new_ts')::timestamptz < v_cutoff THEN -- stay-cold: re-add to Iceberg (DuckDB literal tuple). - ins_arr := ins_arr || ('(' || coldfront._move_row_literal(payload, full_cols, full_types, v_partcol) || ')'); + ins_arr := ins_arr || ('(' || coldfront._move_row_literal(payload, full_cols, full_types, v_partcol, + p_view_schema, p_view_name) || ')'); ELSE -- cold→hot: add to the heap (PG literal tuple, partition column = e). heap_arr := heap_arr || ('(' || coldfront._move_pg_row_literal(payload, full_cols, v_partcol) || ')'); @@ -1145,7 +1218,8 @@ BEGIN v_hot_schema, v_hot_relname, v_pc, v_cut_lit, p_where, p_newpc) LOOP payload := to_jsonb(rec); - ins_arr := ins_arr || ('(' || coldfront._move_row_literal(payload, full_cols, full_types, v_partcol) || ')'); + ins_arr := ins_arr || ('(' || coldfront._move_row_literal(payload, full_cols, full_types, v_partcol, + p_view_schema, p_view_name) || ')'); END LOOP; -- ── Hot heap (plain PG) ─────────────────────────────────────────────────── @@ -1184,7 +1258,8 @@ $fn$; -- else is a quoted literal DuckDB coerces to the storage type. Mirrors -- _tiered_insert_cold's per-row serialiser. CREATE FUNCTION coldfront._move_row_literal( - p_payload jsonb, p_cols text[], p_types text[], p_partcol text + p_payload jsonb, p_cols text[], p_types text[], p_partcol text, + p_schema text, p_view text ) RETURNS text LANGUAGE plpgsql IMMUTABLE AS $$ DECLARE @@ -1192,6 +1267,8 @@ DECLARE col text; val_text text; i int; + vec_cols text[] := '{}'; + vec_exprs text[] := '{}'; BEGIN FOR i IN 1 .. array_length(p_cols, 1) LOOP col := p_cols[i]; @@ -1200,15 +1277,26 @@ BEGIN row_lit := row_lit || quote_literal(p_payload->>'cf_new_ts'); ELSIF p_payload ? col AND jsonb_typeof(p_payload->col) <> 'null' THEN val_text := p_payload->>col; - IF p_types[i] = 'bytea' THEN - row_lit := row_lit || format('from_hex(%L)', substr(val_text, 3)); + IF coldfront._is_vector_type(p_types[i]) THEN + vec_cols := vec_cols || col; + vec_exprs := vec_exprs || coldfront._render_cold_value(val_text, p_types[i]); + row_lit := row_lit || vec_exprs[cardinality(vec_exprs)]; ELSE - row_lit := row_lit || quote_literal(val_text); + row_lit := row_lit || coldfront._render_cold_value(val_text, p_types[i]); END IF; ELSE + -- A NULL vector still owns its prefix slot: the Iceberg schema + -- declares one cluster column per vector column unconditionally, + -- so a shorter prefix would misalign the positional tuple. + IF coldfront._is_vector_type(p_types[i]) THEN + vec_cols := vec_cols || col; + vec_exprs := vec_exprs || 'NULL'::text; + END IF; row_lit := row_lit || 'NULL'; END IF; END LOOP; + -- The cluster leads the tuple, derived from this row's own vector literal. + row_lit := coldfront._vec_list_prefix(p_schema, p_view, vec_cols, vec_exprs) || row_lit; RETURN row_lit; END; $$; @@ -1265,6 +1353,8 @@ DECLARE batch_size int := 1000; pk_list text; col_list text; + col_types text[]; + scratch_proj text; visibility text; scratch_tbl text; scratch_qual text; @@ -1273,13 +1363,15 @@ DECLARE BEGIN -- Resolve column / PK order ONCE per procedure call (stable for the -- lifetime of the partition's archive cycle). - SELECT array_agg(a.attname ORDER BY a.attnum) - INTO col_names + SELECT array_agg(a.attname ORDER BY a.attnum), + array_agg(format_type(a.atttypid, a.atttypmod) ORDER BY a.attnum) + INTO col_names, col_types FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = p_schema AND c.relname = p_part - AND a.attnum > 0 AND NOT a.attisdropped; + AND a.attnum > 0 AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); SELECT array_agg(a.attname ORDER BY x.ord) INTO pk_names @@ -1292,8 +1384,22 @@ BEGIN SELECT string_agg(quote_ident(name), ', ' ORDER BY ord) INTO pk_list FROM unnest(pk_names) WITH ORDINALITY AS u(name, ord); - SELECT string_agg(quote_ident(name), ', ' ORDER BY ord) INTO col_list - FROM unnest(col_names) WITH ORDINALITY AS u(name, ord); + -- Two projections over the same columns. The scratch reads the delta through + -- PostgreSQL, so a vector is cast to real[] there: DuckDB reads the scratch + -- over libpq and cannot scan the pgvector type at all. The Iceberg INSERT is + -- positional and its target leads with the cluster columns. + SELECT string_agg(quote_ident(name), ', ' ORDER BY ord), + string_agg(CASE WHEN coldfront._is_vector_type(typ) + THEN format('%I::real[] AS %I', name, name) + ELSE quote_ident(name) END, ', ' ORDER BY ord) + INTO col_list, scratch_proj + FROM unnest(col_names, col_types) WITH ORDINALITY AS u(name, typ, ord); + + -- The clusters lead the Iceberg INSERT, derived set-based from the scratch's + -- own columns. Keyed on the ref, since the configuration names the user's table + -- and this procedure is handed one of its partitions. + col_list := COALESCE(coldfront._vec_list_prefix_for_ref(p_iceberg_ref, ''), '') + || col_list; -- Snapshot filter: in replay, skip rows still visible in the bulk-copy -- snapshot (they're already in the bulk export). cutover_archive's @@ -1319,8 +1425,8 @@ BEGIN -- the COMMIT, which is what we need. EXECUTE format( 'CREATE UNLOGGED TABLE %s AS - SELECT * FROM %s WHERE %s LIMIT %s', - scratch_qual, delta_tbl, visibility, batch_size); + SELECT %s, coldfront_is_deleted, coldfront_xid FROM %s WHERE %s LIMIT %s', + scratch_qual, scratch_proj, delta_tbl, visibility, batch_size); EXECUTE format('SELECT count(*) FROM %s', scratch_qual) INTO n_applied; @@ -1501,6 +1607,968 @@ $$; -- rejected at create time. See ARCHITECTURE_DECOUPLED.md for the full table. -- ============================================================================ +-- pgvector's vector/halfvec, with or without the dimension typmod. Both maps +-- below call this so they cannot disagree about what counts as a vector: the +-- storage type and the view cast have to move together or a column stores one +-- way and reads another. sparsevec is excluded deliberately (densifying it is a +-- 100x storage blowup), so it falls through to the unsupported-type error. +CREATE OR REPLACE FUNCTION coldfront._is_vector_type(p_pg_type text) +RETURNS boolean +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT lower(trim(p_pg_type)) IN ('vector', 'halfvec') + OR lower(trim(p_pg_type)) LIKE 'vector(%' + OR lower(trim(p_pg_type)) LIKE 'halfvec(%'; +$$; + +-- Train a centroid set for one vector column and store it as a new generation. +-- +-- Lloyd iterations over a reservoir sample, run as DuckDB statements: both the +-- assignment and the mean are distance work over the cold corpus, which is what +-- DuckDB is for and what pulling the vectors into PostgreSQL would waste. The +-- sample and the working tables live in the session's own DuckDB instance, so the +-- whole loop has to run in one call. +-- +-- A PROCEDURE, not a function, and that is a hard requirement rather than a +-- preference: pg_duckdb refuses to execute a DuckDB query inside a function +-- ("DuckDB execution is not supported inside functions"), which is the only way +-- to read the trained centroids back. raw_query runs in a function but is a bare +-- DuckDB channel with no access to PostgreSQL tables, so a function can move data +-- in neither direction. CALL is what makes reading the result possible. +-- +-- Empty clusters simply do not come back from the mean, so the stored count can be +-- below p_nlist; it is recorded rather than padded, since a centroid nothing was +-- assigned to routes nothing. +CREATE PROCEDURE coldfront.vector_train( + p_schema text, + p_table text, + p_column text, + p_nlist int DEFAULT NULL, + p_sample int DEFAULT 20000, + p_iterations int DEFAULT 8 +) +LANGUAGE plpgsql AS $$ +DECLARE + v_ice text; + v_nlist int; + v_gen int; + v_n int; + v_dim int; + i int; +BEGIN + PERFORM coldfront._reject_on_standby('train vector centroids'); + + SELECT iceberg_table INTO v_ice + FROM coldfront.tiered_views + WHERE schema_name = p_schema AND relname = p_table; + IF v_ice IS NULL THEN + RAISE EXCEPTION 'coldfront.vector_train: "%.%" is not a registered tiered table', + p_schema, p_table; + END IF; + + SELECT COALESCE(p_nlist, nlist), generation INTO v_nlist, v_gen + FROM coldfront.vector_config + WHERE schema_name = p_schema AND table_name = p_table AND column_name = p_column; + IF v_nlist IS NULL THEN + RAISE EXCEPTION 'coldfront.vector_train: no configuration for "%.%"."%"', + p_schema, p_table, p_column + USING HINT = 'INSERT a coldfront.vector_config row first: it holds the ' + 'generation pointer this writes, so p_nlist alone is not enough.'; + END IF; + + SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; + PERFORM coldfront.ensure_attached(); + + -- The sample, then the seeds drawn from it. Both seeds are fixed so a retrain + -- on unchanged data reproduces the same centroids. + PERFORM duckdb.raw_query(format( + 'CREATE OR REPLACE TABLE memory.main.cf_samp AS ' + 'SELECT row_number() OVER () AS id, v FROM (' + ' SELECT %I AS v FROM %s WHERE %I IS NOT NULL ' + ' USING SAMPLE reservoir(%s ROWS) REPEATABLE (42))', + p_column, v_ice, p_column, p_sample)); + + SELECT r['n']::int, r['d']::int INTO v_n, v_dim + FROM duckdb.query('SELECT count(*) AS n, max(len(v)) AS d FROM memory.main.cf_samp') AS t(r); + IF COALESCE(v_n, 0) = 0 THEN + RAISE EXCEPTION 'coldfront.vector_train: "%.%"."%" has no cold rows to train on', + p_schema, p_table, p_column; + END IF; + + -- k-means++ seeding: one seed at random, then each next drawn with probability + -- proportional to its squared distance from the nearest seed already chosen, so + -- the seeds spread out instead of clumping. Lloyd's iterations below only move + -- centroids locally, so they cannot repair a clumped start. + -- + -- It matters most where dense clumps exist, which is the lower dimensionalities + -- many embedding models produce; at 1024 dimensions distances concentrate and the + -- spread start makes little difference. Seeding is a one-time training cost and + -- nothing a query pays. + -- + -- The draw is an exponential race: for weights w, the minimum of -ln(u)/w is + -- distributed exactly as a weighted draw, in one pass and with no cumulative sum. + -- d is maintained incrementally, folding in only the seed just added, so a round + -- is one pass over the sample rather than one per seed chosen so far. + PERFORM duckdb.raw_query( + 'CREATE OR REPLACE TABLE memory.main.cf_seed AS ' + 'SELECT 1 AS seq, id, v FROM memory.main.cf_samp ' + 'USING SAMPLE reservoir(1 ROWS) REPEATABLE (7)'); + PERFORM duckdb.raw_query( + 'CREATE OR REPLACE TABLE memory.main.cf_seed_d AS ' + 'SELECT s.id, s.v, ' + '(SELECT min(list_cosine_distance(s.v, p.v)) FROM memory.main.cf_seed p) AS d ' + 'FROM memory.main.cf_samp s'); + FOR i IN 2 .. v_nlist LOOP + -- A sample smaller than nlist, or one whose remaining rows all duplicate a + -- seed, leaves d = 0 everywhere and the draw returns nothing. The centroid + -- count recorded below is whatever came back, so that resolves itself. + PERFORM duckdb.raw_query(format( + 'INSERT INTO memory.main.cf_seed ' + 'SELECT %s, id, v FROM memory.main.cf_seed_d ' + 'WHERE d > 0 ORDER BY -ln(random()) / (d * d) LIMIT 1', i)); + -- Keyed on seq, because a sample id says nothing about insertion order. + PERFORM duckdb.raw_query(format( + 'UPDATE memory.main.cf_seed_d SET d = least(d, ' + 'list_cosine_distance(v, (SELECT p.v FROM memory.main.cf_seed p ' + 'WHERE p.seq = %s)))', i)); + END LOOP; + PERFORM duckdb.raw_query( + 'CREATE OR REPLACE TABLE memory.main.cf_cent AS ' + 'SELECT row_number() OVER (ORDER BY seq) AS cid, v FROM memory.main.cf_seed'); + + -- Assign, then recompute each cluster's mean. The mean unnests the vector + -- against a matching range so the two lists advance together: DuckDB has no + -- WITH ORDINALITY, and position is what makes the average element-wise. + FOR i IN 1 .. p_iterations LOOP + PERFORM duckdb.raw_query( + 'CREATE OR REPLACE TABLE memory.main.cf_asg AS ' + 'SELECT s.id, arg_min(c.cid, list_cosine_distance(s.v, c.v)) AS cid ' + 'FROM memory.main.cf_samp s CROSS JOIN memory.main.cf_cent c GROUP BY s.id'); + PERFORM duckdb.raw_query(format( + 'CREATE OR REPLACE TABLE memory.main.cf_cent AS ' + 'SELECT cid, list(m ORDER BY i)::FLOAT[] AS v FROM (' + ' SELECT cid, i, avg(x) AS m FROM (' + ' SELECT a.cid, unnest(range(1, %s)) AS i, unnest(s.v) AS x ' + ' FROM memory.main.cf_samp s JOIN memory.main.cf_asg a USING (id))' + ' GROUP BY cid, i) GROUP BY cid', v_dim + 1)); + END LOOP; + + -- The centroids land in a temporary heap first. A single INSERT reading a + -- DuckDB scan is planned as a DuckDB statement, and DuckDB cannot write to a + -- PostgreSQL table, so the read and the write have to be separate statements. + -- The drop names pg_temp: an unqualified name would resolve through + -- search_path and could hit a permanent table of the same name. + IF to_regclass('pg_temp.cf_cent_pg') IS NOT NULL THEN + DROP TABLE pg_temp.cf_cent_pg; + END IF; + CREATE TEMP TABLE cf_cent_pg ON COMMIT DROP AS + SELECT r['cid']::int AS cid, r['v']::real[] AS v + FROM duckdb.query('SELECT cid, v FROM memory.main.cf_cent') AS t(r); + + -- A generation is immutable, so this writes a new one and moves the pointer. + v_gen := COALESCE(v_gen, 0) + 1; + INSERT INTO coldfront.vector_centroids + (schema_name, table_name, column_name, generation, centroid_id, centroid) + SELECT p_schema, p_table, p_column, v_gen, cid, v FROM cf_cent_pg; + + GET DIAGNOSTICS v_n = ROW_COUNT; + -- The trained count can fall below the configured nprobe (empty clusters do + -- not come back from the mean); clamp it in the same statement or the row + -- would fail vc_nprobe_fit and abort the whole training transaction. + UPDATE coldfront.vector_config + SET generation = v_gen, nlist = v_n, nprobe = LEAST(nprobe, v_n) + WHERE schema_name = p_schema AND table_name = p_table AND column_name = p_column; + + RAISE NOTICE 'coldfront: trained % centroids for "%.%"."%" as generation %', + v_n, p_schema, p_table, p_column, v_gen; +END; +$$; + +-- Give a cluster to the cold rows that have none. +-- +-- Training writes centroids and every write after it is assigned in the statement +-- that writes it, so the rows without an assignment are exactly those that predate +-- the generation. A probe reads all of them whatever clusters it looks in, so on a +-- corpus tiered before training they are the whole cost of the search. +-- +-- One claimed UPDATE, and nothing new: the SET item is the same generator a cold +-- UPDATE uses when a caller changes an embedding, applied to the embedding already +-- there. Serialised through the bakery like every other cold write, so a concurrent +-- writer cannot land a row assigned under a different generation partway through. +-- +-- What it leaves behind is a merge-on-read delete per rewritten row, and rows in +-- the order the update produced rather than in cluster order. Compaction resolves +-- both: it applies the deletes and merges the result on the sort key. So the +-- sequence is train, assign, compact. +-- +-- Fails rather than no-ops without a live generation. The lookup would resolve to +-- NULL for every row, leaving the table exactly as it was after a full rewrite, and +-- a wrong or absent cluster is invisible in a way a probe never reports. +CREATE PROCEDURE coldfront.vector_assign( + p_schema text, + p_table text, + p_column text +) +LANGUAGE plpgsql AS $$ +DECLARE + v_ice text; + v_gen int; + v_item text; + v_col text := quote_ident(coldfront._vec_list_col(p_column)); + v_n bigint; +BEGIN + PERFORM coldfront._reject_on_standby('assign vector clusters'); + + SELECT iceberg_table INTO v_ice + FROM coldfront.tiered_views + WHERE schema_name = p_schema AND relname = p_table + AND p_column = ANY (COALESCE(vec_columns, '{}')); + IF v_ice IS NULL THEN + RAISE EXCEPTION 'coldfront.vector_assign: "%.%"."%" is not a registered clustered column', + p_schema, p_table, p_column; + END IF; + + SELECT NULLIF(generation, 0) INTO v_gen + FROM coldfront.vector_config + WHERE schema_name = p_schema AND table_name = p_table AND column_name = p_column; + IF v_gen IS NULL THEN + RAISE EXCEPTION 'coldfront.vector_assign: "%.%"."%" has no trained generation', + p_schema, p_table, p_column + USING HINT = 'CALL coldfront.vector_train(...) first: without centroids ' + 'every row would be assigned NULL, which is what it already is.'; + END IF; + + v_item := coldfront._vec_list_set_item(v_ice, p_column, quote_ident(p_column)); + + SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; + PERFORM coldfront.ensure_attached(); + + -- Counted before the write, so the notice reports what this call did rather than + -- what the table looks like afterwards. + -- + -- Through EXECUTE, which is what makes a dynamic table name work here: a bare + -- duckdb.query(format(...)) is not a constant at plan time and is refused, while + -- staging the count in a memory.main table the way vector_train stages its + -- centroids would spend this transaction's one database on memory and leave the + -- UPDATE below unable to write ice at all ("a single transaction can only modify + -- one database"). Built as dynamic SQL, the argument is a literal again. + EXECUTE format('SELECT t.r[%L]::bigint FROM duckdb.query(%L) AS t(r)', 'n', + format('SELECT count(*) AS n FROM %s WHERE %s IS NULL', v_ice, v_col)) + INTO v_n; + + IF COALESCE(v_n, 0) = 0 THEN + RAISE NOTICE 'coldfront: every cold row of "%.%"."%" is already assigned', + p_schema, p_table, p_column; + RETURN; + END IF; + + PERFORM coldfront._exec_iceberg_with_claim(v_ice, format( + 'UPDATE %s SET %s WHERE %s IS NULL', v_ice, v_item, v_col)); + + RAISE NOTICE 'coldfront: assigned % cold row(s) of "%.%"."%" to generation %; ' + 'compact the table to apply the deletes and restore cluster order', + v_n, p_schema, p_table, p_column, v_gen; +END; +$$; + +-- What a decision about a clustered column needs, and nothing else available. +-- +-- The headline is probe_fraction: the share of the corpus a probe reads on +-- average, which is the cost the whole layout exists to lower. Everything beside +-- it is there to explain that number when it is disappointing. rows_unassigned is +-- the part no probe can skip, because a row with no cluster is read by every one +-- of them. clusters_below_row_group counts occupied clusters holding fewer rows +-- than a row group: past that point extra clusters cut the rows scored without +-- cutting the rows read, which is the measured floor on nlist and the reason it is +-- a floor rather than a formula. +-- +-- Deliberately absent: file count, bytes, and row-group structure. Reaching those +-- means resolving a table's metadata location, which is a Lakekeeper HTTP call, +-- and the SQL layer does not make HTTP calls. The compactor already reports files +-- and bytes on every pass, and file count is the wrong health signal anyway: a +-- merge bounds it without changing what a probe reads. +-- +-- A PROCEDURE writing a temporary table, for the same reason vector_train is one: +-- pg_duckdb refuses to execute a DuckDB query inside a function, and a single +-- INSERT ... SELECT over a DuckDB scan is planned as DuckDB's, which cannot write +-- a PostgreSQL table. Nothing is staged on the way, though: DuckDB aggregates the +-- distribution and hands back one row (see the EXECUTE below). +CREATE PROCEDURE coldfront.vector_status( + p_schema text DEFAULT NULL, + p_table text DEFAULT NULL +) +LANGUAGE plpgsql AS $$ +DECLARE + vt record; + v_schemas text[]; + v_names text[]; + v_cols text[]; + v_rows bigint; + v_unasg bigint; + v_occ int; + v_min bigint; + v_max bigint; + v_p50 bigint; + v_p99 bigint; + v_floor int; + v_trained int; + v_added int; + v_i int; +BEGIN + -- Dropped by name only when it is there. IF EXISTS would do the same and add a + -- NOTICE to every caller's output for the ordinary case of a first call. + IF to_regclass('pg_temp.cf_vector_status') IS NOT NULL THEN + DROP TABLE pg_temp.cf_vector_status; + END IF; + CREATE TEMP TABLE cf_vector_status ( + schema_name text, + table_name text, + column_name text, + -- False for a table's second and later vector columns: their probe filters + -- the rows it scores but prunes no row groups, so their probe_fraction is + -- what they score rather than what they read. + prunes boolean, + generation int, + nlist int, + nprobe int, + clusters_trained int, + clusters_occupied int, + additions int, + addition_cap int, + rows_total bigint, + rows_unassigned bigint, + rows_per_cluster_min bigint, + rows_per_cluster_max bigint, + -- The pair a query's cost is actually bounded by. min/max are reported + -- because they are free, but max/min is not a health signal: one tiny + -- cluster sets it, and a real corpus has one. + rows_per_cluster_p50 bigint, + rows_per_cluster_p99 bigint, + clusters_below_row_group int, + probe_fraction numeric, + advice text + ); + -- Session lifetime, not ON COMMIT DROP: a bare CALL is its own transaction, so + -- a table dropped at commit would be gone before the caller could select from + -- it. Re-running the procedure replaces it. + + SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; + PERFORM coldfront.ensure_attached(); + + -- The work list is a pair of key arrays walked by index, not a query the loop + -- iterates: a plpgsql FOR over a query holds a portal open for its body, and + -- pg_duckdb refuses a DuckDB read while one is ("DuckDB execution is not + -- supported inside functions"). An integer loop opens none. + -- One entry per (table, vector column), since a table may carry several and each + -- has its own centroids, generation and distribution. + SELECT array_agg(k.schema_name ORDER BY k.schema_name, k.relname, k.ord), + array_agg(k.relname ORDER BY k.schema_name, k.relname, k.ord), + array_agg(k.col ORDER BY k.schema_name, k.relname, k.ord) + INTO v_schemas, v_names, v_cols + FROM (SELECT tv.schema_name, tv.relname, c.col, c.ord + FROM coldfront.tiered_views tv + CROSS JOIN LATERAL unnest(tv.vec_columns) WITH ORDINALITY AS c(col, ord) + WHERE tv.vec_columns IS NOT NULL + AND (p_schema IS NULL OR tv.schema_name = p_schema) + AND (p_table IS NULL OR tv.relname = p_table)) k; + + FOR v_i IN 1 .. COALESCE(array_length(v_schemas, 1), 0) LOOP + SELECT tv.schema_name, tv.relname, v_cols[v_i] AS vec_column, tv.iceberg_table, + vc.nlist, vc.nprobe, NULLIF(vc.generation, 0) AS generation, + vc.addition_cap, + v_cols[v_i] = tv.vec_columns[1] AS prunes + INTO vt + FROM coldfront.tiered_views tv + LEFT JOIN coldfront.vector_config vc + ON vc.schema_name = tv.schema_name + AND vc.table_name = tv.relname + AND vc.column_name = v_cols[v_i] + WHERE tv.schema_name = v_schemas[v_i] AND tv.relname = v_names[v_i]; + + -- One pass over the cold table: DuckDB groups by cluster and aggregates the + -- grouping, so what crosses back is a single row of scalars. + -- + -- Through EXECUTE because the table name is dynamic and duckdb.query needs a + -- constant at plan time, not a literal in the source. Staging the grouping in + -- a memory.main table the way vector_train stages its centroids would work + -- here too, but it is a table to name, drop and read back for a result that + -- fits in one row, and on any path that also writes Iceberg it would spend + -- the transaction's one writable database (see vector_assign). + EXECUTE format( + 'SELECT t.r[%L]::bigint, t.r[%L]::bigint, t.r[%L]::int, ' + 't.r[%L]::bigint, t.r[%L]::bigint, t.r[%L]::bigint, ' + 't.r[%L]::bigint, t.r[%L]::int ' + 'FROM duckdb.query(%L) AS t(r)', + 'rows_total', 'rows_unasg', 'occupied', 'min_n', 'max_n', + 'p50_n', 'p99_n', 'below', + format( + 'WITH d AS (SELECT %I AS cl, count(*) AS n FROM %s GROUP BY 1) ' + 'SELECT coalesce(sum(n), 0) AS rows_total, ' + 'coalesce(sum(n) FILTER (WHERE cl IS NULL), 0) AS rows_unasg, ' + 'count(*) FILTER (WHERE cl IS NOT NULL) AS occupied, ' + 'min(n) FILTER (WHERE cl IS NOT NULL) AS min_n, ' + 'max(n) FILTER (WHERE cl IS NOT NULL) AS max_n, ' + 'quantile_cont(n, 0.5) FILTER (WHERE cl IS NOT NULL) AS p50_n, ' + 'quantile_cont(n, 0.99) FILTER (WHERE cl IS NOT NULL) AS p99_n, ' + 'count(*) FILTER (WHERE cl IS NOT NULL AND n < ' || coldfront._vec_row_group_limit() || ') AS below ' + 'FROM d', + coldfront._vec_list_col(vt.vec_column), vt.iceberg_table)) + INTO v_rows, v_unasg, v_occ, v_min, v_max, v_p50, v_p99, v_floor; + + SELECT count(*), count(*) FILTER (WHERE parent_id IS NOT NULL) + INTO v_trained, v_added + FROM coldfront.vector_centroids c + WHERE c.schema_name = vt.schema_name AND c.table_name = vt.relname + AND c.column_name = vt.vec_column AND c.generation = vt.generation; + + INSERT INTO cf_vector_status ( + schema_name, table_name, column_name, prunes, + generation, nlist, nprobe, + clusters_trained, clusters_occupied, additions, addition_cap, + rows_total, rows_unassigned, + rows_per_cluster_min, rows_per_cluster_max, + rows_per_cluster_p50, rows_per_cluster_p99, + clusters_below_row_group, probe_fraction, advice + ) VALUES ( + vt.schema_name, vt.relname, vt.vec_column, vt.prunes, + vt.generation, vt.nlist, vt.nprobe, + v_trained, v_occ, v_added, vt.addition_cap, + v_rows, v_unasg, v_min, v_max, v_p50, v_p99, v_floor, + -- What a probe reads on average: its share of the occupied clusters, + -- plus every unassigned row, which it always reads. + CASE WHEN v_rows = 0 THEN NULL + ELSE round(LEAST(1.0, ( + COALESCE(LEAST(vt.nprobe, v_occ)::numeric / NULLIF(v_occ, 0), 1) + * (v_rows - v_unasg) + v_unasg) / v_rows), 4) + END, + CASE + WHEN vt.generation IS NULL THEN + 'no trained generation: every row is unassigned and every probe reads the whole table' + WHEN v_rows > 0 AND v_unasg::numeric / v_rows > 0.5 THEN + 'over half the rows predate training, and a probe reads all of them' + WHEN v_occ > 0 AND v_floor::numeric / v_occ > 0.5 THEN + 'over half the occupied clusters hold less than one row group: retrain with a smaller nlist' + -- Against the median, not the minimum: one tiny cluster sets max/min + -- and every real corpus has one, so a rule on max/min fires on every + -- table. The upper tail against the typical case is what a query + -- landing in a large cluster actually pays. + WHEN v_p50 IS NOT NULL AND v_p50 > 0 AND v_p99 > v_p50 * 4 THEN + 'the largest clusters hold over 4x the median, so a query landing ' + 'in one reads that much more than probe_fraction suggests' + END); + END LOOP; + + IF NOT EXISTS (SELECT 1 FROM cf_vector_status) THEN + RAISE NOTICE 'coldfront: no registered table has a clustered vector column'; + END IF; +END; +$$; + +-- Routing state replicates cluster-wide, for the reason the assignment itself +-- exists: every node has to resolve a vector to the same cluster id, or a row +-- written on one node is invisible to a probe issued on another. Gated on spock +-- like the claims tables, so vanilla is a no-op. +CREATE FUNCTION coldfront._ensure_vector_state_replicated() +RETURNS void LANGUAGE plpgsql AS $$ +DECLARE + t text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'spock') THEN + RETURN; + END IF; + FOREACH t IN ARRAY ARRAY['coldfront.vector_config', 'coldfront.vector_centroids'] LOOP + PERFORM spock.repset_add_table('default', t::regclass, false) + WHERE NOT EXISTS ( + SELECT 1 FROM spock.replication_set rs + JOIN spock.replication_set_table rst ON rst.set_id = rs.set_id + WHERE rs.set_name = 'default' AND rst.set_reloid = t::regclass + ); + END LOOP; +END; +$$; + +-- The distance operators a caller writes, on the real[] the view exposes. +-- +-- Each function is named for the DuckDB function it has to become: pg_duckdb +-- resolves an operator by its implementing function's NAME in DuckDB's catalog, so +-- the cold side gets DuckDB's own list_cosine_distance / list_distance / +-- list_negative_inner_product. The PostgreSQL bodies are real implementations, not +-- stubs, because a read that stays in PostgreSQL executes them: they delegate to +-- pgvector, which is exact and SIMD-accelerated, rather than hand-rolling the +-- arithmetic. Scoring a query vector against the centroid table goes through the +-- same functions, so a probe and an assignment cannot disagree on the metric. +-- +-- They are created in pgvector's own schema, whatever that is. A caller who has a +-- vector column has that schema on their search_path already, so `<=>` resolves +-- unqualified without ColdFront claiming public. Installed at onboarding rather +-- than at CREATE EXTENSION, since pgvector need not be present until a table +-- actually carries a vector. +CREATE OR REPLACE FUNCTION coldfront.install_vector_ops() +RETURNS void +LANGUAGE plpgsql AS $$ +DECLARE + v_nsp text; + r record; +BEGIN + PERFORM coldfront._ensure_vector_state_replicated(); + SELECT n.nspname INTO v_nsp + FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = 'vector'; + + -- Install pgvector rather than demand it. A tiered table cannot have declared a + -- vector column without it, but a decoupled table names its types in jsonb, so + -- the type need never have existed until now. + IF v_nsp IS NULL THEN + CREATE EXTENSION IF NOT EXISTS vector; + SELECT n.nspname INTO v_nsp + FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = 'vector'; + END IF; + IF v_nsp IS NULL THEN + RAISE EXCEPTION 'coldfront: pgvector is required for a vector column and could not be installed' + USING HINT = 'Install the pgvector package, then CREATE EXTENSION vector;'; + END IF; + + FOR r IN + SELECT * FROM (VALUES + ('list_cosine_distance', '<=>'), + ('list_distance', '<->'), + ('list_negative_inner_product', '<#>') + ) AS v(fn, op) + LOOP + EXECUTE format( + 'CREATE OR REPLACE FUNCTION %I.%I(real[], real[]) RETURNS double precision ' + 'LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS ' + '$fn$ SELECT $1::%I.vector OPERATOR(%I.%s) $2::%I.vector $fn$', + v_nsp, r.fn, v_nsp, v_nsp, r.op, v_nsp); + + IF NOT EXISTS ( + SELECT 1 FROM pg_operator o + JOIN pg_namespace n ON n.oid = o.oprnamespace + WHERE o.oprname = r.op + AND n.nspname = v_nsp + AND o.oprleft = 'real[]'::regtype + AND o.oprright = 'real[]'::regtype) + THEN + EXECUTE format( + 'CREATE OPERATOR %I.%s (LEFTARG = real[], RIGHTARG = real[], FUNCTION = %I.%I)', + v_nsp, r.op, v_nsp, r.fn); + END IF; + END LOOP; +END; +$$; + +-- The hot-side column a vector is read through. pg_duckdb rejects a pgvector +-- column while it builds the plan, before a cast in the projection can apply, so +-- the hot table carries a generated real[] column and every hot-side read goes +-- through that. The archiver derives the same name in Go (vecCompanion); the two +-- have to agree or the view reads a column that is not there. +CREATE OR REPLACE FUNCTION coldfront._vec_companion(p_col text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT '_cf_vec_' || p_col; +$$; + +-- True for the generated column above. It belongs to the hot table alone, so no +-- column list describing the user's table includes it: not the Iceberg schema, not +-- the view's projection, not an INSERT's column list (a generated column cannot be +-- written). starts_with rather than LIKE, since the prefix is full of underscores. +CREATE OR REPLACE FUNCTION coldfront._is_vec_companion(p_attname name, p_attgenerated "char") +RETURNS boolean +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT p_attgenerated <> '' AND starts_with(p_attname::text, coldfront._vec_companion('')); +$$; + +-- The Iceberg-only column carrying a row's cluster assignment for one vector +-- column. It is in the Iceberg schema and nowhere else: not on the hot table, not +-- in either branch of the view, so no query written against the view can name it. +-- view.VecListColumn is the Go twin. +-- +-- Named after the column it describes, because a table may carry several vector +-- columns and each needs its own assignment. The read-path rewrite derives this +-- name from whichever column a query orders by, so it is the contract between the +-- writer and the reader rather than an internal detail. +-- +-- These lead the schema rather than trailing it. Iceberg evolution appends, and a +-- cold INSERT is positional because Iceberg rejects a targeted one, so a column +-- added later has to land after everything both sides already agree on. +-- +-- The prefix extends _cf_vec_, which _is_vec_companion matches. There is no +-- collision: a companion is a generated column on the hot heap and a cluster column +-- exists only in Iceberg, so the two never appear in one relation. +CREATE OR REPLACE FUNCTION coldfront._vec_list_col(p_column text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT '_cf_vec_list_' || p_column; +$$; + +-- The table's vector columns in Iceberg-schema order, or an empty array when it has +-- none. One lookup so no generator reads the registry column directly and they +-- cannot disagree on the order, which a positional cold write depends on. +CREATE OR REPLACE FUNCTION coldfront._vec_columns(p_schema text, p_table text) +RETURNS text[] +LANGUAGE sql STABLE AS $$ + SELECT COALESCE(tv.vec_columns, '{}') + FROM coldfront.tiered_views tv + WHERE tv.schema_name = p_schema AND tv.relname = p_table; +$$; + +-- The one whose probe prunes: the first vector column, or NULL when there is none. +CREATE OR REPLACE FUNCTION coldfront._vec_sorted_column(p_schema text, p_table text) +RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT (coldfront._vec_columns(p_schema, p_table))[1]; +$$; + +-- True when a column-type list contains a vector, which is what decides whether a +-- cold write has to attach pglocal at all: only the cluster lookup reads it, so a +-- table without a vector must not pay for the attach. +CREATE OR REPLACE FUNCTION coldfront._types_have_vector(p_types text[]) +RETURNS boolean +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT EXISTS (SELECT 1 FROM unnest(p_types) t WHERE coldfront._is_vector_type(t)); +$$; + +-- The one place a cluster assignment is defined. Given the text of an expression +-- yielding the vector as DuckDB sees it, returns the DuckDB expression that +-- assigns the cluster. Every cold write path emits this and none derives its own, +-- because a row whose cluster disagrees with its vector is invisible to its own +-- search and reports no error. +-- +-- Two properties are deliberate. The centroids are read over pglocal, so the +-- PostgreSQL table is the only copy and no path has to inline a centroid set or +-- keep a session copy it has no way to check. And the generation is resolved by +-- the emitted SQL rather than baked into it, so a statement generated once (a +-- trigger body) keeps assigning against the live generation after a retrain +-- instead of filtering on a generation that no longer exists. +-- +-- Before any training the config carries no generation, the inner query matches +-- nothing, and the expression yields NULL: unassigned, which a probe reads +-- through the null arm of its predicate rather than missing. A retrain cannot +-- interleave with a cold write, since optimize() holds the table's claim for its +-- duration and every cold write serialises on that same claim. +CREATE OR REPLACE FUNCTION coldfront._vec_list_expr( + p_schema text, p_table text, p_column text, p_vec_expr text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT format( + '(SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, %s)) ' + 'FROM pglocal.coldfront.vector_centroids c ' + 'WHERE c.schema_name = %L AND c.table_name = %L AND c.column_name = %L ' + 'AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc ' + 'WHERE vc.schema_name = %L AND vc.table_name = %L ' + 'AND vc.column_name = %L))', + p_vec_expr, p_schema, p_table, p_column, p_schema, p_table, p_column); +$$; + +-- The cluster columns for an Iceberg ref, quoted and comma-joined in schema order. +-- A targeted cold INSERT names them, so the order has to match the expressions +-- _vec_list_prefix_for_ref emits beside it. +CREATE OR REPLACE FUNCTION coldfront._vec_list_cols_for_ref(p_iceberg_ref text) +RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT string_agg(quote_ident(coldfront._vec_list_col(c)), ', ' ORDER BY u.ord) + FROM coldfront.tiered_views tv + CROSS JOIN LATERAL unnest(tv.vec_columns) WITH ORDINALITY AS u(c, ord) + WHERE tv.iceberg_table = p_iceberg_ref; +$$; + +-- The same, for a caller holding the Iceberg ref rather than the view's name: the +-- C rewrite and the archiver's replay drain both know the ref they are writing to. +CREATE OR REPLACE FUNCTION coldfront._vec_list_prefix_for_ref( + p_iceberg_ref text, p_alias text) +RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT coldfront._vec_list_prefix(tv.schema_name, tv.relname, tv.vec_columns, + ARRAY(SELECT p_alias || quote_ident(c) FROM unnest(tv.vec_columns) c)) + FROM coldfront.tiered_views tv + WHERE tv.iceberg_table = p_iceberg_ref AND tv.vec_columns IS NOT NULL; +$$; + +-- The row-group size a clustered layout is built around, in rows: the pruning +-- granule. One definition, because the writer is told this number +-- (_vec_layout_props) and vector_status's floor advice measures clusters against +-- it; disagreeing would mistune exactly the signal the advice exists to give. +CREATE OR REPLACE FUNCTION coldfront._vec_row_group_limit() +RETURNS int +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT 2048; +$$; + +-- The Iceberg table properties a clustered table is created with, as a CREATE +-- TABLE WITH clause, or '' for a table with no vector column. +-- +-- Row groups are the pruning granularity, and the reader skips them on the cluster +-- column's statistics. The row count is what cuts them here, and it is read by +-- iceberg-go alone: DuckDB reads only write.parquet.row-group-size-bytes, and +-- setting that makes DuckDB refuse every write to the table ("does not work while +-- preserving insertion order"). So a trickle write lands one row group per file +-- and compaction is what establishes the layout, which is the division of labour +-- the compactor already has: it creates nothing, it preserves what it finds. +-- +-- The file target is large because on object storage every file a query touches is +-- a billed round trip. The sort key is what lets a compaction concatenate a +-- group's files in key order rather than scrambling them. +-- +-- Set at CREATE TABLE: the catalog takes properties there, and this build has no +-- ALTER for them, so a table that predates its vector column keeps the defaults. +CREATE OR REPLACE FUNCTION coldfront._vec_layout_props(p_sort_key text) +RETURNS text +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT CASE WHEN p_sort_key IS NULL OR p_sort_key = '' THEN '' + ELSE format( + ' WITH (%L=%L, %L=%L, %L=%L)', + 'write.parquet.row-group-limit', coldfront._vec_row_group_limit()::text, + 'write.target-file-size-bytes', '536870912', + 'coldfront.sort-key', p_sort_key) + END; +$$; + +-- The sort key for a clustered table: one cluster column first, since that is what +-- prunes, then the primary key as a tiebreak for determinism. The pk is not a +-- pruning aid; sorting by cluster scatters a cluster's rows through id space. +-- +-- One vector column gets this and the rest do not, because a Parquet file has a +-- single physical row order. Ordering by a second cluster column after the first +-- would scatter its values inside every band of the first, leaving its row-group +-- statistics bounding the whole file. So a second vector column's probe filters the +-- rows it scores and prunes nothing. +CREATE OR REPLACE FUNCTION coldfront._vec_sort_key(p_column text, p_pk_cols text[]) +RETURNS text +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT CASE WHEN p_column IS NULL THEN NULL + ELSE concat_ws(',', coldfront._vec_list_col(p_column), + NULLIF(array_to_string(COALESCE(p_pk_cols, '{}'), ','), '')) + END; +$$; + +-- The SET item a cold UPDATE adds when it sets the embedding, or NULL when the +-- table has no clustered vector column. A row whose embedding changes while its +-- cluster does not is permanently invisible to its own probe, silently, so this +-- rides in the same statement rather than in a follow-up. +CREATE OR REPLACE FUNCTION coldfront._vec_list_set_item( + p_iceberg_ref text, p_column text, p_vec_expr text) +RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT quote_ident(coldfront._vec_list_col(p_column)) || ' = ' + || coldfront._vec_list_expr(tv.schema_name, tv.relname, p_column, p_vec_expr) + FROM coldfront.tiered_views tv + WHERE tv.iceberg_table = p_iceberg_ref + AND p_column = ANY (coldfront._vec_columns(tv.schema_name, tv.relname)); +$$; + +-- The leading entries every positional cold write needs, one per vector column in +-- the order the Iceberg schema declares them, or '' for a table with no vector +-- column. A cold INSERT supplies values by position, so a prefix short of the +-- schema's cluster columns would land every following value in the wrong column. +-- That is why a length mismatch between the two arrays is an error and not a +-- best-effort join: the caller has lost track of its own column list. +CREATE OR REPLACE FUNCTION coldfront._vec_list_prefix( + p_schema text, p_table text, p_columns text[], p_vec_exprs text[]) +RETURNS text +LANGUAGE plpgsql IMMUTABLE AS $$ +DECLARE + v_out text := ''; + i int; +BEGIN + IF p_columns IS NULL OR cardinality(p_columns) = 0 THEN + RETURN ''; + END IF; + IF cardinality(p_columns) <> cardinality(COALESCE(p_vec_exprs, '{}')) THEN + RAISE EXCEPTION 'coldfront: % vector column(s) but % expression(s)', + cardinality(p_columns), cardinality(COALESCE(p_vec_exprs, '{}')); + END IF; + FOR i IN 1 .. cardinality(p_columns) LOOP + v_out := v_out + || coldfront._vec_list_expr(p_schema, p_table, p_columns[i], p_vec_exprs[i]) + || ', '; + END LOOP; + RETURN v_out; +END; +$$; + +-- The probe set for one query vector: the ids of the nearest centroids in the live +-- generation, ascending, or NULL when there is nothing to probe against. +-- +-- NULL is not a failure and this is the one place in the vector path that fails +-- open. A read that loses its predicate scans exactly, which is correct and is what +-- the product did before any of this existed; only a WRITE that cannot resolve a +-- generation has to fail, because a wrong cluster id makes a row invisible to its +-- own probe and says nothing. So an unconfigured table, an untrained one, and a +-- database whose distance shim was never installed all decline here. +-- +-- Scored through that shim rather than in arithmetic here: it delegates to pgvector, +-- which is exact and SIMD-accelerated over the few hundred centroids a generation +-- holds, and it is the same function the assignment uses, so a probe cannot end up +-- on a different metric from the clusters it is searching. +-- +-- Ascending ids, not distance order, because the predicate they become is a set +-- membership test: sorted ids make one generated qual for one probe set whatever +-- order the distances came back in. +CREATE OR REPLACE FUNCTION coldfront._vec_probe_ids( + p_schema text, p_table text, p_column text, p_vec real[], + p_nprobe int DEFAULT NULL) +RETURNS int[] +LANGUAGE plpgsql STABLE AS $$ +DECLARE + v_gen int; + v_nprobe int; + v_fn text; + v_ids int[]; +BEGIN + SELECT NULLIF(vc.generation, 0), COALESCE(p_nprobe, vc.nprobe) + INTO v_gen, v_nprobe + FROM coldfront.vector_config vc + WHERE vc.schema_name = p_schema AND vc.table_name = p_table + AND vc.column_name = p_column; + IF v_gen IS NULL THEN + RETURN NULL; + END IF; + + -- install_vector_ops put this in pgvector's schema, wherever that is; its + -- absence means no table here has ever carried a vector. + SELECT p.oid::regproc::text INTO v_fn + FROM pg_proc p + WHERE p.proname = 'list_cosine_distance' + AND p.pronargs = 2 AND p.proargtypes[0] = 'real[]'::regtype; + IF v_fn IS NULL THEN + RETURN NULL; + END IF; + + EXECUTE format( + 'SELECT array_agg(s.centroid_id ORDER BY s.centroid_id) FROM (' + 'SELECT c.centroid_id FROM coldfront.vector_centroids c ' + 'WHERE c.schema_name = $1 AND c.table_name = $2 AND c.column_name = $3 ' + 'AND c.generation = $4 ' + 'ORDER BY %s(c.centroid, $5) LIMIT $6) s', v_fn) + INTO v_ids + USING p_schema, p_table, p_column, v_gen, p_vec, v_nprobe; + RETURN v_ids; +END; +$$; + +-- The predicate a probe set becomes on the cold arm, or NULL for an empty set. +-- +-- The null arm is not optional. Rows another engine appended straight to Iceberg +-- carry no assignment, and a bare IN drops them silently. It is also not expensive: +-- the reader prunes on each row group's null count, so unassigned rows are read in +-- proportion to their own size rather than the table's. +-- +-- Cast on both arms, matching the cutoff qual the view generator already emits. The +-- subscript yields duckdb.unresolved_type, and the cast is what makes this an +-- integer comparison the Parquet reader can take. +CREATE OR REPLACE FUNCTION coldfront._vec_probe_qual( + p_column text, p_ids int[], p_alias text DEFAULT 'r') +RETURNS text +LANGUAGE sql IMMUTABLE AS $$ + WITH c(ref) AS ( + SELECT format('%s[%L]::integer', p_alias, coldfront._vec_list_col(p_column))) + SELECT format('(%s IN (%s) OR %s IS NULL)', + c.ref, array_to_string(p_ids, ', '), c.ref) + FROM c + WHERE cardinality(p_ids) > 0; +$$; + +-- The view's own definition with a probe predicate on its cold arm, or NULL when +-- there is no cold arm to probe. The read rewrite substitutes this for the view +-- reference, and that substitution is what keeps the cluster column out of the +-- view: the predicate is added where the column already exists, instead of the +-- view exposing a column so a caller's query can name it. +-- +-- Appended, not spliced. The generator puts the cold arm last and gives the view +-- neither ORDER BY nor LIMIT, so the end of the definition is the end of the cold +-- arm: of its WHERE for a tiered view, which always carries the cutoff qual, and of +-- its FROM for a decoupled one, which carries no qual at all. The registry says +-- which, so nothing here parses the deparsed text to find out. The regress test's +-- expected output locks the shape. +-- +-- A tiered view with no cutoff has no cold arm at all, only the hot heap, so there +-- is nothing to probe and the caller keeps its query. +CREATE OR REPLACE FUNCTION coldfront._vec_probed_viewdef( + p_schema text, p_view text, p_qual text) +RETURNS text +LANGUAGE plpgsql STABLE AS $$ +DECLARE + v_iceberg_only boolean; + v_has_cutoff boolean; + v_body text; +BEGIN + IF p_qual IS NULL THEN + RETURN NULL; + END IF; + + SELECT tv.is_iceberg_only, + EXISTS (SELECT 1 FROM coldfront.archive_watermark w + WHERE w.schema_name = p_schema AND w.table_name = p_view) + INTO v_iceberg_only, v_has_cutoff + FROM coldfront.tiered_views tv + WHERE tv.schema_name = p_schema AND tv.relname = p_view + AND tv.vec_columns IS NOT NULL; + IF v_iceberg_only IS NULL OR NOT (v_iceberg_only OR v_has_cutoff) THEN + RETURN NULL; + END IF; + + v_body := rtrim(pg_get_viewdef(format('%I.%I', p_schema, p_view)::regclass), + E' \t\r\n;'); + RETURN v_body + || CASE WHEN v_iceberg_only THEN ' WHERE ' ELSE ' AND ' END + || p_qual; +END; +$$; + +-- Cold-value rendering, shared so the write paths cannot disagree. DuckDB's list +-- cast accepts [1,2,3] and rejects PG's {1,2,3}, and whitespace between elements +-- is fine, which is what splits the two helpers below: a value taken from a jsonb +-- payload is already bracketed (jsonb spells a vector as a string and a real[] as +-- an array), while NEW.col::text on the view's real[] column is brace-delimited. + +-- The literal for a value already serialised to text. Callers: the cursor loop in +-- _tiered_insert_cold and _move_row_literal, both reading a jsonb payload. +CREATE OR REPLACE FUNCTION coldfront._render_cold_value(p_val_text text, p_pg_type text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT CASE + -- p_val_text is PG bytea text '\xHEX' (callers pin bytea_output to hex). + -- DuckDB mis-parses the \x escape into a BLOB, so rebuild the bytes from + -- the hex digits. + WHEN p_pg_type = 'bytea' THEN format('from_hex(%L)', substr(p_val_text, 3)) + -- The Iceberg column is FLOAT[]; without the cast the literal stays a + -- VARCHAR and the INSERT fails. + WHEN coldfront._is_vector_type(p_pg_type) THEN format('CAST(%L AS FLOAT[])', p_val_text) + ELSE quote_literal(p_val_text) + END; +$$; + +-- The INSTEAD OF trigger's cold INSERT is a format() template plus its args. +-- These two spell one column's half of each. Callers: create_iceberg_table for a +-- decoupled table and _rebuild_write_trigger for a tiered one. Identity columns are the +-- caller's business: they take NULL, since Iceberg has no sequences. +CREATE OR REPLACE FUNCTION coldfront._cold_placeholder(p_pg_type text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT CASE + WHEN coldfront._iceberg_storage_type(p_pg_type) = 'BLOB' THEN 'from_hex(%L)' + WHEN coldfront._is_vector_type(p_pg_type) THEN 'CAST(%L AS FLOAT[])' + ELSE '%L' + END; +$$; + +CREATE OR REPLACE FUNCTION coldfront._cold_value(p_col text, p_pg_type text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT CASE + WHEN coldfront._iceberg_storage_type(p_pg_type) = 'BLOB' + THEN format('encode(NEW.%I,%L)', p_col, 'hex') + -- The view exposes a vector as real[], so ::text yields {1,2,3}. + WHEN coldfront._is_vector_type(p_pg_type) + THEN format('translate(NEW.%I::text,%L,%L)', p_col, '{}', '[]') + -- VARCHAR-backed rich types are stored as their text form. + WHEN coldfront._iceberg_view_cast_type(p_pg_type) IN ('json', 'interval') + THEN format('NEW.%I::text', p_col) + -- Everything else round-trips through %L, double precision included. + ELSE format('NEW.%I', p_col) + END; +$$; + -- Map a PG type name (canonical or common alias) to the DuckDB/Iceberg -- storage type used in CREATE TABLE on the attached catalog. Raises on any -- type that cannot round-trip cleanly — silent VARCHAR fallback would lose @@ -1542,12 +2610,14 @@ BEGIN END IF; -- View-cast types: stored as VARCHAR, surfaced via wrapper view as native PG type IF t IN ('jsonb', 'json', 'interval') THEN RETURN 'VARCHAR'; END IF; + -- pgvector: both widen losslessly to float4 and store as list. + IF coldfront._is_vector_type(t) THEN RETURN 'FLOAT[]'; END IF; -- inet/cidr/oid are NOT supported: pg_duckdb rejects them (inet Oid 869, -- oid Oid 26) in any query it plans, and every Iceberg-backed view read is -- planned by pg_duckdb, so no cast makes them readable. Store IP data as -- text and oid values as bigint instead. - RAISE EXCEPTION 'coldfront: PG type % has no Iceberg-compatible mapping. Supported: bigint, integer, smallint, real, double precision, boolean, timestamptz, timestamp, date, time, uuid, text, varchar(N), char(N), bytea, numeric(P,S), jsonb, json, interval. inet/cidr/oid unsupported (store IP data as text, oid values as bigint)', p_pg_type; + RAISE EXCEPTION 'coldfront: PG type % has no Iceberg-compatible mapping. Supported: bigint, integer, smallint, real, double precision, boolean, timestamptz, timestamp, date, time, uuid, text, varchar(N), char(N), bytea, numeric(P,S), jsonb, json, interval, vector(N), halfvec(N). inet/cidr/oid unsupported (store IP data as text, oid values as bigint); sparsevec unsupported (keep it in the hot tier)', p_pg_type; END; $$; @@ -1558,22 +2628,25 @@ $$; CREATE OR REPLACE FUNCTION coldfront._iceberg_view_cast_type(p_pg_type text) RETURNS text LANGUAGE sql IMMUTABLE STRICT AS $$ - SELECT CASE lower(trim(p_pg_type)) - WHEN 'jsonb' THEN 'json' -- DuckDB has no jsonb, surface as json - WHEN 'json' THEN 'json' - WHEN 'interval' THEN 'interval' + SELECT CASE + WHEN t IN ('jsonb', 'json') THEN 'json' -- DuckDB has no jsonb, surface as json + WHEN t = 'interval' THEN 'interval' -- PG has no bare "double" type, so the view's cold cast r['col']::DOUBLE -- (the Iceberg storage name) won't parse. Surface via "double precision". - WHEN 'double precision' THEN 'double precision' - WHEN 'float8' THEN 'double precision' + WHEN t IN ('double precision', 'float8') THEN 'double precision' -- BLOB is not a PG-parseable cast name; surface bytea via "bytea". - WHEN 'bytea' THEN 'bytea' + WHEN t = 'bytea' THEN 'bytea' + -- FLOAT[] would parse in PG as double precision[] (FLOAT is an alias), + -- so the two branches would disagree on the column type. real[] is the + -- one spelling both engines read as 4-byte floats. + WHEN coldfront._is_vector_type(t) THEN 'real[]' -- Everything else (incl. smallint→INTEGER widening) has a storage type -- that is itself a PG-parseable surface; the view casts BOTH branches -- to that storage type, so no separate surface cast is needed and -- bootstrap/post-cutover view column types still agree. ELSE '' - END; + END + FROM (SELECT lower(trim(p_pg_type))) AS s(t); $$; -- create_iceberg_table: provision an iceberg-only table end-to-end. @@ -1603,6 +2676,9 @@ DECLARE ice_ref text := format('ice.%I.%I', p_schema, p_table); iceberg_cols text := ''; view_proj text := ''; + v_vec_cols text[] := '{}'; + v_cluster_cols text; + v_props text := ''; placeholders text := ''; new_refs text := ''; n int := 0; @@ -1632,6 +2708,13 @@ BEGIN IF col_name IS NULL OR pg_type IS NULL THEN RAISE EXCEPTION 'coldfront.create_iceberg_table: each p_columns element needs both "name" and "type"'; END IF; + -- A declared vector needs pgvector present before the view exposes real[] + -- columns a caller compares with <=>, and nothing else here guarantees it: + -- a decoupled table names its types, it does not hold the type. + IF coldfront._is_vector_type(pg_type) THEN + PERFORM coldfront.install_vector_ops(); + v_vec_cols := v_vec_cols || col_name; + END IF; storage_type := coldfront._iceberg_storage_type(pg_type); cast_type := coldfront._iceberg_view_cast_type(pg_type); @@ -1657,24 +2740,26 @@ BEGIN END IF; -- INSERT trigger: format('INSERT INTO ice... VALUES ()', ). - -- * json/interval (VARCHAR-backed): NEW.col::text. - -- * bytea (BLOB): from_hex(%L) + encode(NEW.col,'hex') — %L renders a - -- bytea as PG's '\xcafe' text which DuckDB mis-parses into a BLOB; - -- round-tripping the hex through from_hex() rebuilds the exact bytes. - -- * everything else (incl. double, '2.5' text round-trips): NEW.col. - -- Mirrors _rebuild_tiered_view and the archiver export. - IF storage_type = 'BLOB' THEN - placeholders := placeholders || 'from_hex(%L)'; - new_refs := new_refs || format('encode(NEW.%I,%L)', col_name, 'hex'); - ELSIF cast_type IN ('json', 'interval') THEN - placeholders := placeholders || '%L'; - new_refs := new_refs || format('NEW.%I::text', col_name); - ELSE - placeholders := placeholders || '%L'; - new_refs := new_refs || format('NEW.%I', col_name); - END IF; + -- One shared decision with _rebuild_write_trigger. + placeholders := placeholders || coldfront._cold_placeholder(pg_type); + new_refs := new_refs || coldfront._cold_value(col_name, pg_type); END LOOP; + -- The cluster columns lead the Iceberg schema, one per vector column and in the + -- order they were declared; the view projection deliberately skips them. A + -- decoupled INSERT is rewritten in C, which is where their values come from. + -- Only the first gets the sort order, so only its probe prunes. + IF cardinality(v_vec_cols) > 0 THEN + SELECT string_agg(quote_ident(coldfront._vec_list_col(c)) || ' INTEGER, ', '' + ORDER BY ord) + INTO v_cluster_cols + FROM unnest(v_vec_cols) WITH ORDINALITY AS u(c, ord); + iceberg_cols := v_cluster_cols || iceberg_cols; + -- No primary key is declared in this mode, so the cluster column alone. + v_props := coldfront._vec_layout_props( + coldfront._vec_sort_key(v_vec_cols[1], NULL)); + END IF; + -- TODO: pg_duckdb v1.1.1 + duckdb-iceberg do not accept PARTITIONED BY -- in CREATE TABLE for attached Iceberg catalogs. The Iceberg spec -- supports partition specs, but the DuckDB SQL surface for declaring @@ -1693,9 +2778,7 @@ BEGIN PERFORM coldfront.ensure_attached(); PERFORM duckdb.raw_query(format('CREATE SCHEMA IF NOT EXISTS ice.%I', p_schema)); PERFORM duckdb.raw_query(format( - 'CREATE TABLE IF NOT EXISTS %s (%s)', - ice_ref, iceberg_cols - )); + 'CREATE TABLE IF NOT EXISTS %s (%s)%s', ice_ref, iceberg_cols, v_props)); -- 2. PG-side wrapper view. Source is duckdb.query('SELECT * FROM ice...') -- rather than iceberg_scan('ice...'). The pg_duckdb planner folds both @@ -1726,13 +2809,14 @@ BEGIN -- 4. Registry row — is_iceberg_only=true tells the C hook to short-circuit -- classify_tier to TIER_COLD for any INSERT/UPDATE/DELETE on this view. - INSERT INTO coldfront.tiered_views (schema_name, relname, hot_table, iceberg_table, partition_col, is_iceberg_only) - VALUES (p_schema, p_table, NULL, ice_ref, NULL, true) + INSERT INTO coldfront.tiered_views (schema_name, relname, hot_table, iceberg_table, partition_col, is_iceberg_only, vec_columns) + VALUES (p_schema, p_table, NULL, ice_ref, NULL, true, NULLIF(v_vec_cols, '{}')) ON CONFLICT (schema_name, relname) DO UPDATE SET hot_table = NULL, iceberg_table = EXCLUDED.iceberg_table, partition_col = NULL, - is_iceberg_only = true; + is_iceberg_only = true, + vec_columns = EXCLUDED.vec_columns; -- 5. Prime the table so current-snapshot-id is non-null. Without this, -- the first concurrent N writers against an empty Iceberg table can @@ -1751,7 +2835,8 @@ BEGIN PERFORM duckdb.raw_query(format( 'INSERT INTO %s VALUES (%s); DELETE FROM %s', ice_ref, - array_to_string(array_fill('NULL'::text, ARRAY[n]), ', '), + array_to_string(array_fill('NULL'::text, + ARRAY[n + cardinality(v_vec_cols)]), ', '), ice_ref)); -- 6. Ensure claims is in Spock's default replication set so @@ -2403,6 +3488,13 @@ DECLARE -- used (always safe). Never a 409 either way. v_async boolean := coldfront._iceberg_async_active(); BEGIN + -- A cluster assignment reads the centroids over pglocal, so a statement that + -- names it needs pglocal attached in this backend. Tested on the statement + -- rather than the table: this wrapper is the one chokepoint every C-generated + -- cold write passes through, and only the ones carrying a lookup pay for it. + IF strpos(p_sql, 'pglocal.') > 0 THEN + PERFORM coldfront.ensure_pg_attached(); + END IF; -- This wrapper is the single-statement cold path; _tiered_insert_cold and -- _cross_tier_move issue their own multi-statement cold writes and carry the -- same guard. A hot write hits a PG heap, which PG rejects natively. @@ -2508,6 +3600,7 @@ DECLARE v_hot text; v_iceberg_only boolean; v_hot_reg regclass; + v_companion name; BEGIN SELECT hot_table, is_iceberg_only INTO v_hot, v_iceberg_only @@ -2537,6 +3630,16 @@ BEGIN IF NOT v_iceberg_only AND v_hot IS NOT NULL THEN v_hot_reg := to_regclass(v_hot); IF v_hot_reg IS NOT NULL THEN + -- The generated companions exist only to make a vector scannable + -- through the view. With the view gone the table is a plain table + -- again, so it goes back the shape its owner gave it. + FOR v_companion IN + SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = v_hot_reg AND a.attnum > 0 AND NOT a.attisdropped + AND coldfront._is_vec_companion(a.attname, a.attgenerated) + LOOP + EXECUTE format('ALTER TABLE %s DROP COLUMN %I', v_hot_reg::text, v_companion); + END LOOP; EXECUTE format('ALTER TABLE %s RENAME TO %I', v_hot_reg::text, p_table); END IF; END IF; @@ -2686,12 +3789,193 @@ CREATE FUNCTION coldfront._rename_tiered_view( WHERE schema_name = p_schema AND table_name = p_old_view_name; $$; +-- coldfront._rebuild_write_trigger: (re)build a tiered view's INSTEAD OF INSERT +-- trigger from the registry, the watermark and the hot table's live columns. +-- +-- The one generator of the trigger's positional lists: the archiver calls it at +-- bootstrap and _rebuild_tiered_view calls it after DDL, so the pairing of the +-- inner format()'s placeholders with their arguments is decided in exactly one +-- place, for every column type alike. CREATE OR REPLACE on both the function and +-- the trigger keeps it idempotent: the archiver re-runs bootstrap each cycle +-- against a view that already carries the previous trigger. +-- +-- Iceberg-only views are a no-op: their trigger is create_iceberg_table's, built +-- from the declared jsonb columns rather than a hot heap. +CREATE FUNCTION coldfront._rebuild_write_trigger( + p_schema text, + p_view_name text +) +RETURNS void +LANGUAGE plpgsql AS $$ +DECLARE + v_hot_table text; -- stored quoted, e.g. "public"."_events" + v_iceberg text; -- DuckDB ref, e.g. ice.myapp.events + v_partcol text; + v_is_ice_only boolean; + v_hot_schema text; + v_hot_relname text; + v_cutoff timestamptz; + v_cutoff_lit text; -- UTC text literal of the cutoff + v_has_cutoff boolean; + + v_col_list text := ''; -- hot INSERT target columns (non-identity) + v_hot_vals text := ''; -- NEW."col" refs (non-identity) + v_cold_vals text := ''; -- NEW."col"[::text] refs (non-identity) + v_placeholders text := ''; -- %L / NULL per column, positional + v_vec_cols text[] := '{}'; + v_vec_placeholders text[] := '{}'; + v_vec_refs text[] := '{}'; + + v_func_sql text; + v_funcname text; -- coldfront."_write" + v_trigname text; -- "_write_trigger" + r record; + iter int := 0; +BEGIN + SELECT tv.hot_table, tv.iceberg_table, tv.partition_col, tv.is_iceberg_only + INTO v_hot_table, v_iceberg, v_partcol, v_is_ice_only + FROM coldfront.tiered_views tv + WHERE tv.schema_name = p_schema AND tv.relname = p_view_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'coldfront._rebuild_write_trigger: view %.% not registered', + p_schema, p_view_name; + END IF; + IF v_is_ice_only OR v_hot_table IS NULL OR v_partcol IS NULL THEN + RETURN; + END IF; + + -- hot_table is stored as a quoted identifier; parse_ident handles the + -- quoting/escaping. No EXCEPTION wrapper: pg_duckdb forbids subtxns. + v_hot_schema := (parse_ident(v_hot_table))[1]; + v_hot_relname := (parse_ident(v_hot_table))[2]; + + SELECT cutoff_time INTO v_cutoff + FROM coldfront.archive_watermark + WHERE schema_name = p_schema AND table_name = p_view_name; + v_has_cutoff := (v_cutoff IS NOT NULL); + IF v_has_cutoff THEN + v_cutoff_lit := to_char(v_cutoff AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS+00'); + END IF; + + -- Placeholders are positional over ALL live columns incl. identity and + -- generated (NULL for those, %L otherwise): the cold INSERT supplies the + -- full tuple. An identity or generated column also stays out of the hot + -- INSERT list: PostgreSQL rejects a supplied value for either. + FOR r IN + SELECT a.attname, + format_type(a.atttypid, a.atttypmod) AS pg_type, + a.attidentity, + a.attgenerated + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace nn ON nn.oid = c.relnamespace + WHERE nn.nspname = v_hot_schema + AND c.relname = v_hot_relname + AND a.attnum > 0 + AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated) + ORDER BY a.attnum + LOOP + IF coldfront._is_vector_type(r.pg_type) THEN + v_vec_cols := v_vec_cols || r.attname; + v_vec_placeholders := v_vec_placeholders || coldfront._cold_placeholder(r.pg_type); + v_vec_refs := v_vec_refs || coldfront._cold_value(r.attname, r.pg_type); + END IF; + + IF iter > 0 THEN + v_placeholders := v_placeholders || ', '; + END IF; + iter := iter + 1; + + IF r.attidentity = 'a' OR r.attgenerated <> '' THEN + v_placeholders := v_placeholders || 'NULL'; + ELSE + IF v_col_list <> '' THEN + v_col_list := v_col_list || ', '; + v_hot_vals := v_hot_vals || ', '; + v_cold_vals := v_cold_vals || ', '; + END IF; + v_col_list := v_col_list || quote_ident(r.attname); + v_hot_vals := v_hot_vals || 'NEW.' || quote_ident(r.attname); + -- Cold-INSERT value and its format() placeholder: one shared decision + -- with create_iceberg_table. + v_placeholders := v_placeholders || coldfront._cold_placeholder(r.pg_type); + v_cold_vals := v_cold_vals || coldfront._cold_value(r.attname, r.pg_type); + END IF; + END LOOP; + + IF iter = 0 THEN + RAISE EXCEPTION 'coldfront._rebuild_write_trigger: hot table %.% has no live columns', + v_hot_schema, v_hot_relname; + END IF; + + -- The cluster columns lead the Iceberg schema, so they lead this positional + -- VALUES list too, and each lookup's own %L makes that vector's value lead the + -- argument list with it. + IF cardinality(v_vec_cols) > 0 THEN + v_placeholders := coldfront._vec_list_prefix(p_schema, p_view_name, v_vec_cols, + v_vec_placeholders) || v_placeholders; + v_cold_vals := array_to_string(v_vec_refs, ', ') || ', ' || v_cold_vals; + END IF; + + v_funcname := format('coldfront.%I', p_view_name || '_write'); + v_trigname := p_view_name || '_write_trigger'; + + -- Double-formatted: the outer format() builds the function body; the body + -- itself calls format(...) at trigger time to fill %L placeholders with + -- NEW values. The INSERT template, placeholders, and iceberg ref must + -- survive THIS format() literally, so they are assembled by concatenation. + v_func_sql := format( +$fn$CREATE OR REPLACE FUNCTION %s() RETURNS trigger AS $body$ +DECLARE + cutoff timestamptz; +BEGIN + SELECT cutoff_time INTO cutoff FROM coldfront.archive_watermark WHERE schema_name = %L AND table_name = %L; + IF cutoff IS NULL THEN + cutoff := %s; + END IF; + + IF TG_OP = 'INSERT' THEN + IF NEW.%I < cutoff THEN + PERFORM coldfront.ensure_attached();%s + PERFORM duckdb.raw_query(format( + %L, + %s + )); + RETURN NEW; + END IF; + INSERT INTO %I.%I (%s) VALUES (%s); + RETURN NEW; + END IF; + RETURN NULL; +END; +$body$ LANGUAGE plpgsql$fn$, + v_funcname, + p_schema, p_view_name, -- watermark key literals (schema, name) + CASE WHEN v_has_cutoff + THEN quote_literal(v_cutoff_lit) || '::timestamptz' + ELSE '''-infinity''::timestamptz' END, -- default cutoff + v_partcol, -- NEW. + CASE WHEN cardinality(v_vec_cols) = 0 THEN '' + ELSE E'\n PERFORM coldfront.ensure_pg_attached();' END, + 'INSERT INTO ' || v_iceberg || ' VALUES (' || v_placeholders || ')', + v_cold_vals, -- args to inner format() + v_hot_schema, v_hot_relname, v_col_list, v_hot_vals); -- hot INSERT + + EXECUTE v_func_sql; + EXECUTE format( + 'CREATE OR REPLACE TRIGGER %I INSTEAD OF INSERT ON %I.%I FOR EACH ROW EXECUTE FUNCTION %s()', + v_trigname, p_schema, p_view_name, v_funcname); +END; +$$; + -- coldfront._rebuild_tiered_view: regenerate the transparent UNION-ALL view -- and its INSTEAD OF INSERT trigger after a RENAME TABLE (hot heap) or RENAME --- VIEW. Driven entirely from pg_catalog so it is the runtime equivalent of --- internal/view/view.go's GenerateViewSQL / GenerateTriggerFuncSQL / --- GenerateTriggerSQL. (Also rebuilt after a mirrored column-shape change, so the --- view's column set follows the hot heap; and after a hot-table or view rename.) +-- VIEW. Driven entirely from pg_catalog; the view projection is the runtime +-- equivalent of internal/view/view.go's GenerateViewSQL, and the trigger comes +-- from _rebuild_write_trigger, the same builder the archiver uses. (Also rebuilt +-- after a mirrored column-shape change, so the view's column set follows the hot +-- heap; and after a hot-table or view rename.) -- -- Called by the coldfront DDL hook for tiered views (rows with a non-NULL -- hot_table). Iceberg-only views (is_iceberg_only = true, hot_table NULL) are @@ -2728,21 +4012,12 @@ DECLARE v_hot_proj text := ''; -- hot SELECT list v_cold_proj text := ''; -- cold SELECT list - v_col_list text := ''; -- INSERT target columns (non-identity) - v_hot_vals text := ''; -- NEW."col" refs (non-identity) - v_cold_vals text := ''; -- NEW."col"[::text] refs (non-identity) - v_placeholders text := ''; -- %L / NULL per column, positional - v_view_sql text; - v_func_sql text; - v_funcname text; -- coldfront."_write" - v_trigname text; -- "_write_trigger" r record; n int := 0; -- live-column counter for projections cast_type text; cold_type text; - iter int := 0; -- raw attribute counter (placeholder ordering) BEGIN -- 1. View identity IS the registry key (schema, relname). Resolve the -- registry columns by it; the row persists across the DROP+CREATE below @@ -2781,8 +4056,8 @@ BEGIN END IF; -- 3. Post-DDL column list from the HOT table, attnum order, live columns. - -- Build hot/cold projections and trigger lists in one pass — mirrors - -- view.go's single loop over cfg.Columns. + -- Build the hot/cold projections; the trigger's lists are + -- _rebuild_write_trigger's own pass. FOR r IN SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS pg_type, @@ -2794,18 +4069,26 @@ BEGIN AND c.relname = v_hot_relname AND a.attnum > 0 AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated) ORDER BY a.attnum LOOP cast_type := coldfront._iceberg_view_cast_type(r.pg_type); cold_type := coldfront._iceberg_storage_type(r.pg_type); -- Iceberg storage (BLOB, INTEGER, …) - -- VIEW PROJECTIONS (view.go ~184-203). + -- VIEW PROJECTIONS (view.go GenerateViewSQL). IF n > 0 THEN v_hot_proj := v_hot_proj || ', '; v_cold_proj := v_cold_proj || ', '; END IF; - IF cast_type <> '' THEN + IF coldfront._is_vector_type(r.pg_type) THEN + -- The hot branch reads the generated companion, aliased to the user's + -- column name, so the pgvector column is never scanned. + v_hot_proj := v_hot_proj || format('%I::%s AS %I', + coldfront._vec_companion(r.attname), + cast_type, r.attname); + v_cold_proj := v_cold_proj || format('r[%L]::%s', r.attname, cast_type); + ELSIF cast_type <> '' THEN -- VARCHAR-backed rich types (json/interval): cast both branches to -- the surface type so bootstrap and post-cutover views agree. v_hot_proj := v_hot_proj || quote_ident(r.attname) || '::' || cast_type; @@ -2820,46 +4103,6 @@ BEGIN v_cold_proj := v_cold_proj || format('r[%L]::%s', r.attname, cold_type); END IF; n := n + 1; - - -- TRIGGER LISTS (view.go insertCols / coldInsertVals / - -- coldInsertPlaceholders). Placeholders are positional over ALL - -- columns incl. identity (NULL for identity, %L otherwise) because - -- DuckDB/Iceberg has no targeted insert. - IF iter > 0 THEN - v_placeholders := v_placeholders || ', '; - END IF; - iter := iter + 1; - - IF r.attidentity = 'a' THEN - v_placeholders := v_placeholders || 'NULL'; - ELSE - IF v_col_list <> '' THEN - v_col_list := v_col_list || ', '; - v_hot_vals := v_hot_vals || ', '; - v_cold_vals := v_cold_vals || ', '; - END IF; - v_col_list := v_col_list || quote_ident(r.attname); - v_hot_vals := v_hot_vals || 'NEW.' || quote_ident(r.attname); - -- Cold-INSERT value, serialised through format()'s %L: - -- * json/interval (VARCHAR-backed): NEW.col::text. - -- * bytea (BLOB): from_hex(%L) placeholder + encode(NEW.col,'hex') - -- value. %L renders a bytea as PG's '\xcafe' text, which DuckDB - -- MIS-parses into a BLOB; round-tripping the hex through DuckDB's - -- from_hex() rebuilds the exact bytes. (double precision round- - -- trips fine as '2.5' text, so it stays a plain %L.) - -- * everything else: NEW.col as-is. - -- Consistent with create_iceberg_table and the archiver export. - IF cold_type = 'BLOB' THEN - v_placeholders := v_placeholders || 'from_hex(%L)'; - v_cold_vals := v_cold_vals || format('encode(NEW.%I,%L)', r.attname, 'hex'); - ELSIF cast_type IN ('json', 'interval') THEN - v_placeholders := v_placeholders || '%L'; - v_cold_vals := v_cold_vals || 'NEW.' || quote_ident(r.attname) || '::text'; - ELSE - v_placeholders := v_placeholders || '%L'; - v_cold_vals := v_cold_vals || 'NEW.' || quote_ident(r.attname); - END IF; - END IF; END LOOP; IF n = 0 THEN @@ -2892,57 +4135,9 @@ $ddl$CREATE VIEW %I.%I AS EXECUTE format('DROP VIEW IF EXISTS %I.%I CASCADE', v_schema, v_view_name); EXECUTE v_view_sql; - -- 5. Rebuild the INSTEAD OF INSERT trigger function + trigger. - v_funcname := format('coldfront.%I', v_view_name || '_write'); - v_trigname := v_view_name || '_write_trigger'; - - -- Double-formatted: the outer format() builds the function body; the body - -- itself calls format(...) at trigger time to fill %L placeholders with - -- NEW values. The INSERT template, placeholders, and iceberg ref must - -- survive THIS format() literally — assembled by concatenation below. - v_func_sql := format( -$fn$CREATE OR REPLACE FUNCTION %s() RETURNS trigger AS $body$ -DECLARE - cutoff timestamptz; -BEGIN - SELECT cutoff_time INTO cutoff FROM coldfront.archive_watermark WHERE schema_name = %L AND table_name = %L; - IF cutoff IS NULL THEN - cutoff := %s; - END IF; - - IF TG_OP = 'INSERT' THEN - IF NEW.%I < cutoff THEN - PERFORM coldfront.ensure_attached(); - PERFORM duckdb.raw_query(format( - %L, - %s - )); - RETURN NEW; - END IF; - INSERT INTO %I.%I (%s) VALUES (%s); - RETURN NEW; - END IF; - RETURN NULL; -END; -$body$ LANGUAGE plpgsql$fn$, - v_funcname, - v_schema, v_view_name, -- watermark key literals (schema, name) - CASE WHEN v_has_cutoff - THEN quote_literal(v_cutoff_lit) || '::timestamptz' - ELSE '''-infinity''::timestamptz' END, -- default cutoff - v_partcol, -- NEW. - 'INSERT INTO ' || v_iceberg || ' VALUES (' || v_placeholders || ')', - v_cold_vals, -- args to inner format() - v_hot_schema, v_hot_relname, v_col_list, v_hot_vals); -- hot INSERT - - EXECUTE v_func_sql; - - -- The view was just dropped + recreated fresh above, so no stale trigger - -- exists — create directly (no DROP TRIGGER IF EXISTS, which would only - -- emit a spurious NOTICE). - EXECUTE format( - 'CREATE TRIGGER %I INSTEAD OF INSERT ON %I.%I FOR EACH ROW EXECUTE FUNCTION %s()', - v_trigname, v_schema, v_view_name, v_funcname); + -- 5. The trigger, from the shared builder (the DROP above removed it with + -- the view). + PERFORM coldfront._rebuild_write_trigger(p_schema, p_view_name); -- 6. The registry key (schema, relname) is unchanged by the DROP+CREATE -- above (the view name is stable), so there is nothing to re-point. The diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index f415e32..a6662b0 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -48,11 +48,13 @@ #include "executor/executor.h" #include "executor/spi.h" #include "lib/stringinfo.h" +#include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "optimizer/optimizer.h" #include "parser/analyze.h" +#include "parser/parsetree.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" #include "utils/builtins.h" @@ -190,6 +192,16 @@ static bool coldfront_ice_attached = false; static bool coldfront_allow_mixed_writes = true; static int coldfront_cold_write_batch_size = 10000; +/* + * GUCs: the two levers over a probed vector search. Off gives the exact scan the + * product performed before there was a layout to probe, and a positive nprobe + * overrides the table's configured one. Both exist because the acceptance test for + * the rewrite is that the same query answers identically with the probe disabled + * and with it exhaustive, which needs a way to say each in one session. + */ +static bool coldfront_vector_probe = true; +static int coldfront_vector_nprobe = 0; + /* * GUCs: the deployment-config endpoint/DSN strings that ensure_attached() / * ensure_pg_attached() feed to DuckDB's ATTACH. Those helpers are SECURITY @@ -235,9 +247,21 @@ typedef struct { char *partition_col; /* e.g. "ts"; NULL when is_iceberg_only */ bool has_cutoff; /* false → nothing archived yet */ bool is_iceberg_only; /* true → table lives entirely in Iceberg, no hot tier */ + bool has_vector; /* the table carries clustered vector columns; + * which ones is SQL's to answer (per-column + * lookups keyed on the ref or the query). */ TimestampTz cutoff; /* archive watermark */ } TieredViewInfo; +static char *insert_targetlist_collist(Query *query); +static const char *skip_leading_collist(const char *rest); +static char *build_iceberg_only_insert_with_cluster(Query *query, + TieredViewInfo *info, + const char *source, + const char *col_list); +static char *add_cluster_set_item(Query *query, RangeTblEntry *rte, + TieredViewInfo *info, const char *cold_dml); + /* * Which tier a DML statement targets, based on its WHERE clause predicate on * the partition column. TIER_AMBIGUOUS means we cannot prove the predicate @@ -268,7 +292,7 @@ lookup_tiered_view(Oid relid, const char *vname, TieredViewInfo *info) initStringInfo(&sql); appendStringInfo(&sql, "SELECT tv.hot_table, tv.iceberg_table, tv.partition_col, " - " tv.is_iceberg_only, aw.cutoff_time " + " tv.is_iceberg_only, aw.cutoff_time, tv.vec_columns IS NOT NULL " "FROM coldfront.tiered_views tv " "LEFT JOIN coldfront.archive_watermark aw ON aw.table_name = %s " "WHERE tv.schema_name = %s AND tv.relname = %s", @@ -302,6 +326,9 @@ lookup_tiered_view(Oid relid, const char *vname, TieredViewInfo *info) if (!isnull) info->cutoff = DatumGetTimestampTz(d); + d = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 6, &isnull); + info->has_vector = !isnull && DatumGetBool(d); + MemoryContextSwitchTo(oldcxt); found = true; } @@ -649,7 +676,9 @@ classify_tier(Query *query, TieredViewInfo *info) /* ---------- string helpers -------------------------------------------- */ -typedef struct { const char *pg; const char *duck; } CfSubst; +/* drop_typmod: after substituting, skip a following "(...)" so a typmod that is + * valid on the PG spelling but not on the DuckDB one does not survive. */ +typedef struct { const char *pg; const char *duck; bool drop_typmod; } CfSubst; /* * Cold-WRITE substitutions. The deparsed cold DML is handed to DuckDB inside a @@ -662,13 +691,19 @@ typedef struct { const char *pg; const char *duck; } CfSubst; * json_set) errors in DuckDB — its boundary, not a rewrite coldfront withholds. */ static const CfSubst cf_write_subst[] = { - { "::timestamp with time zone", "::timestamptz" }, - { "::timestamp without time zone", "::timestamp" }, - { "::character varying", "::varchar" }, - { "::double precision", "::double" }, - { "jsonb_build_object(", "json_object(" }, - { "jsonb_build_array(", "json_array(" }, - { "to_jsonb(", "to_json(" }, + { "::timestamp with time zone", "::timestamptz", false }, + { "::timestamp without time zone", "::timestamp", false }, + { "::character varying", "::varchar", false }, + { "::double precision", "::double", false }, + /* pgvector's types are unknown to DuckDB, and the Iceberg column is FLOAT[]. + * The dimension typmod goes with the name: FLOAT[](3) is not a type. The + * cast's operand is already bracketed here, since a vector Const deparses + * through pgvector's own output function. */ + { "::vector", "::FLOAT[]", true }, + { "::halfvec", "::FLOAT[]", true }, + { "jsonb_build_object(", "json_object(", false }, + { "jsonb_build_array(", "json_array(", false }, + { "to_jsonb(", "to_json(", false }, }; /* @@ -684,8 +719,8 @@ static const CfSubst cf_write_subst[] = { * verified identical ([10,20,30]→3, []→0) and exists in both. */ static const CfSubst cf_read_subst[] = { - { "::jsonb", "::json" }, - { "jsonb_array_length(", "json_array_length(" }, + { "::jsonb", "::json", false }, + { "jsonb_array_length(", "json_array_length(", false }, }; /* @@ -753,6 +788,13 @@ cf_apply_subst(const char *sql, const CfSubst *map, int map_len, bool jsonb_catc { appendStringInfoString(&buf, map[i].duck); p += plen; + if (map[i].drop_typmod && *p == '(') + { + while (*p && *p != ')') + p++; + if (*p == ')') + p++; + } replaced = true; break; } @@ -916,6 +958,26 @@ deparse_and_find_prefix(Query *query, DeparseResult *dr) dr->rest = at + strlen(matched); /* nosemgrep */ } +/* + * Prepend the statement's leading WITH clause (dr->head_len bytes, before the + * verb) to a row source. Emitters that wrap the source in a parenthesised + * derived table need this: that subquery is the only scope the CTEs are + * visible from, on either engine. Returns source unchanged when there is no + * leading clause. + */ +static const char * +fold_leading_with(const DeparseResult *dr, const char *source) +{ + StringInfoData sb; + + if (dr->head_len == 0) + return source; + initStringInfo(&sb); + appendBinaryStringInfo(&sb, dr->orig_sql, dr->head_len); + appendStringInfoString(&sb, source); + return sb.data; +} + /* * Build a DML string targeting info->hot_table. Preserves any RETURNING. */ @@ -936,13 +998,31 @@ build_hot_dml(DeparseResult *dr, TieredViewInfo *info) * deparse_and_find_prefix(), so no RETURNING clause appears in dr->rest. */ static char * -build_cold_dml(DeparseResult *dr, TieredViewInfo *info) +build_cold_dml(DeparseResult *dr, TieredViewInfo *info, Query *query) { StringInfoData buf; + char *sql; + initStringInfo(&buf); appendBinaryStringInfo(&buf, dr->orig_sql, dr->head_len); /* leading WITH, if any */ appendStringInfo(&buf, "%s%s %s", dr->verb, info->iceberg_table, dr->rest); - return normalize_casts_for_duckdb(buf.data); + sql = normalize_casts_for_duckdb(buf.data); + + /* A cold UPDATE that sets the embedding re-derives the cluster in the same + * statement. Here rather than in a caller because both the cold and the + * dual path build their cold half through this one function, and a row whose + * embedding changes while its cluster does not is permanently invisible to + * its own probe, silently. */ + if (query != NULL && query->commandType == CMD_UPDATE + && info->has_vector) + { + RangeTblEntry *rte = (RangeTblEntry *) list_nth(query->rtable, + query->resultRelation - 1); + char *restamped = add_cluster_set_item(query, rte, info, sql); + if (restamped != NULL) + sql = restamped; + } + return sql; } /* @@ -963,7 +1043,9 @@ build_cold_dml(DeparseResult *dr, TieredViewInfo *info) * duckdb_target selects value rendering. true: the string reaches * duckdb.raw_query (emit_cold / emit_dual / the fast tiered-INSERT path), so it * mirrors the INSTEAD-OF trigger's DuckDB literals — bytea -> from_hex(%P$L) / - * encode($K,'hex'); json/jsonb/interval -> %P$L / $K::text; else %P$L / $K. + * encode($K,'hex'); real[] (a vector column's view type) -> + * CAST(%P$L AS FLOAT[]) / translate($K::text,'{}','[]'); + * json/jsonb/interval -> %P$L / $K::text; else %P$L / $K. * false: the string is embedded in a PostgreSQL cursor by * coldfront._tiered_insert_cold (the slow IDENTITY-omit path), executed by PG * not DuckDB. Most params render as plain %P$L / $K (PG coerces by the column @@ -1026,6 +1108,13 @@ cold_sql_arg(const char *cold_dml, ColdParamSet *ps, bool duckdb_target) * is irrelevant); both targets reconstruct the exact * bytes: DuckDB via from_hex, PG via decode. */ appendStringInfo(&args, ", encode($%d,'hex')", id); + else if (duckdb_target && t == FLOAT4ARRAYOID) + /* A vector column reaches this path as real[], the type the + * view exposes. PG spells that {1,2,3} and DuckDB's list + * cast takes [1,2,3], so translate rewrites the delimiters + * and the template supplies the cast. PG's own target needs + * neither: it coerces the literal by the column type. */ + appendStringInfo(&args, ", translate($%d::text,'{}','[]')", id); else if (duckdb_target && (t == JSONOID || t == JSONBOID || t == INTERVALOID)) appendStringInfo(&args, ", $%d::text", id); @@ -1034,6 +1123,8 @@ cold_sql_arg(const char *cold_dml, ColdParamSet *ps, bool duckdb_target) } if (t == BYTEAOID && duckdb_target) appendStringInfo(&tmpl, "from_hex(%%%d$L)", pos_of_id[id - 1]); + else if (t == FLOAT4ARRAYOID && duckdb_target) + appendStringInfo(&tmpl, "CAST(%%%d$L AS FLOAT[])", pos_of_id[id - 1]); else if (t == BYTEAOID) /* native PG (the _tiered_insert_cold cursor): rebuild a real * bytea from the hex arg so the projected column is bytea @@ -1321,7 +1412,21 @@ emit_cold(Query *query, TieredViewInfo *info, ColdParamSet *ps, bool in_plpgsql) deparse_and_find_prefix(query, &dr); query->returningList = saved_returning; - cold_dml = build_cold_dml(&dr, info); + cold_dml = build_cold_dml(&dr, info, query); + + /* An iceberg-only INSERT into a clustered table is re-emitted so the cluster + * is derived in the same statement: a row whose cluster disagrees with its + * vector is invisible to its own search and reports no error. */ + if (query->commandType == CMD_INSERT && info->is_iceberg_only + && info->has_vector) + { + char *col_list = insert_targetlist_collist(query); + char *with_cluster = build_iceberg_only_insert_with_cluster( + query, info, + fold_leading_with(&dr, skip_leading_collist(dr.rest)), col_list); + if (with_cluster != NULL) + cold_dml = normalize_casts_for_duckdb(with_cluster); + } /* INSERT … SELECT FROM pg_table needs each non-result PG-table * reference prefixed with pglocal. so DuckDB can resolve via the @@ -1395,7 +1500,7 @@ emit_dual(Query *query, TieredViewInfo *info, ColdParamSet *ps, bool in_plpgsql) query->returningList = NIL; deparse_and_find_prefix(query, &dr_cold); query->returningList = saved_returning; - cold_dml = build_cold_dml(&dr_cold, info); + cold_dml = build_cold_dml(&dr_cold, info, query); call = cold_exec_call(info->iceberg_table, cold_sql_arg(cold_dml, ps, true)); initStringInfo(&buf); @@ -1477,9 +1582,10 @@ skip_leading_collist(const char *rest) /* * Build the cold-side SELECT list for the fast pglocal-streaming path, - * projecting every underlying-table column in attnum order. DuckDB- - * iceberg's INSERT is positional and rejects column lists, so we must - * emit the full tuple. For each underlying column: + * projecting every underlying-table column in attnum order: the cold INSERT + * supplies the full tuple, and a PG-side DEFAULT expression exists nowhere in + * the Iceberg schema, so whatever fills one fills it here. For each + * underlying column: * * - If it appears in the user's INSERT targetList → emit the bare * identifier (gets value from `coldfront_src` alias). @@ -1527,10 +1633,12 @@ append_cold_projection(StringInfo sel, bool in_target, const char *attname, } static char * -build_cold_select_list(const char *hot_qualified, List *targeted) +build_cold_select_list(const char *hot_qualified, const char *iceberg_ref, + List *targeted) { StringInfoData sql, sel; bool first = true; + char *vec_prefix = NULL; initStringInfo(&sql); appendStringInfo(&sql, @@ -1543,6 +1651,7 @@ build_cold_select_list(const char *hot_qualified, List *targeted) "WHERE n.nspname = (parse_ident(%s))[1] " "AND c.relname = (parse_ident(%s))[2] " "AND a.attnum > 0 AND NOT a.attisdropped " + "AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated) " "ORDER BY a.attnum", quote_literal_cstr(hot_qualified), quote_literal_cstr(hot_qualified)); @@ -1568,10 +1677,30 @@ build_cold_select_list(const char *hot_qualified, List *targeted) default_expr); first = false; } + /* The Iceberg schema leads with the cluster column and this INSERT + * is positional, so the projection leads with it too. The + * expression comes from the extension rather than being spelled + * here: a row whose cluster disagrees with its vector is invisible + * to its own search and reports no error, so every write path + * derives from one definition. Still inside the SPI call, and + * inside CurTransactionContext, so the result outlives SPI_finish. */ + { + StringInfoData q; + initStringInfo(&q); + appendStringInfo(&q, + "SELECT coldfront._vec_list_prefix_for_ref(%s, '')", + quote_literal_cstr(iceberg_ref)); + if (SPI_execute(q.data, true, 1) == SPI_OK_SELECT + && SPI_processed == 1) + vec_prefix = SPI_getvalue(SPI_tuptable->vals[0], + SPI_tuptable->tupdesc, 1); + } MemoryContextSwitchTo(oldcxt); } SPI_finish(); } + if (vec_prefix != NULL) + return psprintf("%s%s", vec_prefix, sel.data); return sel.data; } @@ -1666,6 +1795,176 @@ build_tiered_hot_dml(const char *hot_table, const char *col_list, return hot.data; } +/* + * Find the statement's own WHERE in deparsed DML: the first " WHERE " at paren + * depth zero and outside every literal. Quotes are tracked before parens + * because a literal can hold an unbalanced one, and depth matters because a + * sublink carries its own WHERE a level down. NULL when there is none, meaning + * the insertion point is the end of the statement. + */ +static const char * +find_toplevel_where(const char *sql) +{ + const char *p; + bool in_squote = false, in_dquote = false; + int depth = 0; + + for (p = sql; *p; p++) + { + if (in_squote) { if (*p == '\'') in_squote = false; continue; } + if (in_dquote) { if (*p == '"') in_dquote = false; continue; } + if (*p == '\'') { in_squote = true; continue; } + if (*p == '"') { in_dquote = true; continue; } + if (*p == '(') { depth++; continue; } + if (*p == ')') { depth--; continue; } + if (depth == 0 && pg_strncasecmp(p, " WHERE ", 7) == 0) + return p; + } + return NULL; +} + +/* + * Add one SET item to a deparsed UPDATE, before the statement's own WHERE or at + * the end when it has none. Returns palloc'd. + */ +static char * +add_set_item(const char *cold_dml, const char *set_item) +{ + const char *w = find_toplevel_where(cold_dml); + StringInfoData buf; + + initStringInfo(&buf); + if (w == NULL) + appendStringInfo(&buf, "%s, %s", cold_dml, set_item); + else + { + appendBinaryStringInfo(&buf, cold_dml, w - cold_dml); + appendStringInfo(&buf, ", %s%s", set_item, w); + } + return buf.data; +} + +/* + * Re-stamp the cluster on a cold UPDATE that sets an embedding. + * + * The derivation takes the text of the new embedding expression rather than a + * value, which is what makes this tractable: this path never sees row contents. + * The expression is evaluated twice, once for the column and once for the + * cluster. Returns the extended statement, or NULL to leave it untouched. + * + * Every SET column is offered to coldfront._vec_list_set_item, which answers + * NULL for one that is not a clustered vector column: which columns qualify is + * the registry's knowledge, consulted where it lives instead of copied here. + * Every vector column the UPDATE sets gets its own item, because each has its + * own cluster column and leaving one stale would make those rows invisible to + * that column's probe while the others stayed correct. + */ +static char * +add_cluster_set_item(Query *query, RangeTblEntry *rte, TieredViewInfo *info, + const char *cold_dml) +{ + char *out = NULL; + ListCell *lc; + + foreach(lc, query->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + char *colname; + List *dpcontext; + char *e_text, *item = NULL; + StringInfoData q; + + if (tle->resjunk) + continue; + colname = get_attname(rte->relid, tle->resno, true); + if (colname == NULL) + continue; + + dpcontext = deparse_context_for(get_rel_name(rte->relid), rte->relid); + e_text = deparse_expression((Node *) tle->expr, dpcontext, false, false); + + initStringInfo(&q); + appendStringInfo(&q, "SELECT coldfront._vec_list_set_item(%s, %s, %s)", + quote_literal_cstr(info->iceberg_table), + quote_literal_cstr(colname), + quote_literal_cstr(e_text)); + if (SPI_connect() == SPI_OK_CONNECT) + { + if (SPI_execute(q.data, true, 1) == SPI_OK_SELECT && SPI_processed == 1) + { + bool itemnull; + + (void) SPI_getbinval(SPI_tuptable->vals[0], + SPI_tuptable->tupdesc, 1, &itemnull); + if (!itemnull) + { + MemoryContext oldcxt = MemoryContextSwitchTo(CurTransactionContext); + item = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + MemoryContextSwitchTo(oldcxt); + } + } + SPI_finish(); + } + if (item == NULL) + continue; + + /* The item carries the caller's own PG spelling of the new embedding. */ + out = add_set_item(out ? out : cold_dml, normalize_casts_for_duckdb(item)); + } + return out; +} + +/* + * Re-emit an iceberg-only INSERT so it carries the cluster assignment. + * + * The deparsed statement is targeted (`INSERT INTO t (cols) VALUES …`), so the + * column and its value are added by naming both rather than by editing each + * tuple: the source becomes a derived table and the assignment reads the vector + * from it, which is the same shape the tiered cold half already writes. Returns + * NULL when the table has no clustered vector column, leaving today's statement + * untouched. + */ +static char * +build_iceberg_only_insert_with_cluster(Query *query, TieredViewInfo *info, + const char *source, const char *col_list) +{ + StringInfoData sql, q; + char *prefix = NULL; + char *list_cols = NULL; /* already quoted and comma-joined */ + + if (!info->has_vector) + return NULL; + + /* The assignment reads the vector out of the derived table. */ + initStringInfo(&q); + appendStringInfo(&q, + "SELECT coldfront._vec_list_cols_for_ref(%s), " + " coldfront._vec_list_prefix_for_ref(%s, %s)", + quote_literal_cstr(info->iceberg_table), + quote_literal_cstr(info->iceberg_table), + quote_literal_cstr("coldfront_src.")); + if (SPI_connect() == SPI_OK_CONNECT) + { + if (SPI_execute(q.data, true, 1) == SPI_OK_SELECT && SPI_processed == 1) + { + MemoryContext oldcxt = MemoryContextSwitchTo(CurTransactionContext); + list_cols = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + prefix = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2); + MemoryContextSwitchTo(oldcxt); + } + SPI_finish(); + } + if (prefix == NULL || list_cols == NULL) + return NULL; + + initStringInfo(&sql); + appendStringInfo(&sql, + "INSERT INTO %s (%s, %s) SELECT %s%s FROM (%s) AS coldfront_src(%s)", + info->iceberg_table, list_cols, col_list, + prefix, col_list, source, col_list); + return sql.data; +} + /* * Build the slow-path cold call (IDENTITY-omitted case): the target-name * ARRAY[...] plus the coldfront._tiered_insert_cold(...) call. Its source SQL @@ -1733,7 +2032,8 @@ build_cold_bulk_call(Query *query, TieredViewInfo *info, const char *source, if (!tle->resjunk && tle->resname != NULL) targeted_names = lappend(targeted_names, tle->resname); } - cold_select = build_cold_select_list(info->hot_table, targeted_names); + cold_select = build_cold_select_list(info->hot_table, info->iceberg_table, + targeted_names); initStringInfo(&cold); appendStringInfo(&cold, @@ -1743,6 +2043,11 @@ build_cold_bulk_call(Query *query, TieredViewInfo *info, const char *source, info->iceberg_table, cold_select, source, col_list, quote_identifier(info->partition_col), cutoff_lit); + /* Ordered by cluster when there is one, so this write's own row groups each + * hold roughly one cluster and a probe skips the rest of the file. The + * cluster leads the projection, hence ordinal 1. */ + if (info->has_vector) + appendStringInfoString(&cold, " ORDER BY 1"); cold_pfx = prefix_pg_tables_with_pglocal(query, cold.data); cold_norm = normalize_casts_for_duckdb(cold_pfx); @@ -1826,22 +2131,9 @@ emit_tiered_insert(Query *query, TieredViewInfo *info, ColdParamSet *ps, bool in query->returningList = saved_returning; col_list = insert_targetlist_collist(query); - source = skip_leading_collist(dr.rest); + source = fold_leading_with(&dr, skip_leading_collist(dr.rest)); cutoff_lit = format_timestamptz_literal(info->cutoff); - /* A leading WITH clause (dr.head_len bytes, before "INSERT INTO ") must - * reach both halves, which each read `source` inside a parenthesised derived - * table. Fold it into source so its CTEs scope to that subquery on each engine - * (PG hot, DuckDB cold) — no top-level WITH to merge. */ - if (dr.head_len > 0) - { - StringInfoData sb; - initStringInfo(&sb); - appendBinaryStringInfo(&sb, dr.orig_sql, dr.head_len); - appendStringInfoString(&sb, source); - source = sb.data; - } - { RangeTblEntry *rte = (RangeTblEntry *) list_nth(query->rtable, query->resultRelation - 1); @@ -2162,14 +2454,303 @@ cf_normalize_read_jsonb(Query *query) cf_reparse_and_replace(query, norm, &ps); } +/* + * Resolve a node through the grouping RTE. A grouped query's sort expression + * references grouping expressions as Vars of RTE_GROUP, and the expression the + * shape check needs is the one they stand for. On releases without the grouping + * RTE, grouped queries carry the base-relation Vars directly, so this is the + * identity there. + */ +static Node * +cf_unwrap_group_var(Query *query, Node *node) +{ +#if PG_VERSION_NUM >= 180000 + if (node != NULL && IsA(node, Var)) + { + Var *var = (Var *) node; + + if (var->varlevelsup == 0 && + var->varno >= 1 && var->varno <= list_length(query->rtable)) + { + RangeTblEntry *rte = rt_fetch(var->varno, query->rtable); + + if (rte->rtekind == RTE_GROUP) + return (Node *) list_nth(rte->groupexprs, var->varattno - 1); + } + } +#endif + return node; +} + +/* True for a reference to a column of the query's single range-table entry. + * InvalidAttrNumber matches any column, for a caller that wants to learn which. */ +static bool +cf_is_single_rel_var(Node *node, AttrNumber attno) +{ + Var *var; + + if (node == NULL || !IsA(node, Var)) + return false; + var = (Var *) node; + if (var->varno != 1 || var->varlevelsup != 0) + return false; + return attno == InvalidAttrNumber || var->varattno == attno; +} + +/* + * The query vector as a PostgreSQL literal, or NULL if this expression is not one. + * + * The vector has to be inlined: pg_duckdb converts neither a `vector` nor a + * `real[]` bound parameter, on a custom plan as much as a generic one, so a probe + * that could only be computed from a parameter's value could not have run the + * search either. Constant-folded first, because the caller writes ARRAY[…]::real[] + * (an ArrayExpr of constants) or a `vector` literal the operator's implicit cast + * wraps, and neither is a Const until it is folded. + */ +static char * +cf_query_vector_literal(Node *expr) +{ + Const *c; + Oid typoutput; + bool typisvarlena; + + c = (Const *) expression_planner((Expr *) copyObject(expr)); + if (!IsA(c, Const) || c->constisnull || c->consttype != FLOAT4ARRAYOID) + return NULL; + getTypeOutputInfo(c->consttype, &typoutput, &typisvarlena); + return OidOutputFunctionCall(typoutput, c->constvalue); +} + +/* + * The shape the probe rewrite recognises: a single-relation SELECT on a + * registered view with a clustered vector column, ordered by one cosine distance + * between that column and a constant, with a LIMIT. Grouping, aggregation, + * windows and DISTINCT above that ORDER BY are part of the shape; they compute + * over whatever the narrowed scan reads. + * + * The LIMIT is not a detail: a probe trades recall for reads, which is the + * bargain a top-k asks for and not one to impose on a query that asked for every + * row in order. Cosine only, because the centroids were trained under cosine and + * ordering by another metric would route to the wrong clusters and report + * nothing. Which column is being searched comes from the query rather than the + * registry, because a table may carry several vector columns; a column with no + * configuration resolves to no probe set downstream and the rewrite declines. + * + * On a match, *vec_name is the searched column and *vec_lit the query vector's + * PostgreSQL literal. + */ +static bool +cf_probe_match(Query *query, char **vec_name, char **vec_lit) +{ + RangeTblEntry *view_rte; + TieredViewInfo info; + SortGroupClause *sgc; + TargetEntry *tle = NULL; + ListCell *lc; + Node *expr; + List *args; + Oid funcid; + char *fname; + AttrNumber vec_attno; + Node *lhs, *rhs, *other; + int nrte; + + nrte = list_length(query->rtable); +#if PG_VERSION_NUM >= 180000 + /* A grouped query carries an RTE_GROUP entry holding the grouping + * expressions. It adds no second scan, so it does not disqualify the shape; + * its Vars are unwrapped where the sort expression is matched. */ + if (nrte == 2 && + ((RangeTblEntry *) lsecond(query->rtable))->rtekind == RTE_GROUP) + nrte = 1; +#endif + if (query->cteList || query->setOperations || query->hasSubLinks || + query->rowMarks || nrte != 1 || + list_length(query->sortClause) != 1 || query->limitCount == NULL) + return false; + + view_rte = (RangeTblEntry *) linitial(query->rtable); + if (view_rte->rtekind != RTE_RELATION || + get_rel_relkind(view_rte->relid) != RELKIND_VIEW) + return false; + if (!lookup_tiered_view(view_rte->relid, get_rel_name(view_rte->relid), &info)) + return false; + if (!info.has_vector) + return false; + + /* The single sort key, which may be resjunk (ORDER BY an unselected expr). */ + sgc = (SortGroupClause *) linitial(query->sortClause); + foreach(lc, query->targetList) + { + TargetEntry *t = (TargetEntry *) lfirst(lc); + + if (t->ressortgroupref == sgc->tleSortGroupRef) + { + tle = t; + break; + } + } + if (tle == NULL) + return false; + + /* Written as an operator or as the function behind it; both name the same. */ + expr = cf_unwrap_group_var(query, (Node *) tle->expr); + if (IsA(expr, OpExpr)) + { + funcid = ((OpExpr *) expr)->opfuncid; + args = ((OpExpr *) expr)->args; + } + else if (IsA(expr, FuncExpr)) + { + funcid = ((FuncExpr *) expr)->funcid; + args = ((FuncExpr *) expr)->args; + } + else + return false; + if (list_length(args) != 2) + return false; + fname = get_func_name(funcid); + if (fname == NULL || strcmp(fname, "list_cosine_distance") != 0) /* nosemgrep */ + return false; + + /* One side is a column of the view, the other is the query vector. */ + lhs = cf_unwrap_group_var(query, (Node *) linitial(args)); + rhs = cf_unwrap_group_var(query, (Node *) lsecond(args)); + if (cf_is_single_rel_var(lhs, InvalidAttrNumber)) + { + vec_attno = ((Var *) lhs)->varattno; + other = rhs; + } + else if (cf_is_single_rel_var(rhs, InvalidAttrNumber)) + { + vec_attno = ((Var *) rhs)->varattno; + other = lhs; + } + else + return false; + *vec_name = get_attname(view_rte->relid, vec_attno, true); + if (*vec_name == NULL) + return false; + *vec_lit = cf_query_vector_literal(other); + return *vec_lit != NULL; +} + +/* + * Probe injection. A top-k similarity search over a clustered tiered view + * (cf_probe_match) is rewritten to read only the clusters nearest the query + * vector, which is what turns the layout every write maintains into a shorter + * read. Every other shape is left exactly as it was: that is an exact scan, + * which is correct. + * + * The predicate cannot be added to the caller's query, because the column it + * tests is deliberately in no branch of the view (see coldfront._vec_list_col). + * So the view reference is replaced by the view's own definition carrying the + * predicate on its cold arm, which puts the test where the column exists and + * leaves the caller's query surface alone. Nothing here is text surgery on the + * caller's SQL: the substitution swaps one range-table entry for a subquery and + * PostgreSQL deparses the result. + * + * Declining is silent and total. A table with no centroid generation, a probe + * set that resolves to nothing, a view with no cold arm: all of them keep + * today's query. The read is then slower than it could be, never wrong, which is + * the right direction for a read to fail in (a WRITE that cannot resolve a + * generation has to fail loudly instead). + */ +static void +cf_maybe_inject_probe(Query *query) +{ + RangeTblEntry *view_rte; + char *vec_name; + char *vec_lit; + char *body = NULL; + Query *clone; + RangeTblEntry *crte; + char *sql; + ColdParamSet ps; + StringInfoData q; + + if (!coldfront_vector_probe) + return; + if (!cf_probe_match(query, &vec_name, &vec_lit)) + return; + view_rte = (RangeTblEntry *) linitial(query->rtable); + + /* Resolve the probe set and the definition that carries it, in one round trip. */ + initStringInfo(&q); + appendStringInfo(&q, + "SELECT coldfront._vec_probed_viewdef(%s, %s, " + "coldfront._vec_probe_qual(%s, coldfront._vec_probe_ids(" + "%s, %s, %s, %s::real[], %s)))", + quote_literal_cstr(get_namespace_name( + get_rel_namespace(view_rte->relid))), + quote_literal_cstr(get_rel_name(view_rte->relid)), + quote_literal_cstr(vec_name), + quote_literal_cstr(get_namespace_name( + get_rel_namespace(view_rte->relid))), + quote_literal_cstr(get_rel_name(view_rte->relid)), + quote_literal_cstr(vec_name), + quote_literal_cstr(vec_lit), + coldfront_vector_nprobe > 0 + ? psprintf("%d", coldfront_vector_nprobe) : "NULL"); + if (SPI_connect() == SPI_OK_CONNECT) + { + if (SPI_execute(q.data, true, 1) == SPI_OK_SELECT && SPI_processed == 1) + { + MemoryContext oldcxt = MemoryContextSwitchTo(CurTransactionContext); + + body = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + MemoryContextSwitchTo(oldcxt); + } + SPI_finish(); + } + if (body == NULL) + return; + + /* Stand the probed definition in for the view, then deparse and reparse so the + * whole statement is planned against it. */ + clone = copyObject(query); + crte = (RangeTblEntry *) linitial(clone->rtable); + + coldfront_in_rewrite = true; + PG_TRY(); + { + List *parsetree_list = pg_parse_query(body); + RawStmt *raw = linitial_node(RawStmt, parsetree_list); + + crte->subquery = parse_analyze_fixedparams(raw, body, NULL, 0, NULL); + } + PG_FINALLY(); + { + coldfront_in_rewrite = false; + } + PG_END_TRY(); + + crte->rtekind = RTE_SUBQUERY; + crte->relid = InvalidOid; + crte->relkind = 0; + crte->rellockmode = NoLock; + crte->inh = false; + /* The caller's query names its columns by the view's name, so the subquery + * answers to it too. */ + crte->alias = makeAlias(crte->eref->aliasname, NIL); +#if PG_VERSION_NUM >= 160000 + crte->perminfoindex = 0; +#endif + + sql = pg_get_querydef(clone, false); + collect_cold_params(query, &ps); + cf_reparse_and_replace(query, sql, &ps); +} + /* * Read path for a SELECT that touches a registered tiered view. First try to * reroute a provably-hot read to the heap (runs in plain PG). Otherwise the read - * spans the cold tier: lazily attach 'ice' (once per session) so the view body's - * iceberg_scan('ice...') resolves — the version-agnostic cold-read attach (PG - * 16/17/18) — and normalize the whitelisted jsonb spellings so DuckDB (which runs - * the whole view query) accepts them. The relkind check inside - * query_reads_tiered_view keeps plain queries off the SPI path. + * spans the cold tier: lazily attach + * 'ice' (once per session) so the view body's iceberg_scan('ice...') resolves — + * the version-agnostic cold-read attach (PG 16/17/18) — narrow a recognised + * similarity search to its probed clusters, and normalize the whitelisted jsonb + * spellings so DuckDB (which runs the whole view query) accepts them. The relkind + * check inside query_reads_tiered_view keeps plain queries off the SPI path. */ static void cf_maybe_attach_for_read(Query *query) @@ -2180,6 +2761,7 @@ cf_maybe_attach_for_read(Query *query) return; /* rewritten to the hot heap; runs in plain PostgreSQL */ if (!coldfront_ice_attached) ensure_ice_attached_once(); + cf_maybe_inject_probe(query); cf_normalize_read_jsonb(query); } @@ -3390,6 +3972,34 @@ register_gucs(void) 0, NULL, NULL, NULL); + DefineCustomBoolVariable( + "coldfront.vector_probe", + "Restrict a recognised similarity search to its nearest clusters.", + "When on (default), a top-k search on a clustered vector column reads only " + "the clusters nearest the query vector, which is approximate in the way " + "every vector index is. Off gives an exact scan of the whole corpus: " + "slower, and the reference a recall measurement compares against.", + &coldfront_vector_probe, + true, /* boot_val */ + PGC_USERSET, + 0, /* flags */ + NULL, NULL, NULL); + + DefineCustomIntVariable( + "coldfront.vector_nprobe", + "Clusters a similarity search reads, overriding the table's own setting.", + "0 (default) uses the nprobe recorded for the column in " + "coldfront.vector_config. A value at or above that column's nlist reads " + "every cluster, which is exact and is how an approximate result is " + "compared against ground truth without turning the rewrite off.", + &coldfront_vector_nprobe, + 0, /* boot_val: defer to vector_config */ + 0, /* min */ + PG_INT32_MAX, /* max */ + PGC_USERSET, + 0, + NULL, NULL, NULL); + /* * Deployment-config endpoint/DSN GUCs. PGC_SUSET so a non-superuser cannot * redirect the SECURITY DEFINER ensure_attached()/ensure_pg_attached() diff --git a/extension/coldfront/test/expected/cast_normalize.out b/extension/coldfront/test/expected/cast_normalize.out index da37aec..8d7aea0 100644 --- a/extension/coldfront/test/expected/cast_normalize.out +++ b/extension/coldfront/test/expected/cast_normalize.out @@ -16,8 +16,10 @@ SET TIME ZONE 'UTC'; SET coldfront.warehouse = ''; SET coldfront.lakekeeper_endpoint = ''; -- evt_jsonb's name embeds "jsonb": it must survive the jsonb→json rewrite intact. +CREATE EXTENSION IF NOT EXISTS vector; CREATE TABLE public._events (id int, ts timestamptz, c_tsn timestamp, - c_vc varchar, c_dp double precision, evt_jsonb jsonb); + c_vc varchar, c_dp double precision, evt_jsonb jsonb, + emb vector(3)); CREATE VIEW public.events AS SELECT * FROM public._events; INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) VALUES ('public', 'events', 'public._events', 'ice.default.events', 'ts'); @@ -41,6 +43,19 @@ EXPLAIN (COSTS OFF, VERBOSE) Output: _exec_iceberg_with_claim('ice.default.events'::text, 'UPDATE ice.default.events SET c_tsn = ''Mon Jun 01 00:00:00 2020''::timestamp, c_vc = ''x''::varchar, c_dp = (1.5)::double, evt_jsonb = json_set(json_object(''k'', 1), ''{x}''::text[], ''"v"''::json) WHERE (ts < ''Tue Jan 01 00:00:00 2019 UTC''::timestamptz)'::text) (2 rows) +-- (A2) pgvector's cast: DuckDB knows no `vector`, and the Iceberg column is +-- FLOAT[]. The dimension typmod must go with the name, since FLOAT[](3) is not a +-- type. The literal needs no rewriting here: a vector Const deparses through +-- pgvector's own output function, which is already bracket-delimited. +EXPLAIN (COSTS OFF, VERBOSE) + UPDATE public.events SET emb = '[1,2,3]'::vector(3) + WHERE ts < '2019-01-01'::timestamp with time zone; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: _exec_iceberg_with_claim('ice.default.events'::text, 'UPDATE ice.default.events SET emb = ''[1,2,3]''::FLOAT[] WHERE (ts < ''Tue Jan 01 00:00:00 2019 UTC''::timestamptz)'::text) +(2 rows) + -- (B) Parity against the live DuckDB: the rewrite targets are accepted; `jsonb` -- is rejected (which is why the map rewrites it). A void row = accepted. SELECT duckdb.raw_query($$ SELECT NULL::json $$); @@ -49,6 +64,18 @@ SELECT duckdb.raw_query($$ SELECT NULL::json $$); (1 row) +SELECT duckdb.raw_query($$ SELECT '[1,2,3]'::FLOAT[] $$); + raw_query +----------- + +(1 row) + +SELECT duckdb.raw_query($$ SELECT NULL::vector $$); +ERROR: (PGDuckDB/pgduckdb_raw_query_cpp) Catalog Error: Type with name vector does not exist! +Did you mean "dec"? + +LINE 1: SELECT NULL::vector + ^ SELECT duckdb.raw_query($$ SELECT json_object('k', 1) $$); raw_query ----------- diff --git a/extension/coldfront/test/expected/cte_on_insert.out b/extension/coldfront/test/expected/cte_on_insert.out index 2b1b269..941eab8 100644 --- a/extension/coldfront/test/expected/cte_on_insert.out +++ b/extension/coldfront/test/expected/cte_on_insert.out @@ -51,3 +51,24 @@ DELETE FROM coldfront.tiered_views; DELETE FROM coldfront.archive_watermark; DROP VIEW public.events; DROP TABLE public._events; +-- The clustered iceberg-only INSERT is re-emitted so the cluster is derived in +-- the same statement, and the CTE must survive that re-emission too: the WITH +-- folds into the derived table the assignment reads from, the only scope its +-- CTEs are visible from. +CREATE TABLE public._vec_base (id int, ts timestamptz, embedding real[]); +CREATE VIEW public.icevec AS SELECT * FROM public._vec_base; +INSERT INTO coldfront.tiered_views(schema_name, relname, iceberg_table, is_iceberg_only, vec_columns) +VALUES ('public', 'icevec', 'ice.default.icevec', true, ARRAY['embedding']); +EXPLAIN (COSTS OFF, VERBOSE) + WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, ARRAY[1,0,0]::real[] AS embedding) + INSERT INTO public.icevec SELECT id, ts, embedding FROM s; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding)'::text) +(2 rows) + +-- Cleanup. +DELETE FROM coldfront.tiered_views WHERE relname = 'icevec'; +DROP VIEW public.icevec; +DROP TABLE public._vec_base; diff --git a/extension/coldfront/test/expected/ddl_alter_column.out b/extension/coldfront/test/expected/ddl_alter_column.out index b67c91b..748adb8 100644 --- a/extension/coldfront/test/expected/ddl_alter_column.out +++ b/extension/coldfront/test/expected/ddl_alter_column.out @@ -26,8 +26,8 @@ VALUES ('public', 'events', 'public._events', 'ice.default.events', 'ts'); -- 1. Unsupported column type is rejected up front (originator path), before any -- Iceberg I/O; the hot ALTER rolls back with it, so _events is unchanged. ALTER TABLE public._events ADD COLUMN bad inet; -ERROR: coldfront: PG type inet has no Iceberg-compatible mapping. Supported: bigint, integer, smallint, real, double precision, boolean, timestamptz, timestamp, date, time, uuid, text, varchar(N), char(N), bytea, numeric(P,S), jsonb, json, interval. inet/cidr/oid unsupported (store IP data as text, oid values as bigint) -CONTEXT: PL/pgSQL function _iceberg_storage_type(text) line 41 at RAISE +ERROR: coldfront: PG type inet has no Iceberg-compatible mapping. Supported: bigint, integer, smallint, real, double precision, boolean, timestamptz, timestamp, date, time, uuid, text, varchar(N), char(N), bytea, numeric(P,S), jsonb, json, interval, vector(N), halfvec(N). inet/cidr/oid unsupported (store IP data as text, oid values as bigint); sparsevec unsupported (keep it in the hot tier) +CONTEXT: PL/pgSQL function _iceberg_storage_type(text) line 43 at RAISE PL/pgSQL function _mirror_iceberg_alter(text,text,jsonb) line 36 at assignment SQL statement "SELECT coldfront._mirror_iceberg_alter('ice.default.events', 'public._events', jsonb_build_array(jsonb_build_object('op', 'add', 'col', 'bad')))" SELECT attname FROM pg_attribute diff --git a/extension/coldfront/test/expected/drop_iceberg_table.out b/extension/coldfront/test/expected/drop_iceberg_table.out index 4be60ea..11857c5 100644 --- a/extension/coldfront/test/expected/drop_iceberg_table.out +++ b/extension/coldfront/test/expected/drop_iceberg_table.out @@ -123,6 +123,6 @@ SELECT relname, relkind FROM pg_class SELECT coldfront._unregister_iceberg('public', 'nosuch'); ERROR: coldfront: "public.nosuch" is not a registered Iceberg table HINT: Only tables registered in coldfront.tiered_views can be unregistered. -CONTEXT: PL/pgSQL function _unregister_iceberg(text,text) line 13 at RAISE +CONTEXT: PL/pgSQL function _unregister_iceberg(text,text) line 14 at RAISE -- Cleanup. DROP TABLE public.events; diff --git a/extension/coldfront/test/expected/vector_assign.out b/extension/coldfront/test/expected/vector_assign.out new file mode 100644 index 0000000..46874d1 --- /dev/null +++ b/extension/coldfront/test/expected/vector_assign.out @@ -0,0 +1,44 @@ +-- vector_assign refuses before it writes. Both checks run ahead of any DuckDB +-- statement, which is what lets pg_regress reach them with no Iceberg attached; the +-- assignment itself is asserted in ci/journey.sh against a real cold tier. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +-- A column nothing registered. Naming the caller's own arguments back is what makes +-- a typo in a scripted call readable. +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); +ERROR: coldfront.vector_assign: "public.chunks"."embedding" is not a registered clustered column +CONTEXT: PL/pgSQL function vector_assign(text,text,text) line 16 at RAISE +CREATE TABLE public._chunks (id bigint, ts timestamptz, embedding vector(3)); +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'chunks', 'public._chunks', 'ice.default.chunks', 'ts', ARRAY['embedding']); +-- Registered, but the column named is not the clustered one. +CALL coldfront.vector_assign('public', 'chunks', 'other'); +ERROR: coldfront.vector_assign: "public.chunks"."other" is not a registered clustered column +CONTEXT: PL/pgSQL function vector_assign(text,text,text) line 16 at RAISE +-- Registered and clustered, with nothing trained. Assigning here would rewrite +-- every row to the NULL it already holds, so it refuses rather than reporting +-- success over a full rewrite that changed nothing. +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); +ERROR: coldfront.vector_assign: "public.chunks"."embedding" has no trained generation +HINT: CALL coldfront.vector_train(...) first: without centroids every row would be assigned NULL, which is what it already is. +CONTEXT: PL/pgSQL function vector_assign(text,text,text) line 24 at RAISE +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'chunks', 'embedding', 2, 1, 0); +-- Generation 0 is the same case spelled differently: a configuration row exists but +-- no generation was ever written. +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); +ERROR: coldfront.vector_assign: "public.chunks"."embedding" has no trained generation +HINT: CALL coldfront.vector_train(...) first: without centroids every row would be assigned NULL, which is what it already is. +CONTEXT: PL/pgSQL function vector_assign(text,text,text) line 24 at RAISE +-- Cleanup. +DELETE FROM coldfront.vector_config WHERE table_name = 'chunks'; +DELETE FROM coldfront.tiered_views WHERE relname = 'chunks'; +DROP TABLE public._chunks; diff --git a/extension/coldfront/test/expected/vector_centroids.out b/extension/coldfront/test/expected/vector_centroids.out new file mode 100644 index 0000000..4ab1502 --- /dev/null +++ b/extension/coldfront/test/expected/vector_centroids.out @@ -0,0 +1,100 @@ +-- The routing state Phase 2 assigns and probes against: one centroid set per +-- vector column, plus the per-table settings that describe it. +-- +-- It lives in PostgreSQL because three consumers need it there: the read-path +-- rewrite scores a query against it in the same backend, the write paths resolve +-- an assignment inside the statement doing the write, and an adaptive addition is +-- inserted in the same transaction as the row that triggered it. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +-- Deliberately no pgvector here: these tables ship with the extension, so they +-- must exist on a database that has no vectors and may never have any. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +-- Name-keyed, like partition_config: a mesh replicates it by value, so every node +-- assigns identical cluster ids without sharing OIDs. +SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type, a.attnotnull + FROM pg_attribute a + WHERE a.attrelid = 'coldfront.vector_centroids'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; + attname | type | attnotnull +-------------+---------+------------ + schema_name | text | t + table_name | text | t + column_name | text | t + generation | integer | t + centroid_id | integer | t + parent_id | integer | f + centroid | real[] | t +(7 rows) + +SELECT conname, pg_get_constraintdef(oid) AS def + FROM pg_constraint + WHERE conrelid = 'coldfront.vector_centroids'::regclass + ORDER BY conname; + conname | def +---------------------------------------+----------------------------------------------------------------------------- + vcent_gen_pos | CHECK ((generation >= 1)) + vector_centroids_centroid_id_not_null | NOT NULL centroid_id + vector_centroids_centroid_not_null | NOT NULL centroid + vector_centroids_column_name_not_null | NOT NULL column_name + vector_centroids_generation_not_null | NOT NULL generation + vector_centroids_pkey | PRIMARY KEY (schema_name, table_name, column_name, generation, centroid_id) + vector_centroids_schema_name_not_null | NOT NULL schema_name + vector_centroids_table_name_not_null | NOT NULL table_name +(8 rows) + +SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type + FROM pg_attribute a + WHERE a.attrelid = 'coldfront.vector_config'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; + attname | type +--------------+--------- + schema_name | text + table_name | text + column_name | text + nlist | integer + nprobe | integer + generation | integer + addition_cap | integer +(7 rows) + +-- A restored node must re-attach to the same cold tier without retraining, and a +-- generation is meaningless without the centroids that defined it. +SELECT c.relname + FROM pg_class c + WHERE c.oid = ANY (SELECT unnest(extconfig) FROM pg_extension WHERE extname = 'coldfront') + AND c.relname IN ('vector_centroids', 'vector_config') + ORDER BY c.relname; + relname +------------------ + vector_centroids + vector_config +(2 rows) + +-- A generation is immutable: rows are inserted, never updated in place, so a +-- query that resolved a generation keeps meaning the same thing. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) +VALUES ('public', 'chunks', 'embedding', 500, 20); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'chunks', 'embedding', 1, 0, ARRAY[1,0,0]::real[]), + ('public', 'chunks', 'embedding', 1, 1, ARRAY[0,1,0]::real[]); +SELECT count(*) AS centroids FROM coldfront.vector_centroids WHERE table_name = 'chunks'; + centroids +----------- + 2 +(1 row) + +-- The same centroid id twice in one generation would make an assignment ambiguous. +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'chunks', 'embedding', 1, 0, ARRAY[9,9,9]::real[]); +ERROR: duplicate key value violates unique constraint "vector_centroids_pkey" +DETAIL: Key (schema_name, table_name, column_name, generation, centroid_id)=(public, chunks, embedding, 1, 0) already exists. +-- Cleanup. +DELETE FROM coldfront.vector_centroids WHERE table_name = 'chunks'; +DELETE FROM coldfront.vector_config WHERE table_name = 'chunks'; diff --git a/extension/coldfront/test/expected/vector_cold_render.out b/extension/coldfront/test/expected/vector_cold_render.out new file mode 100644 index 0000000..1b2dd44 --- /dev/null +++ b/extension/coldfront/test/expected/vector_cold_render.out @@ -0,0 +1,84 @@ +-- Four cold-write paths render a value for DuckDB, and they share two decisions +-- so a value cannot round-trip through one path and corrupt through another: +-- _cold_placeholder / _cold_value for the INSTEAD OF trigger (create_iceberg_table +-- for a decoupled table, _rebuild_tiered_view for a tiered one, view.go for the Go +-- twin), and _render_cold_value for the per-row loops that read a jsonb payload. +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared +-- regress db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +-- The trigger's pair. The view exposes a vector as real[], whose text form is +-- PG's {1,2,3}; DuckDB's list cast takes [1,2,3] only, so translate rewrites the +-- delimiters and the placeholder supplies the Iceberg column's type. +SELECT t, + coldfront._cold_placeholder(t) AS placeholder, + coldfront._cold_value('embedding', t) AS value + FROM unnest(ARRAY['vector(3)', 'halfvec(3)']) AS t; + t | placeholder | value +------------+---------------------+------------------------------------------ + vector(3) | CAST(%L AS FLOAT[]) | translate(NEW.embedding::text,'{}','[]') + halfvec(3) | CAST(%L AS FLOAT[]) | translate(NEW.embedding::text,'{}','[]') +(2 rows) + +-- Every other type keeps the spelling it already had. +SELECT t, + coldfront._cold_placeholder(t) AS placeholder, + coldfront._cold_value('c', t) AS value + FROM unnest(ARRAY['bytea', 'jsonb', 'json', 'interval', 'bigint', + 'double precision', 'text']) AS t; + t | placeholder | value +------------------+--------------+--------------------- + bytea | from_hex(%L) | encode(NEW.c,'hex') + jsonb | %L | NEW.c::text + json | %L | NEW.c::text + interval | %L | NEW.c::text + bigint | %L | NEW.c + double precision | %L | NEW.c + text | %L | NEW.c +(7 rows) + +-- The per-row loops read their value out of a jsonb payload, which spells a +-- vector as a string and a real[] as an array, so both arrive bracketed already +-- and only the cast is added. Whitespace between elements is accepted by DuckDB. +SELECT coldfront._render_cold_value('[1,2,3]', 'vector(3)') AS vec_tight, + coldfront._render_cold_value('[1, 2, 3]', 'vector(3)') AS vec_spaced; + vec_tight | vec_spaced +----------------------------+------------------------------ + CAST('[1,2,3]' AS FLOAT[]) | CAST('[1, 2, 3]' AS FLOAT[]) +(1 row) + +-- bytea arrives as PG's '\xHEX'; the hex digits are what DuckDB rebuilds from. +SELECT coldfront._render_cold_value('\xcafe', 'bytea') AS blob, + coldfront._render_cold_value('2.5', 'double precision') AS dbl, + coldfront._render_cold_value('a''b', 'text') AS quoted; + blob | dbl | quoted +------------------+-------+-------- + from_hex('cafe') | '2.5' | 'a''b' +(1 row) + +-- The per-row serialiser keeps a NULL vector's positional slot. The Iceberg schema +-- declares one cluster column per vector column unconditionally, so the prefix +-- carries one assignment per vector column whatever this row holds; an entry +-- missing from the prefix would shift every following value one column left. +SELECT coldfront._move_row_literal( + '{"id": 8, "cf_new_ts": "2026-06-01 00:00:00+00", "embedding": "[1,2,3]"}'::jsonb, + ARRAY['id','ts','embedding'], + ARRAY['bigint','timestamp with time zone','vector(3)'], + 'ts', 'public', 'chunks') AS vec_row; + vec_row +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[1,2,3]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'chunks' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'chunks' AND vc.column_name = 'embedding')), '8', '2026-06-01 00:00:00+00', CAST('[1,2,3]' AS FLOAT[]) +(1 row) + +SELECT coldfront._move_row_literal( + '{"id": 7, "cf_new_ts": "2026-06-01 00:00:00+00", "embedding": null}'::jsonb, + ARRAY['id','ts','embedding'], + ARRAY['bigint','timestamp with time zone','vector(3)'], + 'ts', 'public', 'chunks') AS null_vec_row; + null_vec_row +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, NULL)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'chunks' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'chunks' AND vc.column_name = 'embedding')), '7', '2026-06-01 00:00:00+00', NULL +(1 row) + diff --git a/extension/coldfront/test/expected/vector_multicolumn.out b/extension/coldfront/test/expected/vector_multicolumn.out new file mode 100644 index 0000000..83b96a1 --- /dev/null +++ b/extension/coldfront/test/expected/vector_multicolumn.out @@ -0,0 +1,192 @@ +-- A table may carry more than one vector column. Every one of them gets a cluster +-- column and an assignment on every write path; only the FIRST gets the file sort +-- order, because a Parquet file has one physical row order. So the first column's +-- probe prunes row groups and the others only cut the rows scored. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +-- The probe scores against the centroids through the distance shim, so it has to be +-- installed before _vec_probe_ids can resolve anything. +SELECT coldfront.install_vector_ops(); + install_vector_ops +-------------------- + +(1 row) + +-- One cluster column per vector column, named after it. The name is what the +-- read-path rewrite derives from the column a query orders by, so it is the contract +-- between the writer and the reader. +SELECT coldfront._vec_list_col('embedding') AS embedding_cluster, + coldfront._vec_list_col('summary') AS summary_cluster; + embedding_cluster | summary_cluster +------------------------+---------------------- + _cf_vec_list_embedding | _cf_vec_list_summary +(1 row) + +-- The sort key names one cluster column and the primary key. Passing a second vector +-- column would not help: within one value of the first, the second's values are +-- scattered, so its statistics stop bounding anything. +SELECT coldfront._vec_sort_key('embedding', ARRAY['id','ts']) AS sort_key; + sort_key +------------------------------ + _cf_vec_list_embedding,id,ts +(1 row) + +-- The positional prefix carries every vector column, in the order the Iceberg schema +-- declares them, because a cold INSERT supplies values by position. +SELECT coldfront._vec_list_prefix('public', 'docs', + ARRAY['embedding','summary'], + ARRAY['s.embedding','s.summary']) AS prefix; + prefix +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'embedding')), (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.summary)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'summary' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'summary')), +(1 row) + +-- A single vector column yields a prefix of one expression. +SELECT coldfront._vec_list_prefix('public', 'docs', + ARRAY['embedding'], ARRAY['s.embedding']) AS one_column; + one_column +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'embedding')), +(1 row) + +-- No vector column, no prefix: a table without vectors pays nothing. +SELECT coldfront._vec_list_prefix('public', 'docs', ARRAY[]::text[], ARRAY[]::text[]) = '' + AS no_vectors; + no_vectors +------------ + t +(1 row) + +-- Mismatched arrays are a caller bug, and a silently short prefix would write a +-- positional INSERT that lands values in the wrong columns. +SELECT coldfront._vec_list_prefix('public', 'docs', ARRAY['a','b'], ARRAY['s.a']); +ERROR: coldfront: 2 vector column(s) but 1 expression(s) +CONTEXT: PL/pgSQL function _vec_list_prefix(text,text,text[],text[]) line 10 at RAISE +-- The probe predicate names the column being searched, so two searches on one table +-- filter on different cluster columns. +SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS embedding_qual, + coldfront._vec_probe_qual('summary', ARRAY[7]) AS summary_qual; + embedding_qual | summary_qual +--------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------- + (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) | (r['_cf_vec_list_summary']::integer IN (7) OR r['_cf_vec_list_summary']::integer IS NULL) +(1 row) + +-- Each column carries its own configuration and its own generation. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'docs', 'embedding', 4, 2, 1), + ('public', 'docs', 'summary', 2, 1, 1); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public','docs','embedding',1,0,ARRAY[1,0,0]::real[]), + ('public','docs','embedding',1,1,ARRAY[0,1,0]::real[]), + ('public','docs','summary', 1,0,ARRAY[1,1]::real[]), + ('public','docs','summary', 1,1,ARRAY[-1,1]::real[]); +SELECT coldfront._vec_probe_ids('public','docs','embedding', ARRAY[0.9,0.1,0]::real[]) AS embedding_probe, + coldfront._vec_probe_ids('public','docs','summary', ARRAY[-1,1]::real[]) AS summary_probe; + embedding_probe | summary_probe +-----------------+--------------- + {0,1} | {1} +(1 row) + +-- The write trigger, from its one builder, on a real two-vector hot table. This +-- is the bootstrap shape: registered, no watermark yet, so the trigger carries +-- the -infinity default and every insert routes hot until the first cutover. +CREATE TABLE public._docs ( + id bigint GENERATED ALWAYS AS IDENTITY, + ts timestamptz, + body text, + body_len int GENERATED ALWAYS AS (length(body)) STORED, + embedding vector(3), + summary vector(2) +); +ALTER TABLE public._docs + ADD COLUMN "_cf_vec_embedding" real[] GENERATED ALWAYS AS ("embedding"::real[]) STORED, + ADD COLUMN "_cf_vec_summary" real[] GENERATED ALWAYS AS ("summary"::real[]) STORED; +CREATE VIEW public.docs AS SELECT id, ts, body, body_len, + "_cf_vec_embedding"::real[] AS embedding, "_cf_vec_summary"::real[] AS summary + FROM public._docs; +INSERT INTO coldfront.tiered_views (schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'docs', 'public._docs', 'ice.default.docs', 'ts', ARRAY['embedding','summary']); +SELECT coldfront._rebuild_write_trigger('public', 'docs'); + _rebuild_write_trigger +------------------------ + +(1 row) + +-- The C rewrite asks these two for a targeted decoupled INSERT; the same registry +-- row answers both, in schema order. +SELECT coldfront._vec_list_cols_for_ref('ice.default.docs') AS cluster_cols; + cluster_cols +---------------------------------------------- + _cf_vec_list_embedding, _cf_vec_list_summary +(1 row) + +SELECT coldfront._vec_list_prefix_for_ref('ice.default.docs', 's.') IS NOT NULL AS prefix_resolves; + prefix_resolves +----------------- + t +(1 row) + +-- The UPDATE re-stamp offers every SET column; only a clustered vector column +-- answers with an item, which is what keeps the column list out of C entirely. +SELECT coldfront._vec_list_set_item('ice.default.docs', 'embedding', 'NEW_EXPR') IS NOT NULL AS vector_answers, + coldfront._vec_list_set_item('ice.default.docs', 'body', 'NEW_EXPR') IS NULL AS non_vector_declines; + vector_answers | non_vector_declines +----------------+--------------------- + t | t +(1 row) + +-- The trigger function is the contract. Both cluster expressions lead, each +-- pairing with its own column's value, in column order; the identity column and +-- the user-written generated column are positional NULLs the hot INSERT skips, +-- since neither accepts a supplied value; the watermark is read at fire time +-- with the bootstrap default. +-- Needles containing apostrophes are dollar-quoted: inside the trigger body the +-- cold-INSERT template is itself a quoted literal, so its own apostrophes arrive +-- doubled in the function definition. +SELECT (length(def) - length(replace(def, 'arg_min', ''))) / length('arg_min') AS cluster_lookups, + (length(def) - length(replace(def, 'CAST(%L AS FLOAT[])', ''))) / length('CAST(%L AS FLOAT[])') AS float_placeholders, + strpos(def, '_cf_vec_list') = 0 AS positional_not_named, + strpos(def, $n$column_name = ''embedding''$n$) < strpos(def, $n$column_name = ''summary''$n$) AS expressions_in_column_order, + strpos(def, 'translate(NEW.embedding') < strpos(def, 'translate(NEW.summary') AS arguments_in_column_order, + strpos(def, 'translate(NEW.summary') < strpos(def, ', NEW.ts') AS arguments_lead_the_list, + strpos(def, 'VALUES (NULL') = 0 AS prefix_before_identity_null, + strpos(def, ', NULL, %L') > 0 AS identity_positional_null, + strpos(def, '%L, NULL, CAST(') > 0 AS generated_positional_null, + def LIKE '%FROM coldfront.archive_watermark%' AS watermark_read_at_fire_time, + def LIKE '%''-infinity''::timestamptz%' AS bootstrap_default, + def LIKE '%INSERT INTO public._docs (ts, body, embedding, summary)%' AS hot_insert_skips_generated + FROM pg_get_functiondef('coldfront.docs_write()'::regprocedure) AS d(def); + cluster_lookups | float_placeholders | positional_not_named | expressions_in_column_order | arguments_in_column_order | arguments_lead_the_list | prefix_before_identity_null | identity_positional_null | generated_positional_null | watermark_read_at_fire_time | bootstrap_default | hot_insert_skips_generated +-----------------+--------------------+----------------------+-----------------------------+---------------------------+-------------------------+-----------------------------+--------------------------+---------------------------+-----------------------------+-------------------+---------------------------- + 2 | 4 | t | t | t | t | t | t | t | t | t | t +(1 row) + +-- Idempotent: the archiver re-runs bootstrap every cycle against a view that +-- already carries the trigger. +SELECT coldfront._rebuild_write_trigger('public', 'docs'); + _rebuild_write_trigger +------------------------ + +(1 row) + +SELECT count(*) AS triggers FROM pg_trigger + WHERE tgrelid = 'public.docs'::regclass AND NOT tgisinternal; + triggers +---------- + 1 +(1 row) + +-- Cleanup. Unregister before dropping: the DDL hook blocks DROP of a registered +-- tiered table/view. +DELETE FROM coldfront.tiered_views WHERE relname = 'docs'; +DELETE FROM coldfront.vector_centroids WHERE table_name = 'docs'; +DELETE FROM coldfront.vector_config WHERE table_name = 'docs'; +DROP VIEW public.docs; +DROP TABLE public._docs; +DROP FUNCTION coldfront.docs_write(); diff --git a/extension/coldfront/test/expected/vector_ops.out b/extension/coldfront/test/expected/vector_ops.out new file mode 100644 index 0000000..3c17ddb --- /dev/null +++ b/extension/coldfront/test/expected/vector_ops.out @@ -0,0 +1,104 @@ +-- The distance operators a caller writes against the real[] the view exposes. +-- Installed into pgvector's own schema so `<=>` resolves unqualified, with each +-- function named for the DuckDB function it becomes on the cold side. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +SELECT coldfront.install_vector_ops(); + install_vector_ops +-------------------- + +(1 row) + +-- Unqualified resolution is the point: these are written the way a caller writes +-- them, with no schema and no cast to vector. +SELECT round((ARRAY[1,0,0]::real[] <=> ARRAY[0,1,0]::real[])::numeric, 6) AS cosine_orthogonal, + round((ARRAY[1,0,0]::real[] <=> ARRAY[1,0,0]::real[])::numeric, 6) AS cosine_same, + round((ARRAY[1,0,0]::real[] <-> ARRAY[0,1,0]::real[])::numeric, 6) AS l2, + round((ARRAY[1,2,3]::real[] <#> ARRAY[1,2,3]::real[])::numeric, 6) AS neg_inner; + cosine_orthogonal | cosine_same | l2 | neg_inner +-------------------+-------------+----------+------------ + 1.000000 | 0.000000 | 1.414214 | -14.000000 +(1 row) + +-- The names are what pg_duckdb hands to DuckDB, so they are part of the contract. +SELECT p.proname, pg_get_function_arguments(p.oid) AS args + FROM pg_proc p + WHERE p.proname IN ('list_cosine_distance', 'list_distance', 'list_negative_inner_product') + AND pg_get_function_arguments(p.oid) = 'real[], real[]' + ORDER BY p.proname; + proname | args +-----------------------------+---------------- + list_cosine_distance | real[], real[] + list_distance | real[], real[] + list_negative_inner_product | real[], real[] +(3 rows) + +-- Idempotent: onboarding runs it for every vector table, repeatedly. +SELECT coldfront.install_vector_ops(); + install_vector_ops +-------------------- + +(1 row) + +SELECT count(*) AS operators FROM pg_operator + WHERE oprname IN ('<=>', '<->', '<#>') + AND oprleft = 'real[]'::regtype AND oprright = 'real[]'::regtype; + operators +----------- + 3 +(1 row) + +-- A vector-typed argument needs no cast from the caller: pgvector's vector -> real[] +-- cast is implicit, so operator resolution reaches the real[] shim. +SELECT round((ARRAY[1,0,0]::real[] <=> '[0,1,0]'::vector)::numeric, 6) AS mixed_operands; + mixed_operands +---------------- + 1.000000 +(1 row) + +-- A same-shaped operator in an unrelated schema does not satisfy install: the +-- caller resolves unqualified through pgvector's schema, so the operator has to +-- exist there. Drop one installed operator, plant a foreign clash, reinstall. +CREATE SCHEMA cf_opclash; +CREATE FUNCTION cf_opclash.zero_dist(real[], real[]) RETURNS double precision +LANGUAGE sql IMMUTABLE AS 'SELECT 0::double precision'; +DO $$ +DECLARE v_nsp text; +BEGIN + SELECT n.nspname INTO v_nsp + FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = 'vector'; + EXECUTE format('DROP OPERATOR %I.<-> (real[], real[])', v_nsp); + CREATE OPERATOR cf_opclash.<-> (LEFTARG = real[], RIGHTARG = real[], FUNCTION = cf_opclash.zero_dist); +END $$; +SELECT coldfront.install_vector_ops(); + install_vector_ops +-------------------- + +(1 row) + +SELECT count(*) AS l2_in_vector_schema + FROM pg_operator o + WHERE o.oprname = '<->' AND o.oprleft = 'real[]'::regtype AND o.oprright = 'real[]'::regtype + AND o.oprnamespace = (SELECT t.typnamespace FROM pg_type t WHERE t.typname = 'vector'); + l2_in_vector_schema +--------------------- + 1 +(1 row) + +-- Unqualified resolution lands on the reinstalled operator, not the clash. +SELECT round((ARRAY[1,0,0]::real[] <-> ARRAY[0,1,0]::real[])::numeric, 6) AS l2_after_reinstall; + l2_after_reinstall +-------------------- + 1.414214 +(1 row) + +DROP OPERATOR cf_opclash.<-> (real[], real[]); +DROP FUNCTION cf_opclash.zero_dist(real[], real[]); +DROP SCHEMA cf_opclash; diff --git a/extension/coldfront/test/expected/vector_param_render.out b/extension/coldfront/test/expected/vector_param_render.out new file mode 100644 index 0000000..96a49ab --- /dev/null +++ b/extension/coldfront/test/expected/vector_param_render.out @@ -0,0 +1,44 @@ +-- A bound parameter carrying an embedding into a cold write. The view exposes the +-- column as real[], so that is the parameter's type by the time the rewrite sees +-- it, and %L would spell it PG's way ({1,2,3}) which DuckDB's list cast rejects. +-- +-- White-box, like param_cold_via_plpgsql: EXPLAIN VERBOSE shows the rewritten cold +-- SQL and nothing touches Iceberg (warehouse/endpoint left ''). force_generic_plan +-- keeps $N from folding to a Const so the format() call stays visible. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +-- The cold SQL embeds the cutoff as a literal, so its spelling follows DateStyle. +-- Pin it for the same reason the timezone is pinned: the assertion is the rewrite, +-- not the session's formatting. +SET DateStyle = 'ISO, MDY'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SET plan_cache_mode = force_generic_plan; +CREATE TABLE public._chunks (id int, ts timestamptz, embedding vector(3)); +CREATE VIEW public.chunks AS SELECT id, ts, embedding::real[] AS embedding FROM public._chunks; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'chunks', 'public._chunks', 'ice.default.chunks', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'chunks', '2026-03-01'::timestamptz); +-- The parameter must reach DuckDB as CAST(%1$L AS FLOAT[]) with a +-- translate($1::text,'{}','[]') argument. +PREPARE cold_vec(real[]) AS + UPDATE public.chunks SET embedding = $1 WHERE ts < '2026-03-01'; +EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_vec('{1,2,3}'::real[]); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: _exec_iceberg_with_claim('ice.default.chunks'::text, format('UPDATE ice.default.chunks SET embedding = CAST(%1$L AS FLOAT[]) WHERE (ts < ''2026-03-01 00:00:00+00''::timestamptz)'::text, translate(($1)::text, '{}'::text, '[]'::text))) +(2 rows) + +-- Cleanup: this suite shares one database. Unregister first, since DROP on a +-- table that still has a registered cold tier is blocked by design. +DEALLOCATE cold_vec; +DELETE FROM coldfront.tiered_views WHERE relname = 'chunks'; +DELETE FROM coldfront.archive_watermark WHERE table_name = 'chunks'; +DROP VIEW public.chunks; +DROP TABLE public._chunks; diff --git a/extension/coldfront/test/expected/vector_probe.out b/extension/coldfront/test/expected/vector_probe.out new file mode 100644 index 0000000..e0690d8 --- /dev/null +++ b/extension/coldfront/test/expected/vector_probe.out @@ -0,0 +1,184 @@ +-- What a probed read is made of: which clusters to look in, the predicate that +-- says so, and the view definition the predicate is added to. Nothing here +-- executes a cold read (pg_regress has no Iceberg attached); the assertions are on +-- the generated SQL, and ci/journey.sh runs it against a real cold tier. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +-- The cutoff literal is deparsed into the view definition asserted below, so both +-- of the settings that render a timestamptz are pinned. +SET DateStyle = 'ISO, MDY'; +-- White-box: checks the generated SQL, not Iceberg I/O. +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT coldfront.install_vector_ops(); + install_vector_ops +-------------------- + +(1 row) + +-- Three orthogonal centroids, and a query vector plainly nearest the second. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'chunks', 'embedding', 3, 2, 1); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'chunks', 'embedding', 1, 0, ARRAY[1,0,0]::real[]), + ('public', 'chunks', 'embedding', 1, 1, ARRAY[0,1,0]::real[]), + ('public', 'chunks', 'embedding', 1, 2, ARRAY[0,0,1]::real[]); +-- nprobe comes from the configuration unless the caller overrides it, and the ids +-- come back ascending so one probe set makes one predicate whatever order the +-- distances arrived in. +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[]) AS nprobe_from_config; + nprobe_from_config +-------------------- + {1,2} +(1 row) + +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[], 1) AS nearest_only; + nearest_only +-------------- + {1} +(1 row) + +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[], 3) AS exhaustive; + exhaustive +------------ + {0,1,2} +(1 row) + +-- Nothing to probe against is not an error: the read loses its predicate and scans +-- exactly, which is correct. An unconfigured table, and a configured but untrained +-- one. +SELECT coldfront._vec_probe_ids('public', 'absent', 'embedding', ARRAY[1,0,0]::real[]) IS NULL AS unconfigured; + unconfigured +-------------- + t +(1 row) + +UPDATE coldfront.vector_config SET generation = 0 WHERE table_name = 'chunks'; +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[1,0,0]::real[]) IS NULL AS untrained; + untrained +----------- + t +(1 row) + +UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'chunks'; +-- The predicate. The null arm is not optional: a row another engine appended +-- straight to Iceberg carries no assignment, and a bare IN would drop it. +SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS qual; + qual +-------------------------------------------------------------------------------------------------- + (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) +(1 row) + +SELECT coldfront._vec_probe_qual('embedding', coldfront._vec_probe_ids('public', 'chunks', 'embedding', + ARRAY[0.1,0.9,0.2]::real[])) AS qual_from_probe; + qual_from_probe +-------------------------------------------------------------------------------------------------- + (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) +(1 row) + +SELECT coldfront._vec_probe_qual('embedding', NULL) IS NULL AS no_probe_set, + coldfront._vec_probe_qual('embedding', '{}') IS NULL AS empty_probe_set; + no_probe_set | empty_probe_set +--------------+----------------- + t | t +(1 row) + +-- A real tiered view, built by the generator, so the definition the rewrite +-- appends to is the one the product actually creates. +CREATE TABLE public._chunks (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz, body text, embedding vector(3)); +ALTER TABLE public._chunks ADD COLUMN IF NOT EXISTS "_cf_vec_embedding" real[] GENERATED ALWAYS AS ("embedding"::real[]) STORED; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'chunks', 'public._chunks', 'ice.default.chunks', 'ts', ARRAY['embedding']); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'chunks', '2026-03-01'::timestamptz); +SELECT coldfront._rebuild_tiered_view('public', 'chunks'); + _rebuild_tiered_view +---------------------- + +(1 row) + +-- Neither branch of the view projects the cluster column: a caller's query cannot +-- name it, which is why the predicate has to be added inside the definition. +SELECT count(*) AS cluster_column_in_view + FROM pg_attribute + WHERE attrelid = 'public.chunks'::regclass AND attname = coldfront._vec_list_col('embedding'); + cluster_column_in_view +------------------------ + 0 +(1 row) + +-- The probed definition. The tail is what matters: the qual lands inside the cold +-- arm's WHERE, after the cutoff comparison, and the hot arm is untouched. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])); + _vec_probed_viewdef +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + SELECT _chunks.id, + + _chunks.ts, + + (_chunks.body)::character varying AS body, + + _chunks._cf_vec_embedding AS embedding + + FROM _chunks + + WHERE (_chunks.ts >= '2026-03-01 00:00:00+00'::timestamp with time zone) + + UNION ALL + + SELECT (r.r['id'::text])::bigint AS id, + + (r.r['ts'::text])::timestamp with time zone AS ts, + + (r.r['body'::text])::character varying AS body, + + (r.r['embedding'::text])::real[] AS embedding + + FROM iceberg_scan('ice.default.chunks'::text) r(r) + + WHERE (r.r['ts'::text] < '2026-03-01 00:00:00+00'::timestamp with time zone) AND (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) +(1 row) + +-- It reparses, which is the whole contract: the rewrite substitutes this for the +-- view reference and PostgreSQL has to accept it. IN becomes = ANY on the way in. +DO $do$ +BEGIN + EXECUTE format('CREATE VIEW public.chunks_probed AS %s', + coldfront._vec_probed_viewdef('public', 'chunks', + coldfront._vec_probe_qual('embedding', ARRAY[1,2]))); +END +$do$; +SELECT right(pg_get_viewdef('public.chunks_probed'::regclass), 120) AS reparsed_tail; + reparsed_tail +-------------------------------------------------------------------------------------------------------------------------- + vec_list_embedding'::text])::integer = ANY (ARRAY[1, 2])) OR ((r.r['_cf_vec_list_embedding'::text])::integer IS NULL))); +(1 row) + +-- Declining is silent. No probe set, and a table with no vector column. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', NULL) IS NULL AS no_qual; + no_qual +--------- + t +(1 row) + +UPDATE coldfront.tiered_views SET vec_columns = NULL WHERE relname = 'chunks'; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS no_vector_column; + no_vector_column +------------------ + t +(1 row) + +UPDATE coldfront.tiered_views SET vec_columns = ARRAY['embedding'] WHERE relname = 'chunks'; +-- A tiered view with no cutoff is hot-only: it has no cold arm, so there is nothing +-- to probe. +DELETE FROM coldfront.archive_watermark WHERE table_name = 'chunks'; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS hot_only; + hot_only +---------- + t +(1 row) + +-- Cleanup. Unregister before dropping: the DDL hook blocks DROP of a registered +-- tiered table/view. +DROP VIEW public.chunks_probed; +DELETE FROM coldfront.tiered_views WHERE relname = 'chunks'; +DELETE FROM coldfront.vector_centroids WHERE table_name = 'chunks'; +DELETE FROM coldfront.vector_config WHERE table_name = 'chunks'; +DROP VIEW public.chunks; +DROP TABLE public._chunks; +DROP FUNCTION coldfront.chunks_write(); diff --git a/extension/coldfront/test/expected/vector_status.out b/extension/coldfront/test/expected/vector_status.out new file mode 100644 index 0000000..65ace6f --- /dev/null +++ b/extension/coldfront/test/expected/vector_status.out @@ -0,0 +1,71 @@ +-- What vector_status reports is an API, so its column list is asserted here. The +-- numbers are not: filling them reads the cold table through DuckDB, which +-- pg_regress has no Iceberg to do, so the distribution is asserted in ci/journey.sh +-- against a real cold tier. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +-- White-box: no Iceberg here, so ensure_attached() must be a no-op. +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +-- With nothing clustered the loop body never runs, which is the case that says so +-- rather than reporting an empty table and leaving the caller to guess. +CALL coldfront.vector_status(); +NOTICE: coldfront: no registered table has a clustered vector column +-- The reported columns. File count and bytes are deliberately absent: reaching them +-- means resolving a metadata location over HTTP, which this layer does not do, and +-- the compactor already reports both. +SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type + FROM pg_attribute a + WHERE a.attrelid = 'cf_vector_status'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; + attname | type +--------------------------+--------- + schema_name | text + table_name | text + column_name | text + prunes | boolean + generation | integer + nlist | integer + nprobe | integer + clusters_trained | integer + clusters_occupied | integer + additions | integer + addition_cap | integer + rows_total | bigint + rows_unassigned | bigint + rows_per_cluster_min | bigint + rows_per_cluster_max | bigint + rows_per_cluster_p50 | bigint + rows_per_cluster_p99 | bigint + clusters_below_row_group | integer + probe_fraction | numeric + advice | text +(20 rows) + +-- Re-running replaces the previous result rather than appending to it. +CALL coldfront.vector_status(); +NOTICE: coldfront: no registered table has a clustered vector column +SELECT count(*) AS reported FROM cf_vector_status; + reported +---------- + 0 +(1 row) + +-- Naming a table that is not registered reports nothing, and is not an error: a +-- caller scripting this against a list of tables should not have to know which of +-- them carry vectors. +CALL coldfront.vector_status('public', 'nosuchtable'); +NOTICE: coldfront: no registered table has a clustered vector column +SELECT count(*) AS reported FROM cf_vector_status; + reported +---------- + 0 +(1 row) + +DROP TABLE cf_vector_status; diff --git a/extension/coldfront/test/expected/vector_type_map.out b/extension/coldfront/test/expected/vector_type_map.out new file mode 100644 index 0000000..f35872d --- /dev/null +++ b/extension/coldfront/test/expected/vector_type_map.out @@ -0,0 +1,41 @@ +-- pgvector must be present in the image: a tiered vector table carries the type +-- on its hot side, and both the cold read and cold write paths rely on +-- pgvector's implicit vector -> real[] cast. +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared +-- regress db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +-- The decoupled path's type map is the twin of the archiver's Go map +-- (pgFormatTypeToDuckDB). Both must return the same pair or a column stores one +-- way and reads another, so these same literals are asserted on the Go side in +-- TestPgFormatTypeToDuckDB. +SELECT t, + coldfront._iceberg_storage_type(t) AS storage, + coldfront._iceberg_view_cast_type(t) AS view_cast + FROM unnest(ARRAY['vector(1536)', 'vector', 'halfvec(768)', 'halfvec']) AS t; + t | storage | view_cast +--------------+---------+----------- + vector(1536) | FLOAT[] | real[] + vector | FLOAT[] | real[] + halfvec(768) | FLOAT[] | real[] + halfvec | FLOAT[] | real[] +(4 rows) + +-- sparsevec is refused rather than densified: float4[65536] is a 100x storage +-- blowup, so it stays hot-only. +SELECT coldfront._iceberg_storage_type('sparsevec(65536)'); +ERROR: coldfront: PG type sparsevec(65536) has no Iceberg-compatible mapping. Supported: bigint, integer, smallint, real, double precision, boolean, timestamptz, timestamp, date, time, uuid, text, varchar(N), char(N), bytea, numeric(P,S), jsonb, json, interval, vector(N), halfvec(N). inet/cidr/oid unsupported (store IP data as text, oid values as bigint); sparsevec unsupported (keep it in the hot tier) +CONTEXT: PL/pgSQL function _iceberg_storage_type(text) line 43 at RAISE +-- format_type output is what both maps actually receive, dimension included. +CREATE TABLE vec_map_probe (id bigint, embedding vector(3)); +SELECT format_type(atttypid, atttypmod) AS format_type + FROM pg_attribute + WHERE attrelid = 'vec_map_probe'::regclass AND attname = 'embedding'; + format_type +------------- + vector(3) +(1 row) + +DROP TABLE vec_map_probe; diff --git a/extension/coldfront/test/sql/cast_normalize.sql b/extension/coldfront/test/sql/cast_normalize.sql index 24980bd..c1abae1 100644 --- a/extension/coldfront/test/sql/cast_normalize.sql +++ b/extension/coldfront/test/sql/cast_normalize.sql @@ -19,8 +19,10 @@ SET coldfront.warehouse = ''; SET coldfront.lakekeeper_endpoint = ''; -- evt_jsonb's name embeds "jsonb": it must survive the jsonb→json rewrite intact. +CREATE EXTENSION IF NOT EXISTS vector; CREATE TABLE public._events (id int, ts timestamptz, c_tsn timestamp, - c_vc varchar, c_dp double precision, evt_jsonb jsonb); + c_vc varchar, c_dp double precision, evt_jsonb jsonb, + emb vector(3)); CREATE VIEW public.events AS SELECT * FROM public._events; INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) VALUES ('public', 'events', 'public._events', 'ice.default.events', 'ts'); @@ -40,9 +42,19 @@ EXPLAIN (COSTS OFF, VERBOSE) evt_jsonb = jsonb_set(jsonb_build_object('k', 1), '{x}', '"v"'::jsonb) WHERE ts < '2019-01-01'::timestamp with time zone; +-- (A2) pgvector's cast: DuckDB knows no `vector`, and the Iceberg column is +-- FLOAT[]. The dimension typmod must go with the name, since FLOAT[](3) is not a +-- type. The literal needs no rewriting here: a vector Const deparses through +-- pgvector's own output function, which is already bracket-delimited. +EXPLAIN (COSTS OFF, VERBOSE) + UPDATE public.events SET emb = '[1,2,3]'::vector(3) + WHERE ts < '2019-01-01'::timestamp with time zone; + -- (B) Parity against the live DuckDB: the rewrite targets are accepted; `jsonb` -- is rejected (which is why the map rewrites it). A void row = accepted. SELECT duckdb.raw_query($$ SELECT NULL::json $$); +SELECT duckdb.raw_query($$ SELECT '[1,2,3]'::FLOAT[] $$); +SELECT duckdb.raw_query($$ SELECT NULL::vector $$); SELECT duckdb.raw_query($$ SELECT json_object('k', 1) $$); SELECT duckdb.raw_query($$ SELECT NULL::timestamp $$); SELECT duckdb.raw_query($$ SELECT NULL::varchar $$); diff --git a/extension/coldfront/test/sql/cte_on_insert.sql b/extension/coldfront/test/sql/cte_on_insert.sql index 5fbf972..919717c 100644 --- a/extension/coldfront/test/sql/cte_on_insert.sql +++ b/extension/coldfront/test/sql/cte_on_insert.sql @@ -30,3 +30,21 @@ DELETE FROM coldfront.tiered_views; DELETE FROM coldfront.archive_watermark; DROP VIEW public.events; DROP TABLE public._events; + +-- The clustered iceberg-only INSERT is re-emitted so the cluster is derived in +-- the same statement, and the CTE must survive that re-emission too: the WITH +-- folds into the derived table the assignment reads from, the only scope its +-- CTEs are visible from. +CREATE TABLE public._vec_base (id int, ts timestamptz, embedding real[]); +CREATE VIEW public.icevec AS SELECT * FROM public._vec_base; +INSERT INTO coldfront.tiered_views(schema_name, relname, iceberg_table, is_iceberg_only, vec_columns) +VALUES ('public', 'icevec', 'ice.default.icevec', true, ARRAY['embedding']); + +EXPLAIN (COSTS OFF, VERBOSE) + WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, ARRAY[1,0,0]::real[] AS embedding) + INSERT INTO public.icevec SELECT id, ts, embedding FROM s; + +-- Cleanup. +DELETE FROM coldfront.tiered_views WHERE relname = 'icevec'; +DROP VIEW public.icevec; +DROP TABLE public._vec_base; diff --git a/extension/coldfront/test/sql/vector_assign.sql b/extension/coldfront/test/sql/vector_assign.sql new file mode 100644 index 0000000..db14c2d --- /dev/null +++ b/extension/coldfront/test/sql/vector_assign.sql @@ -0,0 +1,41 @@ +-- vector_assign refuses before it writes. Both checks run ahead of any DuckDB +-- statement, which is what lets pg_regress reach them with no Iceberg attached; the +-- assignment itself is asserted in ci/journey.sh against a real cold tier. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; + +-- A column nothing registered. Naming the caller's own arguments back is what makes +-- a typo in a scripted call readable. +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); + +CREATE TABLE public._chunks (id bigint, ts timestamptz, embedding vector(3)); +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'chunks', 'public._chunks', 'ice.default.chunks', 'ts', ARRAY['embedding']); + +-- Registered, but the column named is not the clustered one. +CALL coldfront.vector_assign('public', 'chunks', 'other'); + +-- Registered and clustered, with nothing trained. Assigning here would rewrite +-- every row to the NULL it already holds, so it refuses rather than reporting +-- success over a full rewrite that changed nothing. +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); + +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'chunks', 'embedding', 2, 1, 0); + +-- Generation 0 is the same case spelled differently: a configuration row exists but +-- no generation was ever written. +CALL coldfront.vector_assign('public', 'chunks', 'embedding'); + +-- Cleanup. +DELETE FROM coldfront.vector_config WHERE table_name = 'chunks'; +DELETE FROM coldfront.tiered_views WHERE relname = 'chunks'; +DROP TABLE public._chunks; diff --git a/extension/coldfront/test/sql/vector_centroids.sql b/extension/coldfront/test/sql/vector_centroids.sql new file mode 100644 index 0000000..2ea22c1 --- /dev/null +++ b/extension/coldfront/test/sql/vector_centroids.sql @@ -0,0 +1,60 @@ +-- The routing state Phase 2 assigns and probes against: one centroid set per +-- vector column, plus the per-table settings that describe it. +-- +-- It lives in PostgreSQL because three consumers need it there: the read-path +-- rewrite scores a query against it in the same backend, the write paths resolve +-- an assignment inside the statement doing the write, and an adaptive addition is +-- inserted in the same transaction as the row that triggered it. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +-- Deliberately no pgvector here: these tables ship with the extension, so they +-- must exist on a database that has no vectors and may never have any. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; + +-- Name-keyed, like partition_config: a mesh replicates it by value, so every node +-- assigns identical cluster ids without sharing OIDs. +SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type, a.attnotnull + FROM pg_attribute a + WHERE a.attrelid = 'coldfront.vector_centroids'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; + +SELECT conname, pg_get_constraintdef(oid) AS def + FROM pg_constraint + WHERE conrelid = 'coldfront.vector_centroids'::regclass + ORDER BY conname; + +SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type + FROM pg_attribute a + WHERE a.attrelid = 'coldfront.vector_config'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; + +-- A restored node must re-attach to the same cold tier without retraining, and a +-- generation is meaningless without the centroids that defined it. +SELECT c.relname + FROM pg_class c + WHERE c.oid = ANY (SELECT unnest(extconfig) FROM pg_extension WHERE extname = 'coldfront') + AND c.relname IN ('vector_centroids', 'vector_config') + ORDER BY c.relname; + +-- A generation is immutable: rows are inserted, never updated in place, so a +-- query that resolved a generation keeps meaning the same thing. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) +VALUES ('public', 'chunks', 'embedding', 500, 20); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'chunks', 'embedding', 1, 0, ARRAY[1,0,0]::real[]), + ('public', 'chunks', 'embedding', 1, 1, ARRAY[0,1,0]::real[]); +SELECT count(*) AS centroids FROM coldfront.vector_centroids WHERE table_name = 'chunks'; + +-- The same centroid id twice in one generation would make an assignment ambiguous. +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'chunks', 'embedding', 1, 0, ARRAY[9,9,9]::real[]); + +-- Cleanup. +DELETE FROM coldfront.vector_centroids WHERE table_name = 'chunks'; +DELETE FROM coldfront.vector_config WHERE table_name = 'chunks'; diff --git a/extension/coldfront/test/sql/vector_cold_render.sql b/extension/coldfront/test/sql/vector_cold_render.sql new file mode 100644 index 0000000..c3af0e2 --- /dev/null +++ b/extension/coldfront/test/sql/vector_cold_render.sql @@ -0,0 +1,52 @@ +-- Four cold-write paths render a value for DuckDB, and they share two decisions +-- so a value cannot round-trip through one path and corrupt through another: +-- _cold_placeholder / _cold_value for the INSTEAD OF trigger (create_iceberg_table +-- for a decoupled table, _rebuild_tiered_view for a tiered one, view.go for the Go +-- twin), and _render_cold_value for the per-row loops that read a jsonb payload. +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared +-- regress db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; + +-- The trigger's pair. The view exposes a vector as real[], whose text form is +-- PG's {1,2,3}; DuckDB's list cast takes [1,2,3] only, so translate rewrites the +-- delimiters and the placeholder supplies the Iceberg column's type. +SELECT t, + coldfront._cold_placeholder(t) AS placeholder, + coldfront._cold_value('embedding', t) AS value + FROM unnest(ARRAY['vector(3)', 'halfvec(3)']) AS t; + +-- Every other type keeps the spelling it already had. +SELECT t, + coldfront._cold_placeholder(t) AS placeholder, + coldfront._cold_value('c', t) AS value + FROM unnest(ARRAY['bytea', 'jsonb', 'json', 'interval', 'bigint', + 'double precision', 'text']) AS t; + +-- The per-row loops read their value out of a jsonb payload, which spells a +-- vector as a string and a real[] as an array, so both arrive bracketed already +-- and only the cast is added. Whitespace between elements is accepted by DuckDB. +SELECT coldfront._render_cold_value('[1,2,3]', 'vector(3)') AS vec_tight, + coldfront._render_cold_value('[1, 2, 3]', 'vector(3)') AS vec_spaced; + +-- bytea arrives as PG's '\xHEX'; the hex digits are what DuckDB rebuilds from. +SELECT coldfront._render_cold_value('\xcafe', 'bytea') AS blob, + coldfront._render_cold_value('2.5', 'double precision') AS dbl, + coldfront._render_cold_value('a''b', 'text') AS quoted; + +-- The per-row serialiser keeps a NULL vector's positional slot. The Iceberg schema +-- declares one cluster column per vector column unconditionally, so the prefix +-- carries one assignment per vector column whatever this row holds; an entry +-- missing from the prefix would shift every following value one column left. +SELECT coldfront._move_row_literal( + '{"id": 8, "cf_new_ts": "2026-06-01 00:00:00+00", "embedding": "[1,2,3]"}'::jsonb, + ARRAY['id','ts','embedding'], + ARRAY['bigint','timestamp with time zone','vector(3)'], + 'ts', 'public', 'chunks') AS vec_row; +SELECT coldfront._move_row_literal( + '{"id": 7, "cf_new_ts": "2026-06-01 00:00:00+00", "embedding": null}'::jsonb, + ARRAY['id','ts','embedding'], + ARRAY['bigint','timestamp with time zone','vector(3)'], + 'ts', 'public', 'chunks') AS null_vec_row; diff --git a/extension/coldfront/test/sql/vector_multicolumn.sql b/extension/coldfront/test/sql/vector_multicolumn.sql new file mode 100644 index 0000000..ce205dd --- /dev/null +++ b/extension/coldfront/test/sql/vector_multicolumn.sql @@ -0,0 +1,130 @@ +-- A table may carry more than one vector column. Every one of them gets a cluster +-- column and an assignment on every write path; only the FIRST gets the file sort +-- order, because a Parquet file has one physical row order. So the first column's +-- probe prunes row groups and the others only cut the rows scored. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +-- The probe scores against the centroids through the distance shim, so it has to be +-- installed before _vec_probe_ids can resolve anything. +SELECT coldfront.install_vector_ops(); + +-- One cluster column per vector column, named after it. The name is what the +-- read-path rewrite derives from the column a query orders by, so it is the contract +-- between the writer and the reader. +SELECT coldfront._vec_list_col('embedding') AS embedding_cluster, + coldfront._vec_list_col('summary') AS summary_cluster; + +-- The sort key names one cluster column and the primary key. Passing a second vector +-- column would not help: within one value of the first, the second's values are +-- scattered, so its statistics stop bounding anything. +SELECT coldfront._vec_sort_key('embedding', ARRAY['id','ts']) AS sort_key; + +-- The positional prefix carries every vector column, in the order the Iceberg schema +-- declares them, because a cold INSERT supplies values by position. +SELECT coldfront._vec_list_prefix('public', 'docs', + ARRAY['embedding','summary'], + ARRAY['s.embedding','s.summary']) AS prefix; + +-- A single vector column yields a prefix of one expression. +SELECT coldfront._vec_list_prefix('public', 'docs', + ARRAY['embedding'], ARRAY['s.embedding']) AS one_column; + +-- No vector column, no prefix: a table without vectors pays nothing. +SELECT coldfront._vec_list_prefix('public', 'docs', ARRAY[]::text[], ARRAY[]::text[]) = '' + AS no_vectors; + +-- Mismatched arrays are a caller bug, and a silently short prefix would write a +-- positional INSERT that lands values in the wrong columns. +SELECT coldfront._vec_list_prefix('public', 'docs', ARRAY['a','b'], ARRAY['s.a']); + +-- The probe predicate names the column being searched, so two searches on one table +-- filter on different cluster columns. +SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS embedding_qual, + coldfront._vec_probe_qual('summary', ARRAY[7]) AS summary_qual; + +-- Each column carries its own configuration and its own generation. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'docs', 'embedding', 4, 2, 1), + ('public', 'docs', 'summary', 2, 1, 1); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public','docs','embedding',1,0,ARRAY[1,0,0]::real[]), + ('public','docs','embedding',1,1,ARRAY[0,1,0]::real[]), + ('public','docs','summary', 1,0,ARRAY[1,1]::real[]), + ('public','docs','summary', 1,1,ARRAY[-1,1]::real[]); +SELECT coldfront._vec_probe_ids('public','docs','embedding', ARRAY[0.9,0.1,0]::real[]) AS embedding_probe, + coldfront._vec_probe_ids('public','docs','summary', ARRAY[-1,1]::real[]) AS summary_probe; + +-- The write trigger, from its one builder, on a real two-vector hot table. This +-- is the bootstrap shape: registered, no watermark yet, so the trigger carries +-- the -infinity default and every insert routes hot until the first cutover. +CREATE TABLE public._docs ( + id bigint GENERATED ALWAYS AS IDENTITY, + ts timestamptz, + body text, + body_len int GENERATED ALWAYS AS (length(body)) STORED, + embedding vector(3), + summary vector(2) +); +ALTER TABLE public._docs + ADD COLUMN "_cf_vec_embedding" real[] GENERATED ALWAYS AS ("embedding"::real[]) STORED, + ADD COLUMN "_cf_vec_summary" real[] GENERATED ALWAYS AS ("summary"::real[]) STORED; +CREATE VIEW public.docs AS SELECT id, ts, body, body_len, + "_cf_vec_embedding"::real[] AS embedding, "_cf_vec_summary"::real[] AS summary + FROM public._docs; +INSERT INTO coldfront.tiered_views (schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'docs', 'public._docs', 'ice.default.docs', 'ts', ARRAY['embedding','summary']); + +SELECT coldfront._rebuild_write_trigger('public', 'docs'); + +-- The C rewrite asks these two for a targeted decoupled INSERT; the same registry +-- row answers both, in schema order. +SELECT coldfront._vec_list_cols_for_ref('ice.default.docs') AS cluster_cols; +SELECT coldfront._vec_list_prefix_for_ref('ice.default.docs', 's.') IS NOT NULL AS prefix_resolves; + +-- The UPDATE re-stamp offers every SET column; only a clustered vector column +-- answers with an item, which is what keeps the column list out of C entirely. +SELECT coldfront._vec_list_set_item('ice.default.docs', 'embedding', 'NEW_EXPR') IS NOT NULL AS vector_answers, + coldfront._vec_list_set_item('ice.default.docs', 'body', 'NEW_EXPR') IS NULL AS non_vector_declines; + +-- The trigger function is the contract. Both cluster expressions lead, each +-- pairing with its own column's value, in column order; the identity column and +-- the user-written generated column are positional NULLs the hot INSERT skips, +-- since neither accepts a supplied value; the watermark is read at fire time +-- with the bootstrap default. +-- Needles containing apostrophes are dollar-quoted: inside the trigger body the +-- cold-INSERT template is itself a quoted literal, so its own apostrophes arrive +-- doubled in the function definition. +SELECT (length(def) - length(replace(def, 'arg_min', ''))) / length('arg_min') AS cluster_lookups, + (length(def) - length(replace(def, 'CAST(%L AS FLOAT[])', ''))) / length('CAST(%L AS FLOAT[])') AS float_placeholders, + strpos(def, '_cf_vec_list') = 0 AS positional_not_named, + strpos(def, $n$column_name = ''embedding''$n$) < strpos(def, $n$column_name = ''summary''$n$) AS expressions_in_column_order, + strpos(def, 'translate(NEW.embedding') < strpos(def, 'translate(NEW.summary') AS arguments_in_column_order, + strpos(def, 'translate(NEW.summary') < strpos(def, ', NEW.ts') AS arguments_lead_the_list, + strpos(def, 'VALUES (NULL') = 0 AS prefix_before_identity_null, + strpos(def, ', NULL, %L') > 0 AS identity_positional_null, + strpos(def, '%L, NULL, CAST(') > 0 AS generated_positional_null, + def LIKE '%FROM coldfront.archive_watermark%' AS watermark_read_at_fire_time, + def LIKE '%''-infinity''::timestamptz%' AS bootstrap_default, + def LIKE '%INSERT INTO public._docs (ts, body, embedding, summary)%' AS hot_insert_skips_generated + FROM pg_get_functiondef('coldfront.docs_write()'::regprocedure) AS d(def); + +-- Idempotent: the archiver re-runs bootstrap every cycle against a view that +-- already carries the trigger. +SELECT coldfront._rebuild_write_trigger('public', 'docs'); +SELECT count(*) AS triggers FROM pg_trigger + WHERE tgrelid = 'public.docs'::regclass AND NOT tgisinternal; + +-- Cleanup. Unregister before dropping: the DDL hook blocks DROP of a registered +-- tiered table/view. +DELETE FROM coldfront.tiered_views WHERE relname = 'docs'; +DELETE FROM coldfront.vector_centroids WHERE table_name = 'docs'; +DELETE FROM coldfront.vector_config WHERE table_name = 'docs'; +DROP VIEW public.docs; +DROP TABLE public._docs; +DROP FUNCTION coldfront.docs_write(); diff --git a/extension/coldfront/test/sql/vector_ops.sql b/extension/coldfront/test/sql/vector_ops.sql new file mode 100644 index 0000000..28e1ed7 --- /dev/null +++ b/extension/coldfront/test/sql/vector_ops.sql @@ -0,0 +1,63 @@ +-- The distance operators a caller writes against the real[] the view exposes. +-- Installed into pgvector's own schema so `<=>` resolves unqualified, with each +-- function named for the DuckDB function it becomes on the cold side. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; + +SELECT coldfront.install_vector_ops(); + +-- Unqualified resolution is the point: these are written the way a caller writes +-- them, with no schema and no cast to vector. +SELECT round((ARRAY[1,0,0]::real[] <=> ARRAY[0,1,0]::real[])::numeric, 6) AS cosine_orthogonal, + round((ARRAY[1,0,0]::real[] <=> ARRAY[1,0,0]::real[])::numeric, 6) AS cosine_same, + round((ARRAY[1,0,0]::real[] <-> ARRAY[0,1,0]::real[])::numeric, 6) AS l2, + round((ARRAY[1,2,3]::real[] <#> ARRAY[1,2,3]::real[])::numeric, 6) AS neg_inner; + +-- The names are what pg_duckdb hands to DuckDB, so they are part of the contract. +SELECT p.proname, pg_get_function_arguments(p.oid) AS args + FROM pg_proc p + WHERE p.proname IN ('list_cosine_distance', 'list_distance', 'list_negative_inner_product') + AND pg_get_function_arguments(p.oid) = 'real[], real[]' + ORDER BY p.proname; + +-- Idempotent: onboarding runs it for every vector table, repeatedly. +SELECT coldfront.install_vector_ops(); +SELECT count(*) AS operators FROM pg_operator + WHERE oprname IN ('<=>', '<->', '<#>') + AND oprleft = 'real[]'::regtype AND oprright = 'real[]'::regtype; + +-- A vector-typed argument needs no cast from the caller: pgvector's vector -> real[] +-- cast is implicit, so operator resolution reaches the real[] shim. +SELECT round((ARRAY[1,0,0]::real[] <=> '[0,1,0]'::vector)::numeric, 6) AS mixed_operands; + +-- A same-shaped operator in an unrelated schema does not satisfy install: the +-- caller resolves unqualified through pgvector's schema, so the operator has to +-- exist there. Drop one installed operator, plant a foreign clash, reinstall. +CREATE SCHEMA cf_opclash; +CREATE FUNCTION cf_opclash.zero_dist(real[], real[]) RETURNS double precision +LANGUAGE sql IMMUTABLE AS 'SELECT 0::double precision'; +DO $$ +DECLARE v_nsp text; +BEGIN + SELECT n.nspname INTO v_nsp + FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = 'vector'; + EXECUTE format('DROP OPERATOR %I.<-> (real[], real[])', v_nsp); + CREATE OPERATOR cf_opclash.<-> (LEFTARG = real[], RIGHTARG = real[], FUNCTION = cf_opclash.zero_dist); +END $$; +SELECT coldfront.install_vector_ops(); +SELECT count(*) AS l2_in_vector_schema + FROM pg_operator o + WHERE o.oprname = '<->' AND o.oprleft = 'real[]'::regtype AND o.oprright = 'real[]'::regtype + AND o.oprnamespace = (SELECT t.typnamespace FROM pg_type t WHERE t.typname = 'vector'); +-- Unqualified resolution lands on the reinstalled operator, not the clash. +SELECT round((ARRAY[1,0,0]::real[] <-> ARRAY[0,1,0]::real[])::numeric, 6) AS l2_after_reinstall; +DROP OPERATOR cf_opclash.<-> (real[], real[]); +DROP FUNCTION cf_opclash.zero_dist(real[], real[]); +DROP SCHEMA cf_opclash; diff --git a/extension/coldfront/test/sql/vector_param_render.sql b/extension/coldfront/test/sql/vector_param_render.sql new file mode 100644 index 0000000..de4d67d --- /dev/null +++ b/extension/coldfront/test/sql/vector_param_render.sql @@ -0,0 +1,43 @@ +-- A bound parameter carrying an embedding into a cold write. The view exposes the +-- column as real[], so that is the parameter's type by the time the rewrite sees +-- it, and %L would spell it PG's way ({1,2,3}) which DuckDB's list cast rejects. +-- +-- White-box, like param_cold_via_plpgsql: EXPLAIN VERBOSE shows the rewritten cold +-- SQL and nothing touches Iceberg (warehouse/endpoint left ''). force_generic_plan +-- keeps $N from folding to a Const so the format() call stays visible. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; + +SET TIME ZONE 'UTC'; +-- The cold SQL embeds the cutoff as a literal, so its spelling follows DateStyle. +-- Pin it for the same reason the timezone is pinned: the assertion is the rewrite, +-- not the session's formatting. +SET DateStyle = 'ISO, MDY'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SET plan_cache_mode = force_generic_plan; + +CREATE TABLE public._chunks (id int, ts timestamptz, embedding vector(3)); +CREATE VIEW public.chunks AS SELECT id, ts, embedding::real[] AS embedding FROM public._chunks; + +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'chunks', 'public._chunks', 'ice.default.chunks', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'chunks', '2026-03-01'::timestamptz); + +-- The parameter must reach DuckDB as CAST(%1$L AS FLOAT[]) with a +-- translate($1::text,'{}','[]') argument. +PREPARE cold_vec(real[]) AS + UPDATE public.chunks SET embedding = $1 WHERE ts < '2026-03-01'; +EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_vec('{1,2,3}'::real[]); + +-- Cleanup: this suite shares one database. Unregister first, since DROP on a +-- table that still has a registered cold tier is blocked by design. +DEALLOCATE cold_vec; +DELETE FROM coldfront.tiered_views WHERE relname = 'chunks'; +DELETE FROM coldfront.archive_watermark WHERE table_name = 'chunks'; +DROP VIEW public.chunks; +DROP TABLE public._chunks; diff --git a/extension/coldfront/test/sql/vector_probe.sql b/extension/coldfront/test/sql/vector_probe.sql new file mode 100644 index 0000000..16a7446 --- /dev/null +++ b/extension/coldfront/test/sql/vector_probe.sql @@ -0,0 +1,104 @@ +-- What a probed read is made of: which clusters to look in, the predicate that +-- says so, and the view definition the predicate is added to. Nothing here +-- executes a cold read (pg_regress has no Iceberg attached); the assertions are on +-- the generated SQL, and ci/journey.sh runs it against a real cold tier. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +-- The cutoff literal is deparsed into the view definition asserted below, so both +-- of the settings that render a timestamptz are pinned. +SET DateStyle = 'ISO, MDY'; +-- White-box: checks the generated SQL, not Iceberg I/O. +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; + +SELECT coldfront.install_vector_ops(); + +-- Three orthogonal centroids, and a query vector plainly nearest the second. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'chunks', 'embedding', 3, 2, 1); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'chunks', 'embedding', 1, 0, ARRAY[1,0,0]::real[]), + ('public', 'chunks', 'embedding', 1, 1, ARRAY[0,1,0]::real[]), + ('public', 'chunks', 'embedding', 1, 2, ARRAY[0,0,1]::real[]); + +-- nprobe comes from the configuration unless the caller overrides it, and the ids +-- come back ascending so one probe set makes one predicate whatever order the +-- distances arrived in. +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[]) AS nprobe_from_config; +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[], 1) AS nearest_only; +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[], 3) AS exhaustive; + +-- Nothing to probe against is not an error: the read loses its predicate and scans +-- exactly, which is correct. An unconfigured table, and a configured but untrained +-- one. +SELECT coldfront._vec_probe_ids('public', 'absent', 'embedding', ARRAY[1,0,0]::real[]) IS NULL AS unconfigured; +UPDATE coldfront.vector_config SET generation = 0 WHERE table_name = 'chunks'; +SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[1,0,0]::real[]) IS NULL AS untrained; +UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'chunks'; + +-- The predicate. The null arm is not optional: a row another engine appended +-- straight to Iceberg carries no assignment, and a bare IN would drop it. +SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS qual; +SELECT coldfront._vec_probe_qual('embedding', coldfront._vec_probe_ids('public', 'chunks', 'embedding', + ARRAY[0.1,0.9,0.2]::real[])) AS qual_from_probe; +SELECT coldfront._vec_probe_qual('embedding', NULL) IS NULL AS no_probe_set, + coldfront._vec_probe_qual('embedding', '{}') IS NULL AS empty_probe_set; + +-- A real tiered view, built by the generator, so the definition the rewrite +-- appends to is the one the product actually creates. +CREATE TABLE public._chunks (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz, body text, embedding vector(3)); +ALTER TABLE public._chunks ADD COLUMN IF NOT EXISTS "_cf_vec_embedding" real[] GENERATED ALWAYS AS ("embedding"::real[]) STORED; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'chunks', 'public._chunks', 'ice.default.chunks', 'ts', ARRAY['embedding']); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'chunks', '2026-03-01'::timestamptz); +SELECT coldfront._rebuild_tiered_view('public', 'chunks'); + +-- Neither branch of the view projects the cluster column: a caller's query cannot +-- name it, which is why the predicate has to be added inside the definition. +SELECT count(*) AS cluster_column_in_view + FROM pg_attribute + WHERE attrelid = 'public.chunks'::regclass AND attname = coldfront._vec_list_col('embedding'); + +-- The probed definition. The tail is what matters: the qual lands inside the cold +-- arm's WHERE, after the cutoff comparison, and the hot arm is untouched. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])); + +-- It reparses, which is the whole contract: the rewrite substitutes this for the +-- view reference and PostgreSQL has to accept it. IN becomes = ANY on the way in. +DO $do$ +BEGIN + EXECUTE format('CREATE VIEW public.chunks_probed AS %s', + coldfront._vec_probed_viewdef('public', 'chunks', + coldfront._vec_probe_qual('embedding', ARRAY[1,2]))); +END +$do$; +SELECT right(pg_get_viewdef('public.chunks_probed'::regclass), 120) AS reparsed_tail; + +-- Declining is silent. No probe set, and a table with no vector column. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', NULL) IS NULL AS no_qual; +UPDATE coldfront.tiered_views SET vec_columns = NULL WHERE relname = 'chunks'; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS no_vector_column; +UPDATE coldfront.tiered_views SET vec_columns = ARRAY['embedding'] WHERE relname = 'chunks'; + +-- A tiered view with no cutoff is hot-only: it has no cold arm, so there is nothing +-- to probe. +DELETE FROM coldfront.archive_watermark WHERE table_name = 'chunks'; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS hot_only; + +-- Cleanup. Unregister before dropping: the DDL hook blocks DROP of a registered +-- tiered table/view. +DROP VIEW public.chunks_probed; +DELETE FROM coldfront.tiered_views WHERE relname = 'chunks'; +DELETE FROM coldfront.vector_centroids WHERE table_name = 'chunks'; +DELETE FROM coldfront.vector_config WHERE table_name = 'chunks'; +DROP VIEW public.chunks; +DROP TABLE public._chunks; +DROP FUNCTION coldfront.chunks_write(); diff --git a/extension/coldfront/test/sql/vector_status.sql b/extension/coldfront/test/sql/vector_status.sql new file mode 100644 index 0000000..af10d25 --- /dev/null +++ b/extension/coldfront/test/sql/vector_status.sql @@ -0,0 +1,39 @@ +-- What vector_status reports is an API, so its column list is asserted here. The +-- numbers are not: filling them reads the cold table through DuckDB, which +-- pg_regress has no Iceberg to do, so the distribution is asserted in ci/journey.sh +-- against a real cold tier. +-- +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +-- White-box: no Iceberg here, so ensure_attached() must be a no-op. +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; + +-- With nothing clustered the loop body never runs, which is the case that says so +-- rather than reporting an empty table and leaving the caller to guess. +CALL coldfront.vector_status(); + +-- The reported columns. File count and bytes are deliberately absent: reaching them +-- means resolving a metadata location over HTTP, which this layer does not do, and +-- the compactor already reports both. +SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type + FROM pg_attribute a + WHERE a.attrelid = 'cf_vector_status'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; + +-- Re-running replaces the previous result rather than appending to it. +CALL coldfront.vector_status(); +SELECT count(*) AS reported FROM cf_vector_status; + +-- Naming a table that is not registered reports nothing, and is not an error: a +-- caller scripting this against a list of tables should not have to know which of +-- them carry vectors. +CALL coldfront.vector_status('public', 'nosuchtable'); +SELECT count(*) AS reported FROM cf_vector_status; + +DROP TABLE cf_vector_status; diff --git a/extension/coldfront/test/sql/vector_type_map.sql b/extension/coldfront/test/sql/vector_type_map.sql new file mode 100644 index 0000000..aad9172 --- /dev/null +++ b/extension/coldfront/test/sql/vector_type_map.sql @@ -0,0 +1,29 @@ +-- pgvector must be present in the image: a tiered vector table carries the type +-- on its hot side, and both the cold read and cold write paths rely on +-- pgvector's implicit vector -> real[] cast. +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared +-- regress db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +CREATE EXTENSION IF NOT EXISTS vector; + +-- The decoupled path's type map is the twin of the archiver's Go map +-- (pgFormatTypeToDuckDB). Both must return the same pair or a column stores one +-- way and reads another, so these same literals are asserted on the Go side in +-- TestPgFormatTypeToDuckDB. +SELECT t, + coldfront._iceberg_storage_type(t) AS storage, + coldfront._iceberg_view_cast_type(t) AS view_cast + FROM unnest(ARRAY['vector(1536)', 'vector', 'halfvec(768)', 'halfvec']) AS t; + +-- sparsevec is refused rather than densified: float4[65536] is a 100x storage +-- blowup, so it stays hot-only. +SELECT coldfront._iceberg_storage_type('sparsevec(65536)'); + +-- format_type output is what both maps actually receive, dimension included. +CREATE TABLE vec_map_probe (id bigint, embedding vector(3)); +SELECT format_type(atttypid, atttypmod) AS format_type + FROM pg_attribute + WHERE attrelid = 'vec_map_probe'::regclass AND attname = 'embedding'; +DROP TABLE vec_map_probe; diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..1cf67c6 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,8 @@ +// Package version provides build version information. +package version + +// Version and BuildTime are set via ldflags at build time. +var ( + Version = "unknown" + BuildTime = "unknown" +) diff --git a/internal/version/version_test.go b/internal/version/version_test.go new file mode 100644 index 0000000..302f723 --- /dev/null +++ b/internal/version/version_test.go @@ -0,0 +1,11 @@ +package version + +import "testing" + +// The defaults are placeholders the Makefile overrides; a binary must never +// print an empty version line. +func TestDefaultsNonEmpty(t *testing.T) { + if Version == "" || BuildTime == "" { + t.Errorf("Version = %q, BuildTime = %q; want non-empty defaults", Version, BuildTime) + } +} diff --git a/internal/view/view.go b/internal/view/view.go index 0c9a053..64632db 100644 --- a/internal/view/view.go +++ b/internal/view/view.go @@ -49,10 +49,66 @@ type Column struct { Name string Type string // storage / DuckDB-CREATE-TABLE type, e.g. "BIGINT", "VARCHAR", "DECIMAL(20,5)" ViewCastType string // optional surface-type cast emitted by the view, e.g. "json", "interval", "bytea" + HotSource string // hot-side column to read instead of Name; "" means Name itself IsIdentity bool // pg_attribute.attidentity = 'a' (GENERATED ALWAYS) — skip from INSERT IsPK bool // participates in primary key (pg_index.indisprimary) } +// HotRef returns the quoted hot-side identifier this column is read from, and +// whether that differs from the column's own name. +// +// Only a vector differs. pg_duckdb rejects a pgvector column while it builds the +// plan, before a cast in the projection can apply, so the hot table carries a +// generated real[] companion and every hot-side read goes through that instead. +func (c Column) HotRef() (ref string, aliased bool) { + if c.HotSource != "" && c.HotSource != c.Name { + return pgx.Identifier{c.HotSource}.Sanitize(), true + } + return pgx.Identifier{c.Name}.Sanitize(), false +} + +// VecListColumn names the Iceberg-only column carrying a row's cluster assignment +// for one vector column. Such a column exists in the Iceberg schema and nowhere +// else: not on the hot table, and not in either branch of the view, so no query +// written against the view can name it. coldfront._vec_list_col is the SQL twin. +// +// They lead the schema rather than trailing it. Iceberg evolution appends, and a +// cold INSERT is positional because Iceberg rejects a targeted one, so a column +// added later must land after everything both sides already agree on. +func VecListColumn(col string) string { return "_cf_vec_list_" + col } + +// IsVector reports whether this column's Iceberg storage is a list of floats, +// which is how a pgvector vector/halfvec is stored. +// +// Two consequences follow, and both have callers. The view exposes the column as +// real[], so every path that renders a value for DuckDB converts PG's {1,2,3} +// array text into DuckDB's [1,2,3] list literal. And pg_duckdb cannot scan the +// pgvector type at all, so every hot-side read goes through the generated +// companion instead (see HotRef). +func (c Column) IsVector() bool { return c.Type == "FLOAT[]" } + +// ExportCast returns the cast the archiver's bulk-export projection applies to +// this column on the PostgreSQL side, before pg_duckdb reads it, or "" when the +// column exports as-is. +// +// Two cases need one, for different reasons. A VARCHAR-backed rich type +// (jsonb/json/interval) exports ::text because its Iceberg column is VARCHAR. A +// vector exports ::real[] because pg_duckdb's PG reader cannot scan the pgvector +// type at all, and ::text would stringify it into the wrong column type. +// +// Types whose Iceberg storage is native (BLOB, DOUBLE) carry a ViewCastType only +// to give the view a PG-parseable spelling, and must export unchanged: +// ::text-casting a bytea would write '\xdeadbeef' into a binary column. +func (c Column) ExportCast() string { + switch { + case c.Type == "VARCHAR" && c.ViewCastType != "": + return "text" + case c.IsVector(): + return "real[]" + } + return "" +} + // Generator creates and replaces the view and triggers. type Generator struct { db DBTX @@ -93,80 +149,6 @@ func (c ViewConfig) hotTable() string { return pgx.Identifier{"_" + c.SourceTable}.Sanitize() } -// insertCols returns column names and NEW."col" refs for INSERT. -// Skips GENERATED ALWAYS AS IDENTITY columns (cannot accept explicit values). -// No per-type cast: the view exposes each column in its native PG type (jsonb -// stays jsonb, etc.), so NEW.col arrives at the trigger already matching the -// underlying _events column type. -func (c ViewConfig) insertCols() (colList, valList string) { - var cols, vals []string - for _, col := range c.Columns { - if col.IsIdentity { - continue - } - q := pgx.Identifier{col.Name}.Sanitize() - cols = append(cols, q) - vals = append(vals, "NEW."+q) - } - return strings.Join(cols, ", "), strings.Join(vals, ", ") -} - -// coldInsertVals returns the format() args for the cold INSERT via -// duckdb.raw_query(format(...)). Identity columns are excluded (same as hot). -// Each arg pairs positionally with a %L placeholder from coldInsertPlaceholders. -// -// - VARCHAR-backed rich types (jsonb/json/interval — Type=="VARCHAR" with a -// ViewCastType): serialised via ::text, since their Iceberg column is VARCHAR. -// - bytea (Type=="BLOB"): emitted as encode(NEW.col,'hex') and wrapped by a -// from_hex(%L) placeholder. The cold INSERT goes through %L, which renders a -// bytea as PG's '\xcafe' text — which DuckDB then MIS-parses into a BLOB -// (\xca → 1 byte, fe → 2 literal bytes = 3 bytes, corruption). Round-tripping -// the hex string through DuckDB's from_hex() rebuilds the exact bytes. -// - everything else (incl. double precision, whose '2.5' text round-trips -// cleanly through DuckDB): NEW.col as-is. -// -// This must stay consistent with the archiver's bulk-export path, which writes -// bytea natively because pg_duckdb scans it directly (no %L stringification). -func (c ViewConfig) coldInsertVals() string { - var vals []string - for _, col := range c.Columns { - if col.IsIdentity { - continue - } - q := pgx.Identifier{col.Name}.Sanitize() - switch { - case col.Type == "BLOB": - vals = append(vals, "encode(NEW."+q+",'hex')") - case col.Type == "VARCHAR" && col.ViewCastType != "": - vals = append(vals, "NEW."+q+"::text") - default: - vals = append(vals, "NEW."+q) - } - } - return strings.Join(vals, ", ") -} - -// coldInsertPlaceholders returns positional value placeholders for the cold -// INSERT via PG's format() call. Identity columns use literal NULL (Iceberg -// has no sequences); bytea uses from_hex(%L) so DuckDB reconstructs the exact -// bytes from the hex string (see coldInsertVals); all other columns use %L so -// format() quotes them safely. DuckDB/Iceberg does not support targeted inserts -// (INSERT INTO t(col) ...), so we emit a positional INSERT INTO t VALUES (...). -func (c ViewConfig) coldInsertPlaceholders() string { - var ph []string - for _, col := range c.Columns { - switch { - case col.IsIdentity: - ph = append(ph, "NULL") - case col.Type == "BLOB": - ph = append(ph, "from_hex(%L)") - default: - ph = append(ph, "%L") - } - } - return strings.Join(ph, ", ") -} - // GenerateSwapSQL generates the conditional rename of the source table to _{source}. // Idempotent: only renames if the source is still a regular table (not already a view). // @@ -226,7 +208,11 @@ func GenerateViewSQL(cfg ViewConfig) string { if c.ViewCastType != "" { surface = c.ViewCastType } - hotCols[i] = hotName + "::" + surface + hotRef, aliased := c.HotRef() + hotCols[i] = hotRef + "::" + surface + if aliased { + hotCols[i] += " AS " + hotName + } coldCols[i] = fmt.Sprintf("r['%s']::%s", coldKey, surface) } @@ -259,86 +245,54 @@ func GenerateViewSQL(cfg ViewConfig) string { coldColKey, cutoff) } -// GenerateTriggerFuncSQL generates the INSTEAD OF INSERT trigger function for -// the unified view. Hot inserts go to _{source}; cold inserts are forwarded -// to Iceberg via duckdb.raw_query. UPDATE/DELETE are handled by the -// coldfront C extension's post_parse_analyze_hook rewrite, not this trigger. -func GenerateTriggerFuncSQL(cfg ViewConfig) string { - funcName := pgx.Identifier{"coldfront", cfg.SourceTable + "_write"}.Sanitize() - fqHot := cfg.fqHot() - col := pgx.Identifier{cfg.PartitionColumn}.Sanitize() - - cutoff := "'-infinity'::timestamptz" - if cfg.hasCutoff() { - cutoff = fmt.Sprintf("'%s'::timestamptz", cfg.cutoffLiteral()) +// GenerateVecCompanionSQL adds each vector column's generated real[] companion to +// the hot table. Empty when no column needs one. +// +// The companion is what every hot-side read goes through, since pg_duckdb rejects +// a pgvector column while it builds the plan. It is a column of the hot table only: +// the view does not project it, and getColumns does not enumerate it, so the user's +// surface carries exactly one embedding column. Adding it to a table that already +// holds rows rewrites those rows once, at onboarding. +func GenerateVecCompanionSQL(cfg ViewConfig) string { + var b strings.Builder + for _, c := range cfg.Columns { + ref, aliased := c.HotRef() + if !aliased { + continue + } + fmt.Fprintf(&b, + "ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s real[] GENERATED ALWAYS AS (%s::real[]) STORED;\n", + cfg.fqHot(), ref, pgx.Identifier{c.Name}.Sanitize()) } - - colList, hotVals := cfg.insertCols() - coldPlaceholders := cfg.coldInsertPlaceholders() - coldVals := cfg.coldInsertVals() - - // cfg.IcebergTable is the ref DuckDB parses (not a PG identifier); embed - // it in the format() template as-is, apostrophe-escaped so it survives - // PG's outer string-literal scan. - iceRef := strings.ReplaceAll(cfg.IcebergTable, "'", "''") - - return fmt.Sprintf(`CREATE OR REPLACE FUNCTION %s() RETURNS trigger AS $fn$ -DECLARE - cutoff timestamptz; -BEGIN - SELECT cutoff_time INTO cutoff FROM coldfront.archive_watermark WHERE schema_name = %s AND table_name = %s; - IF cutoff IS NULL THEN - cutoff := %s; - END IF; - - IF TG_OP = 'INSERT' THEN - IF NEW.%s < cutoff THEN - PERFORM coldfront.ensure_attached(); - PERFORM duckdb.raw_query(format( - 'INSERT INTO %s VALUES (%s)', - %s - )); - RETURN NEW; - END IF; - INSERT INTO %s (%s) VALUES (%s); - RETURN NEW; - END IF; - RETURN NULL; -END; -$fn$ LANGUAGE plpgsql`, - funcName, - sqlutil.Literal(cfg.SourceSchema), sqlutil.Literal(cfg.SourceTable), - cutoff, - col, - iceRef, coldPlaceholders, coldVals, - fqHot, colList, hotVals) + return b.String() } -// GenerateTriggerSQL generates the DROP + CREATE TRIGGER on the unified view. -// INSERT-only: UPDATE/DELETE are rewritten by the coldfront hook before -// they reach the view, so no trigger is needed for those operations. -func GenerateTriggerSQL(cfg ViewConfig) string { - trigName := pgx.Identifier{cfg.SourceTable + "_write_trigger"}.Sanitize() - viewName := cfg.fqSource() - funcName := pgx.Identifier{"coldfront", cfg.SourceTable + "_write"}.Sanitize() - - return fmt.Sprintf(`DROP TRIGGER IF EXISTS %s ON %s; -CREATE TRIGGER %s - INSTEAD OF INSERT ON %s - FOR EACH ROW EXECUTE FUNCTION %s()`, - trigName, viewName, - trigName, viewName, funcName) +// GenerateVectorOpsSQL installs the distance operators a caller writes against the +// view's real[] columns. Empty when the table carries no vector column. +func GenerateVectorOpsSQL(cfg ViewConfig) string { + for _, c := range cfg.Columns { + if c.IsVector() { + return "SELECT coldfront.install_vector_ops()" + } + } + return "" } -// Recreate performs the table→view swap (if needed) and recreates the view + triggers. +// Recreate performs the table→view swap (if needed) and recreates the view. The +// INSTEAD OF INSERT trigger is not built here: coldfront._rebuild_write_trigger +// is its one generator, and the archiver calls it after registering the view, +// since the builder reads the registry. func (g *Generator) Recreate(ctx context.Context, cfg ViewConfig) error { stmts := []string{ GenerateSwapSQL(cfg), + GenerateVecCompanionSQL(cfg), // after the swap: it targets the renamed hot table + GenerateVectorOpsSQL(cfg), GenerateViewSQL(cfg), - GenerateTriggerFuncSQL(cfg), - GenerateTriggerSQL(cfg), } for _, sql := range stmts { + if sql == "" { + continue + } if _, err := g.db.Exec(ctx, sql); err != nil { // nosemgrep return fmt.Errorf("recreate view: %w", err) } diff --git a/internal/view/view_test.go b/internal/view/view_test.go index dc0b917..fdedd85 100644 --- a/internal/view/view_test.go +++ b/internal/view/view_test.go @@ -37,6 +37,76 @@ var testCfg = ViewConfig{ }, } +// A tiered table carrying an embedding. The view exposes the column as real[], +// so that is what NEW.embedding is inside the INSTEAD OF trigger. +var vectorCfg = ViewConfig{ + SourceSchema: "public", + SourceTable: "chunks", + IcebergTable: "ice.default.chunks", + CutoffTime: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC), + PartitionColumn: "ts", + Columns: []Column{ + {Name: "id", Type: "BIGINT", IsPK: true}, + {Name: "ts", Type: "TIMESTAMPTZ"}, + {Name: "embedding", Type: "FLOAT[]", ViewCastType: "real[]"}, + }, +} + +// pg_duckdb rejects a pgvector column while it builds the plan, before any cast in +// the projection can apply, so the hot table carries a generated real[] companion +// and the hot branch reads that under the user's own column name. The view still +// exposes one embedding column, and the companion is never a column of the view. +func TestGenerateViewSQL_VectorHotSourceIsTheCompanion(t *testing.T) { + cfg := vectorCfg + cfg.Columns = []Column{ + {Name: "id", Type: "BIGINT", IsPK: true}, + {Name: "ts", Type: "TIMESTAMPTZ"}, + {Name: "embedding", Type: "FLOAT[]", ViewCastType: "real[]", HotSource: "_cf_vec_embedding"}, + } + sql := GenerateViewSQL(cfg) + assert.Contains(t, sql, `"_cf_vec_embedding"::real[] AS "embedding"`, + "hot branch reads the companion, aliased to the user's column name") + assert.NotContains(t, sql, `"embedding"::real[] FROM`, + "the pgvector column itself must not be scanned") + assert.Contains(t, sql, "r['embedding']::real[]", "the cold branch is unchanged") +} + +// A column without a HotSource is read by its own name, which is every column that +// is not a vector. +func TestGenerateViewSQL_NoHotSourceReadsItsOwnName(t *testing.T) { + assert.Contains(t, GenerateViewSQL(testCfg), `"status"::VARCHAR`) +} + +// The distance operators are installed only for a table that carries a vector. +func TestGenerateVectorOpsSQL(t *testing.T) { + cfg := vectorCfg + assert.Equal(t, "SELECT coldfront.install_vector_ops()", GenerateVectorOpsSQL(cfg)) + assert.Equal(t, "", GenerateVectorOpsSQL(testCfg), "no vector column, no operators") +} + +// The companion is added to the renamed hot table, idempotently, and only for a +// column that needs one. +func TestGenerateVecCompanionSQL(t *testing.T) { + cfg := vectorCfg + cfg.Columns = []Column{ + {Name: "id", Type: "BIGINT"}, + {Name: "embedding", Type: "FLOAT[]", ViewCastType: "real[]", HotSource: "_cf_vec_embedding"}, + } + sql := GenerateVecCompanionSQL(cfg) + assert.Contains(t, sql, `ALTER TABLE "public"."_chunks" ADD COLUMN IF NOT EXISTS "_cf_vec_embedding" real[]`) + assert.Contains(t, sql, `GENERATED ALWAYS AS ("embedding"::real[]) STORED`) + assert.Equal(t, "", GenerateVecCompanionSQL(testCfg), "no vector column, no companion") +} + +// The view exposes real[] on both branches: FLOAT[] would parse in PG as +// double precision[] and the two branches would disagree. +func TestGenerateViewSQL_VectorSurfaceIsRealArray(t *testing.T) { + sql := GenerateViewSQL(vectorCfg) + assert.Contains(t, sql, `"embedding"::real[]`, "hot side casts to real[]") + assert.Contains(t, sql, "r['embedding']::real[]", "cold side casts to real[]") + assert.NotContains(t, sql, "::FLOAT[]", "FLOAT[] is double precision[] in PG") +} + func TestGenerateSwapSQL(t *testing.T) { sql := GenerateSwapSQL(testCfg) // Identifier positions are double-quoted; literal-string positions keep @@ -73,29 +143,6 @@ func TestGenerateViewSQL_NoCutoff(t *testing.T) { assert.NotContains(t, sql, "UNION ALL") } -// Trigger is INSERT-only: the C extension handles UPDATE/DELETE via CTE rewrite. -func TestGenerateTriggerFuncSQL_InsertOnly(t *testing.T) { - sql := GenerateTriggerFuncSQL(testCfg) - assert.Contains(t, sql, `CREATE OR REPLACE FUNCTION "coldfront"."events_write"`) - assert.Contains(t, sql, "RETURNS trigger") - assert.Contains(t, sql, "TG_OP = 'INSERT'") - assert.NotContains(t, sql, "TG_OP = 'UPDATE'") - assert.NotContains(t, sql, "TG_OP = 'DELETE'") -} - -// Cold INSERT routes to duckdb.raw_query, not RAISE EXCEPTION. -func TestGenerateTriggerFuncSQL_ColdInsertRoutesToRawQuery(t *testing.T) { - sql := GenerateTriggerFuncSQL(testCfg) - assert.Contains(t, sql, "duckdb.raw_query") - assert.NotContains(t, sql, "RAISE EXCEPTION") - assert.NotContains(t, sql, "Cannot insert into archived range") - // Cold INSERT must reference the Iceberg table - assert.Contains(t, sql, "ice.default.events") - // jsonb must be serialized to text on the cold-INSERT path (Iceberg - // stores jsonb as VARCHAR). Independent of the view's read-side cast. - assert.Contains(t, sql, `NEW."data"::text`) -} - // nativeCfg has columns whose Iceberg storage is NATIVE (BLOB / DOUBLE) but // which carry a ViewCastType only for the view's PG-parseable hot-side cast. var nativeCfg = ViewConfig{ @@ -112,19 +159,28 @@ var nativeCfg = ViewConfig{ }, } -// Cold-INSERT serialises each value through format()'s %L. bytea must go -// through from_hex(encode(NEW.col,'hex')) — %L renders a bytea as PG's '\xcafe' -// text which DuckDB mis-parses into a BLOB; round-tripping the hex string -// rebuilds the exact bytes. double precision round-trips fine as '2.5' text, so -// it stays native (no ::text). Only VARCHAR-backed json is ::text-serialised. -func TestGenerateTriggerFuncSQL_ColdInsertBlobViaFromHex(t *testing.T) { - sql := GenerateTriggerFuncSQL(nativeCfg) - assert.Contains(t, sql, "from_hex(%L)", "bytea placeholder rebuilds bytes in DuckDB") - assert.Contains(t, sql, `encode(NEW."blob",'hex')`, "bytea value is sent as hex") - assert.NotContains(t, sql, `NEW."blob"::text`, "bytea must not be ::text-stringified") - assert.Contains(t, sql, `NEW."amt"`, "double inserted as-is (text round-trips)") - assert.NotContains(t, sql, `NEW."amt"::text`, "double needs no ::text") - assert.Contains(t, sql, `NEW."doc"::text`, "json IS VARCHAR-backed; serialise to text") +// Only two storage forms need a PG-side cast on the way out to pg_duckdb: the +// VARCHAR-backed rich types and the vector. Everything else exports as-is, and +// a ViewCastType alone does not imply a cast (bytea and double carry one purely +// so the view has a PG-parseable spelling). +func TestColumn_ExportCast(t *testing.T) { + for _, tt := range []struct { + name string + col Column + want string + }{ + {"jsonb is VARCHAR-backed", Column{Type: "VARCHAR", ViewCastType: "json"}, "text"}, + {"interval is VARCHAR-backed", Column{Type: "VARCHAR", ViewCastType: "interval"}, "text"}, + {"plain text needs nothing", Column{Type: "VARCHAR"}, ""}, + {"bytea exports as bytes", Column{Type: "BLOB", ViewCastType: "bytea"}, ""}, + {"double exports as-is", Column{Type: "DOUBLE", ViewCastType: "double precision"}, ""}, + {"integer exports as-is", Column{Type: "INTEGER"}, ""}, + {"vector exports as real[]", Column{Type: "FLOAT[]", ViewCastType: "real[]"}, "real[]"}, + } { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.col.ExportCast()) + }) + } } // The view casts BLOB→bytea / DOUBLE→double precision on BOTH UNION branches @@ -157,126 +213,17 @@ func TestGenerateViewSQL_BootstrapMatchesCutoverHotCasts(t *testing.T) { } } -// Cold INSERT checks the watermark before routing. -func TestGenerateTriggerFuncSQL_ColdInsertWatermarkCheck(t *testing.T) { - sql := GenerateTriggerFuncSQL(testCfg) - assert.Contains(t, sql, "archive_watermark") - assert.Contains(t, sql, "schema_name = 'public'") // watermark lookup is schema-scoped - assert.Contains(t, sql, `NEW."ts" < cutoff`) -} - -// Hot INSERT still goes to _events. -func TestGenerateTriggerFuncSQL_HotInsert(t *testing.T) { - sql := GenerateTriggerFuncSQL(testCfg) - assert.Contains(t, sql, `INSERT INTO "public"."_events"`) - // INSERT column list excludes the identity column - assert.Contains(t, sql, `"ts", "status", "data"`) - // NEW.data arrives as jsonb natively (view exposes jsonb now), so no - // cast is needed on the hot-INSERT path. - assert.Contains(t, sql, `NEW."ts", NEW."status", NEW."data"`) - assert.NotContains(t, sql, `NEW."data"::jsonb`, "NEW.data already jsonb, redundant cast removed") - assert.NotContains(t, sql, `NEW."id"`) -} - -// Non-identity, non-PK column named `id`: included in INSERT col list. -func TestGenerateTriggerFuncSQL_IdIsNotIdentity(t *testing.T) { - cfg := ViewConfig{ - SourceSchema: "public", - SourceTable: "events", - IcebergTable: "ice.default.events", - CutoffTime: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC), - PartitionColumn: "ts", - Columns: []Column{ - {Name: "event_id", Type: "UUID", IsPK: true}, - {Name: "id", Type: "BIGINT"}, - {Name: "ts", Type: "TIMESTAMPTZ"}, - {Name: "status", Type: "VARCHAR"}, - }, - } - sql := GenerateTriggerFuncSQL(cfg) - assert.Contains(t, sql, `"event_id", "id", "ts", "status"`) - assert.Contains(t, sql, `NEW."event_id", NEW."id", NEW."ts", NEW."status"`) -} - -// Composite PK: identity column excluded from INSERT col list. -func TestGenerateTriggerFuncSQL_CompositePK(t *testing.T) { - cfg := ViewConfig{ - SourceSchema: "public", - SourceTable: "events", - IcebergTable: "ice.default.events", - CutoffTime: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC), - PartitionColumn: "ts", - Columns: []Column{ - {Name: "tenant_id", Type: "INTEGER", IsPK: true}, - {Name: "event_id", Type: "BIGINT", IsPK: true, IsIdentity: true}, - {Name: "ts", Type: "TIMESTAMPTZ"}, - {Name: "status", Type: "VARCHAR"}, - }, - } - sql := GenerateTriggerFuncSQL(cfg) - assert.Contains(t, sql, `"tenant_id", "ts", "status"`) - assert.NotContains(t, sql, `NEW."event_id"`) -} - -// No PK: all non-identity columns in INSERT col list. -func TestGenerateTriggerFuncSQL_NoPK(t *testing.T) { - cfg := ViewConfig{ - SourceSchema: "public", - SourceTable: "events", - IcebergTable: "ice.default.events", - CutoffTime: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC), - PartitionColumn: "ts", - Columns: []Column{ - {Name: "ts", Type: "TIMESTAMPTZ"}, - {Name: "status", Type: "VARCHAR"}, - }, - } - sql := GenerateTriggerFuncSQL(cfg) - assert.Contains(t, sql, `"ts", "status"`) - assert.NotContains(t, sql, `"id"`) -} - -// Identity without PK: excluded from INSERT, trigger is still INSERT-only. -func TestGenerateTriggerFuncSQL_IdentityNoPK(t *testing.T) { - cfg := ViewConfig{ - SourceSchema: "public", - SourceTable: "events", - IcebergTable: "ice.default.events", - CutoffTime: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC), - PartitionColumn: "ts", - Columns: []Column{ - {Name: "id", Type: "BIGINT", IsIdentity: true}, - {Name: "ts", Type: "TIMESTAMPTZ"}, - {Name: "status", Type: "VARCHAR"}, - {Name: "data", Type: "VARCHAR", ViewCastType: "json"}, - }, - } - sql := GenerateTriggerFuncSQL(cfg) - assert.NotContains(t, sql, `NEW."id"`) - assert.Contains(t, sql, `"ts", "status", "data"`) -} - -// Trigger fires on INSERT only — no UPDATE or DELETE. -func TestGenerateTriggerSQL_InsertOnly(t *testing.T) { - sql := GenerateTriggerSQL(testCfg) - assert.Contains(t, sql, `DROP TRIGGER IF EXISTS "events_write_trigger" ON "public"."events"`) - assert.Contains(t, sql, `CREATE TRIGGER "events_write_trigger"`) - assert.Contains(t, sql, `INSTEAD OF INSERT ON "public"."events"`) - assert.NotContains(t, sql, "UPDATE") - assert.NotContains(t, sql, "DELETE") -} - func TestRecreate(t *testing.T) { db := &mockDB{} g := NewGenerator(db) err := g.Recreate(context.Background(), testCfg) require.NoError(t, err) - require.Len(t, db.execSQL, 4) + // Swap and view only: the write trigger is coldfront._rebuild_write_trigger's, + // built by the archiver after registration. + require.Len(t, db.execSQL, 2) assert.Contains(t, db.execSQL[0], `ALTER TABLE "public"."events" RENAME TO "_events"`) assert.Contains(t, db.execSQL[1], `"public"."_events"`) assert.Contains(t, db.execSQL[1], "iceberg_scan") - assert.Contains(t, db.execSQL[2], "CREATE OR REPLACE FUNCTION") - assert.Contains(t, db.execSQL[3], "CREATE TRIGGER") } // Complex identifiers: mixed case, hyphens, reserved keywords, embedded diff --git a/mkdocs.yml b/mkdocs.yml index 1695dcc..7eed389 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,8 +65,10 @@ nav: - Overview: architecture.md - Tiered Mode: architecture_tiered.md - Decoupled Mode: architecture_decoupled.md + - Vector Storage: architecture_vectors.md - Using ColdFront: usage.md + - Embeddings: usage_vectors.md - Compaction: compaction.md - Developer Resources: