diff --git a/.changeset/pglite-pg-partman-initial.md b/.changeset/pglite-pg-partman-initial.md new file mode 100644 index 000000000..3bacbf5c5 --- /dev/null +++ b/.changeset/pglite-pg-partman-initial.md @@ -0,0 +1,5 @@ +--- +'@electric-sql/pglite-pg_partman': patch +--- + +pglite-pg_partman: initial deployment diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index bdac57238..28481f418 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -78,6 +78,7 @@ jobs: packages/pglite-postgis/release packages/pglite-pgvector/release packages/pglite-pgmq/release + packages/pglite-pg_partman/release key: wasm-build-${{ steps.postgres-pglite-commit-id.outputs.hash }}-pglite-${{ steps.pglite-version.outputs.version }} - name: Build all if necessary @@ -187,6 +188,14 @@ jobs: path: ./packages/pglite-pgmq/release/** retention-days: 60 + - name: Upload pglite-pg_partman build artifacts to Github artifacts + id: upload-pglite-pg-partman-release-files + uses: actions/upload-artifact@v7 + with: + name: pglite-pg-partman-release-files-node-v20.x + path: ./packages/pglite-pg_partman/release/** + retention-days: 60 + - name: Typecheck pglite working-directory: ${{ github.workspace }}/packages/pglite run: pnpm typecheck @@ -394,6 +403,12 @@ jobs: with: name: pglite-pgmq-release-files-node-v20.x path: ./packages/pglite-pgmq/release/ + + - name: Download pglite-pg_partman build artifacts + uses: actions/download-artifact@v8 + with: + name: pglite-pg-partman-release-files-node-v20.x + path: ./packages/pglite-pg_partman/release/ - name: Install dependencies run: pnpm install --frozen-lockfile @@ -483,6 +498,13 @@ jobs: name: pglite-pgmq-dist-node-v${{ matrix.node }} path: ./packages/pglite-pgmq/dist/* + - name: Upload pglite-pg_partman distribution artifact + id: upload-pglite-pg-partman-dist + uses: actions/upload-artifact@v7 + with: + name: pglite-pg-partman-dist-node-v${{ matrix.node }} + path: ./packages/pglite-pg_partman/dist/* + publish-website-with-demos: name: Publish website with demos runs-on: blacksmith-4vcpu-ubuntu-2204 @@ -576,6 +598,12 @@ jobs: name: pglite-pgmq-dist-node-v20.x path: ./packages/pglite-pgmq/dist/ + - name: Download pglite-pg_partman dist artifacts + uses: actions/download-artifact@v8 + with: + name: pglite-pg-partman-dist-node-v20.x + path: ./packages/pglite-pg_partman/dist/ + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -732,6 +760,12 @@ jobs: name: pglite-pgmq-release-files-node-v20.x path: ./packages/pglite-pgmq/release/ + - name: Download pglite-pg_partman build artifacts + uses: actions/download-artifact@v8 + with: + name: pglite-pg-partman-release-files-node-v20.x + path: ./packages/pglite-pg_partman/release/ + - name: Download pglite-age build artifacts uses: actions/download-artifact@v8 with: diff --git a/docs/extensions/extensions.data.ts b/docs/extensions/extensions.data.ts index 7d098743a..d4bd7cc7f 100644 --- a/docs/extensions/extensions.data.ts +++ b/docs/extensions/extensions.data.ts @@ -681,6 +681,24 @@ const baseExtensions: Extension[] = [ size: 41908, npmjsUrl: 'https://www.npmjs.com/package/@electric-sql/pglite-pgmq', }, + { + name: 'pg_partman', + description: ` + Automated creation and maintenance of time-based and serial-based table + partition sets. The background worker is not included; call + run_maintenance() from your application instead. + + This is an **external** extension, distributed as a separate package at https://www.npmjs.com/package/@electric-sql/pglite-pg_partman + `, + shortDescription: + 'Automated creation and maintenance of table partition sets.', + docs: 'https://github.com/pgpartman/pg_partman', + tags: ['postgres extension', 'external'], + importPath: '@electric-sql/pglite-pg_partman', + importName: 'pg_partman', + size: 1270062, + npmjsUrl: 'https://www.npmjs.com/package/@electric-sql/pglite-pg_partman', + }, ] const tags = [ diff --git a/docs/package.json b/docs/package.json index 34dd712f3..300406c9c 100644 --- a/docs/package.json +++ b/docs/package.json @@ -29,6 +29,7 @@ "@electric-sql/pglite-pg_uuidv7": "workspace:*", "@electric-sql/pglite-pgtap": "workspace:*", "@electric-sql/pglite-pgmq": "workspace:*", + "@electric-sql/pglite-pg_partman": "workspace:*", "@uiw/codemirror-theme-github": "^4.23.0", "dedent": "^1.5.3" } diff --git a/docs/repl/allExtensions.ts b/docs/repl/allExtensions.ts index 30f363551..66eecac80 100644 --- a/docs/repl/allExtensions.ts +++ b/docs/repl/allExtensions.ts @@ -41,3 +41,4 @@ export { uuid_ossp } from '@electric-sql/pglite/contrib/uuid_ossp' export { vector } from '@electric-sql/pglite-pgvector' export { pgmq } from '@electric-sql/pglite-pgmq' export { moddatetime } from '@electric-sql/pglite/contrib/moddatetime' +export { pg_partman } from '@electric-sql/pglite-pg_partman' diff --git a/package.json b/package.json index 36c1eec4c..ae287de9a 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "ts:build": "pnpm -r --filter \"./packages/**\" build", "ts:build:debug": "DEBUG=true pnpm ts:build", "wasm:copy-pgmq": "mkdir -p ./packages/pglite-pgmq/release && cp ./postgres-pglite/dist/extensions/other/pgmq-extension.tar.gz ./packages/pglite-pgmq/release/pgmq.tar.gz", + "wasm:copy-pg_partman": "mkdir -p ./packages/pglite-pg_partman/release && cp ./postgres-pglite/dist/extensions/other/pg_partman.tar.gz ./packages/pglite-pg_partman/release", "wasm:copy-pgvector": "mkdir -p ./packages/pglite-pgvector/release && cp ./postgres-pglite/dist/extensions/other/vector.tar.gz ./packages/pglite-pgvector/release", "wasm:copy-pg_uuidv7": "mkdir -p ./packages/pglite-pg_uuidv7/release && cp ./postgres-pglite/dist/extensions/other/pg_uuidv7.tar.gz ./packages/pglite-pg_uuidv7/release", "wasm:copy-pgtap": "mkdir -p ./packages/pglite-pgtap/release && cp ./postgres-pglite/dist/extensions/other/pgtap.tar.gz ./packages/pglite-pgtap/release", @@ -21,7 +22,7 @@ "wasm:copy-age": "mkdir -p ./packages/pglite-age/release && cp ./postgres-pglite/dist/extensions/other/age.tar.gz ./packages/pglite-age/release", "wasm:copy-pg_textsearch": "mkdir -p ./packages/pglite-pg_textsearch/release && cp ./postgres-pglite/dist/extensions/other/pg_textsearch.tar.gz ./packages/pglite-pg_textsearch/release", "wasm:copy-postgis": "mkdir -p ./packages/pglite-postgis/release && cp ./postgres-pglite/dist/extensions/other/postgis.tar.gz ./packages/pglite-postgis/release", - "wasm:copy-other_extensions": "pnpm wasm:copy-postgis; pnpm wasm:copy-pg_textsearch; pnpm wasm:copy-age; pnpm wasm:copy-pg_hashids; pnpm wasm:copy-pg_ivm; pnpm wasm:copy-pgtap; pnpm wasm:copy-pg_uuidv7; pnpm wasm:copy-pgvector; pnpm wasm:copy-pgmq", + "wasm:copy-other_extensions": "pnpm wasm:copy-postgis; pnpm wasm:copy-pg_textsearch; pnpm wasm:copy-age; pnpm wasm:copy-pg_hashids; pnpm wasm:copy-pg_ivm; pnpm wasm:copy-pgtap; pnpm wasm:copy-pg_uuidv7; pnpm wasm:copy-pgvector; pnpm wasm:copy-pgmq; pnpm wasm:copy-pg_partman", "wasm:copy-initdb": "mkdir -p ./packages/pglite/release && cp ./postgres-pglite/dist/bin/initdb.* ./packages/pglite/release", "wasm:copy-pgdump": "mkdir -p ./packages/pglite-tools/release && cp ./postgres-pglite/dist/bin/pg_dump.* ./packages/pglite-tools/release", "wasm:copy-pglite": "mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/", diff --git a/packages/pglite-pg_partman/.gitignore b/packages/pglite-pg_partman/.gitignore new file mode 100644 index 000000000..ef8abb0e7 --- /dev/null +++ b/packages/pglite-pg_partman/.gitignore @@ -0,0 +1,2 @@ +release/* +dist \ No newline at end of file diff --git a/packages/pglite-pg_partman/README.md b/packages/pglite-pg_partman/README.md new file mode 100644 index 000000000..2f0f6e229 --- /dev/null +++ b/packages/pglite-pg_partman/README.md @@ -0,0 +1,70 @@ +# @electric-sql/pglite-pg_partman + +[pg_partman](https://github.com/pgpartman/pg_partman) extension for +[PGlite](https://pglite.dev). + +Automated creation and maintenance of time-based and serial-based table +partition sets, running entirely in the browser or Node via WASM. + +## Installation + +```bash +npm install @electric-sql/pglite-pg_partman +``` + +## Usage + +```typescript +import { PGlite } from '@electric-sql/pglite' +import { pg_partman } from '@electric-sql/pglite-pg_partman' + +const pg = new PGlite({ + extensions: { + pg_partman, + }, +}) + +// pg_partman is not relocatable and is conventionally installed +// into its own schema +await pg.exec(` + CREATE SCHEMA partman; + CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; +`) + +// register a partitioned table with pg_partman +await pg.exec(` + CREATE TABLE public.events ( + id int, + payload text, + created_at timestamptz NOT NULL DEFAULT now() + ) PARTITION BY RANGE (created_at); + + SELECT partman.create_parent('public.events', 'created_at', '1 day'); +`) + +// periodically create new partitions / apply retention +await pg.exec('SELECT partman.run_maintenance();') +``` + +## Differences from pg_partman on a Postgres server + +- **No background worker.** pg_partman's optional `pg_partman_bgw` scheduler + is a Postgres background worker, which cannot exist in single-process WASM. + This package ships the (officially supported) SQL-only build; the embedding + application decides when to call `run_maintenance()`, for example from a + timer. +- **Run your session in UTC.** pg_partman computes partition boundaries using + the session timezone, and in a browser that defaults to the user's local + timezone. Upstream strongly recommends UTC: + `SET TIME ZONE 'UTC';` +- Procedures that manage their own transactions (`run_maintenance_proc()`, + `partition_data_proc()`, ...) work when `CALL`ed outside a transaction + block, but not inside `pg.transaction()`. The plain function equivalents + (`run_maintenance()`, `partition_data_time()`, ...) work everywhere. + +See the [pg_partman documentation](https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md) +for the full API. + +## License + +Apache-2.0 diff --git a/packages/pglite-pg_partman/eslint.config.js b/packages/pglite-pg_partman/eslint.config.js new file mode 100644 index 000000000..e1dc3262d --- /dev/null +++ b/packages/pglite-pg_partman/eslint.config.js @@ -0,0 +1,21 @@ +import globals from 'globals' +import rootConfig from '../../eslint.config.js' + +export default [ + ...rootConfig, + { + ignores: ['release/**/*', 'dist/**/*'], + }, + { + languageOptions: { + globals: { + ...globals.browser, + ...globals.node, + }, + }, + rules: { + ...rootConfig.rules, + '@typescript-eslint/no-explicit-any': 'off', + }, + }, +] \ No newline at end of file diff --git a/packages/pglite-pg_partman/package.json b/packages/pglite-pg_partman/package.json new file mode 100644 index 000000000..de94e3dcb --- /dev/null +++ b/packages/pglite-pg_partman/package.json @@ -0,0 +1,64 @@ +{ + "name": "@electric-sql/pglite-pg_partman", + "version": "0.0.1", + "description": "pg_partman extension for PGlite", + "author": "Electric DB Limited", + "homepage": "https://pglite.dev", + "license": "Apache-2.0", + "keywords": [ + "postgres", + "sql", + "database", + "wasm", + "pglite", + "pg_partman" + ], + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/electric-sql/pglite.git", + "directory": "packages/pglite-pg_partman" + }, + "scripts": { + "build": "tsup && tsx scripts/bundle-wasm.ts", + "check:exports": "attw . --pack --profile node16", + "lint": "eslint ./src ./tests --report-unused-disable-directives --max-warnings 0", + "format": "prettier --write ./src ./tests", + "typecheck": "tsc", + "stylecheck": "pnpm lint && prettier --check ./src ./tests", + "test": "vitest", + "prepublishOnly": "pnpm check:exports" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.18.1", + "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-utils": "workspace:*", + "@types/node": "^20.16.11", + "vitest": "^2.1.2" + }, + "peerDependencies": { + "@electric-sql/pglite": "workspace:*" + } +} diff --git a/packages/pglite-pg_partman/scripts/bundle-wasm.ts b/packages/pglite-pg_partman/scripts/bundle-wasm.ts new file mode 100644 index 000000000..0c17886e1 --- /dev/null +++ b/packages/pglite-pg_partman/scripts/bundle-wasm.ts @@ -0,0 +1,8 @@ +import { copyFiles, findAndReplaceInDir } from '@electric-sql/pglite-utils/scripts/fileUtils' + +async function main() { + await copyFiles('./release', './dist') + await findAndReplaceInDir('./dist', /\.\.\/release\//g, './', ['.js', '.cjs']) +} + +await main() diff --git a/packages/pglite-pg_partman/src/index.ts b/packages/pglite-pg_partman/src/index.ts new file mode 100644 index 000000000..258053a53 --- /dev/null +++ b/packages/pglite-pg_partman/src/index.ts @@ -0,0 +1,17 @@ +import type { + Extension, + ExtensionSetupResult, + PGliteInterface, +} from '@electric-sql/pglite' + +const setup = async (_pg: PGliteInterface, emscriptenOpts: any) => { + return { + emscriptenOpts, + bundlePath: new URL('../release/pg_partman.tar.gz', import.meta.url), + } satisfies ExtensionSetupResult +} + +export const pg_partman = { + name: 'pg_partman', + setup, +} satisfies Extension diff --git a/packages/pglite-pg_partman/tests/pg_partman.test.ts b/packages/pglite-pg_partman/tests/pg_partman.test.ts new file mode 100644 index 000000000..a8ddd0540 --- /dev/null +++ b/packages/pglite-pg_partman/tests/pg_partman.test.ts @@ -0,0 +1,155 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { PGlite } from '@electric-sql/pglite' +import { pg_partman } from '../src/index.js' + +describe('pg_partman', () => { + let pg: PGlite + let dataDirArchive: File | Blob + beforeEach(async () => { + // The first test pays initdb + extension install once and snapshots the + // cluster; restoring it is faster and proves the extension survives a dump/restore. + if (!dataDirArchive) { + pg = await PGlite.create({ + extensions: { pg_partman }, + }) + await pg.exec( + 'CREATE SCHEMA partman; CREATE EXTENSION pg_partman SCHEMA partman;', + ) + dataDirArchive = await pg.dumpDataDir('gzip') + } else { + pg = await PGlite.create({ + extensions: { pg_partman }, + loadDataDir: dataDirArchive, + }) + } + // pg_partman derives partition boundaries from the session timezone; + // pin it so child-table suffixes are deterministic + await pg.exec(`SET TIME ZONE 'UTC';`) + }) + afterEach(async () => { + if (!pg.closed) { + await pg.close() + } + }) + + const countPartitions = async (parent: string) => { + const res = await pg.query<{ n: number }>( + `SELECT count(*)::int AS n FROM partman.show_partitions('${parent}')`, + ) + return res.rows[0].n + } + + it('can load extension', async () => { + const res = await pg.query<{ extname: string; extversion: string }>(` + SELECT extname, extversion + FROM pg_extension + WHERE extname = 'pg_partman' + `) + + expect(res.rows).toHaveLength(1) + expect(res.rows[0].extversion).toBe('5.5.0') + }) + + it('creates a time-partitioned set with premade children, a default partition and a template table', async () => { + await pg.exec(` + CREATE SCHEMA partman_test; + CREATE TABLE partman_test.time_tbl + (col1 int, col2 text, col3 timestamptz NOT NULL DEFAULT now()) + PARTITION BY RANGE (col3); + SELECT partman.create_parent('partman_test.time_tbl', 'col3', '1 day'); + `) + + // default premake is 4: four days back, today, four days ahead + expect(await countPartitions('partman_test.time_tbl')).toBe(9) + + const defaultPartition = await pg.query(` + SELECT 1 FROM pg_tables + WHERE schemaname = 'partman_test' AND tablename = 'time_tbl_default' + `) + expect(defaultPartition.rows).toHaveLength(1) + + const templateTable = await pg.query(` + SELECT 1 FROM pg_tables + WHERE schemaname = 'partman' AND tablename = 'template_partman_test_time_tbl' + `) + expect(templateTable.rows).toHaveLength(1) + }) + + it('run_maintenance premakes new time partitions as data arrives', async () => { + await pg.exec(` + CREATE SCHEMA partman_test; + CREATE TABLE partman_test.time_tbl + (col1 int, col3 timestamptz NOT NULL DEFAULT now()) + PARTITION BY RANGE (col3); + SELECT partman.create_parent('partman_test.time_tbl', 'col3', '1 day'); + INSERT INTO partman_test.time_tbl (col1) VALUES (1); + UPDATE partman.part_config SET premake = 6 + WHERE parent_table = 'partman_test.time_tbl'; + SELECT partman.run_maintenance(); + `) + + expect(await countPartitions('partman_test.time_tbl')).toBe(11) + }) + + it('run_maintenance detaches and drops partitions past the retention window', async () => { + await pg.exec(` + CREATE SCHEMA partman_test; + CREATE TABLE partman_test.time_tbl + (col1 int, col3 timestamptz NOT NULL DEFAULT now()) + PARTITION BY RANGE (col3); + SELECT partman.create_parent('partman_test.time_tbl', 'col3', '1 day'); + `) + const before = await countPartitions('partman_test.time_tbl') + + await pg.exec(` + UPDATE partman.part_config + SET retention = '2 days', retention_keep_table = false + WHERE parent_table = 'partman_test.time_tbl'; + SELECT partman.run_maintenance(); + `) + + // of the four premade days in the past, the two beyond the window go away + expect(await countPartitions('partman_test.time_tbl')).toBe(before - 2) + }) + + it('creates an id-partitioned set and premakes ahead of the max id', async () => { + await pg.exec(` + CREATE SCHEMA partman_test; + CREATE TABLE partman_test.id_tbl + (col1 bigint NOT NULL, col2 text) + PARTITION BY RANGE (col1); + SELECT partman.create_parent('partman_test.id_tbl', 'col1', '10'); + `) + const before = await countPartitions('partman_test.id_tbl') + expect(before).toBeGreaterThan(0) + + await pg.exec(` + INSERT INTO partman_test.id_tbl + SELECT g, 'row ' || g FROM generate_series(1, 35) g; + SELECT partman.run_maintenance(); + `) + + expect(await countPartitions('partman_test.id_tbl')).toBeGreaterThan(before) + }) + + it('show_partition_name locates the child table for a value', async () => { + await pg.exec(` + CREATE SCHEMA partman_test; + CREATE TABLE partman_test.id_tbl + (col1 bigint NOT NULL, col2 text) + PARTITION BY RANGE (col1); + SELECT partman.create_parent('partman_test.id_tbl', 'col1', '10'); + `) + + const res = await pg.query<{ + partition_table: string + table_exists: boolean + }>(` + SELECT partition_table, table_exists + FROM partman.show_partition_name('partman_test.id_tbl', '15') + `) + + expect(res.rows[0].partition_table).toBe('id_tbl_p10') + expect(res.rows[0].table_exists).toBe(true) + }) +}) diff --git a/packages/pglite-pg_partman/tsconfig.json b/packages/pglite-pg_partman/tsconfig.json new file mode 100644 index 000000000..27611a8fd --- /dev/null +++ b/packages/pglite-pg_partman/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src", "tsup.config.ts", "vitest.config.ts"] +} \ No newline at end of file diff --git a/packages/pglite-pg_partman/tsup.config.ts b/packages/pglite-pg_partman/tsup.config.ts new file mode 100644 index 000000000..cf946c789 --- /dev/null +++ b/packages/pglite-pg_partman/tsup.config.ts @@ -0,0 +1,28 @@ +import { cpSync } from 'fs' +import { resolve } from 'path' +import { defineConfig } from 'tsup' + +const entryPoints = ['src/index.ts'] + +const minify = process.env.DEBUG === 'true' ? false : true + +export default defineConfig([ + { + entry: entryPoints, + sourcemap: true, + dts: { + entry: entryPoints, + resolve: true, + }, + clean: true, + minify: minify, + shims: true, + format: ['esm', 'cjs'], + onSuccess: async () => { + cpSync( + resolve('release/pg_partman.tar.gz'), + resolve('dist/pg_partman.tar.gz'), + ) + }, + }, +]) diff --git a/packages/pglite-pg_partman/vitest.config.ts b/packages/pglite-pg_partman/vitest.config.ts new file mode 100644 index 000000000..378f02808 --- /dev/null +++ b/packages/pglite-pg_partman/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + testTimeout: 30000, + }, +}) \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5873937f0..ef968cf16 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: '@electric-sql/pglite-pg_ivm': specifier: workspace:* version: link:../packages/pglite-pg_ivm + '@electric-sql/pglite-pg_partman': + specifier: workspace:* + version: link:../packages/pglite-pg_partman '@electric-sql/pglite-pg_textsearch': specifier: workspace:* version: link:../packages/pglite-pg_textsearch @@ -313,6 +316,24 @@ importers: specifier: ^2.1.2 version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-pg_partman: + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.18.1 + version: 0.18.1 + '@electric-sql/pglite': + specifier: workspace:* + version: link:../pglite + '@electric-sql/pglite-utils': + specifier: workspace:* + version: link:../pglite-utils + '@types/node': + specifier: ^20.16.11 + version: 20.16.11 + vitest: + specifier: ^2.1.2 + version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-pg_textsearch: devDependencies: '@arethetypeswrong/cli':