From dbc5d07acbcf8dc386b55cf43bab1ccf66c8325c Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:09:43 +0200 Subject: [PATCH 1/7] test: add integration suite against real PostgreSQL and MySQL Executes queries built by every public API (params, glue, map, quoteIdent, unsafe, nested statements, append, nulls, special chars, injection-as-literal) against live PostgreSQL and MySQL (both mysql and mysql2 drivers), proving the generated SQL is valid and that interpolated values are stored as literals. - docker-compose: add mysql:8.0 service + healthchecks, bump postgres to 17 - package.json: add mysql/mysql2 devDeps and db:up/db:down/test:integration scripts - ci: add MySQL service and integration test step - README: document the integration commands --- .github/workflows/ci.yml | 15 ++ README.md | 26 ++- docker-compose.yml | 24 ++- integration/SQL.mysql.integration.test.js | 18 ++ integration/SQL.mysql2.integration.test.js | 18 ++ integration/SQL.pg.integration.test.js | 18 ++ integration/helpers/config.js | 26 +++ integration/helpers/db.js | 114 +++++++++++++ integration/shared/featureSuite.js | 186 +++++++++++++++++++++ package.json | 6 + 10 files changed, 443 insertions(+), 8 deletions(-) create mode 100644 integration/SQL.mysql.integration.test.js create mode 100644 integration/SQL.mysql2.integration.test.js create mode 100644 integration/SQL.pg.integration.test.js create mode 100644 integration/helpers/config.js create mode 100644 integration/helpers/db.js create mode 100644 integration/shared/featureSuite.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c759340..ea3ac07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,11 +35,24 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: sqlmap + ports: + - 3306:3306 + options: --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent" + --health-interval 10s --health-timeout 5s --health-retries 10 env: PGPASS: postgres PGUSER: postgres PGDB: sqlmap PGHOST: localhost + MYSQLHOST: 127.0.0.1 + MYSQLUSER: root + MYSQLPASS: mysql + MYSQLDB: sqlmap steps: - name: Checkout source code uses: actions/checkout@v6 @@ -53,6 +66,8 @@ jobs: run: npm run lint - name: Test run: npm test + - name: Test Integration + run: npm run test:integration - name: Test Security run: npm run test:security automerge: diff --git a/README.md b/README.md index 4452f1d..f725729 100644 --- a/README.md +++ b/README.md @@ -289,13 +289,29 @@ const pascalOrCamelToSnake = str => This module can be tested and reported on in a variety of ways... ```sh -npm run test # runs tap based unit test suite. -npm run test:security # runs sqlmap security tests. -npm run test:typescript # runs type definition tests. -npm run coverage # generates a coverage report in docs dir. -npm run lint # lints via standardJS. +npm run test # runs the node:test unit test suite. +npm run test:integration # runs integration tests against running PostgreSQL & MySQL. +npm run test:integration:docker # spins up DBs via docker compose, runs integration tests, tears down. +npm run test:security # runs sqlmap security tests. +npm run test:typescript # runs type definition tests. +npm run coverage # generates a coverage report in docs dir. +npm run lint # lints via standardJS. ``` +The integration suite executes queries built by every public API against real +PostgreSQL and MySQL instances (the latter via both the `mysql` and `mysql2` +drivers), verifying the generated SQL is valid and that interpolated values are +stored as literals. To run it locally: + +```sh +npm run db:up # start postgres + mysql via docker compose (waits for healthy) +npm run test:integration # run the suite +npm run db:down # tear down +``` + +Connection settings default to the docker-compose services and can be overridden +with `PGHOST/PGPORT/PGUSER/PGPASS/PGDB` and `MYSQLHOST/MYSQLPORT/MYSQLUSER/MYSQLPASS/MYSQLDB`. + ## Benchmark Find more about `@nearform/sql` speed [here](benchmark) diff --git a/docker-compose.yml b/docker-compose.yml index caf1663..355eed7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,28 @@ -version: '3.4' +version: '3.9' services: - db: - image: postgres:9-alpine + postgres: + image: postgres:17-alpine environment: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres POSTGRES_DB: sqlmap ports: - 5432:5432 + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U postgres'] + interval: 5s + timeout: 5s + retries: 10 + + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: sqlmap + ports: + - 3306:3306 + healthcheck: + test: ['CMD-SHELL', 'mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent'] + interval: 5s + timeout: 5s + retries: 10 diff --git a/integration/SQL.mysql.integration.test.js b/integration/SQL.mysql.integration.test.js new file mode 100644 index 0000000..da6c3e2 --- /dev/null +++ b/integration/SQL.mysql.integration.test.js @@ -0,0 +1,18 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { withMysql, createUsersTable } = require('./helpers/db') +const runFeatureSuite = require('./shared/featureSuite') + +const ctx = { db: null } + +before(async () => { + ctx.db = await withMysql() + await createUsersTable(ctx.db) +}) + +after(async () => { + if (ctx.db) await ctx.db.end() +}) + +runFeatureSuite(test, () => ctx.db) diff --git a/integration/SQL.mysql2.integration.test.js b/integration/SQL.mysql2.integration.test.js new file mode 100644 index 0000000..358e02b --- /dev/null +++ b/integration/SQL.mysql2.integration.test.js @@ -0,0 +1,18 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { withMysql2, createUsersTable } = require('./helpers/db') +const runFeatureSuite = require('./shared/featureSuite') + +const ctx = { db: null } + +before(async () => { + ctx.db = await withMysql2() + await createUsersTable(ctx.db) +}) + +after(async () => { + if (ctx.db) await ctx.db.end() +}) + +runFeatureSuite(test, () => ctx.db) diff --git a/integration/SQL.pg.integration.test.js b/integration/SQL.pg.integration.test.js new file mode 100644 index 0000000..ee69126 --- /dev/null +++ b/integration/SQL.pg.integration.test.js @@ -0,0 +1,18 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { withPg, createUsersTable } = require('./helpers/db') +const runFeatureSuite = require('./shared/featureSuite') + +const ctx = { db: null } + +before(async () => { + ctx.db = await withPg() + await createUsersTable(ctx.db) +}) + +after(async () => { + if (ctx.db) await ctx.db.end() +}) + +runFeatureSuite(test, () => ctx.db) diff --git a/integration/helpers/config.js b/integration/helpers/config.js new file mode 100644 index 0000000..39a44d2 --- /dev/null +++ b/integration/helpers/config.js @@ -0,0 +1,26 @@ +'use strict' + +// PostgreSQL connection config. +// Mirrors the env-var convention already used by sqlmap/config.js so the same +// CI environment variables drive both suites. +const pg = { + user: process.env.PGUSER || 'postgres', + host: process.env.PGHOST || 'localhost', + database: process.env.PGDB || 'sqlmap', + password: process.env.PGPASS || 'postgres', + port: Number(process.env.PGPORT) || 5432 +} + +// MySQL connection config, shared by both the `mysql` and `mysql2` drivers. +const mysql = { + host: process.env.MYSQLHOST || 'localhost', + port: Number(process.env.MYSQLPORT) || 3306, + user: process.env.MYSQLUSER || 'root', + password: process.env.MYSQLPASS || 'mysql', + database: process.env.MYSQLDB || 'sqlmap', + // utf8mb4 so 4-byte characters (e.g. emoji) round-trip on both the + // `mysql` driver (defaults to 3-byte utf8) and `mysql2`. + charset: 'utf8mb4' +} + +module.exports = { pg, mysql } diff --git a/integration/helpers/db.js b/integration/helpers/db.js new file mode 100644 index 0000000..d829aa4 --- /dev/null +++ b/integration/helpers/db.js @@ -0,0 +1,114 @@ +'use strict' + +const config = require('./config') + +// Each adapter exposes the same shape so the shared feature suite is +// driver-agnostic: +// - dialect: 'pg' | 'mysql' (drives dialect-specific SQL in the suite) +// - query(stmt): runs a SqlStatement, returns the result rows +// - raw(text): runs a plain SQL string (used for DDL / cleanup) +// - end(): closes the connection + +// PostgreSQL via `pg`. A SqlStatement is passed straight to client.query() +// because it exposes `.text` and `.values` getters — the documented usage. +async function withPg () { + const { Client } = require('pg') + const client = new Client(config.pg) + await client.connect() + return { + dialect: 'pg', + async query (stmt) { + const res = await client.query(stmt) + return res.rows + }, + async raw (text) { + const res = await client.query(text) + return res.rows + }, + async end () { + await client.end() + } + } +} + +// MySQL via `mysql2/promise`. The driver reads `sql` and `values` off the +// options object, which map directly onto a SqlStatement's `.sql`/`.values`. +async function withMysql2 () { + const mysql = require('mysql2/promise') + const conn = await mysql.createConnection(config.mysql) + return { + dialect: 'mysql', + async query (stmt) { + const [rows] = await conn.query({ sql: stmt.sql, values: stmt.values }) + return rows + }, + async raw (text) { + const [rows] = await conn.query(text) + return rows + }, + async end () { + await conn.end() + } + } +} + +// The legacy `mysql` driver cannot speak MySQL 8's default +// `caching_sha2_password` auth. mysql2 can, so we use it to switch the account +// to `mysql_native_password` first. This works identically locally and in CI +// without needing a container `command` override (unsupported by GitHub +// Actions service containers). +async function ensureNativePassword () { + const mysql = require('mysql2/promise') + const conn = await mysql.createConnection(config.mysql) + try { + // We connect over TCP, so the account is `@'%'`. `?` placeholders are + // escaped client-side (text protocol) into a valid `'user'@'%'` account. + await conn.query( + "ALTER USER ?@'%' IDENTIFIED WITH mysql_native_password BY ?", + [config.mysql.user, config.mysql.password] + ) + } finally { + await conn.end() + } +} + +// MySQL via the original callback-based `mysql` driver, promisified. +async function withMysql () { + await ensureNativePassword() + const mysql = require('mysql') + const conn = mysql.createConnection(config.mysql) + await new Promise((resolve, reject) => + conn.connect(err => (err ? reject(err) : resolve())) + ) + const run = (sql, values) => + new Promise((resolve, reject) => + conn.query(sql, values, (err, rows) => (err ? reject(err) : resolve(rows))) + ) + return { + dialect: 'mysql', + async query (stmt) { + return run(stmt.sql, stmt.values) + }, + async raw (text) { + return run(text) + }, + async end () { + await new Promise(resolve => conn.end(() => resolve())) + } + } +} + +// Fresh, portable `users` table. Explicit (non-auto) integer ids keep the +// inserts identical across dialects. +async function createUsersTable (db) { + await db.raw('DROP TABLE IF EXISTS users') + await db.raw(`CREATE TABLE users ( + id INTEGER PRIMARY KEY, + username VARCHAR(255) NOT NULL, + email VARCHAR(255), + password VARCHAR(255), + metadata VARCHAR(255) + )`) +} + +module.exports = { withPg, withMysql, withMysql2, createUsersTable } diff --git a/integration/shared/featureSuite.js b/integration/shared/featureSuite.js new file mode 100644 index 0000000..2dc52bf --- /dev/null +++ b/integration/shared/featureSuite.js @@ -0,0 +1,186 @@ +'use strict' + +const assert = require('node:assert') +const SQL = require('../../SQL') + +// Runs the full @nearform/sql feature matrix against a live database. +// Every case builds a query with the public API and EXECUTES it, then asserts +// on the rows read back — proving the generated SQL is both valid and safe. +// +// test - the node:test `test` function from the calling file +// getDb - returns the connected adapter (see helpers/db.js). It's a getter +// because the connection is opened in the file's `before` hook, +// after this function has registered its subtests. +module.exports = function runFeatureSuite (test, getDb) { + const reset = db => db.raw('DELETE FROM users') + + test('basic parameterized insert/select round-trips', async () => { + const db = getDb() + await reset(db) + await db.query(SQL` + INSERT INTO users (id, username, email, password) + VALUES (${1}, ${'alice'}, ${'alice@example.com'}, ${'secret'}) + `) + const rows = await db.query(SQL` + SELECT id, username, email, password FROM users WHERE id = ${1} + `) + assert.equal(rows.length, 1) + assert.equal(rows[0].username, 'alice') + assert.equal(rows[0].email, 'alice@example.com') + assert.equal(rows[0].password, 'secret') + }) + + test('interpolated values are stored as literals, not executed (injection)', async () => { + const db = getDb() + await reset(db) + const evil = "Robert'); DROP TABLE users;--" + await db.query(SQL`INSERT INTO users (id, username) VALUES (${2}, ${evil})`) + const rows = await db.query(SQL`SELECT username FROM users WHERE id = ${2}`) + assert.equal(rows[0].username, evil) + // The table must still exist and hold exactly the row we inserted. + const all = await db.query(SQL`SELECT id FROM users`) + assert.equal(all.length, 1) + }) + + test('glue builds a working IN clause', async () => { + const db = getDb() + await reset(db) + for (const id of [1, 2, 3, 4]) { + await db.query(SQL`INSERT INTO users (id, username) VALUES (${id}, ${'u' + id})`) + } + const ids = [1, 3] + const rows = await db.query(SQL` + SELECT id FROM users + WHERE id IN (${SQL.glue(ids.map(id => SQL`${id}`), ' , ')}) + ORDER BY id + `) + assert.deepEqual(rows.map(r => Number(r.id)), [1, 3]) + }) + + test('glue builds a working batch insert', async () => { + const db = getDb() + await reset(db) + const users = [ + { id: 10, name: 'u10' }, + { id: 11, name: 'u11' }, + { id: 12, name: 'u12' } + ] + await db.query(SQL` + INSERT INTO users (id, username) + VALUES ${SQL.glue(users.map(u => SQL`(${u.id}, ${u.name})`), ' , ')} + `) + const rows = await db.query(SQL`SELECT id, username FROM users ORDER BY id`) + assert.equal(rows.length, 3) + assert.deepEqual(rows.map(r => r.username), ['u10', 'u11', 'u12']) + }) + + test('map builds a working IN clause (default and object mapper)', async () => { + const db = getDb() + await reset(db) + for (const id of [21, 22, 23, 24]) { + await db.query(SQL`INSERT INTO users (id, username) VALUES (${id}, ${'u' + id})`) + } + // default mapper over an array of scalars + const rows = await db.query(SQL` + SELECT id FROM users WHERE id IN (${SQL.map([21, 23])}) ORDER BY id + `) + assert.deepEqual(rows.map(r => Number(r.id)), [21, 23]) + + // explicit mapper over an array of objects + const objs = [{ id: 22 }, { id: 24 }] + const rows2 = await db.query(SQL` + SELECT id FROM users WHERE id IN (${SQL.map(objs, o => o.id)}) ORDER BY id + `) + assert.deepEqual(rows2.map(r => Number(r.id)), [22, 24]) + }) + + test('quoteIdent produces valid quoted identifiers (incl. escaping)', async () => { + const db = getDb() + // dynamic column + table name on the existing table + await reset(db) + await db.query(SQL`INSERT INTO ${SQL.quoteIdent('users')} (id, username) VALUES (${80}, ${'quoted'})`) + const rows = await db.query(SQL` + SELECT ${SQL.quoteIdent('username')} FROM ${SQL.quoteIdent('users')} WHERE id = ${80} + `) + assert.equal(rows[0].username, 'quoted') + + // an identifier that is INVALID unquoted (contains a space) and one that + // contains the dialect's own quote char — exercises the escaping path. + const weird = db.dialect === 'pg' ? 'we"ird table' : 'we`ird table' + await db.raw(`DROP TABLE IF EXISTS ${quoteRaw(db.dialect, weird)}`) + await db.query(SQL`CREATE TABLE ${SQL.quoteIdent(weird)} (id INTEGER PRIMARY KEY)`) + try { + await db.query(SQL`INSERT INTO ${SQL.quoteIdent(weird)} (id) VALUES (${99})`) + const wrows = await db.query(SQL`SELECT id FROM ${SQL.quoteIdent(weird)} WHERE id = ${99}`) + assert.equal(Number(wrows[0].id), 99) + } finally { + await db.raw(`DROP TABLE IF EXISTS ${quoteRaw(db.dialect, weird)}`) + } + }) + + test('unsafe interpolates a trusted fragment literally', async () => { + const db = getDb() + await reset(db) + const columns = ['id', 'username'] // trusted, not user input + await db.query(SQL` + INSERT INTO users (${SQL.unsafe(columns.join(', '))}) + VALUES (${30}, ${'unsafe-user'}) + `) + const rows = await db.query(SQL`SELECT username FROM users WHERE id = ${30}`) + assert.equal(rows[0].username, 'unsafe-user') + }) + + test('nested SqlStatement keeps placeholder numbering correct', async () => { + const db = getDb() + await reset(db) + await db.query(SQL` + INSERT INTO users (id, username, email) + VALUES (${40}, ${'bob'}, ${'bob@example.com'}) + `) + // For pg this generates `... id = $1 AND (username = $2 AND email = $3)`; + // executing it proves the $N offset logic is correct against a real server. + const condition = SQL`username = ${'bob'} AND email = ${'bob@example.com'}` + const rows = await db.query(SQL` + SELECT id FROM users WHERE id = ${40} AND (${condition}) + `) + assert.equal(rows.length, 1) + assert.equal(Number(rows[0].id), 40) + }) + + test('deprecated append builds an executable query', async () => { + const db = getDb() + await reset(db) + await db.query(SQL`INSERT INTO users (id, username) VALUES (${50}, ${'appended'})`) + const sql = SQL`SELECT id, username FROM users WHERE username = ${'appended'}` + sql.append(SQL` AND id = ${50}`) + const rows = await db.query(sql) + assert.equal(rows.length, 1) + assert.equal(Number(rows[0].id), 50) + }) + + test('null values round-trip', async () => { + const db = getDb() + await reset(db) + await db.query(SQL` + INSERT INTO users (id, username, metadata) VALUES (${60}, ${'nulluser'}, ${null}) + `) + const rows = await db.query(SQL`SELECT metadata FROM users WHERE id = ${60}`) + assert.equal(rows[0].metadata, null) + }) + + test('special characters and unicode round-trip intact', async () => { + const db = getDb() + await reset(db) + const weirdVal = 'o\'brien "the\\great" 😀 ®' + await db.query(SQL`INSERT INTO users (id, username) VALUES (${70}, ${weirdVal})`) + const rows = await db.query(SQL`SELECT username FROM users WHERE id = ${70}`) + assert.equal(rows[0].username, weirdVal) + }) +} + +// Quote an identifier for raw (non-SqlStatement) DDL — same rule as +// quoteIdentifier.js, used only for setup/teardown of the "weird" table. +function quoteRaw (dialect, value) { + const q = dialect === 'mysql' ? '`' : '"' + return q + value.replace(new RegExp(q, 'g'), q + q) + q +} diff --git a/package.json b/package.json index 82cf3c6..69c9070 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ "scripts": { "test": "node --test *.test.js", "posttest": "tstyche", + "db:up": "docker compose up -d --wait", + "db:down": "docker compose down -v", + "test:integration": "node --test --test-concurrency=1 integration/**/*.test.js", + "test:integration:docker": "npm run db:up && npm run test:integration; npm run db:down", "test:security": "node ./sqlmap/sqlmap.js", "test:typescript": "tstyche", "pretest:security": "git clone --depth=1 https://github.com/sqlmapproject/sqlmap node_modules/sqlmap && node ./sqlmap/db-init.js", @@ -29,6 +33,8 @@ "benchmark": "^2.1.4", "fastify": "^5.8.5", "jsonfile": "^6.2.1", + "mysql": "^2.18.1", + "mysql2": "^3.11.0", "pg": "^8.20.0", "sql-template-strings": "^2.2.2", "standard": "^17.1.2", From 20f794f0f487b5d3c96300389debfa089a45cd32 Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:11:14 +0200 Subject: [PATCH 2/7] fix(deps): update mysql2, pg, and tstyche dependencies --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 69c9070..b5223ea 100644 --- a/package.json +++ b/package.json @@ -34,11 +34,11 @@ "fastify": "^5.8.5", "jsonfile": "^6.2.1", "mysql": "^2.18.1", - "mysql2": "^3.11.0", - "pg": "^8.20.0", + "mysql2": "^3.22.5", + "pg": "^8.21.0", "sql-template-strings": "^2.2.2", "standard": "^17.1.2", - "tstyche": "^7.1.0" + "tstyche": "^7.2.1" }, "standard": { "ignore": [ From ddf5e52f83c384c4a02bf74cc9403079caa42bed Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:14:31 +0200 Subject: [PATCH 3/7] fix(ci): update Node.js versions in CI matrix --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea3ac07..af8878a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,8 @@ jobs: strategy: matrix: node-version: - - 20 - - 22 - 24 + - 26 os: - ubuntu-latest postgres-version: From 4e1b06f8f92389f21435a767ca5fd3653512d493 Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:17:53 +0200 Subject: [PATCH 4/7] ci: split PostgreSQL and MySQL integration tests into separate matrices Run pg and mysql integration suites in two independent jobs, each with its own DB service and matrix, instead of spinning up MySQL for every Postgres version. Add granular test:integration:pg / test:integration:mysql scripts. --- .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++---------- package.json | 2 ++ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af8878a..afd5909 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: branches: - master jobs: - build-lint-test: + postgres: runs-on: ${{ matrix.os }} strategy: matrix: @@ -34,8 +34,46 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + env: + PGPASS: postgres + PGUSER: postgres + PGDB: sqlmap + PGHOST: localhost + steps: + - name: Checkout source code + uses: actions/checkout@v6 + - name: Use Node.js + uses: actions/setup-node@v6 + with: + node-version: ${{ matrix.node-version }} + - name: Install + run: npm install + - name: Lint + run: npm run lint + - name: Test + run: npm test + - name: Test Integration (PostgreSQL) + run: npm run test:integration:pg + - name: Test Security + run: npm run test:security + mysql: + runs-on: ${{ matrix.os }} + strategy: + matrix: + node-version: + - 24 + - 26 + os: + - ubuntu-latest + # Pinned to 8.0: the legacy `mysql` driver requires the + # mysql_native_password plugin, which 8.4+ disables by default and + # can't be re-enabled via a service container (no command override). + mysql-version: + - "8.0" + name: "Node ${{ matrix.node-version }} - MySQL ${{ matrix.mysql-version }}" + services: mysql: - image: mysql:8.0 + image: mysql:${{ matrix.mysql-version }} env: MYSQL_ROOT_PASSWORD: mysql MYSQL_DATABASE: sqlmap @@ -44,10 +82,6 @@ jobs: options: --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent" --health-interval 10s --health-timeout 5s --health-retries 10 env: - PGPASS: postgres - PGUSER: postgres - PGDB: sqlmap - PGHOST: localhost MYSQLHOST: 127.0.0.1 MYSQLUSER: root MYSQLPASS: mysql @@ -61,16 +95,12 @@ jobs: node-version: ${{ matrix.node-version }} - name: Install run: npm install - - name: Lint - run: npm run lint - - name: Test - run: npm test - - name: Test Integration - run: npm run test:integration - - name: Test Security - run: npm run test:security + - name: Test Integration (MySQL) + run: npm run test:integration:mysql automerge: - needs: build-lint-test + needs: + - postgres + - mysql runs-on: ubuntu-latest permissions: pull-requests: write diff --git a/package.json b/package.json index b5223ea..9a6c830 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "db:up": "docker compose up -d --wait", "db:down": "docker compose down -v", "test:integration": "node --test --test-concurrency=1 integration/**/*.test.js", + "test:integration:pg": "node --test integration/SQL.pg.integration.test.js", + "test:integration:mysql": "node --test --test-concurrency=1 integration/SQL.mysql.integration.test.js integration/SQL.mysql2.integration.test.js", "test:integration:docker": "npm run db:up && npm run test:integration; npm run db:down", "test:security": "node ./sqlmap/sqlmap.js", "test:typescript": "tstyche", From cca3c8e8b06b98427e70490c3ce5d49f5f0c601f Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:19:17 +0200 Subject: [PATCH 5/7] fix(ci): update MySQL version in CI and docker-compose to 9 --- .github/workflows/ci.yml | 3 ++- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afd5909..6e7ecd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,8 @@ jobs: # mysql_native_password plugin, which 8.4+ disables by default and # can't be re-enabled via a service container (no command override). mysql-version: - - "8.0" + - 8 + - 9 name: "Node ${{ matrix.node-version }} - MySQL ${{ matrix.mysql-version }}" services: mysql: diff --git a/docker-compose.yml b/docker-compose.yml index 355eed7..0e5c432 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: retries: 10 mysql: - image: mysql:8.0 + image: mysql:9 environment: MYSQL_ROOT_PASSWORD: mysql MYSQL_DATABASE: sqlmap From 2fc0faa64250e7d805c0fb61a997ce21f3c2f643 Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:27:45 +0200 Subject: [PATCH 6/7] ci: pin legacy mysql driver to MySQL 8.0, run mysql2 on 8 & 9 The legacy `mysql` driver only supports mysql_native_password, which is removed/disabled in MySQL 8.4+/9, so it cannot connect there. Give it its own connection namespace (MYSQLLEGACY*) pointing at a dedicated MySQL 8.0 server, and split CI into separate mysql2 (8 & 9 matrix) and legacy mysql (8.0) jobs. Add a mysql8 service to docker-compose for local runs. --- .github/workflows/ci.yml | 53 +++++++++++++++++++++++++++++++---- README.md | 31 +++++++++++++------- docker-compose.yml | 17 +++++++++++ integration/helpers/config.js | 21 +++++++++++--- integration/helpers/db.js | 16 +++++------ package.json | 3 +- 6 files changed, 112 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e7ecd6..ddc6d1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,9 @@ jobs: run: npm run test:integration:pg - name: Test Security run: npm run test:security - mysql: + # mysql2 supports modern caching_sha2_password auth, so it runs against the + # current MySQL releases. + mysql2: runs-on: ${{ matrix.os }} strategy: matrix: @@ -65,13 +67,10 @@ jobs: - 26 os: - ubuntu-latest - # Pinned to 8.0: the legacy `mysql` driver requires the - # mysql_native_password plugin, which 8.4+ disables by default and - # can't be re-enabled via a service container (no command override). mysql-version: - 8 - 9 - name: "Node ${{ matrix.node-version }} - MySQL ${{ matrix.mysql-version }}" + name: "Node ${{ matrix.node-version }} - mysql2 - MySQL ${{ matrix.mysql-version }}" services: mysql: image: mysql:${{ matrix.mysql-version }} @@ -96,11 +95,53 @@ jobs: node-version: ${{ matrix.node-version }} - name: Install run: npm install - - name: Test Integration (MySQL) + - name: Test Integration (mysql2) + run: npm run test:integration:mysql2 + # The legacy `mysql` driver only supports mysql_native_password, which is + # removed/disabled in MySQL 8.4+/9, so it is pinned to MySQL 8.0. + mysql: + runs-on: ${{ matrix.os }} + strategy: + matrix: + node-version: + - 24 + - 26 + os: + - ubuntu-latest + name: "Node ${{ matrix.node-version }} - mysql - MySQL 8.0" + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: sqlmap + ports: + - 3306:3306 + options: --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent" + --health-interval 10s --health-timeout 5s --health-retries 10 + env: + # The legacy driver reads the MYSQLLEGACY* namespace; point it at the + # 8.0 service on the default port. + MYSQLLEGACYHOST: 127.0.0.1 + MYSQLLEGACYPORT: 3306 + MYSQLLEGACYUSER: root + MYSQLLEGACYPASS: mysql + MYSQLLEGACYDB: sqlmap + steps: + - name: Checkout source code + uses: actions/checkout@v6 + - name: Use Node.js + uses: actions/setup-node@v6 + with: + node-version: ${{ matrix.node-version }} + - name: Install + run: npm install + - name: Test Integration (mysql legacy) run: npm run test:integration:mysql automerge: needs: - postgres + - mysql2 - mysql runs-on: ubuntu-latest permissions: diff --git a/README.md b/README.md index f725729..ca7adbb 100644 --- a/README.md +++ b/README.md @@ -289,13 +289,16 @@ const pascalOrCamelToSnake = str => This module can be tested and reported on in a variety of ways... ```sh -npm run test # runs the node:test unit test suite. -npm run test:integration # runs integration tests against running PostgreSQL & MySQL. -npm run test:integration:docker # spins up DBs via docker compose, runs integration tests, tears down. -npm run test:security # runs sqlmap security tests. -npm run test:typescript # runs type definition tests. -npm run coverage # generates a coverage report in docs dir. -npm run lint # lints via standardJS. +npm run test # runs the node:test unit test suite. +npm run test:integration # runs all integration tests against running PostgreSQL & MySQL. +npm run test:integration:pg # PostgreSQL only (pg driver). +npm run test:integration:mysql2 # MySQL only (mysql2 driver). +npm run test:integration:mysql # MySQL only (legacy mysql driver, requires MySQL 8.0). +npm run test:integration:docker # spins up DBs via docker compose, runs all integration tests, tears down. +npm run test:security # runs sqlmap security tests. +npm run test:typescript # runs type definition tests. +npm run coverage # generates a coverage report in docs dir. +npm run lint # lints via standardJS. ``` The integration suite executes queries built by every public API against real @@ -304,13 +307,21 @@ drivers), verifying the generated SQL is valid and that interpolated values are stored as literals. To run it locally: ```sh -npm run db:up # start postgres + mysql via docker compose (waits for healthy) +npm run db:up # start postgres + mysql (9) + mysql8 (8.0) via docker compose npm run test:integration # run the suite npm run db:down # tear down ``` -Connection settings default to the docker-compose services and can be overridden -with `PGHOST/PGPORT/PGUSER/PGPASS/PGDB` and `MYSQLHOST/MYSQLPORT/MYSQLUSER/MYSQLPASS/MYSQLDB`. +> **Note:** the legacy `mysql` driver only supports `mysql_native_password`, +> which is removed/disabled in MySQL 8.4+/9. It therefore runs against a +> dedicated MySQL 8.0 service (the `mysql8` container, exposed on port `3307`), +> while `mysql2` runs against the modern `mysql` container. + +Connection settings default to the docker-compose services and can be overridden: + +- PostgreSQL — `PGHOST/PGPORT/PGUSER/PGPASS/PGDB` +- MySQL (mysql2) — `MYSQLHOST/MYSQLPORT/MYSQLUSER/MYSQLPASS/MYSQLDB` (default port `3306`) +- MySQL (legacy mysql) — `MYSQLLEGACYHOST/MYSQLLEGACYPORT/MYSQLLEGACYUSER/MYSQLLEGACYPASS/MYSQLLEGACYDB` (default port `3307`) ## Benchmark diff --git a/docker-compose.yml b/docker-compose.yml index 0e5c432..2264982 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: timeout: 5s retries: 10 + # Modern MySQL for the mysql2 driver. mysql: image: mysql:9 environment: @@ -26,3 +27,19 @@ services: interval: 5s timeout: 5s retries: 10 + + # MySQL 8.0 for the legacy `mysql` driver, which only supports + # mysql_native_password (removed/disabled in 8.4+/9). Exposed on 3307 so it + # can run alongside the modern instance above (see MYSQLLEGACY* env vars). + mysql8: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: sqlmap + ports: + - 3307:3306 + healthcheck: + test: ['CMD-SHELL', 'mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent'] + interval: 5s + timeout: 5s + retries: 10 diff --git a/integration/helpers/config.js b/integration/helpers/config.js index 39a44d2..208da17 100644 --- a/integration/helpers/config.js +++ b/integration/helpers/config.js @@ -11,16 +11,29 @@ const pg = { port: Number(process.env.PGPORT) || 5432 } -// MySQL connection config, shared by both the `mysql` and `mysql2` drivers. +// MySQL connection config used by the `mysql2` driver. Targets a modern MySQL +// (8.x/9.x) — mysql2 speaks the default caching_sha2_password auth. const mysql = { host: process.env.MYSQLHOST || 'localhost', port: Number(process.env.MYSQLPORT) || 3306, user: process.env.MYSQLUSER || 'root', password: process.env.MYSQLPASS || 'mysql', database: process.env.MYSQLDB || 'sqlmap', - // utf8mb4 so 4-byte characters (e.g. emoji) round-trip on both the - // `mysql` driver (defaults to 3-byte utf8) and `mysql2`. + // utf8mb4 so 4-byte characters (e.g. emoji) round-trip. charset: 'utf8mb4' } -module.exports = { pg, mysql } +// Separate config for the legacy `mysql` driver. It only supports +// mysql_native_password, which is removed/disabled in MySQL 8.4+/9, so it must +// point at a MySQL 8.0 server (own port locally; its own CI job). Its own +// env namespace lets it differ from the mysql2 target when both run together. +const mysqlLegacy = { + host: process.env.MYSQLLEGACYHOST || 'localhost', + port: Number(process.env.MYSQLLEGACYPORT) || 3307, + user: process.env.MYSQLLEGACYUSER || 'root', + password: process.env.MYSQLLEGACYPASS || 'mysql', + database: process.env.MYSQLLEGACYDB || 'sqlmap', + charset: 'utf8mb4' +} + +module.exports = { pg, mysql, mysqlLegacy } diff --git a/integration/helpers/db.js b/integration/helpers/db.js index d829aa4..222b514 100644 --- a/integration/helpers/db.js +++ b/integration/helpers/db.js @@ -52,20 +52,20 @@ async function withMysql2 () { } } -// The legacy `mysql` driver cannot speak MySQL 8's default -// `caching_sha2_password` auth. mysql2 can, so we use it to switch the account -// to `mysql_native_password` first. This works identically locally and in CI -// without needing a container `command` override (unsupported by GitHub -// Actions service containers). +// The legacy `mysql` driver cannot speak MySQL's default `caching_sha2_password` +// auth — it only supports `mysql_native_password`. mysql2 can connect, so we use +// it to switch the account to native_password first. This requires a MySQL 8.0 +// server (config.mysqlLegacy): native_password is removed/disabled in 8.4+/9, so +// the plugin wouldn't even be loaded there. async function ensureNativePassword () { const mysql = require('mysql2/promise') - const conn = await mysql.createConnection(config.mysql) + const conn = await mysql.createConnection(config.mysqlLegacy) try { // We connect over TCP, so the account is `@'%'`. `?` placeholders are // escaped client-side (text protocol) into a valid `'user'@'%'` account. await conn.query( "ALTER USER ?@'%' IDENTIFIED WITH mysql_native_password BY ?", - [config.mysql.user, config.mysql.password] + [config.mysqlLegacy.user, config.mysqlLegacy.password] ) } finally { await conn.end() @@ -76,7 +76,7 @@ async function ensureNativePassword () { async function withMysql () { await ensureNativePassword() const mysql = require('mysql') - const conn = mysql.createConnection(config.mysql) + const conn = mysql.createConnection(config.mysqlLegacy) await new Promise((resolve, reject) => conn.connect(err => (err ? reject(err) : resolve())) ) diff --git a/package.json b/package.json index 9a6c830..69b934a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "db:down": "docker compose down -v", "test:integration": "node --test --test-concurrency=1 integration/**/*.test.js", "test:integration:pg": "node --test integration/SQL.pg.integration.test.js", - "test:integration:mysql": "node --test --test-concurrency=1 integration/SQL.mysql.integration.test.js integration/SQL.mysql2.integration.test.js", + "test:integration:mysql2": "node --test integration/SQL.mysql2.integration.test.js", + "test:integration:mysql": "node --test integration/SQL.mysql.integration.test.js", "test:integration:docker": "npm run db:up && npm run test:integration; npm run db:down", "test:security": "node ./sqlmap/sqlmap.js", "test:typescript": "tstyche", From 6d06b9b65132c7667f8632fe4de113608f687a01 Mon Sep 17 00:00:00 2001 From: Luca Del Puppo Date: Wed, 10 Jun 2026 02:34:04 +0200 Subject: [PATCH 7/7] test: address review feedback on integration suite - config: add allowPublicKeyRetrieval to mysql configs (hardens caching_sha2 auth over non-TLS across MySQL images/versions) - helpers: extract runFile.js to remove triplicated connection-lifecycle boilerplate from the three test files - featureSuite: add coverage for append({ unsafe: true }) literal interpolation - docker-compose: drop obsolete `version` key --- docker-compose.yml | 1 - integration/SQL.mysql.integration.test.js | 18 +++------------- integration/SQL.mysql2.integration.test.js | 18 +++------------- integration/SQL.pg.integration.test.js | 18 +++------------- integration/helpers/config.js | 11 ++++++++-- integration/helpers/runFile.js | 24 ++++++++++++++++++++++ integration/shared/featureSuite.js | 15 ++++++++++++++ 7 files changed, 57 insertions(+), 48 deletions(-) create mode 100644 integration/helpers/runFile.js diff --git a/docker-compose.yml b/docker-compose.yml index 2264982..d4cb24e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.9' services: postgres: image: postgres:17-alpine diff --git a/integration/SQL.mysql.integration.test.js b/integration/SQL.mysql.integration.test.js index da6c3e2..1c818cf 100644 --- a/integration/SQL.mysql.integration.test.js +++ b/integration/SQL.mysql.integration.test.js @@ -1,18 +1,6 @@ 'use strict' -const { test, before, after } = require('node:test') -const { withMysql, createUsersTable } = require('./helpers/db') -const runFeatureSuite = require('./shared/featureSuite') +const { withMysql } = require('./helpers/db') +const runIntegrationFile = require('./helpers/runFile') -const ctx = { db: null } - -before(async () => { - ctx.db = await withMysql() - await createUsersTable(ctx.db) -}) - -after(async () => { - if (ctx.db) await ctx.db.end() -}) - -runFeatureSuite(test, () => ctx.db) +runIntegrationFile(withMysql) diff --git a/integration/SQL.mysql2.integration.test.js b/integration/SQL.mysql2.integration.test.js index 358e02b..17bbb4c 100644 --- a/integration/SQL.mysql2.integration.test.js +++ b/integration/SQL.mysql2.integration.test.js @@ -1,18 +1,6 @@ 'use strict' -const { test, before, after } = require('node:test') -const { withMysql2, createUsersTable } = require('./helpers/db') -const runFeatureSuite = require('./shared/featureSuite') +const { withMysql2 } = require('./helpers/db') +const runIntegrationFile = require('./helpers/runFile') -const ctx = { db: null } - -before(async () => { - ctx.db = await withMysql2() - await createUsersTable(ctx.db) -}) - -after(async () => { - if (ctx.db) await ctx.db.end() -}) - -runFeatureSuite(test, () => ctx.db) +runIntegrationFile(withMysql2) diff --git a/integration/SQL.pg.integration.test.js b/integration/SQL.pg.integration.test.js index ee69126..f2b480e 100644 --- a/integration/SQL.pg.integration.test.js +++ b/integration/SQL.pg.integration.test.js @@ -1,18 +1,6 @@ 'use strict' -const { test, before, after } = require('node:test') -const { withPg, createUsersTable } = require('./helpers/db') -const runFeatureSuite = require('./shared/featureSuite') +const { withPg } = require('./helpers/db') +const runIntegrationFile = require('./helpers/runFile') -const ctx = { db: null } - -before(async () => { - ctx.db = await withPg() - await createUsersTable(ctx.db) -}) - -after(async () => { - if (ctx.db) await ctx.db.end() -}) - -runFeatureSuite(test, () => ctx.db) +runIntegrationFile(withPg) diff --git a/integration/helpers/config.js b/integration/helpers/config.js index 208da17..39ba10c 100644 --- a/integration/helpers/config.js +++ b/integration/helpers/config.js @@ -20,7 +20,11 @@ const mysql = { password: process.env.MYSQLPASS || 'mysql', database: process.env.MYSQLDB || 'sqlmap', // utf8mb4 so 4-byte characters (e.g. emoji) round-trip. - charset: 'utf8mb4' + charset: 'utf8mb4', + // A fresh caching_sha2_password account over a non-TLS connection needs the + // server's public key to complete auth; allow retrieving it. Fine for local/ + // CI test infra (no untrusted network). + allowPublicKeyRetrieval: true } // Separate config for the legacy `mysql` driver. It only supports @@ -33,7 +37,10 @@ const mysqlLegacy = { user: process.env.MYSQLLEGACYUSER || 'root', password: process.env.MYSQLLEGACYPASS || 'mysql', database: process.env.MYSQLLEGACYDB || 'sqlmap', - charset: 'utf8mb4' + charset: 'utf8mb4', + // Used by the mysql2 connection that flips the account to native_password + // (see ensureNativePassword); the legacy `mysql` driver ignores it. + allowPublicKeyRetrieval: true } module.exports = { pg, mysql, mysqlLegacy } diff --git a/integration/helpers/runFile.js b/integration/helpers/runFile.js new file mode 100644 index 0000000..f1d262b --- /dev/null +++ b/integration/helpers/runFile.js @@ -0,0 +1,24 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { createUsersTable } = require('./db') +const runFeatureSuite = require('../shared/featureSuite') + +// Wires the connection lifecycle for one driver's integration test file: +// open the connection and create a fresh table before the suite, close it +// after, then register the shared feature suite. `connect` is one of the +// withPg/withMysql/withMysql2 adapters from helpers/db.js. +module.exports = function runIntegrationFile (connect) { + const ctx = { db: null } + + before(async () => { + ctx.db = await connect() + await createUsersTable(ctx.db) + }) + + after(async () => { + if (ctx.db) await ctx.db.end() + }) + + runFeatureSuite(test, () => ctx.db) +} diff --git a/integration/shared/featureSuite.js b/integration/shared/featureSuite.js index 2dc52bf..6b18e71 100644 --- a/integration/shared/featureSuite.js +++ b/integration/shared/featureSuite.js @@ -158,6 +158,21 @@ module.exports = function runFeatureSuite (test, getDb) { assert.equal(Number(rows[0].id), 50) }) + test('deprecated append with { unsafe: true } interpolates literally', async () => { + const db = getDb() + await reset(db) + await db.query(SQL`INSERT INTO users (id, username) VALUES (${55}, ${'unsafe-append'})`) + // unsafe append inlines the value as a literal (no placeholder) — here a + // trusted column name in an ORDER BY clause. Executing it proves the + // unsafe branch of append (SQL.js) produces valid SQL. + const orderColumn = 'username' // trusted, not user input + const sql = SQL`SELECT id, username FROM users WHERE username = ${'unsafe-append'}` + sql.append(SQL` ORDER BY ${orderColumn}`, { unsafe: true }) + const rows = await db.query(sql) + assert.equal(rows.length, 1) + assert.equal(Number(rows[0].id), 55) + }) + test('null values round-trip', async () => { const db = getDb() await reset(db)