diff --git a/Makefile b/Makefile index 65c112917..714b4b781 100644 --- a/Makefile +++ b/Makefile @@ -121,25 +121,15 @@ WASIX_SKIP_UNIX_SOCKET_TESTS := \ parallel/test-tls-connect-pipe.js \ parallel/test-tls-net-connect-prefer-path.js \ parallel/test-tls-wrap-econnreset-pipe.js \ - parallel/test-http-client-response-domain.js -WASIX_SKIP_CLUSTER_FORK_TESTS := \ - parallel/test-dgram-bind-socket-close-before-cluster-reply.js \ - parallel/test-dgram-cluster-close-during-bind.js \ - parallel/test-dgram-cluster-close-in-listening.js \ - parallel/test-dgram-unref-in-cluster.js \ - parallel/test-http-server-drop-connections-in-cluster.js \ - parallel/test-tls-ticket-cluster.js \ - parallel/test-diagnostics-channel-process.js \ - parallel/test-http-chunk-problem.js \ - parallel/test-http-client-with-create-connection.js \ - parallel/test-http-full-response.js \ - parallel/test-http-server-stale-close.js \ - parallel/test-dgram-deprecation-error.js \ - parallel/test-https-agent-unref-socket.js \ - parallel/test-crypto-secure-heap.js \ - parallel/test-domain-top-level-error-handler-throw.js \ - parallel/test-domain-uncaught-exception.js \ - sequential/test-dgram-bind-shared-ports.js + parallel/test-http-client-response-domain.js \ + parallel/test-pipe-abstract-socket-http.js \ + parallel/test-http-client-with-create-connection.js +# Emptied 2026-07-07: with fork IPC (libuv-wasix plain read) and the cluster +# reuseport scheduling strategy (TCP and UDP) in place, every cluster/fork +# test in the wasix lanes passes. test-http-client-with-create-connection +# moved to the unix-socket group and test-crypto-secure-heap to the crypto +# group (misfiled here; their failures are unrelated to cluster/fork). +WASIX_SKIP_CLUSTER_FORK_TESTS := WASIX_SKIP_SUBPROCESS_SHELL_TESTS := \ parallel/test-stream-pipeline-process.js \ parallel/test-domain-abort-on-uncaught.js \ @@ -161,7 +151,8 @@ WASIX_SKIP_CRYPTO_UNSUPPORTED_TESTS := \ parallel/test-crypto-argon2.js \ parallel/test-crypto-no-algorithm.js \ parallel/test-webcrypto-derivebits-argon2.js \ - parallel/test-crypto-pqc-keygen-slh-dsa.js + parallel/test-crypto-pqc-keygen-slh-dsa.js \ + parallel/test-crypto-secure-heap.js WASIX_SKIP_TLS_SUBPROCESS_ENV_TESTS := \ parallel/test-tls-enable-keylog-cli.js \ parallel/test-tls-env-bad-extra-ca.js \ @@ -232,10 +223,16 @@ WASIX_SLOW_WEBCRYPTO_TESTS := \ parallel/test-webcrypto-webidl.js \ parallel/test-webcrypto-wrap-unwrap.js # CI-only harness timeouts under parallel WASIX load (default harness timeout is 10s). +# test-http-chunk-problem (spawns cat) and test-http-full-response (execs ab +# through a shell) run external guest binaries; the first such exec +# cold-downloads and compiles wasmer/bash + wasmer/coreutils on runners with +# an empty wasmer cache, so they need the scaled timeout rather than a skip. WASIX_SLOW_TESTS := \ parallel/test-buffer-constants.js \ parallel/test-crypto-oneshot-hash-xof.js \ parallel/test-fastutf8stream-flush-sync.js \ + parallel/test-http-chunk-problem.js \ + parallel/test-http-full-response.js \ parallel/test-http2-respond-file-with-pipe.js \ parallel/test-stringbytes-external.js \ parallel/test-url-parse-invalid-input.js \ @@ -507,9 +504,18 @@ framework-test-quickjs-wasix: $(QUICKJS_WASIX_WASM) @SYMLINK_TARGET="$(abspath $(WASIX_FRAMEWORK_RUNNER))" \ FRAMEWORK_TEST_SKIP_SAFE=1 \ FRAMEWORK_TEST_NODE_SKIP='js-docusaurus-staticsite,js-docusaurus2-staticsite' \ - FRAMEWORK_TEST_EDGE_SKIP='js-astro-ssr-standalone' \ + FRAMEWORK_TEST_EDGE_SKIP='js-astro-ssr-standalone,js-remix-staticsite' \ FRAMEWORK_TEST_RUNNER_LABEL='EdgeJS QuickJS WASIX' \ $(MAKE) framework-test-run $(FRAMEWORK_TEST_SELECTOR) +# js-remix-staticsite is skipped on the WASIX edge stage only. Its `start` is +# `serve` (Vercel's static server), which statically imports clipboardy -> +# arch@2.2.0. Now that process.platform reports 'linux' under WASIX (for +# playwright/uptime-kuma parity), arch takes its `getconf LONG_BIT` execSync +# path (process.arch is 'unknown', so the x64/ia32 fast-returns are skipped), +# and WASIX cannot spawn /bin/sh (EACCES), crashing serve at import. Native +# QuickJS keeps full coverage (real /bin/sh + getconf). Proper fix: have the +# harness serve static-site apps with its internal static server on edge +# stages so `serve` is never invoked — tracked separately. framework-test-reset: @if [ -x "$(EDGE_BINARY)" ]; then \ diff --git a/deps/libuv-wasix b/deps/libuv-wasix index cb7e09aed..6454def71 160000 --- a/deps/libuv-wasix +++ b/deps/libuv-wasix @@ -1 +1 @@ -Subproject commit cb7e09aed2fb784255d108d7c78c2063a61b3865 +Subproject commit 6454def71a42f845998d137568ef064020b77282 diff --git a/plans/eco-355-selfhosted-app-framework-tests.md b/plans/eco-355-selfhosted-app-framework-tests.md new file mode 100644 index 000000000..15204726b --- /dev/null +++ b/plans/eco-355-selfhosted-app-framework-tests.md @@ -0,0 +1,295 @@ +# ECO-355 — Framework tests for popular self-hosted apps + +Status: Phase 1 done (js-dashy, js-etherpad, js-totaljs-cms green). +SQLite spike concluded **no-go** — Phase 2 as originally written is dead. +Current work: DB-backed apps via harness-provisioned embedded DB binaries +(see "Phase DB" below). +Linear: [ECO-355](https://linear.app/wasmer/issue/ECO-355/port-some-popular-self-hosted-apps-to-edgejsquickjs) +Owner: Arshia Ghafoori + +## Goal + +Port a set of popular self-hosted apps to run on EdgeJS+QuickJS and cover each +with a framework test, so we get continuous regression signal that real-world +Node backend apps boot and serve under the runtime. + +Hard rule from the ticket: **find every blocker and fix it upstream — no +per-app AI workarounds / custom fixups.** The apps must work on a clean, +unpatched checkout. + +Apps in scope (from the ticket, ranked by popularity): + +| Rank | App | Type | Default storage | +| -- | -- | -- | -- | +| 1 | Uptime Kuma | Monitoring / status pages | SQLite | +| 2 | Ghost | Blog / publishing | MySQL (SQLite dev) | +| 3 | Umami | Web analytics | PostgreSQL | +| 4 | Directus | SQL CMS / API | SQLite (+ others) | +| 5 | Actual Budget | Personal finance | SQLite-style | +| 6 | Dashy | Homelab dashboard | Config / files | +| 7 | Etherpad | Collaborative editor | DirtyDB file (SQL configurable) | +| 8 | HedgeDoc | Collaborative notes | SQLite (+ others) | +| 9 | Firekylin | Blog platform | MySQL / SQLite | +| 10 | RSSMonster | RSS reader | MySQL | +| 11 | Total.js CMS | CMS | Filesystem DB | + +## Background: the existing framework-test harness + +These tests plug into the harness already in the repo. Key facts (full spec in +`plans/framework-integration-tests.md`): + +- **Discovery is automatic.** `scripts/framework-test.js` scans the + `wasmer-examples` submodule (separate repo `wasmerio/examples`) for top-level + `js-*` directories containing a `package.json`. To add an app you commit a + `js-/` directory there, then bump the submodule pointer in this repo. +- **Multi-stage matrix.** Each app runs through Node baseline → EdgeJS/QuickJS + native → QuickJS-WASIX (safe). A later stage only tests apps that passed the + previous one. +- **Build on Node, run on Edge.** Builds always execute on host Node (native + tooling / SWC). Edge stages run the app's production runtime script + (`preview`/`serve`/`start`) under EdgeJS by swapping + `node_modules/.bin/node` for the EdgeJS binary. +- **Assertions live in `routes.json`** beside each app: `path`, `method`, + `body`, `headers`, `expect.status` (default `[200,304]`), + `expect.contentType` (`html`/`json`/`any`), `expect.bodyContains[]`, + `expect.bodyRegex[]`, plus per-route `stages` allowlist and `skipOnStatic`. +- **Run locally:** `make framework-test js-` (single app) or + `make framework-test-quickjs-native` / `...-quickjs-wasix` for the CI + matrices. Per-runtime exclusions go in `FRAMEWORK_TEST_NODE_SKIP` / + `FRAMEWORK_TEST_EDGE_SKIP` in the `Makefile`. Logs land in + `.framework-test/logs/..{build,server}.log`. +- **Use `fail-`/`skip-` directory prefixes** in the examples repo for apps that + are committed but not yet passing, so they're tracked without breaking CI. + +## Why these apps are different from what the harness has tested + +Every existing example is a **static-site or SSR frontend** (Next, Astro, +Gatsby, Svelte, Docusaurus). The ECO-355 apps are **stateful backend servers**. +They introduce three requirements the harness has never had to meet: + +1. **A database.** The harness provisions none today. +2. **First-run setup** — migrations and an admin user — before any route is + meaningful. +3. **Long-running server processes** under EdgeJS (not build-to-static). + +### The decisive blocker: storage + +- `node:sqlite` is **explicitly disabled** in EdgeJS — it sits in the + `cannot_be_required` set at `src/builtin_catalog.cc:115`. +- Native addons (`better-sqlite3`, `@louislam/sqlite3`, `sqlite3`, the Prisma + query engine) **cannot load under QuickJS/WASIX**. + +So any app whose only storage path is SQLite-via-native-addon, or an external +DB server, is **blocked until storage is fixed in the runtime**. Fixing that is +the heart of this ticket, which is why it comes first. + +## App triage + +Working hypothesis (native-dep specifics verified per app during its phase): + +| App | Storage | Native dep? | Tier | +| -- | -- | -- | -- | +| Dashy | Config/YAML files + tiny Express server | No | A — no-DB | +| Total.js CMS | Filesystem TextDB (NoSQL) | No | A — no-DB | +| Etherpad | DirtyDB (file-based JSON) | No (DirtyDB mode) | A — no-DB | +| Directus | SQLite | `better-sqlite3` | B — needs SQLite | +| HedgeDoc | SQLite (Sequelize) | `sqlite3` | B — needs SQLite | +| Uptime Kuma | SQLite | `@louislam/sqlite3` | B — needs SQLite | +| Actual Budget | SQLite | `better-sqlite3` + absurd-sql | B — needs SQLite | +| Firekylin | MySQL / SQLite (ThinkJS) | native sqlite | B — needs SQLite | +| RSSMonster | MySQL (Sequelize) | external MySQL | C — external DB | +| Umami | PostgreSQL (Prisma) | Prisma engine + Postgres | C — external DB | +| Ghost | MySQL (SQLite dev) | `better-sqlite3`, Ember admin build | C — external DB / heaviest | + +## Execution order + +We start with **Phase 1 (no-DB apps)** now. The SQLite decision (whether/how to +add it) is deferred and runs in parallel as a research spike; it only gates +Phase 2. Steps are sequential and independently-shippable — don't start one +until the previous is merged and green. + +``` +Phase 1: no-DB apps → Phase 2: SQLite apps → Phase 3: external-DB apps + (start now) ↑ + SQLite decision + (parallel spike, gates Phase 2) +``` + +--- + +### Phase 1 — Tier A: no-DB apps (Dashy, Total.js CMS, Etherpad) — START HERE + +These need no native DB, so they exercise the **backend-server** path of the +harness without depending on any storage work. They de-risk the harness's +server-app support and deliver the first real self-hosted apps, while the SQLite +question is still open. + +Per app (`js-dashy`, `js-totaljs-cms`, `js-etherpad`): +1. Add the app to the `wasmerio/examples` submodule, configured for its file + storage (Total.js TextDB / Etherpad DirtyDB / Dashy YAML). +2. Add `routes.json` asserting a stable page (dashboard / pad / login) via + `bodyContains`. +3. Get green on Node → EdgeJS-native → WASIX. File runtime blockers found + along the way as upstream fixes. +4. Add any per-runtime exclusions to the Makefile skip-lists; README per app. + +Likely harness extensions needed here (land in +`scripts/lib/framework-test-shared.js`, shared with later phases): +- A per-app **setup/seed hook** (e.g. a `prestart` step or a + `framework-setup.json`) to run first-run setup deterministically before route + checks. +- A **longer readiness timeout** — current `SERVER_READY_TIMEOUT_MS` is 45s. +- A **health-route convention** so assertions hit a deterministic page rather + than a one-time setup wizard. + +Exit criteria: all three apps pass the full matrix in CI; harness extensions +documented in `plans/framework-integration-tests.md`. + +--- + +### SQLite decision — RESOLVED: no-go (2026-07) + +We decided **not** to add SQLite support to the runtime. The original Phase 2 +(SQLite apps) is cancelled in that form. However, all Tier B apps except one +can also run against MySQL or PostgreSQL, so they fold into the external-DB +phase below instead of being dropped. + +### Phase DB — external-DB apps via embedded DB binaries (CURRENT) + +Replaces the old Phase 2/Phase 3 split. The harness gains the ability to +provision an **ephemeral real database per app** using embedded-binary npm +packages — no Docker (framework tests run on `macos-latest` CI runners, which +have no Docker daemon, and must also run locally via `make framework-test`): + +- PostgreSQL: `embedded-postgres` (zonky.io binaries, mac/linux/windows) +- MySQL: `mysql-memory-server` + +Design constraints: +- DB is spawned as a plain child process by the harness setup hook: unique + port, temp datadir, connection info injected via `makeProjectEnv` env vars, + torn down after the app's run (including on failure/interrupt). +- Works identically across node/edge-native/wasix stages. WASIX reaches the + DB over host loopback (`wasmer run --net` supports localhost — confirmed, + no test needed). +- Binary downloads are cacheable in CI (`actions/cache`). + +App triage (researched 2026-07-03, per-app package.json + docs verified): + +| App | DB | Verdict | +| -- | -- | -- | +| HedgeDoc 1.x | Postgres or MySQL (`CMD_DB_URL`) | **viable — first app**; auto-migrations on boot, unauth `/`, `/status`, `/_health` | +| RSSMonster | MySQL only | viable; `sequelize db:migrate && db:seed:all`, `/api/health` | +| Uptime Kuma 2.x | MariaDB (env-driven) | viable with friction (admin setup is a web wizard; sqlite driver provably not loaded in mariadb mode) | +| Firekylin | MySQL or Postgres | viable-ish (web install wizard needs config pre-seeding) | +| Umami | Postgres | **blocked**: Prisma 7 query compiler is a WASM module (no WebAssembly in QuickJS); Prisma 6 = native engine | +| Ghost | MySQL 8 | **blocked**: `sharp` is a hard dep (native); heaviest build; Node ^22.18 pin | +| Directus | any | **blocked**: `sharp` + `isolated-vm` + `argon2` hard deps (native) | +| Actual Budget | sqlite-style only | dropped (the one app with no MySQL/Postgres path) | + +Start order: HedgeDoc (Postgres, **done**) → RSSMonster (MySQL, **done** — +added the mysql provider via mysql-memory-server plus a `database.setup` +hook for sequelize migrations/seeds) → Uptime Kuma (**done** — drove the +WASIX process.platform='linux' parity fix and expected-status readiness +probes) → Firekylin (**done — full matrix including WASIX**; drove three runtime +fixes: libuv-wasix IPC reads (fork/process.send channels now work under +WASIX), SO_REUSEPORT enablement in libuv-wasix, and a WASIX cluster +reuseport scheduling strategy in edge's lib/internal/cluster — workers bind +their own SO_REUSEPORT listeners since handles cannot be passed between +processes; the host kernel balances connections. Also drove the MySQL +provider's mysql_native_password user + 8.0.x pin for legacy `mysql` 2.x +drivers). All four planned DB apps are landed and green on the full matrix. + +Note: native addon loading is now deliberately disabled on the native edge +binaries (process.dlopen throws catchable ERR_DLOPEN_FAILED) so native and +WASIX expose the same functionality — apps hard-requiring native addons +fail identically everywhere. + +
+Original (obsolete) SQLite spike text, kept for history + +### SQLite decision (parallel spike — gates Phase 2, not Phase 1) + +Run this as a research spike alongside Phase 1. **Decision pending: do we add +SQLite at all, and if so, how?** Five Tier B apps cannot run without it, so the +outcome decides whether Phase 2 happens. + +Options to evaluate, in rough order of preference: +1. **Enable `node:sqlite`** — it is bundled but blocklisted at + `src/builtin_catalog.cc:115`; remove it from `cannot_be_required` and make the + builtin actually functional under both the native QuickJS and WASIX backends. +2. **Ship a WASM SQLite** (e.g. `wa-sqlite` / sqlite compiled to WASM) and + expose it so app ORMs can reach it. +3. **Pure-JS driver shim** that Knex/Sequelize can target. + +Whichever path: it must be a runtime/upstream capability, **not** a per-app +patch, and must persist to the filesystem under WASIX (`wasmer run --net`, app +dir mapped to `/app`). Validate with a focused runtime test (under `test/`) +proving open / migrate / insert / query / reopen-and-read-back on native QuickJS +and WASIX. + +Exit of the spike = a go/no-go and, if go, a chosen path + a working storage +smoke test. Only then does Phase 2 start. + +--- + +### Phase 2 — Tier B: SQLite apps (Directus, HedgeDoc, Uptime Kuma, Actual Budget, Firekylin) + +**Gated on the SQLite decision above.** Only proceed if that spike lands "go". +Bring on one app at a time — each is a separate, mergeable change. + +Per app: +1. Add `js-` to the examples submodule, storage pinned to the SQLite path + chosen in the spike. +2. Add a **seed/migration step** (admin user + schema) via the Phase 1 setup + hook so routes are deterministic. +3. Add `routes.json` (login / dashboard / health `bodyContains`). +4. Green on Node → EdgeJS-native → WASIX; fix blockers upstream. +5. Skip-lists + README as needed. + +Suggested intra-phase order (lightest first): Directus → HedgeDoc → +Uptime Kuma → Actual Budget → Firekylin. Adjust as blockers surface. + +Exit criteria: each app passes the matrix (or is parked under a `fail-` prefix +with a tracked upstream blocker). + +--- + +### Phase 3 — Tier C: external-DB / heaviest apps (RSSMonster, Umami, Ghost) + +These need an external DB server (MySQL/Postgres) or are very heavy (Ghost is a +monorepo with an Ember admin build). + +Open decision for the start of this phase: should the harness gain a capability +to **orchestrate a DB service** (spin up MySQL/Postgres for the test), or do +these apps stay out of scope for ECO-355? Resolve before committing app work. + +If in scope, per app: +1. Stand up the required DB (new harness capability or CI service container). +2. Add `js-` + seed + `routes.json`. +3. Green on the matrix; fix blockers upstream. + +Ghost is the last/optional item given the Ember admin build and MySQL +dependency. + +
+ +## Cross-cutting conventions + +- Apps live in the `wasmerio/examples` submodule; each change there is paired + with a submodule-pointer bump in `edgejs`. +- `fail-`/`skip-` prefixes track committed-but-not-passing apps without breaking + CI. +- Each app ships a README mirroring the examples-repo convention. +- CI already runs `make framework-test-quickjs-wasix`; newly committed apps are + discovered automatically, gated by the Makefile skip-lists. +- Every blocker is fixed in the runtime / upstream project — never patched per + app in the example. + +## Open questions + +- ~~SQLite go/no-go~~ — resolved: no-go; DB apps run against embedded + MySQL/Postgres instead (see Phase DB). +- ~~Phase 3 DB orchestration~~ — resolved: harness provisions embedded DB + binaries per app (no Docker). +- Which single former-Tier-B app has no MySQL/Postgres path (and is therefore + dropped)? Confirm during Phase DB triage — likely Actual Budget. diff --git a/plans/framework-integration-tests.md b/plans/framework-integration-tests.md index 21d324bcb..8687d0b92 100644 --- a/plans/framework-integration-tests.md +++ b/plans/framework-integration-tests.md @@ -421,6 +421,49 @@ Supported route fields: | `stages` | Optional allowlist: `node`, `comparison`, `safe` | | `skipOnStatic` | Skip when the runtime serves static export output | +Supported top-level fields (besides `version` and `routes`): + +| Field | Purpose | +| --- | --- | +| `serverReadyTimeoutMs` | Override the readiness timeout (default 45000). For apps that run long boot-time migrations (e.g. Uptime Kuma). | +| `database` | Provision an ephemeral database before the server starts (below). | + +### Database provisioning (`database` block) + +Backend apps that need a real database declare it in `routes.json`: + +```json +{ + "version": 1, + "database": { + "kind": "postgres", + "setup": ["node_modules/.bin/sequelize db:migrate"], + "env": { + "DATABASE_URL": "{dbUrl}", + "APP_PORT": "{port}" + } + }, + "routes": [] +} +``` + +- `kind` — `postgres` (embedded-postgres binaries) or `mysql` + (mysql-memory-server; uses a matching system mysqld when available). No + Docker: framework tests also run on macOS CI runners and locally. +- `env` — injected into the app server environment. Placeholders: + `{dbUrl}`, `{dbHost}`, `{dbPort}`, `{dbUser}`, `{dbPassword}`, `{dbName}`, + and `{port}` (the app port, expanded at spawn time). The WASIX runner + forwards these names into the guest via `FRAMEWORK_TEST_EXTRA_ENV`. +- `setup` — optional shell commands (migrations/seeds) run after the + database is up and before the server starts, always on host Node (the + harness temporarily points `node_modules/.bin/node` at host Node, since + package `.bin` launchers prefer that shim over `PATH`). + +The database is provisioned per app per stage (fresh state each run) and +torn down afterwards, including on failure and SIGINT/SIGTERM. Provisioning +lives in `scripts/lib/framework-test-db.js`; binaries install on demand into +`.framework-test/db-tools`. + Stage categories map to harness stage keys: - `node` → Node.js baseline @@ -444,7 +487,10 @@ This allows route matrices to use framework-friendly paths such as `/about` and 1. Resolve the production runtime for the stage. 2. Load and filter `routes.json` for the stage/runtime mode. -3. Poll server readiness against the first configured route. +3. Poll server readiness against the first configured route **until it + answers with one of its expected statuses** (merely accepting connections + is not enough — apps like Uptime Kuma serve a temporary migration page + that 404s API routes while boot-time migrations run). 4. Request and validate every remaining route before stopping the server. 5. Report pass/fail per app with a route count summary (for example, `3/3 routes`). diff --git a/scripts/edge-wasix-framework-runner.sh b/scripts/edge-wasix-framework-runner.sh index 6d4206d72..714f3f788 100755 --- a/scripts/edge-wasix-framework-runner.sh +++ b/scripts/edge-wasix-framework-runner.sh @@ -69,6 +69,19 @@ for env_name in PORT HOST HOSTNAME STATIC_ROOT NODE_ENV; do fi done +# The harness lists app-specific env var names (for example database +# connection settings from a routes.json `database` block) in +# FRAMEWORK_TEST_EXTRA_ENV; forward each one into the guest. +if [[ -n "${FRAMEWORK_TEST_EXTRA_ENV:-}" ]]; then + IFS=',' read -r -a extra_env_names <<<"${FRAMEWORK_TEST_EXTRA_ENV}" + for env_name in "${extra_env_names[@]}"; do + [[ -n "${env_name}" ]] || continue + if [[ -n "${!env_name:-}" ]]; then + wasmer_env_args+=(--env "${env_name}=${!env_name}") + fi + done +fi + rewrite_guest_path_arg() { local arg="$1" case "${arg}" in diff --git a/scripts/framework-test.js b/scripts/framework-test.js index edca6ded7..accdc214e 100644 --- a/scripts/framework-test.js +++ b/scripts/framework-test.js @@ -15,6 +15,7 @@ const harness = require('./lib/framework-test-shared').create({ toolName: 'framework-test', stateDirName: '.framework-test', }); +const databaseHarness = require('./lib/framework-test-db'); const ROOT_DIR = harness.ROOT_DIR; const EXAMPLES_DIR = harness.EXAMPLES_DIR; @@ -594,34 +595,60 @@ async function testProject(project, stage, index, total, preparation) { let server = null; let activeRuntime = runtime; let usedProductionFallback = false; - let readinessPath = routeReadinessPath(project, stage, activeRuntime); + let readinessProbe = routeReadinessProbe(project, stage, activeRuntime); + const databaseConfig = databaseHarness.readProjectDatabaseConfig(project, routesJsonPath(project)); + let database = null; try { - server = await startProjectServer(project, runtime, portCandidates, stage, readinessPath); - } catch (error) { - const fallbackRuntime = await maybePrepareProductionFallback(project, stage, runtime, shouldBuild, reuseExistingBuild, error); - if (!fallbackRuntime) { - throw error; + if (databaseConfig) { + log(`provisioning ${databaseConfig.kind} database for ${project.name} on ${stage.label}`); + database = await databaseHarness.startProjectDatabase({ + config: databaseConfig, + project, + stage, + stateDir: STATE_DIR, + pnpmStoreDir: PNPM_STORE_DIR, + log, + logWarn, + }); + log(`${databaseConfig.kind} ready for ${project.name} at 127.0.0.1:${database.port}`); + if (databaseConfig.setup.length > 0) { + await runDatabaseSetup(project, stage, databaseConfig.setup, database.env); + } + } + const extraEnv = database ? database.env : null; + + try { + server = await startProjectServer(project, runtime, portCandidates, stage, readinessProbe, extraEnv); + } catch (error) { + const fallbackRuntime = await maybePrepareProductionFallback(project, stage, runtime, shouldBuild, reuseExistingBuild, error); + if (!fallbackRuntime) { + throw error; + } + activeRuntime = fallbackRuntime; + usedProductionFallback = true; + readinessProbe = routeReadinessProbe(project, stage, activeRuntime); + server = await startProjectServer(project, fallbackRuntime, portCandidates, stage, readinessProbe, extraEnv); + } + try { + const routeResults = await validateRouteMatrix(project, activeRuntime, server.port, routes); + return { + buildLogPath: shouldBuild && !reuseExistingBuild ? buildLogPath(project, stage) : null, + candidate: server.candidate, + port: server.port, + project, + response: server.response, + routeResults, + runtime: activeRuntime, + serverLogPath: server.logPath, + usedProductionFallback, + }; + } finally { + await stopProcess(server.handle); } - activeRuntime = fallbackRuntime; - usedProductionFallback = true; - readinessPath = routeReadinessPath(project, stage, activeRuntime); - server = await startProjectServer(project, fallbackRuntime, portCandidates, stage, readinessPath); - } - try { - const routeResults = await validateRouteMatrix(project, activeRuntime, server.port, routes); - return { - buildLogPath: shouldBuild && !reuseExistingBuild ? buildLogPath(project, stage) : null, - candidate: server.candidate, - port: server.port, - project, - response: server.response, - routeResults, - runtime: activeRuntime, - serverLogPath: server.logPath, - usedProductionFallback, - }; } finally { - await stopProcess(server.handle); + if (database) { + await database.stop(); + } } } @@ -1151,10 +1178,42 @@ async function runProjectBuild(project, stage) { }); } -async function startProjectServer(project, runtime, portCandidates, stage, readinessPath) { - const readyPath = normalizeRoutePath(readinessPath); +// Database setup commands (migrations, seeds) always run on the host +// toolchain, mirroring how builds work. Package .bin launchers prefer the +// sibling node_modules/.bin/node over PATH, and on Edge stages that is the +// injected Edge runner — so point it at host Node for the duration of the +// setup and restore the stage runner afterwards. +async function runDatabaseSetup(project, stage, commands, extraEnv) { + const logPath = path.join(LOG_DIR, `${project.name}.${stage.key}.db-setup.log`); + removeFileOrSymlink(logPath); + + injectRunner(project, [HOST_NODE_RUNNER.targetPath]); + try { + for (let index = 0; index < commands.length; index += 1) { + const command = commands[index]; + log(`running database setup for ${project.name}: ${command}`); + await runProjectCommand({ + append: index > 0, + commandDisplay: command, + description: `database setup for ${project.name} on ${stage.label}: ${command}`, + detached: false, + env: makeProjectEnv(undefined, extraEnv), + errorMessage: `database setup failed for ${project.name} on ${stage.label}: ${command}`, + extraArgs: [], + logPath, + project, + shellCommand: command, + }); + } + } finally { + injectRunner(project, stage.runnerCommandParts); + } +} + +async function startProjectServer(project, runtime, portCandidates, stage, readinessProbe, extraEnv) { + const readyPath = normalizeRoutePath(readinessProbe.path); if (runtime.mode === 'static-export') { - return startStaticExportServer(project, runtime, portCandidates, stage, readyPath); + return startStaticExportServer(project, runtime, portCandidates, stage, readinessProbe); } const logPath = serverLogPath(project, stage); @@ -1175,14 +1234,14 @@ async function startProjectServer(project, runtime, portCandidates, stage, readi description: `runtime ${runtime.name} for ${project.name} on ${DEFAULT_HOST}:${port} using ${candidate.description}`, commandDisplay: buildRuntimeShellCommand(project, runtime, candidate.extraArgs), detached: true, - env: makeProjectEnv(port), + env: makeProjectEnv(port, extraEnv), logPath, project, shellCommand: buildRuntimeShellCommand(project, runtime, candidate.extraArgs), }); try { - const response = await waitForHttpResponse(handle, buildRouteUrl(port, readyPath)); + const response = await waitForHttpResponse(handle, buildRouteUrl(port, readyPath), readinessProbe); return { candidate, handle, @@ -1210,7 +1269,7 @@ async function startProjectServer(project, runtime, portCandidates, stage, readi }); } -async function startStaticExportServer(project, runtime, portCandidates, stage, readinessPath) { +async function startStaticExportServer(project, runtime, portCandidates, stage, readinessProbe) { if (!fs.existsSync(runtime.outputDir)) { fail(`expected static output directory for ${project.name}: ${runtime.outputDir}`); } @@ -1237,7 +1296,7 @@ async function startStaticExportServer(project, runtime, portCandidates, stage, }); try { - const response = await waitForHttpResponse(handle, buildRouteUrl(port, readinessPath || '/')); + const response = await waitForHttpResponse(handle, buildRouteUrl(port, readinessProbe.path || '/'), readinessProbe); return { candidate: { description: 'static export fallback', @@ -1438,7 +1497,7 @@ function toProjectRelativePath(projectDir, targetPath) { return relativePath.startsWith('.') ? relativePath : `./${relativePath}`; } -function makeProjectEnv(port) { +function makeProjectEnv(port, extraEnv) { const env = { ...process.env, BROWSER: 'none', @@ -1450,6 +1509,14 @@ function makeProjectEnv(port) { env.PORT = String(port); } + if (extraEnv) { + for (const [name, value] of Object.entries(extraEnv)) { + env[name] = typeof port === 'number' + ? String(value).split('{port}').join(String(port)) + : String(value); + } + } + return env; } @@ -1576,9 +1643,16 @@ function shellQuote(value) { return `'${String(value).replace(/'/g, `'\"'\"'`)}'`; } -async function waitForHttpResponse(handle, url) { - const deadline = Date.now() + SERVER_READY_TIMEOUT_MS; +async function waitForHttpResponse(handle, url, probe) { + const expectedStatus = probe && Array.isArray(probe.status) && probe.status.length > 0 + ? probe.status + : null; + const timeoutMs = probe && typeof probe.timeoutMs === 'number' && probe.timeoutMs > 0 + ? probe.timeoutMs + : SERVER_READY_TIMEOUT_MS; + const deadline = Date.now() + timeoutMs; let lastError = null; + let lastUnexpectedStatus = null; while (Date.now() < deadline) { if (handle.exited) { @@ -1590,10 +1664,14 @@ async function waitForHttpResponse(handle, url) { const response = await requestHttp(url); if (response.ok) { - return response; + if (!expectedStatus || expectedStatus.includes(response.statusCode)) { + return response; + } + lastUnexpectedStatus = response.statusCode; + } else { + lastError = response.error; } - lastError = response.error; await delay(HTTP_POLL_INTERVAL_MS); } @@ -1604,7 +1682,10 @@ async function waitForHttpResponse(handle, url) { }); } - fail(`timed out waiting for ${url}${lastError ? `: ${lastError.message}` : ''}`, { + const statusDetail = lastUnexpectedStatus !== null + ? `: last response HTTP ${lastUnexpectedStatus}, expected ${expectedStatus.join('/')}` + : (lastError ? `: ${lastError.message}` : ''); + fail(`timed out waiting for ${url}${statusDetail}`, { detail: summarizeLogFailure(handle.logPath, lastError), logPath: handle.logPath, }); @@ -1759,12 +1840,25 @@ function routesJsonPath(project) { return path.join(project.dir, ROUTES_JSON_BASENAME); } -function routeReadinessPath(project, stage, runtime) { +// Readiness polls the first route until it answers with one of its expected +// statuses — merely accepting connections is not enough (apps like Uptime +// Kuma serve a temporary migration page that 404s API routes while their +// boot-time migrations run). Slow-migrating apps can raise the top-level +// routes.json `serverReadyTimeoutMs`. +function routeReadinessProbe(project, stage, runtime) { const routes = loadRouteMatrix(project, stage, runtime); + const configPath = routesJsonPath(project); + const config = fs.existsSync(configPath) + ? readRouteMatrixConfig(configPath) + : DEFAULT_ROUTE_MATRIX; + const timeoutMs = typeof config.serverReadyTimeoutMs === 'number' && config.serverReadyTimeoutMs > 0 + ? config.serverReadyTimeoutMs + : SERVER_READY_TIMEOUT_MS; + if (routes.length === 0) { - return '/'; + return { path: '/', status: null, timeoutMs }; } - return routes[0].path; + return { path: routes[0].path, status: routes[0].expect.status.slice(), timeoutMs }; } function loadRouteMatrix(project, stage, runtime) { @@ -2290,6 +2384,14 @@ function removeGeneratedFrameworkArtifacts(project) { continue; } + // Vendored apps may commit prebuilt final artifacts (a release tarball's + // public/build, a prebuilt client dist). Those are part of the example, + // not stale build output — never delete tracked paths. + if (isTrackedExamplePath(project, relativePath)) { + log(`keeping ${targetPath} (tracked in the examples repo)`); + continue; + } + log(`removing ${targetPath}`); fs.rmSync(targetPath, { recursive: true, force: true }); } @@ -2297,22 +2399,26 @@ function removeGeneratedFrameworkArtifacts(project) { maybeRemoveUntrackedPublicDir(project); } -function maybeRemoveUntrackedPublicDir(project) { - const publicDir = path.join(project.dir, 'public'); - if (!fs.existsSync(publicDir)) { - return; - } - - const tracked = spawnSync('git', ['-C', EXAMPLES_DIR, 'ls-files', '--', `${project.name}/public`], { +function isTrackedExamplePath(project, relativePath) { + const tracked = spawnSync('git', ['-C', EXAMPLES_DIR, 'ls-files', '--', `${project.name}/${relativePath}`], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }); if (tracked.error || tracked.status !== 0) { + return false; + } + + return tracked.stdout.trim() !== ''; +} + +function maybeRemoveUntrackedPublicDir(project) { + const publicDir = path.join(project.dir, 'public'); + if (!fs.existsSync(publicDir)) { return; } - if (tracked.stdout.trim() !== '') { + if (isTrackedExamplePath(project, 'public')) { return; } diff --git a/scripts/lib/framework-test-db.js b/scripts/lib/framework-test-db.js new file mode 100644 index 000000000..08df07a75 --- /dev/null +++ b/scripts/lib/framework-test-db.js @@ -0,0 +1,313 @@ +'use strict'; + +// Ephemeral database provisioning for framework tests. +// +// Backend apps declare their database need in a top-level `database` block in +// routes.json: +// +// { +// "version": 1, +// "database": { +// "kind": "postgres", +// "env": { +// "CMD_DB_URL": "{dbUrl}", +// "CMD_PORT": "{port}" +// } +// }, +// "routes": [...] +// } +// +// The harness then starts a real database as a plain child process before the +// app's server starts (no Docker: framework tests also run on macOS CI +// runners, which have no Docker daemon) and injects the `env` block into the +// server environment. Values may reference: +// {dbUrl} {dbHost} {dbPort} {dbUser} {dbPassword} {dbName} — database info +// {port} — the app port, +// expanded later by makeProjectEnv once the harness picks it. +// +// An optional `setup` array lists shell commands (e.g. sequelize migrations +// and seeds) that the harness runs on the host toolchain after the database +// is up and before the app server starts, with the same env injected. +// +// Postgres is provided by the `embedded-postgres` npm package (real zonky.io +// binaries spawned via initdb/pg_ctl); MySQL by `mysql-memory-server` (uses a +// matching system mysqld when available, otherwise downloads official +// binaries). Both install on demand into /db-tools with the same +// pnpm store the project installs use. + +const fs = require('node:fs'); +const net = require('node:net'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { createRequire } = require('node:module'); + +const EMBEDDED_POSTGRES_VERSION = '17.10.0-beta.17'; +const MYSQL_MEMORY_SERVER_VERSION = '1.14.1'; +// Semver range: a matching system mysqld is used as is (Linux CI images and +// most dev machines ship MySQL 8.0), otherwise the newest matching official +// binary is downloaded once and cached (the macOS CI case). Pinned to 8.0.x +// because the harness user authenticates with mysql_native_password (apps on +// the legacy `mysql` 2.x driver — e.g. Firekylin's think-mysql — cannot do +// caching_sha2_password), and 8.4+ disables that plugin by default. +const MYSQL_VERSION_RANGE = '8.0.x'; +const DB_USER = 'framework'; +const DB_PASSWORD = 'framework'; +const DB_NAME = 'app'; +const SUPPORTED_KINDS = ['postgres', 'mysql']; + +const activeDatabases = new Set(); +let signalHandlersInstalled = false; + +function readProjectDatabaseConfig(project, routesJsonPath) { + if (!fs.existsSync(routesJsonPath)) { + return null; + } + + let config = null; + try { + config = JSON.parse(fs.readFileSync(routesJsonPath, 'utf8')); + } catch (error) { + throw new Error(`invalid JSON in ${routesJsonPath}: ${error.message}`); + } + + if (!config || typeof config !== 'object' || config.database == null) { + return null; + } + + const database = config.database; + if (typeof database !== 'object') { + throw new Error(`invalid database block in ${routesJsonPath}: expected an object`); + } + if (!SUPPORTED_KINDS.includes(database.kind)) { + throw new Error(`invalid database block in ${routesJsonPath}: kind must be one of ${SUPPORTED_KINDS.join(', ')}`); + } + if (database.env != null && (typeof database.env !== 'object' || Array.isArray(database.env))) { + throw new Error(`invalid database block in ${routesJsonPath}: env must be an object of string values`); + } + + const env = {}; + for (const [name, value] of Object.entries(database.env || {})) { + if (typeof value !== 'string') { + throw new Error(`invalid database env value for ${name} in ${routesJsonPath}: expected a string`); + } + env[name] = value; + } + + if (database.setup != null && !Array.isArray(database.setup)) { + throw new Error(`invalid database block in ${routesJsonPath}: setup must be an array of shell commands`); + } + const setup = (database.setup || []).map((command) => { + if (typeof command !== 'string' || !command.trim()) { + throw new Error(`invalid database setup command in ${routesJsonPath}: expected a non-empty string`); + } + return command; + }); + + return { kind: database.kind, env, setup }; +} + +function ensureDatabaseTools(stateDir, pnpmStoreDir, log) { + const toolsDir = path.join(stateDir, 'db-tools'); + const manifestPath = path.join(toolsDir, 'package.json'); + const manifest = { + name: 'framework-test-db-tools', + private: true, + dependencies: { + 'embedded-postgres': EMBEDDED_POSTGRES_VERSION, + 'mysql-memory-server': MYSQL_MEMORY_SERVER_VERSION, + }, + }; + const manifestJson = `${JSON.stringify(manifest, null, 2)}\n`; + + fs.mkdirSync(toolsDir, { recursive: true }); + const existing = fs.existsSync(manifestPath) ? fs.readFileSync(manifestPath, 'utf8') : null; + if (existing !== manifestJson) { + fs.writeFileSync(manifestPath, manifestJson); + } + + let needsInstall = existing !== manifestJson; + for (const [name, version] of Object.entries(manifest.dependencies)) { + if (needsInstall) { + break; + } + try { + const installedMarker = path.join(toolsDir, 'node_modules', name, 'package.json'); + const installed = JSON.parse(fs.readFileSync(installedMarker, 'utf8')); + needsInstall = installed.version !== version; + } catch { + needsInstall = true; + } + } + + if (needsInstall) { + log(`installing database tools (${Object.entries(manifest.dependencies).map(([n, v]) => `${n} ${v}`).join(', ')}) into ${toolsDir}`); + const args = ['install', '--no-lockfile', '--config.dangerouslyAllowAllBuilds=true']; + if (pnpmStoreDir) { + args.push('--store-dir', pnpmStoreDir); + } + const result = spawnSync('pnpm', args, { cwd: toolsDir, encoding: 'utf8' }); + if (result.status !== 0) { + const detail = `${result.stdout || ''}${result.stderr || ''}`.trim(); + throw new Error(`pnpm install failed for database tools in ${toolsDir}${detail ? `:\n${detail}` : ''}`); + } + } + + return createRequire(manifestPath); +} + +function allocateFreePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address(); + server.close((error) => (error ? reject(error) : resolve(port))); + }); + }); +} + +function installSignalHandlers() { + if (signalHandlersInstalled) { + return; + } + signalHandlersInstalled = true; + for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => { + stopAllDatabases().finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 143); + }); + }); + } +} + +async function startPostgres(requireTools, project, stage, stateDir) { + const embeddedPostgres = requireTools('embedded-postgres'); + const EmbeddedPostgres = embeddedPostgres.default || embeddedPostgres; + + const port = await allocateFreePort(); + const dataDir = path.join(stateDir, 'db', `${project.name}.${stage.key}`); + fs.rmSync(dataDir, { recursive: true, force: true }); + fs.mkdirSync(dataDir, { recursive: true }); + + const instance = new EmbeddedPostgres({ + databaseDir: dataDir, + user: DB_USER, + password: DB_PASSWORD, + port, + persistent: false, + onLog: () => {}, + onError: () => {}, + }); + + await instance.initialise(); + await instance.start(); + await instance.createDatabase(DB_NAME); + + return { + port, + dataDir, + urlScheme: 'postgres', + stop: () => instance.stop(), + }; +} + +async function startMysql(requireTools) { + const { createDB } = requireTools('mysql-memory-server'); + + // The package creates its `username` user without a password and only for + // 'localhost'; apps connect over TCP with credentials, so create the + // harness user with a password via the init SQL instead. + const instance = await createDB({ + version: MYSQL_VERSION_RANGE, + dbName: DB_NAME, + logLevel: 'ERROR', + downloadBinaryOnce: true, + xEnabled: 'OFF', + initSQLString: [ + `CREATE USER '${DB_USER}'@'%' IDENTIFIED WITH mysql_native_password BY '${DB_PASSWORD}';`, + `GRANT ALL ON *.* TO '${DB_USER}'@'%' WITH GRANT OPTION;`, + ].join('\n'), + }); + + return { + port: instance.port, + dataDir: null, + urlScheme: 'mysql', + stop: () => instance.stop(), + }; +} + +const PROVIDERS = { + postgres: startPostgres, + mysql: startMysql, +}; + +async function startProjectDatabase(options) { + const { config, project, stage, stateDir, pnpmStoreDir, log, logWarn } = options; + const provider = PROVIDERS[config.kind]; + if (!provider) { + throw new Error(`unsupported database kind: ${config.kind}`); + } + + const requireTools = ensureDatabaseTools(stateDir, pnpmStoreDir, log); + const instance = await provider(requireTools, project, stage, stateDir); + + const values = { + dbHost: '127.0.0.1', + dbPort: String(instance.port), + dbUser: DB_USER, + dbPassword: DB_PASSWORD, + dbName: DB_NAME, + dbUrl: `${instance.urlScheme}://${DB_USER}:${DB_PASSWORD}@127.0.0.1:${instance.port}/${DB_NAME}`, + }; + + const env = {}; + for (const [name, template] of Object.entries(config.env)) { + env[name] = Object.entries(values).reduce( + (value, [key, replacement]) => value.split(`{${key}}`).join(replacement), + template, + ); + } + // The WASIX framework runner forwards only an allowlist of env vars into + // the guest; it extends that allowlist with the names listed here. + env.FRAMEWORK_TEST_EXTRA_ENV = Object.keys(env).join(','); + + const handle = { + dataDir: instance.dataDir, + env, + kind: config.kind, + port: instance.port, + stopped: false, + async stop() { + if (handle.stopped) { + return; + } + handle.stopped = true; + activeDatabases.delete(handle); + try { + await instance.stop(); + } catch (error) { + logWarn(`failed to stop ${config.kind} for ${project.name}: ${error.message}`); + } + if (instance.dataDir) { + fs.rmSync(instance.dataDir, { recursive: true, force: true }); + } + }, + }; + + activeDatabases.add(handle); + installSignalHandlers(); + return handle; +} + +async function stopAllDatabases() { + const handles = Array.from(activeDatabases); + await Promise.allSettled(handles.map((handle) => handle.stop())); +} + +module.exports = { + readProjectDatabaseConfig, + startProjectDatabase, + stopAllDatabases, +}; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 63d9eed4a..63a2efbaf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,6 +9,7 @@ set(EDGE_RUNTIME_CORE_SOURCES "${CMAKE_CURRENT_LIST_DIR}/edge_runtime.cc" "${CMAKE_CURRENT_LIST_DIR}/edge_runtime_platform_v8.cc" "${CMAKE_CURRENT_LIST_DIR}/edge_node_compat.cc" + "${CMAKE_CURRENT_LIST_DIR}/edge_cluster_wasix.cc" "${CMAKE_CURRENT_LIST_DIR}/edge_process.cc" "${CMAKE_CURRENT_LIST_DIR}/edge_active_resource.cc" "${CMAKE_CURRENT_LIST_DIR}/edge_worker_env.cc" diff --git a/src/edge_cluster_wasix.cc b/src/edge_cluster_wasix.cc new file mode 100644 index 000000000..09bba981f --- /dev/null +++ b/src/edge_cluster_wasix.cc @@ -0,0 +1,169 @@ +#include "edge_cluster_wasix.h" + +#if defined(__wasi__) + +#include + +#include + +#include "edge_module_loader.h" + +namespace { + +// Invoked with (cluster, net, dgram, sendHelper, UV_TCP_REUSEPORT, +// UV_UDP_REUSEPORT). Kept as an embedded script instead of a lib/ module +// because the Node lib/ tree stays byte-identical to upstream; the behavior +// is edge/WASIX-specific. +// +// TCP and UDP port listens bypass the queryServer round trip entirely: the +// worker binds its own SO_REUSEPORT socket and reports the 'listening' act +// for the primary's bookkeeping (worker state, cluster 'listening' event). +// fd and pipe listens keep the upstream path. Known deviations vs the +// upstream flow: obj._getServerData/_setServerData is not round-tripped +// through the primary (e.g. TLS session ticket keys are per-worker), and +// UDP datagrams are distributed by the kernel's reuseport source hash +// (flows pin to a worker) instead of shared-socket delivery. +constexpr const char kInstallScript[] = R"JS( +(function installWasixClusterReusePort(cluster, net, dgram, sendHelper, + UV_TCP_REUSEPORT, UV_UDP_REUSEPORT) { + 'use strict'; + + if (!cluster.isWorker || typeof cluster._getServer !== 'function') + return; + + const originalGetServer = cluster._getServer; + const ownHandles = new Set(); + let disconnectHookInstalled = false; + + cluster._getServer = function(obj, options, cb) { + const isTcp = options.addressType === 4 || options.addressType === 6; + const isUdp = options.addressType === 'udp4' || + options.addressType === 'udp6'; + // dgram passes the raw bind() arguments through: port can be null, + // undefined, or even the bind callback function (socket.bind(cb)); + // per the bind([port][, address][, callback]) signature all of those + // mean an ephemeral-port listen, not an fd/pipe listen. + const port = (isUdp && typeof options.port !== 'number') ? 0 : options.port; + const isPortListen = + typeof port === 'number' && port >= 0 && + (options.fd == null || options.fd < 0); + + if ((!isTcp && !isUdp) || !isPortListen) + return originalGetServer.call(this, obj, options, cb); + + const rval = isTcp ? + net._createServerHandle(options.address, port, + options.addressType, options.fd, + (options.flags | UV_TCP_REUSEPORT) >>> 0) : + dgram._createSocketHandle(options.address, port, + options.addressType, options.fd, + (options.flags | UV_UDP_REUSEPORT) >>> 0); + if (typeof rval === 'number') { + process.nextTick(cb, rval, null); + return; + } + + ownHandles.add(rval); + const originalClose = rval.close; + rval.close = function() { + ownHandles.delete(rval); + return originalClose.apply(rval, arguments); + }; + + if (!disconnectHookInstalled && cluster.worker) { + disconnectHookInstalled = true; + // Mirror lib/internal/cluster/child.js: close listeners when the + // worker disconnects so the process can drain and exit. + cluster.worker.once('disconnect', () => { + for (const handle of ownHandles) + handle.close(); + ownHandles.clear(); + }); + } + + obj.once('listening', () => { + if (cluster.worker) + cluster.worker.state = 'listening'; + const address = obj.address(); + sendHelper(process, { + act: 'listening', + address: options.address, + port: (address && address.port) || options.port, + addressType: options.addressType, + fd: options.fd, + }, null); + }); + + // Upstream _getServer callbacks always arrive asynchronously (an IPC + // round trip); keep that contract so callers' close-during-bind windows + // behave the same. + process.nextTick(cb, 0, rval); + }; +}) +)JS"; + +void ClearPendingException(napi_env env) { + bool pending = false; + if (napi_is_exception_pending(env, &pending) == napi_ok && pending) { + napi_value ignored = nullptr; + (void)napi_get_and_clear_last_exception(env, &ignored); + } +} + +} // namespace + +void EdgeMaybeInstallWasixClusterReusePort(napi_env env) { + // pre_execution deletes NODE_UNIQUE_ID from process.env, but this runs + // before the main builtin, while the variable is still present. + if (std::getenv("NODE_UNIQUE_ID") == nullptr) return; + + napi_value cluster = nullptr; + napi_value net = nullptr; + napi_value dgram = nullptr; + napi_value utils = nullptr; + if (!EdgeRequireBuiltin(env, "cluster", &cluster) || cluster == nullptr || + !EdgeRequireBuiltin(env, "net", &net) || net == nullptr || + !EdgeRequireBuiltin(env, "internal/dgram", &dgram) || dgram == nullptr || + !EdgeRequireBuiltin(env, "internal/cluster/utils", &utils) || utils == nullptr) { + ClearPendingException(env); + return; + } + + napi_value send_helper = nullptr; + if (napi_get_named_property(env, utils, "sendHelper", &send_helper) != napi_ok || + send_helper == nullptr) { + ClearPendingException(env); + return; + } + + napi_value script = nullptr; + napi_value install_fn = nullptr; + if (napi_create_string_utf8(env, kInstallScript, NAPI_AUTO_LENGTH, &script) != napi_ok || + napi_run_script(env, script, &install_fn) != napi_ok || install_fn == nullptr) { + ClearPendingException(env); + return; + } + + napi_value tcp_reuseport_flag = nullptr; + napi_value udp_reuseport_flag = nullptr; + napi_value global = nullptr; + if (napi_create_uint32(env, static_cast(UV_TCP_REUSEPORT), &tcp_reuseport_flag) != napi_ok || + napi_create_uint32(env, static_cast(UV_UDP_REUSEPORT), &udp_reuseport_flag) != napi_ok || + napi_get_global(env, &global) != napi_ok) { + ClearPendingException(env); + return; + } + + napi_value argv[] = {cluster, net, dgram, send_helper, + tcp_reuseport_flag, udp_reuseport_flag}; + napi_value result = nullptr; + if (napi_call_function(env, global, install_fn, 6, argv, &result) != napi_ok) { + ClearPendingException(env); + } +} + +#else // !defined(__wasi__) + +void EdgeMaybeInstallWasixClusterReusePort(napi_env /*env*/) {} + +#endif // defined(__wasi__) diff --git a/src/edge_cluster_wasix.h b/src/edge_cluster_wasix.h new file mode 100644 index 000000000..acc51023d --- /dev/null +++ b/src/edge_cluster_wasix.h @@ -0,0 +1,22 @@ +#ifndef EDGE_CLUSTER_WASIX_H_ +#define EDGE_CLUSTER_WASIX_H_ + +#include "unofficial_napi.h" + +// Installs the WASIX cluster reuseport scheduling strategy in cluster worker +// processes. No-op on native targets and outside cluster workers. +// +// WASIX cannot pass listen handles between processes (no SCM_RIGHTS over the +// IPC channel), which breaks both of Node cluster's scheduling strategies: +// round robin passes every accepted connection to a worker, and shared-handle +// mode passes the listen handle itself. SO_REUSEPORT works end to end, so TCP +// and UDP port listens in cluster workers bind their own socket instead and +// the host kernel distributes traffic between the workers. +// +// The strategy is implemented by replacing the worker-side cluster._getServer +// (an exported, documented-as-replaceable property) from an embedded script. +// It lives here rather than in lib/ because the Node lib/ tree is kept +// byte-identical to upstream. +void EdgeMaybeInstallWasixClusterReusePort(napi_env env); + +#endif // EDGE_CLUSTER_WASIX_H_ diff --git a/src/edge_intl.cc b/src/edge_intl.cc index 1b18d4f0e..756a649bd 100644 --- a/src/edge_intl.cc +++ b/src/edge_intl.cc @@ -404,6 +404,19 @@ bool InstallConstructor(napi_env env, return true; } +// ECMA-402 legacy behavior: Intl.DateTimeFormat and Intl.NumberFormat may be +// called as plain functions, acting like `new` (ListFormat et al. still +// require `new`). Re-dispatches to the installed constructor. +napi_value ConstructViaNew(napi_env env, const char* name, size_t argc, napi_value* argv) { + napi_value global = nullptr, intl = nullptr, ctor = nullptr, instance = nullptr; + if (napi_get_global(env, &global) != napi_ok || + napi_get_named_property(env, global, "Intl", &intl) != napi_ok || + napi_get_named_property(env, intl, name, &ctor) != napi_ok || ctor == nullptr) + return nullptr; + if (napi_new_instance(env, ctor, argc, argv, &instance) != napi_ok) return nullptr; + return instance; +} + // --------------------------------------------------------------------------- // Intl.ListFormat (ulistfmt_*) // --------------------------------------------------------------------------- @@ -775,7 +788,7 @@ napi_value NumberFormatConstructor(napi_env env, napi_callback_info info) { napi_value new_target = nullptr; if (napi_get_new_target(env, info, &new_target) != napi_ok) return nullptr; - if (new_target == nullptr) return ThrowType(env, "Constructor Intl.NumberFormat requires 'new'"); + if (new_target == nullptr) return ConstructViaNew(env, "NumberFormat", argc, argv); auto* state = new NumberFormatState(); state->locale = argc > 0 ? ResolveIcuLocale(env, argv[0]) : "en_US"; @@ -1133,7 +1146,7 @@ napi_value DateTimeFormatConstructor(napi_env env, napi_callback_info info) { napi_value new_target = nullptr; if (napi_get_new_target(env, info, &new_target) != napi_ok) return nullptr; - if (new_target == nullptr) return ThrowType(env, "Constructor Intl.DateTimeFormat requires 'new'"); + if (new_target == nullptr) return ConstructViaNew(env, "DateTimeFormat", argc, argv); auto* state = new DateTimeFormatState(); state->locale = argc > 0 ? ResolveIcuLocale(env, argv[0]) : "en_US"; diff --git a/src/edge_process.cc b/src/edge_process.cc index e1c166a10..ffabeb864 100644 --- a/src/edge_process.cc +++ b/src/edge_process.cc @@ -94,14 +94,6 @@ std::string g_edge_argv0; std::string g_process_title = "node"; uint32_t g_process_debug_port = 9229; std::mutex g_process_umask_mutex; -std::mutex g_process_dlopen_mutex; -std::map> g_process_dlopen_handles; - -#if defined(__APPLE__) || defined(__linux__) || defined(__sun) || defined(_AIX) -constexpr int kDefaultDlopenFlags = RTLD_LAZY; -#else -constexpr int kDefaultDlopenFlags = 0; -#endif #ifndef EDGE_EMBEDDED_V8_VERSION #define EDGE_EMBEDDED_V8_VERSION "0.0.0-node.0" @@ -136,53 +128,6 @@ std::string GetGlibcCompilerVersion() { #endif } -napi_addon_register_func GetNapiInitializerCallback(uv_lib_t* lib) { - if (lib == nullptr) return nullptr; - void* symbol = nullptr; - if (uv_dlsym(lib, "napi_register_module_v1", &symbol) != 0 || symbol == nullptr) { - return nullptr; - } - return reinterpret_cast(symbol); -} - -std::string BuildDlopenCacheKey(const std::string& filename, int32_t flags) { - return filename + "#" + std::to_string(flags); -} - -int OpenDynamicLibrary(const std::string& filename, int32_t flags, uv_lib_t* lib, std::string* error_out) { - if (lib == nullptr) return UV_EINVAL; - lib->handle = nullptr; - lib->errmsg = nullptr; -#if defined(__APPLE__) || defined(__linux__) || defined(__sun) || defined(_AIX) - lib->handle = dlopen(filename.c_str(), flags); - if (lib->handle != nullptr) return 0; - if (error_out != nullptr) { - const char* error = dlerror(); - *error_out = (error != nullptr && error[0] != '\0') ? error : ("Cannot open shared object file: '" + filename + "'"); - } - return UV_EINVAL; -#else - const int rc = uv_dlopen(filename.c_str(), lib); - if (rc != 0 && error_out != nullptr) { - const char* error = uv_dlerror(lib); - *error_out = (error != nullptr && error[0] != '\0') ? error : ("Cannot open shared object file: '" + filename + "'"); - } - return rc; -#endif -} - -void CloseDynamicLibrary(uv_lib_t* lib) { - if (lib == nullptr) return; -#if defined(__APPLE__) || defined(__linux__) || defined(__sun) || defined(_AIX) - if (lib->handle != nullptr) { - (void)dlclose(lib->handle); - lib->handle = nullptr; - } -#else - uv_dlclose(lib); -#endif -} - #ifndef EDGE_STRINGIFY_HELPER #define EDGE_STRINGIFY_HELPER(x) #x #endif @@ -465,7 +410,11 @@ const char* DetectPlatform() { #elif defined(__linux__) return "linux"; #elif defined(__wasi__) - return "wasi"; + // WASIX emulates Linux syscall semantics, and native and WASIX must expose + // the same functionality: packages that switch on process.platform (e.g. + // playwright-core's registry, which throws on unknown platforms at require + // time) must behave identically on both targets. + return "linux"; #elif defined(__sun) return "sunos"; #elif defined(_AIX) @@ -4363,85 +4312,17 @@ napi_value ProcessMethodsDlopenCallback(napi_env env, napi_callback_info info) { const std::string maybe_name = NapiValueToUtf8(env, argv[1]); if (!maybe_name.empty()) filename = maybe_name; - int32_t flags = kDefaultDlopenFlags; - if (argc > 2 && argv[2] != nullptr) { - if (napi_get_value_int32(env, argv[2], &flags) != napi_ok) { - ThrowTypeErrorWithCode(env, "ERR_INVALID_ARG_TYPE", "flag argument must be an integer."); - return nullptr; - } - } - const std::string cache_key = BuildDlopenCacheKey(filename, flags); - - napi_value module = nullptr; - if (napi_coerce_to_object(env, argv[0], &module) != napi_ok || module == nullptr) { - return nullptr; - } - - napi_value exports_value = nullptr; - if (napi_get_named_property(env, module, "exports", &exports_value) != napi_ok || exports_value == nullptr) { - return nullptr; - } - - napi_value exports = nullptr; - if (napi_coerce_to_object(env, exports_value, &exports) != napi_ok || exports == nullptr) { - return nullptr; - } - - napi_addon_register_func init = nullptr; - uv_lib_t* lib = nullptr; - std::unique_ptr newly_loaded; - bool cache_loaded_library = false; - { - std::lock_guard lock(g_process_dlopen_mutex); - auto it = g_process_dlopen_handles.find(cache_key); - if (it != g_process_dlopen_handles.end()) { - lib = it->second.get(); - } - } - - if (lib == nullptr) { - newly_loaded = std::make_unique(); - std::string message; - if (OpenDynamicLibrary(filename, flags, newly_loaded.get(), &message) != 0) { - ThrowErrorWithCode(env, "ERR_DLOPEN_FAILED", message.c_str()); - return nullptr; - } - lib = newly_loaded.get(); - cache_loaded_library = true; - } - - init = GetNapiInitializerCallback(lib); - - if (init == nullptr) { - const std::string message = "Module did not self-register: '" + filename + "'."; - if (cache_loaded_library && newly_loaded != nullptr) { - CloseDynamicLibrary(newly_loaded.get()); - } - ThrowErrorWithCode(env, "ERR_DLOPEN_FAILED", message.c_str()); - return nullptr; - } - - napi_value addon_exports = init(env, exports); - - bool has_pending = false; - if (napi_is_exception_pending(env, &has_pending) == napi_ok && has_pending) { - return nullptr; - } - - bool same_exports = false; - if (addon_exports != nullptr && - (napi_strict_equals(env, addon_exports, exports, &same_exports) != napi_ok || !same_exports)) { - napi_set_named_property(env, module, "exports", addon_exports); - } - - if (cache_loaded_library && newly_loaded != nullptr) { - std::lock_guard lock(g_process_dlopen_mutex); - g_process_dlopen_handles.emplace(cache_key, std::move(newly_loaded)); - } - - napi_value undefined = nullptr; - napi_get_undefined(env, &undefined); - return undefined; + // EdgeJS deliberately does not load native addons: the native and WASIX + // targets must expose the same functionality, and WASIX has no dynamic + // linking. Fail before any dlopen() so no addon static constructor runs + // (a constructor referencing an unresolved symbol such as + // napi_module_register would abort the whole process uncatchably) and + // optional accelerators like bufferutil can fall back to their pure-JS + // implementations, exactly as they do under WASIX. + const std::string message = + "Loading native addons is not supported: '" + filename + "'"; + ThrowErrorWithCode(env, "ERR_DLOPEN_FAILED", message.c_str()); + return nullptr; } napi_value ProcessMethodsEmptyArrayCallback(napi_env env, napi_callback_info info) { diff --git a/src/edge_runtime.cc b/src/edge_runtime.cc index 9dfd370fa..5f9ae396f 100644 --- a/src/edge_runtime.cc +++ b/src/edge_runtime.cc @@ -50,6 +50,7 @@ #include "edge_crypto.h" #include "edge_encoding.h" #include "edge_http_parser.h" +#include "edge_cluster_wasix.h" #include "edge_module_loader.h" #include "edge_os.h" #include "edge_option_helpers.h" @@ -3025,6 +3026,11 @@ int RunScriptWithGlobals(napi_env env, delete_global_named("__dirname"); } + // Under WASIX, cluster workers get the reuseport scheduling strategy + // installed before the main builtin runs pre-execution (which consumes + // NODE_UNIQUE_ID). No-op elsewhere. + EdgeMaybeInstallWasixClusterReusePort(env); + napi_value result = nullptr; if (selected_main_builtin_id != nullptr && selected_main_builtin_id[0] != '\0') { if (EdgeExecuteBuiltin(env, selected_main_builtin_id, &result)) { diff --git a/src/webassembly/edge_wasm.cc b/src/webassembly/edge_wasm.cc index b0570f5c3..70471b214 100644 --- a/src/webassembly/edge_wasm.cc +++ b/src/webassembly/edge_wasm.cc @@ -51,15 +51,17 @@ struct WasmInstanceObject { struct WasmMemoryObject { WasmObjectBase base; wasm_memory_t *memory = nullptr; + // Cached .buffer ArrayBuffer, detached (and re-minted on next access) when + // the backing wasm memory grows/moves — mirroring the JS API's + // detach-on-grow semantics that wasm-bindgen's view caching relies on. + napi_ref buffer_ref = nullptr; + void *buffer_data = nullptr; + size_t buffer_size = 0; }; struct WasmTableObject { WasmObjectBase base; wasm_table_t *table = nullptr; - bool local_only = false; - uint32_t local_size = 0; - uint32_t local_max = wasm_limits_max_default; - wasm_valkind_t local_element_kind = WASM_FUNCREF; }; struct WasmGlobalObject { @@ -89,6 +91,7 @@ struct WasmState { explicit WasmState(napi_env env_in) : env(env_in) {} ~WasmState() { + DeleteRefIfPresent(env, &externref_values_ref); DeleteRefIfPresent(env, &pending_import_exception_ref); DeleteRefIfPresent(env, &webassembly_ref); DeleteRefIfPresent(env, &module_ctor_ref); @@ -130,6 +133,16 @@ struct WasmState { napi_ref table_ctor_ref = nullptr; napi_ref global_ctor_ref = nullptr; napi_ref pending_import_exception_ref = nullptr; + // Externref registry: JS values passed into wasm as externrefs are rooted + // in a JS array; the array index rides inside the wasm-side foreign object + // as host info. Entries are never released — the wasmer store has no + // reference lifetime management yet (WARP-70 Part B), so the extern objects + // leak until store death regardless. + napi_ref externref_values_ref = nullptr; + uint32_t next_externref_id = 1; + // Live memory objects whose cached buffers must be revalidated at JS↔wasm + // boundaries (wasm-internal memory.grow has no JS-side hook). + std::vector live_memories; }; struct ImportFuncData { @@ -581,16 +594,157 @@ bool ParseValueKind(napi_env env, napi_value value, wasm_valkind_t *out) { return false; } -bool JsToWasmVal(napi_env env, napi_value value, wasm_valkind_t kind, - wasm_val_t *out) { +napi_value CreateFunctionObject(WasmState *state, const std::string &name, + wasm_func_t *owned_func); + +napi_value ExternrefRegistry(WasmState *state) { + if (state == nullptr) + return nullptr; + napi_env env = state->env; + napi_value registry = nullptr; + if (state->externref_values_ref != nullptr) { + if (napi_get_reference_value(env, state->externref_values_ref, + ®istry) != napi_ok) + return nullptr; + return registry; + } + if (napi_create_array(env, ®istry) != napi_ok || registry == nullptr) + return nullptr; + if (napi_create_reference(env, registry, 1, &state->externref_values_ref) != + napi_ok) { + state->externref_values_ref = nullptr; + return nullptr; + } + return registry; +} + +// Mints an owned wasm reference for an arbitrary JS value. Only JS null (and +// an absent value) maps to the null reference; undefined is a real externref +// value (wasm-bindgen roots it in a sentinel table slot). +bool JsToExternRef(WasmState *state, napi_value value, wasm_ref_t **out) { + if (state == nullptr || out == nullptr) + return false; + *out = nullptr; + if (value == nullptr) + return true; + napi_env env = state->env; + napi_valuetype type = napi_undefined; + if (napi_typeof(env, value, &type) != napi_ok) + return false; + if (type == napi_null) + return true; + napi_value registry = ExternrefRegistry(state); + if (registry == nullptr) + return false; + uint32_t id = state->next_externref_id++; + if (napi_set_element(env, registry, id, value) != napi_ok) + return false; + wasm_foreign_t *foreign = wasm_foreign_new(state->store); + if (foreign == nullptr) + return false; + wasm_ref_t *ref = wasm_foreign_as_ref(foreign); + wasm_ref_set_host_info(ref, + reinterpret_cast(static_cast(id))); + *out = ref; + return true; +} + +bool JsToFuncRef(WasmState *state, napi_value value, wasm_ref_t **out) { + if (state == nullptr || out == nullptr) + return false; + *out = nullptr; + if (value == nullptr || IsNullOrUndefined(state->env, value)) + return true; + auto *wrapped = + Unwrap(state->env, value, WasmObjectKind::kFunction); + if (wrapped == nullptr || wrapped->func == nullptr) + return false; + *out = wasm_func_as_ref(wrapped->func); + return *out != nullptr; +} + +bool JsToRef(WasmState *state, napi_value value, wasm_valkind_t kind, + wasm_ref_t **out) { + return kind == WASM_FUNCREF ? JsToFuncRef(state, value, out) + : JsToExternRef(state, value, out); +} + +// Maps a wasm reference back to JS. Externrefs minted by JsToExternRef +// round-trip to the exact same JS value via the registry; funcrefs wrap into +// fresh callable function objects (identity across round-trips is not +// preserved, which the JS API spec permits). Borrows `ref`. +napi_value RefToJs(WasmState *state, wasm_ref_t *ref) { + if (state == nullptr) + return nullptr; + napi_env env = state->env; + if (ref == nullptr) + return Null(env); + if (wasm_func_t *func = wasm_ref_as_func(ref); func != nullptr) + return CreateFunctionObject(state, std::string(), func); + uintptr_t id = reinterpret_cast(wasm_ref_get_host_info(ref)); + if (id == 0) + return Null(env); + napi_value registry = ExternrefRegistry(state); + napi_value out = nullptr; + if (registry == nullptr || + napi_get_element(env, registry, static_cast(id), &out) != + napi_ok) + return Null(env); + return out; +} + +void RefreshMemoryView(WasmMemoryObject *object) { + if (object == nullptr || object->buffer_ref == nullptr || + object->memory == nullptr) + return; + void *data = wasm_memory_data(object->memory); + size_t size = wasm_memory_data_size(object->memory); + if (data == object->buffer_data && size == object->buffer_size) + return; + napi_env env = object->base.state->env; + napi_value buffer = nullptr; + if (napi_get_reference_value(env, object->buffer_ref, &buffer) == napi_ok && + buffer != nullptr) + napi_detach_arraybuffer(env, buffer); + DeleteRefIfPresent(env, &object->buffer_ref); + object->buffer_data = nullptr; + object->buffer_size = 0; +} + +void RefreshMemoryViews(WasmState *state) { + if (state == nullptr) + return; + for (auto *object : state->live_memories) + RefreshMemoryView(object); +} + +bool TableElementKind(wasm_table_t *table, wasm_valkind_t *out) { + if (table == nullptr || out == nullptr) + return false; + wasm_tabletype_t *type = wasm_table_type(table); + if (type == nullptr) + return false; + *out = wasm_valtype_kind(wasm_tabletype_element(type)); + wasm_tabletype_delete(type); + return true; +} + +bool JsToWasmVal(WasmState *state, napi_env env, napi_value value, + wasm_valkind_t kind, wasm_val_t *out) { if (out == nullptr) return false; out->kind = kind; switch (kind) { case WASM_I32: { int32_t number = 0; - if (napi_get_value_int32(env, value, &number) != napi_ok) - return false; + if (napi_get_value_int32(env, value, &number) != napi_ok) { + // ToInt32 coercion, as the JS API demands (wasm-bindgen glue returns + // booleans from predicate imports typed i32). + napi_value coerced = nullptr; + if (napi_coerce_to_number(env, value, &coerced) != napi_ok || + napi_get_value_int32(env, coerced, &number) != napi_ok) + return false; + } out->of.i32 = number; return true; } @@ -608,8 +762,12 @@ bool JsToWasmVal(napi_env env, napi_value value, wasm_valkind_t kind, case WASM_F32: case WASM_F64: { double number = 0; - if (napi_get_value_double(env, value, &number) != napi_ok) - return false; + if (napi_get_value_double(env, value, &number) != napi_ok) { + napi_value coerced = nullptr; + if (napi_coerce_to_number(env, value, &coerced) != napi_ok || + napi_get_value_double(env, coerced, &number) != napi_ok) + return false; + } if (kind == WASM_F32) { out->of.f32 = static_cast(number); } else { @@ -618,17 +776,22 @@ bool JsToWasmVal(napi_env env, napi_value value, wasm_valkind_t kind, return true; } case WASM_EXTERNREF: - case WASM_FUNCREF: - if (!IsNullOrUndefined(env, value)) + case WASM_FUNCREF: { + wasm_ref_t *ref = nullptr; + if (!JsToRef(state, value, kind, &ref)) return false; - out->of.ref = nullptr; + // The wasm_val_t owns the boxed reference; wasm_val_delete / + // wasm_val_vec_delete frees it. + out->of.ref = ref; return true; + } default: return false; } } -napi_value WasmValToJs(napi_env env, const wasm_val_t *value) { +napi_value WasmValToJs(WasmState *state, napi_env env, + const wasm_val_t *value) { if (value == nullptr) return Undefined(env); napi_value out = nullptr; @@ -647,7 +810,7 @@ napi_value WasmValToJs(napi_env env, const wasm_val_t *value) { break; case WASM_EXTERNREF: case WASM_FUNCREF: - out = Null(env); + out = RefToJs(state, value->of.ref); break; default: out = Undefined(env); @@ -705,6 +868,9 @@ wasm_trap_t *JsImportCallback(void *raw, const wasm_val_vec_t *args, } napi_env env = data->env; + // Wasm may have grown its memory since the last JS↔wasm crossing; stale + // cached buffers must read as detached before glue code touches them. + RefreshMemoryViews(data->state); napi_value function = nullptr; if (napi_get_reference_value(env, data->function_ref, &function) != napi_ok || function == nullptr) { @@ -713,7 +879,7 @@ wasm_trap_t *JsImportCallback(void *raw, const wasm_val_vec_t *args, std::vector js_args(args == nullptr ? 0 : args->size); for (size_t i = 0; i < js_args.size(); ++i) { - js_args[i] = WasmValToJs(env, &args->data[i]); + js_args[i] = WasmValToJs(data->state, env, &args->data[i]); } napi_value global = nullptr; @@ -742,7 +908,8 @@ wasm_trap_t *JsImportCallback(void *raw, const wasm_val_vec_t *args, data->state, "WebAssembly import function result type metadata is missing"); } - if (!JsToWasmVal(env, result, data->result_kinds[0], &results->data[0])) { + if (!JsToWasmVal(data->state, env, result, data->result_kinds[0], + &results->data[0])) { return MakeTrap( data->state, "WebAssembly import function returned an incompatible value"); @@ -782,10 +949,16 @@ void InstanceFinalize(napi_env env, void *data, void *) { delete object; } -void MemoryFinalize(napi_env, void *data, void *) { +void MemoryFinalize(napi_env env, void *data, void *) { auto *object = static_cast(data); if (object == nullptr) return; + if (object->base.state != nullptr) { + auto &memories = object->base.state->live_memories; + memories.erase(std::remove(memories.begin(), memories.end(), object), + memories.end()); + } + DeleteRefIfPresent(env, &object->buffer_ref); if (object->memory != nullptr) wasm_memory_delete(object->memory); delete object; @@ -871,10 +1044,15 @@ napi_value CreateFunctionObject(WasmState *state, const std::string &name, wasm_val_vec_t wasm_results; wasm_val_vec_new_uninitialized(&wasm_args, param_count); wasm_val_vec_new_uninitialized(&wasm_results, result_count); + // Zero-fill: wasm_val_delete on a ref-kind val frees of.ref, so + // no slot may hold uninitialized garbage on an early-exit path. + if (param_count > 0) + std::memset(wasm_args.data, 0, param_count * sizeof(wasm_val_t)); bool ok = true; for (size_t i = 0; i < param_count; ++i) { wasm_valkind_t kind = wasm_valtype_kind(params->data[i]); - if (!JsToWasmVal(env, argv[i], kind, &wasm_args.data[i])) { + if (!JsToWasmVal(function->base.state, env, argv[i], kind, + &wasm_args.data[i])) { ok = false; break; } @@ -882,6 +1060,7 @@ napi_value CreateFunctionObject(WasmState *state, const std::string &name, for (size_t i = 0; i < result_count; ++i) { wasm_results.data[i].kind = wasm_valtype_kind(result_types->data[i]); + wasm_results.data[i].of.ref = nullptr; } wasm_functype_delete(type); if (!ok) { @@ -895,6 +1074,7 @@ napi_value CreateFunctionObject(WasmState *state, const std::string &name, wasm_trap_t *trap = wasm_func_call(function->func, &wasm_args, &wasm_results); wasm_val_vec_delete(&wasm_args); + RefreshMemoryViews(function->base.state); if (trap != nullptr) { napi_value pending_exception = nullptr; if (TakePendingImportException(function->base.state, @@ -911,9 +1091,26 @@ napi_value CreateFunctionObject(WasmState *state, const std::string &name, return nullptr; } - napi_value out = result_count == 0 - ? Undefined(env) - : WasmValToJs(env, &wasm_results.data[0]); + napi_value out = nullptr; + if (result_count == 0) { + out = Undefined(env); + } else if (result_count == 1) { + out = WasmValToJs(function->base.state, env, + &wasm_results.data[0]); + } else { + // Multi-value results surface as a JS array, as in the JS API + // (wasm-bindgen's externref ABI relies on this). + if (napi_create_array_with_length(env, result_count, &out) != + napi_ok) { + out = nullptr; + } else { + for (size_t i = 0; i < result_count; ++i) { + napi_set_element(env, out, static_cast(i), + WasmValToJs(function->base.state, env, + &wasm_results.data[i])); + } + } + } wasm_val_vec_delete(&wasm_results); return out; }, @@ -1341,6 +1538,7 @@ napi_value MemoryConstructor(napi_env env, napi_callback_info info) { MemoryFinalize(env, object, nullptr); return nullptr; } + state->live_memories.push_back(object); return this_arg; } @@ -1391,6 +1589,7 @@ napi_value MemoryConstructor(napi_env env, napi_callback_info info) { MemoryFinalize(env, object, nullptr); return nullptr; } + state->live_memories.push_back(object); return this_arg; } @@ -1405,6 +1604,14 @@ napi_value MemoryBufferGetter(napi_env env, napi_callback_info info) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Memory"); return nullptr; } + RefreshMemoryView(object); + if (object->buffer_ref != nullptr) { + napi_value cached = nullptr; + if (napi_get_reference_value(env, object->buffer_ref, &cached) == + napi_ok && + cached != nullptr) + return cached; + } void *data = wasm_memory_data(object->memory); size_t size = wasm_memory_data_size(object->memory); napi_value array_buffer = nullptr; @@ -1416,6 +1623,13 @@ napi_value MemoryBufferGetter(napi_env env, napi_callback_info info) { "Failed to create WebAssembly.Memory buffer"); return nullptr; } + if (napi_create_reference(env, array_buffer, 1, &object->buffer_ref) == + napi_ok) { + object->buffer_data = data; + object->buffer_size = size; + } else { + object->buffer_ref = nullptr; + } return array_buffer; } @@ -1442,6 +1656,7 @@ napi_value MemoryGrow(napi_env env, napi_callback_info info) { napi_throw_range_error(env, nullptr, "WebAssembly.Memory.grow failed"); return nullptr; } + RefreshMemoryView(object); napi_value out = nullptr; napi_create_uint32(env, previous, &out); return out; @@ -1502,20 +1717,32 @@ napi_value TableConstructor(napi_env env, napi_callback_info info) { return nullptr; } - if (argc >= 2 && !IsNullOrUndefined(env, argv[1])) { + wasm_ref_t *init_ref = nullptr; + if (argc >= 2 && !IsNullOrUndefined(env, argv[1]) && + !JsToRef(state, argv[1], element_kind, &init_ref)) { napi_throw_type_error(env, nullptr, - "WebAssembly.Table non-null initial values are not " - "supported by this Wasmer C API build"); + "Invalid WebAssembly.Table initial value"); + return nullptr; + } + + wasm_tabletype_t *table_type = + wasm_tabletype_new(wasm_valtype_new(element_kind), &limits); + wasm_table_t *table = + table_type == nullptr ? nullptr + : wasm_table_new(state->store, table_type, init_ref); + if (table_type != nullptr) + wasm_tabletype_delete(table_type); + if (init_ref != nullptr) + wasm_ref_delete(init_ref); + if (table == nullptr) { + napi_throw_error(env, nullptr, "Failed to create WebAssembly.Table"); return nullptr; } auto *object = new WasmTableObject(); object->base.kind = WasmObjectKind::kTable; object->base.state = state; - object->local_only = true; - object->local_size = limits.min; - object->local_max = limits.max; - object->local_element_kind = element_kind; + object->table = table; if (napi_wrap(env, this_arg, object, TableFinalize, nullptr, nullptr) != napi_ok) { TableFinalize(env, object, nullptr); @@ -1530,15 +1757,12 @@ napi_value TableLengthGetter(napi_env env, napi_callback_info info) { if (!GetCallback(env, info, &argc, nullptr, &this_arg, nullptr)) return nullptr; auto *object = Unwrap(env, this_arg, WasmObjectKind::kTable); - if (object == nullptr || (!object->local_only && object->table == nullptr)) { + if (object == nullptr || object->table == nullptr) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Table"); return nullptr; } napi_value out = nullptr; - napi_create_uint32(env, - object->local_only ? object->local_size - : wasm_table_size(object->table), - &out); + napi_create_uint32(env, wasm_table_size(object->table), &out); return out; } @@ -1549,7 +1773,7 @@ napi_value TableGet(napi_env env, napi_callback_info info) { if (!GetCallback(env, info, &argc, argv, &this_arg, nullptr)) return nullptr; auto *object = Unwrap(env, this_arg, WasmObjectKind::kTable); - if (object == nullptr || (!object->local_only && object->table == nullptr)) { + if (object == nullptr || object->table == nullptr) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Table"); return nullptr; } @@ -1559,14 +1783,16 @@ napi_value TableGet(napi_env env, napi_callback_info info) { "WebAssembly.Table.get expects an index"); return nullptr; } - uint32_t size = - object->local_only ? object->local_size : wasm_table_size(object->table); - if (index >= size) { + if (index >= wasm_table_size(object->table)) { napi_throw_range_error(env, nullptr, "WebAssembly.Table.get index is out of range"); return nullptr; } - return Null(env); + wasm_ref_t *ref = wasm_table_get(object->table, index); + napi_value out = RefToJs(object->base.state, ref); + if (ref != nullptr) + wasm_ref_delete(ref); + return out; } napi_value TableSet(napi_env env, napi_callback_info info) { @@ -1576,7 +1802,7 @@ napi_value TableSet(napi_env env, napi_callback_info info) { if (!GetCallback(env, info, &argc, argv, &this_arg, nullptr)) return nullptr; auto *object = Unwrap(env, this_arg, WasmObjectKind::kTable); - if (object == nullptr || (!object->local_only && object->table == nullptr)) { + if (object == nullptr || object->table == nullptr) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Table"); return nullptr; } @@ -1586,17 +1812,28 @@ napi_value TableSet(napi_env env, napi_callback_info info) { "WebAssembly.Table.set expects an index"); return nullptr; } - if (argc >= 2 && !IsNullOrUndefined(env, argv[1])) { + if (index >= wasm_table_size(object->table)) { + napi_throw_range_error(env, nullptr, + "WebAssembly.Table.set index is out of range"); + return nullptr; + } + wasm_valkind_t element_kind = WASM_FUNCREF; + if (!TableElementKind(object->table, &element_kind)) { + napi_throw_error(env, nullptr, "Failed to read WebAssembly.Table type"); + return nullptr; + } + wasm_ref_t *ref = nullptr; + if (argc >= 2 && + !JsToRef(object->base.state, argv[1], element_kind, &ref)) { napi_throw_type_error(env, nullptr, - "WebAssembly.Table.set only supports null values " - "with this Wasmer C API build"); + "Invalid value for WebAssembly.Table.set"); return nullptr; } - uint32_t size = - object->local_only ? object->local_size : wasm_table_size(object->table); - if (index >= size) { - napi_throw_range_error(env, nullptr, - "WebAssembly.Table.set index is out of range"); + bool ok = wasm_table_set(object->table, index, ref); + if (ref != nullptr) + wasm_ref_delete(ref); + if (!ok) { + napi_throw_range_error(env, nullptr, "WebAssembly.Table.set failed"); return nullptr; } return Undefined(env); @@ -1609,7 +1846,7 @@ napi_value TableGrow(napi_env env, napi_callback_info info) { if (!GetCallback(env, info, &argc, argv, &this_arg, nullptr)) return nullptr; auto *object = Unwrap(env, this_arg, WasmObjectKind::kTable); - if (object == nullptr || (!object->local_only && object->table == nullptr)) { + if (object == nullptr || object->table == nullptr) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Table"); return nullptr; } @@ -1619,21 +1856,23 @@ napi_value TableGrow(napi_env env, napi_callback_info info) { "WebAssembly.Table.grow expects a count"); return nullptr; } - if (argc >= 2 && !IsNullOrUndefined(env, argv[1])) { + wasm_valkind_t element_kind = WASM_FUNCREF; + if (!TableElementKind(object->table, &element_kind)) { + napi_throw_error(env, nullptr, "Failed to read WebAssembly.Table type"); + return nullptr; + } + wasm_ref_t *init_ref = nullptr; + if (argc >= 2 && + !JsToRef(object->base.state, argv[1], element_kind, &init_ref)) { napi_throw_type_error(env, nullptr, - "WebAssembly.Table.grow only supports null initial " - "values with this Wasmer C API build"); + "Invalid initial value for WebAssembly.Table.grow"); return nullptr; } - uint32_t previous = - object->local_only ? object->local_size : wasm_table_size(object->table); - if (object->local_only) { - if (delta > object->local_max || previous > object->local_max - delta) { - napi_throw_range_error(env, nullptr, "WebAssembly.Table.grow failed"); - return nullptr; - } - object->local_size += delta; - } else if (!wasm_table_grow(object->table, delta, nullptr)) { + uint32_t previous = wasm_table_size(object->table); + bool ok = wasm_table_grow(object->table, delta, init_ref); + if (init_ref != nullptr) + wasm_ref_delete(init_ref); + if (!ok) { napi_throw_range_error(env, nullptr, "WebAssembly.Table.grow failed"); return nullptr; } @@ -1704,7 +1943,7 @@ napi_value GlobalConstructor(napi_env env, napi_callback_info info) { initial = zero; } wasm_val_t initial_value; - if (!JsToWasmVal(env, initial, value_kind, &initial_value)) { + if (!JsToWasmVal(state, env, initial, value_kind, &initial_value)) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Global initial value"); return nullptr; @@ -1745,7 +1984,7 @@ napi_value GlobalValueGetter(napi_env env, napi_callback_info info) { } wasm_val_t value; wasm_global_get(object->global, &value); - napi_value out = WasmValToJs(env, &value); + napi_value out = WasmValToJs(object->base.state, env, &value); wasm_val_delete(&value); return out; } @@ -1772,7 +2011,8 @@ napi_value GlobalValueSetter(napi_env env, napi_callback_info info) { } wasm_valkind_t kind = wasm_valtype_kind(wasm_globaltype_content(type)); wasm_val_t value; - bool ok = argc >= 1 && JsToWasmVal(env, argv[0], kind, &value); + bool ok = argc >= 1 && + JsToWasmVal(object->base.state, env, argv[0], kind, &value); wasm_globaltype_delete(type); if (!ok) { napi_throw_type_error(env, nullptr, "Invalid WebAssembly.Global value"); diff --git a/wasmer-examples b/wasmer-examples index 3e160274a..1425ccee5 160000 --- a/wasmer-examples +++ b/wasmer-examples @@ -1 +1 @@ -Subproject commit 3e160274ae644e97774784576cacdd7af60865da +Subproject commit 1425ccee5af02e84adcf15fa2af79057d045f8f0