test(cli-e2e): capability probes + capability-based live gating

apps/cli-e2e/src/tests/env.ts

@@ -50,6 +50,41 @@ export const TARGET_API_URL =
 export const PROJECT_HOST =
   process.env["CLI_E2E_PROJECT_HOST"] ?? (TARGET_ENV === "staging" ? "supabase.red" : "");
 
+// Runtime capabilities a live target can offer the cli. Live tests declare what
+// they need (see `testLiveRequires`) and are skipped when the target can't
+// provide it — so the same suite runs everywhere, skipping only what a given
+// environment genuinely can't do:
+//   - docker         control a container / has a Docker socket
+//   - internet       reach 3rd-party network at runtime (jsr.io, npm, image pulls)
+//   - external-tool  native pg_dump/psql, diff engine (SUPABASE_DB_USE_LOCAL_TOOLS)
+const ALL_CAPABILITIES = ["docker", "internet", "external-tool"] as const;
+export type Capability = (typeof ALL_CAPABILITIES)[number];
+
+// Per-target defaults for what the environment provides. `staging` provides
+// everything — it is the oracle: every probe must pass there, so a supabox
+// skip/red is a genuine gap. `supabox` starts empty (locked down) and is opened
+// up via CLI_E2E_CAPABILITIES as it gains real support; a probe only *runs* on
+// supabox once its capability is declared, and must then pass (vs staging).
+const DEFAULT_CAPABILITIES: Record<CliE2eTargetEnv, readonly Capability[]> = {
+  staging: ALL_CAPABILITIES,
+  supabox: [],
+};
+
+function isCapability(value: string): value is Capability {
+  return (ALL_CAPABILITIES as readonly string[]).includes(value);
+}
+
+// Override the per-target defaults with an explicit comma list, e.g.
+// CLI_E2E_CAPABILITIES=docker,external-tool (an Antithesis run never sets `internet`).
+const CAPABILITIES_OVERRIDE = process.env["CLI_E2E_CAPABILITIES"];
+export const PROVIDED_CAPABILITIES: ReadonlySet<Capability> = new Set(
+  CAPABILITIES_OVERRIDE === undefined
+    ? DEFAULT_CAPABILITIES[TARGET_ENV]
+    : CAPABILITIES_OVERRIDE.split(",")
+        .map((entry) => entry.trim())
+        .filter(isCapability),
+);
+
 // In replay mode the token never reaches a real API, but the Go CLI validates
 // the format before making any request (must match sbp_[a-f0-9]{40}).
 // In record/live mode it must be a valid token for the target env. Falls back to

apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts

@@ -0,0 +1,115 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect } from "vitest";
+import { expectFunctionOk } from "./invoke.ts";
+import { seedFunctions, testLiveRequires } from "./live-context.ts";
+
+// Capability probes: one minimal test per runtime-capability combination the cli
+// needs from its target. Each is a smoke check that FORCES its capability, so a
+// failure (or a target-driven skip) is a precise signal of what an environment
+// can or cannot do. Distilled from the richer feature tests so each isolates a
+// single capability.
+//
+// Staging is the oracle: with CLI_E2E_TARGET_ENV=staging (provides all
+// capabilities) every probe runs and must pass — proving the probe is sound, so
+// any supabox skip/red is a genuine gap. On supabox each probe only runs once
+// its capability is declared via CLI_E2E_CAPABILITIES, then must match staging.
+// See `testLiveRequires` + PROVIDED_CAPABILITIES in ../env.ts.
+describe("capability probes (live)", () => {
+  // C1 — mgmt-api only (no docker, no internet, no external tool). The
+  // provisioned project shows up in `projects list`: a pure control-plane read.
+  testLiveRequires([])(
+    "[C1] mgmt-api only: projects list includes the project",
+    async ({ run, projectRef }) => {
+      const res = await run(["projects", "list", "--output", "json"]);
+      expect(res.exitCode, res.stderr).toBe(0);
+      const refs = (JSON.parse(res.stdout) as Array<{ id?: string; ref?: string }>).map(
+        (project) => project.ref ?? project.id,
+      );
+      expect(refs).toContain(projectRef);
+    },
+  );
+
+  // C3 — external tool, no docker/internet. `db dump` of the remote schema over
+  // the IPv4 pooler using the native pg_dump/psql on PATH
+  // (SUPABASE_DB_USE_LOCAL_TOOLS), i.e. without spawning a supabase/postgres
+  // container. Fails if the external tool is absent.
+  testLiveRequires(["external-tool"])(
+    "[C3] external tool: db dump exports the remote schema",
+    async ({ run, dbUrl, workspace }) => {
+      const file = join(workspace.path, "dump.sql");
+      const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]);
+      expect(res.exitCode, res.stderr).toBe(0);
+      expect(existsSync(file)).toBe(true);
+      expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i);
+    },
+  );
+
+  // C2 — docker control, no runtime internet. `db pull`'s schema diff starts a
+  // shadow postgres *server* (DockerStart) and runs the diff engine in a
+  // container; both use pre-built images (no 3rd-party network). Push first so
+  // local history matches the shared per-run project, then pull. A missing
+  // Docker socket makes DockerStart fail — a genuine docker gate.
+  testLiveRequires(["docker"])(
+    "[C2] docker (offline): db push then db pull round-trips",
+    async ({ run, dbUrl, workspace }) => {
+      const migrations = join(workspace.path, "supabase", "migrations");
+      mkdirSync(migrations, { recursive: true });
+      writeFileSync(
+        join(migrations, "20240101000000_probe_push.sql"),
+        "create table if not exists capability_probe (id int);\n",
+      );
+
+      const pushed = await run(["db", "push", "--db-url", dbUrl, "--yes"]);
+      expect(pushed.exitCode, pushed.stderr).toBe(0);
+
+      const pulled = await run(["db", "pull", "--db-url", dbUrl, "--yes"]);
+      const output = `${pulled.stdout}${pulled.stderr}`;
+      // Distinguish a real docker/connection failure from a benign "no changes".
+      expect(output, "db pull hit a docker/connection error").not.toMatch(
+        /cannot connect to the docker daemon|is the docker daemon running|dial|connection refused|could not connect/i,
+      );
+      expect(pulled.exitCode === 0 || /No schema changes found/i.test(output), pulled.stderr).toBe(
+        true,
+      );
+    },
+  );
+
+  // C4 — runtime 3rd-party internet, no docker. Deploy a function that imports
+  // from jsr.io with the default (server-side) bundler and invoke it; the bundle
+  // path must fetch the import over the network. Fails offline.
+  testLiveRequires(["internet"])(
+    "[C4] internet: deploy a jsr-importing function and invoke",
+    async ({ run, invoke, workspace, projectRef }) => {
+      seedFunctions(workspace.path);
+      const slug = "deploy-e2e-jsr";
+      const deployed = await run(["functions", "deploy", slug, "--project-ref", projectRef]);
+      expect(deployed.exitCode, deployed.stderr).toBe(0);
+      expect(deployed.stdout).toContain("Deployed Functions");
+      expectFunctionOk(await invoke(slug), slug);
+    },
+  );
+
+  // C5 — docker AND runtime internet. Same jsr-importing function, but bundled
+  // locally in a Docker container (`--use-docker`): needs a Docker socket AND the
+  // in-container bundler needs internet to fetch the jsr import. Fails if either
+  // is missing.
+  testLiveRequires(["docker", "internet"])(
+    "[C5] docker + internet: --use-docker deploy of a jsr function",
+    async ({ run, invoke, workspace, projectRef }) => {
+      seedFunctions(workspace.path);
+      const slug = "deploy-e2e-jsr";
+      const deployed = await run([
+        "functions",
+        "deploy",
+        slug,
+        "--project-ref",
+        projectRef,
+        "--use-docker",
+      ]);
+      expect(deployed.exitCode, deployed.stderr).toBe(0);
+      expect(deployed.stdout).toContain("Deployed Functions");
+      expectFunctionOk(await invoke(slug), slug);
+    },
+  );
+});

apps/cli-e2e/src/tests/live/live-context.ts

@@ -8,7 +8,15 @@ import {
   type CLIResult,
   type TempDir,
 } from "@supabase/cli-test-helpers";
-import { ACCESS_TOKEN, isLive, PROJECT_HOST, TARGET, TARGET_API_URL } from "../env.ts";
+import {
+  ACCESS_TOKEN,
+  type Capability,
+  isLive,
+  PROJECT_HOST,
+  PROVIDED_CAPABILITIES,
+  TARGET,
+  TARGET_API_URL,
+} from "../env.ts";
 import { invokeFunction, type InvokeResult } from "./invoke.ts";
 
 type ExecOptions = NonNullable<Parameters<typeof exec>[2]>;
@@ -120,3 +128,14 @@ const base = test.extend<LiveFixtures>({
 /** Live test API — skipped unless CLI_E2E_MODE=live, so files are inert on
  *  replay/PR runs (and globalSetup provisions nothing). */
 export const testLive = base.skipIf(!isLive);
+
+/** Live test API that additionally skips unless the target env provides every
+ *  required runtime capability (docker / internet / external-tool). Lets one
+ *  suite run against staging (all capabilities → runs everything, the oracle),
+ *  supabox (only what it currently supports), and Antithesis (offline subset),
+ *  each skipping only what it genuinely can't do. Put the requirement in the test
+ *  name (e.g. "[C5] … (docker+internet)") so a skip reads clearly in the report. */
+export function testLiveRequires(required: readonly Capability[]): typeof testLive {
+  const missing = required.filter((capability) => !PROVIDED_CAPABILITIES.has(capability));
+  return missing.length === 0 ? testLive : testLive.skip;
+}