diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a3571ced..23892363 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "source": { "source": "npm", "package": "@copilotkit/aimock", - "version": "^1.35.1" + "version": "^1.38.0" }, "description": "Fixture authoring skill for @copilotkit/aimock — LLM, multimedia (image/TTS/transcription/video), MCP, A2A, AG-UI, vector, embeddings, structured output, sequential responses, streaming physics, record/replay, agent loop patterns, and debugging" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 0c719560..26ee361c 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "aimock", - "version": "1.35.1", + "version": "1.38.0", "description": "Fixture authoring guidance for @copilotkit/aimock — LLM, multimedia, MCP, A2A, AG-UI, vector, and service mocking", "author": { "name": "CopilotKit" diff --git a/.github/workflows/publish-pytest.yml b/.github/workflows/publish-pytest.yml deleted file mode 100644 index 4bb7ddbf..00000000 --- a/.github/workflows/publish-pytest.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Publish aimock-pytest -on: - push: - branches: [main] - paths: - - "packages/aimock-pytest/**" - workflow_dispatch: - -permissions: - contents: read - -jobs: - publish: - runs-on: ubuntu-latest - environment: pypi - permissions: - contents: read - id-token: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: { persist-credentials: false } - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install build tools - run: pip install hatch - - - name: Check if version is already published - id: check - run: | - VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/aimock-pytest/pyproject.toml', 'rb'))['project']['version'])") - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - if pip install "aimock-pytest==$VERSION" --dry-run --no-deps 2>/dev/null; then - echo "published=true" >> "$GITHUB_OUTPUT" - else - echo "published=false" >> "$GITHUB_OUTPUT" - fi - - - name: Build - if: steps.check.outputs.published == 'false' - run: cd packages/aimock-pytest && hatch build - - - name: Publish to PyPI - if: steps.check.outputs.published == 'false' - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 - with: - packages-dir: packages/aimock-pytest/dist/ diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index bbd3a841..16a03dc8 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -155,3 +155,53 @@ jobs: curl -s -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" + + publish-pytest: + needs: [build, publish] + if: >- + always() && + (needs.publish.result == 'success' || needs.build.outputs.published == 'true') + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: { persist-credentials: false } + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + package-manager-cache: false + + - name: Verify pinned npm release is published + run: | + VERSION=$(python -c "import sys; sys.path.insert(0, 'packages/aimock-pytest/src'); from aimock_pytest._version import AIMOCK_VERSION; print(AIMOCK_VERSION)") + npm view "@copilotkit/aimock@${VERSION}" version + + - name: Install build tools + run: pip install hatch + + - name: Check if pytest version is already published + id: check + run: | + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/aimock-pytest/pyproject.toml', 'rb'))['project']['version'])") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if pip install "aimock-pytest==$VERSION" --dry-run --no-deps 2>/dev/null; then + echo "published=true" >> "$GITHUB_OUTPUT" + else + echo "published=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build pytest package + if: steps.check.outputs.published == 'false' + run: cd packages/aimock-pytest && hatch build + + - name: Publish pytest package to PyPI + if: steps.check.outputs.published == 'false' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + with: + packages-dir: packages/aimock-pytest/dist/ diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 5f89188d..ba3e0ee5 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -23,4 +23,5 @@ jobs: node-version: ${{ matrix.node-version }} cache: pnpm - run: pnpm install --frozen-lockfile + - run: pnpm build - run: pnpm test diff --git a/CHANGELOG.md b/CHANGELOG.md index ddcf4e9c..324d133c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,14 @@ ## [Unreleased] +## [1.38.0] - 2026-08-03 + ### Added +- OpenAI transcription replay for `gpt-transcribe` HTTP streams and `gpt-live-transcribe` Realtime sessions. Recorded responses retain transcript languages and usage metadata; replay supports progressive transcript events, timing controls, strict fixture matching, and interruption behavior. +- Opt-in inbound API-key validation for HTTP, control, mount, and WebSocket boundaries. Validated test credentials are never forwarded by the generic record/proxy path; auth-enabled proxying requires a configured provider credential. +- API-key egress isolation is request-scoped across generic, AG-UI, fal, and video record paths. Authorization schemes are case-insensitive, and OpenRouter off-origin unsigned URLs never receive a configured provider credential. +- `aimock-pytest` `0.5.2` is released by the Release workflow after npm `@copilotkit/aimock` `1.38.0`; the PyPI job verifies the pinned npm package is available before building a wheel. - **Reasoning `encrypted_content` on the Responses API.** aimock now synthesizes an opaque (base64) `reasoning.encrypted_content` blob on the TERMINAL reasoning item (`response.output_item.done` and non-streaming `output[]`). Emitted only when the request wants it: `include: ["reasoning.encrypted_content"]` OR a stateless request (`store: false` / ZDR). Stored / opted-out replays stay byte-identical. A request carrying the EXPLICIT `include: ["reasoning.encrypted_content"]` opt-in, against a reasoning-CAPABLE model, now gets a reasoning item even when the fixture declares NO `reasoning` summary — a `summary: []` item that exists purely to carry the blob, matching what real OpenAI returns when summaries were never requested. This is the shape a fixture recorded from the real stateless agent-framework flow has, so gating the item on a declared summary starved the exact case the feature exists to serve. The synthesized item leads `output[]` / takes `output_index` 0 and shifts the rest, emits `output_item.added` → `output_item.done` with no summary-text events (there is no summary part to describe), and is suppressed for non-reasoning models (gpt-4o etc.), which have no reasoning channel at all. Note the two gates differ in width on purpose: `store: false` still attaches the blob to a reasoning item that a declared summary already produces, but it does NOT synthesize an item where the fixture declares none — creating an output item that did not previously exist is a bigger behavior change than adding a field to one already on the wire, so it takes the gate that is directly observable in the request. `agent-framework-openai` >= 1.11.0 sends `include` and never `store`, so the narrower gate costs the feature nothing. This lets `agent-framework-openai` >= 1.11.0 replay reasoning-paired tool calls on a stateless reasoning + multi-tool chain (related: microsoft/agent-framework#7233). Applies to both the HTTP and WebSocket Responses transports. Known limitations: the blob is withheld from the in-progress `added` item, which is aimock's own simplification rather than upstream parity — real OpenAI populates `added` too (recorded captures carry a shorter blob there beside an empty summary, re-encrypted by `done`), but the field is `anyOf: [string, null]` and non-required, so omitting it is contract-legal and no known consumer requires it; and aimock skips inbound reasoning items, so it cannot reproduce OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id. - **OpenRouter chat / LLM router simulation.** Requests whose original path starts with `/api/v1/` (point any OpenAI SDK at a `baseURL` ending `/api/v1`) are detected as OpenRouter and shaped to match real OpenRouter bytes: a `gen-` id prefix (a fixture `id` override still wins), a top-level `provider` (default = the winning model slug's author, fixture-overridable via `provider`), both `finish_reason` and `native_finish_reason` on every choice/delta, an always-present `system_fingerprint` and `service_tier` (null by default), `message.reasoning`, and a rich `usage` with a **fixture-scriptable** `cost` + `cost_details` (emitted only when a fixture supplies a cost — never fabricated), plus `is_byok` / `prompt_tokens_details` / `completion_tokens_details` when overridden. Callers on the plain OpenAI `/v1/...` base are byte-for-byte unchanged. - **`models[]` fallback (router failover) simulation.** When the request body carries `models: [...]`, aimock walks `[model, ...models]` in order and serves the first fixture returning a NON-error response; a `429`/`503` error fixture on a candidate simulates a runtime provider failure and falls through to the next. The winning slug is echoed back as the top-level `model`, so a test asserts failover by reading `response.model`. (Deliberate non-goal: an unknown/invalid model is a fixture miss, not OpenRouter's up-front invalid-model 400.) @@ -12,6 +18,10 @@ - **Opt-in `: OPENROUTER PROCESSING` SSE keepalive** (fixture option `openRouterProcessing`, default off): one comment line emitted before the first data frame, matching real OpenRouter streams. - OpenRouter request extensions (`provider`, `models`, `route`, `reasoning`, `plugins`, `prediction`, `usage`) and attribution headers (`HTTP-Referer`, `X-OpenRouter-Title`, legacy `X-Title`) are accepted and journaled, never required or rejected. +### Fixed + +- **`GET /__aimock/journal` no longer leaks accepted inbound credentials.** Journal header redaction is now derived from the same registry that decides which headers are accepted as credentials, so the two can no longer drift apart. Previously only `authorization`, `x-api-key` and `api-key` were redacted while `x-goog-api-key` and `xi-api-key` were accepted as credentials and journaled **in plaintext** — so a real Gemini / Veo / ElevenLabs key sent on those headers appeared verbatim in the journal, which is unauthenticated unless inbound API-key validation is enabled. Affects 1.37.4 and earlier. A test asserts every accepted header redacts, so adding a new accepted header without redacting it now fails. + ## [1.37.4] - 2026-07-20 ### Fixed diff --git a/README.md b/README.md index 3b51d6d3..5a90e25a 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,24 @@ Private and link-local addresses (loopback, RFC1918, CGNAT, cloud metadata, ULA, On replay, `turnIndex` is a non-fatal disambiguator, not a hard reject gate: a content-matching fixture is served even when its scripted `turnIndex` differs from the request's assistant-message count. This kills false "no fixture matched" misses for multi-bubble agent runs (multi-step agents emit several assistant bubbles per logical turn). When a served fixture diverges from its scripted `turnIndex`, the match diagnostic carries `turnIndexRelaxed: true` and aimock logs a one-shot warning (at the `warn` log level — silent by default). To restore the legacy strict behavior where a defined `turnIndex` must equal the assistant count exactly, set `AIMOCK_STRICT_TURN_INDEX=1`. The record path is always strict regardless of this flag. +## API-key validation + +By default aimock accepts all requests. Opt into inbound test-client validation with a programmatic option, top-level `aimock.json` field, or environment-only key list: + +```ts +await createServer(fixtures, { auth: { apiKeys: ["test-key"] } }); +``` + +```json +{ "auth": { "apiKeys": ["test-key"] } } +``` + +```bash +AIMOCK_API_KEYS=test-key,rotated-key npx @copilotkit/aimock --config aimock.json +``` + +Use `Authorization: Bearer `, `Authorization: Key `, `x-api-key`, `x-goog-api-key`, `api-key`, or `xi-api-key`. Every supplied credential must resolve to one configured key; mismatches return `401` with an OpenAI-compatible authentication error. HTTP routes, control APIs, mounts, and WebSocket upgrades are protected. Genuine CORS preflights plus `GET /health`, `GET /ready`, and `GET /metrics` remain public. This is inbound test access control, distinct from `record.providerKeys`; when enabled, proxying strips test credentials and requires a configured static provider credential before egress. + ### aimock-owned upstream keys — `AIMOCK_PROVIDER_*_KEY` In record or `--proxy-only` mode, aimock forwards the caller's auth header to the real provider unchanged. If your tests can only send a dummy placeholder key (e.g. an SDK that refuses to start without a non-empty API key), aimock can inject its own configured upstream key on a fixture-miss passthrough so the proxied call actually authenticates. Each provider has an independent env var, and the key is applied with the provider-correct wire scheme: diff --git a/charts/aimock/Chart.yaml b/charts/aimock/Chart.yaml index d972ea60..aa52da76 100644 --- a/charts/aimock/Chart.yaml +++ b/charts/aimock/Chart.yaml @@ -3,4 +3,4 @@ name: aimock description: Mock infrastructure for AI application testing (OpenAI, Anthropic, Gemini, MCP, A2A, vector) type: application version: 0.1.0 -appVersion: "1.35.1" +appVersion: "1.38.0" diff --git a/package.json b/package.json index d26d1270..e9c8f574 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@copilotkit/aimock", - "version": "1.37.4", + "version": "1.38.0", "description": "Mock infrastructure for AI application testing — LLM APIs, image generation, image editing, text-to-speech, transcription, audio translation, audio generation, video generation, embeddings, MCP tools, A2A agents, AG-UI event streams, vector databases, search, rerank, and moderation. One package, one port, zero dependencies.", "license": "MIT", "keywords": [ diff --git a/packages/aimock-pytest/README.md b/packages/aimock-pytest/README.md index ae099024..c3540626 100644 --- a/packages/aimock-pytest/README.md +++ b/packages/aimock-pytest/README.md @@ -79,9 +79,14 @@ aimock.reset() # alias for reset_fixtures() ``` --aimock-node PATH Path to node binary ---aimock-version VER aimock npm version (default: 1.35.1) +--aimock-version VER aimock npm version (default: 1.38.0) +--aimock-api-key KEY Inbound API key for the aimock child process ``` +## API-key validation + +Pass `pytest --aimock-api-key test-key` to protect the aimock child. The helper sends this key on all control API calls, and the child receives it through `AIMOCK_API_KEYS`, never through process arguments. Direct client calls must use `Authorization: Bearer test-key`. For direct construction, use `AIMockServer(node_manager, api_key="test-key")`. + ## Environment Variables | Variable | Description | @@ -127,8 +132,10 @@ The `test-pytest.yml` workflow: Tests run across a matrix of Python 3.10--3.13 and Node 20/22. -The `publish-pytest.yml` workflow publishes to PyPI on pushes to `main` when -the version in `pyproject.toml` has not already been published. +The Release workflow publishes `aimock-pytest` to PyPI after its npm publish +job succeeds. Its PyPI job verifies that the `AIMOCK_VERSION` pin exists on +npm before building a wheel, so npm publication completes before the +corresponding `aimock-pytest` release. ## License diff --git a/packages/aimock-pytest/pyproject.toml b/packages/aimock-pytest/pyproject.toml index 1cbd98c0..7d30cff1 100644 --- a/packages/aimock-pytest/pyproject.toml +++ b/packages/aimock-pytest/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aimock-pytest" -version = "0.5.0" +version = "0.5.2" description = "pytest fixtures for aimock — mock LLM APIs, multimedia, MCP, A2A, AG-UI, vector DBs, and more" readme = "README.md" requires-python = ">=3.10" diff --git a/packages/aimock-pytest/src/aimock_pytest/_server.py b/packages/aimock-pytest/src/aimock_pytest/_server.py index 07d98a09..27eaf134 100644 --- a/packages/aimock-pytest/src/aimock_pytest/_server.py +++ b/packages/aimock-pytest/src/aimock_pytest/_server.py @@ -27,10 +27,12 @@ def __init__( node_manager: NodeManager, port: int = 0, fixtures_path: str | Path | None = None, + api_key: str | None = None, ) -> None: self.node_manager = node_manager self.port = port self.fixtures_path = fixtures_path + self.api_key = api_key self._proc: subprocess.Popen[str] | None = None self._base_url: str | None = None # Background stdout drainer state. The reader thread continuously @@ -81,11 +83,18 @@ def start(self) -> str: fixtures_arg, ] + child_env = os.environ.copy() + # The plugin option owns child auth. Do not inherit a developer's or + # CI runner's ambient key into ordinary unconfigured fixtures. + child_env.pop("AIMOCK_API_KEYS", None) + if self.api_key is not None: + child_env["AIMOCK_API_KEYS"] = self.api_key self._proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + env=child_env, ) atexit.register(self.stop) @@ -163,6 +172,16 @@ def url(self) -> str: # ── control API methods ───────────────────────────────────────────── + def _control_headers(self) -> dict[str, str]: + api_key = getattr(self, "api_key", None) + return {"Authorization": f"Bearer {api_key}"} if api_key else {} + + def _control_request(self, method: str, path: str, **kwargs: Any) -> requests.Response: + headers = dict(kwargs.pop("headers", {})) + headers.update(self._control_headers()) + request_fn = getattr(requests, method.lower()) + return request_fn(f"{self.base_url}/__aimock{path}", headers=headers, **kwargs) + # Match-level option keys. These belong under the fixture's ``match`` # block: the server reads exactly these fields from ``entry.match`` in # ``entryToFixture`` (src/fixture-loader.ts). This set MUST track that @@ -218,8 +237,7 @@ def add_fixture( fixture_match[key] = value else: fixture[key] = value - r = requests.post( - f"{self.base_url}/__aimock/fixtures", + r = self._control_request("POST", "/fixtures", json={"fixtures": [fixture]}, timeout=5, ) @@ -293,8 +311,7 @@ def load_fixtures(self, path: str | Path) -> AIMockServer: f"got {type(data).__name__}" ) - r = requests.post( - f"{self.base_url}/__aimock/fixtures", + r = self._control_request("POST", "/fixtures", json={"fixtures": fixtures}, timeout=5, ) @@ -303,9 +320,7 @@ def load_fixtures(self, path: str | Path) -> AIMockServer: def clear_fixtures(self) -> AIMockServer: """Delete all fixtures via ``DELETE /__aimock/fixtures``.""" - requests.delete( - f"{self.base_url}/__aimock/fixtures", timeout=5 - ).raise_for_status() + self._control_request("DELETE", "/fixtures", timeout=5).raise_for_status() return self def reset(self) -> AIMockServer: @@ -316,22 +331,18 @@ def reset(self) -> AIMockServer: def reset_fixtures(self) -> AIMockServer: """Clear fixtures + generation state (and journal) via ``POST /__aimock/reset/fixtures``.""" - requests.post( - f"{self.base_url}/__aimock/reset/fixtures", timeout=5 - ).raise_for_status() + self._control_request("POST", "/reset/fixtures", timeout=5).raise_for_status() return self def reset_journal(self) -> AIMockServer: """Clear ONLY the request journal, leaving fixtures intact, via ``POST /__aimock/reset/journal``.""" - requests.post( - f"{self.base_url}/__aimock/reset/journal", timeout=5 - ).raise_for_status() + self._control_request("POST", "/reset/journal", timeout=5).raise_for_status() return self def get_journal(self) -> list[dict[str, Any]]: """Return all recorded journal entries.""" - r = requests.get(f"{self.base_url}/__aimock/journal", timeout=5) + r = self._control_request("GET", "/journal", timeout=5) r.raise_for_status() return r.json() # type: ignore[no-any-return] @@ -346,8 +357,7 @@ def next_error( body: dict[str, Any] | None = None, ) -> AIMockServer: """Queue a one-shot error via ``POST /__aimock/error``.""" - requests.post( - f"{self.base_url}/__aimock/error", + self._control_request("POST", "/error", json={"status": status, "body": body or {}}, timeout=5, ).raise_for_status() @@ -432,9 +442,8 @@ def _wait_for_ready_inner(self, timeout: int) -> str: while time.monotonic() < health_deadline: attempts += 1 try: - r = requests.get( - f"{url}/__aimock/health", timeout=0.5 - ) + headers = self._control_headers() + r = requests.get(f"{url}/__aimock/health", headers=headers, timeout=0.5) if r.status_code == 200: return url except requests.RequestException: diff --git a/packages/aimock-pytest/src/aimock_pytest/_version.py b/packages/aimock-pytest/src/aimock_pytest/_version.py index 7130d640..101c9311 100644 --- a/packages/aimock-pytest/src/aimock_pytest/_version.py +++ b/packages/aimock-pytest/src/aimock_pytest/_version.py @@ -5,7 +5,10 @@ downloads exactly this version when AIMOCK_CLI_PATH is not set, so it must point at a published `@copilotkit/aimock` release and be bumped to the release that contains any new server routes/features the client calls (e.g. the reset-split -control routes ship in the next release). Keep it tracking npm releases. +control routes ship in the next release). Release npm before publishing the +corresponding pytest package. The Release workflow runs the PyPI job after +npm publication and verifies that this pin exists before it builds or uploads +a wheel. Keep it tracking npm releases. """ -AIMOCK_VERSION = "1.37.2" +AIMOCK_VERSION = "1.38.0" diff --git a/packages/aimock-pytest/src/aimock_pytest/plugin.py b/packages/aimock-pytest/src/aimock_pytest/plugin.py index ba9617c0..c3e9926a 100644 --- a/packages/aimock-pytest/src/aimock_pytest/plugin.py +++ b/packages/aimock-pytest/src/aimock_pytest/plugin.py @@ -26,6 +26,11 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=AIMOCK_VERSION, help=f"aimock npm package version to use (default: {AIMOCK_VERSION})", ) + group.addoption( + "--aimock-api-key", + default=None, + help="Inbound API key passed to the aimock child process", + ) @pytest.fixture(scope="session") @@ -38,20 +43,20 @@ def _aimock_node_manager(request: pytest.FixtureRequest) -> NodeManager: @pytest.fixture -def aimock(_aimock_node_manager: NodeManager) -> Generator[AIMockServer, None, None]: +def aimock(request: pytest.FixtureRequest, _aimock_node_manager: NodeManager) -> Generator[AIMockServer, None, None]: """Function-scoped aimock server. A fresh server is started for every test that requests this fixture, and torn down afterwards.""" - server = AIMockServer(_aimock_node_manager, port=0) + server = AIMockServer(_aimock_node_manager, port=0, api_key=request.config.getoption("--aimock-api-key")) server.start() yield server server.stop() @pytest.fixture(scope="session") -def aimock_session(_aimock_node_manager: NodeManager) -> Generator[AIMockServer, None, None]: +def aimock_session(request: pytest.FixtureRequest, _aimock_node_manager: NodeManager) -> Generator[AIMockServer, None, None]: """Session-scoped aimock server. One server is shared across all tests that request this fixture.""" - server = AIMockServer(_aimock_node_manager, port=0) + server = AIMockServer(_aimock_node_manager, port=0, api_key=request.config.getoption("--aimock-api-key")) server.start() yield server server.stop() diff --git a/packages/aimock-pytest/tests/test_basic.py b/packages/aimock-pytest/tests/test_basic.py index ab1a93ac..fb43b4a8 100644 --- a/packages/aimock-pytest/tests/test_basic.py +++ b/packages/aimock-pytest/tests/test_basic.py @@ -1,3 +1,6 @@ +import os +import subprocess +import sys from unittest import mock import requests @@ -38,6 +41,116 @@ def test_server_starts(aimock): assert r.json()["status"] == "ok" +def test_api_key_controls_and_client_requests(_aimock_node_manager): + """The helper authenticates control traffic without bypassing client auth.""" + from aimock_pytest import AIMockServer + + server = AIMockServer(_aimock_node_manager, api_key="pytest-test-key") + try: + server.start() + server.on_message("hello", {"content": "keyed"}) + body = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hello"}], + } + assert requests.post(f"{server.base_url}/v1/chat/completions", json=body).status_code == 401 + response = requests.post( + f"{server.base_url}/v1/chat/completions", + json=body, + headers={"Authorization": "Bearer pytest-test-key"}, + ) + assert response.status_code == 200 + assert requests.post(f"{server.base_url}/__aimock/fixtures", json={"fixtures": []}).status_code == 401 + finally: + server.stop() + + +def test_aimock_api_key_option_authenticates_plugin_fixture_in_a_subprocess(tmp_path): + """The pytest option reaches the child and the fixture's control client.""" + test_file = tmp_path / "test_keyed_fixture.py" + test_file.write_text( + """ +import requests + + +def test_keyed_fixture(aimock): + aimock.on_message("hello", {"content": "keyed fixture"}) + body = {"model": "gpt-4", "messages": [{"role": "user", "content": "hello"}]} + assert requests.post(f"{aimock.base_url}/v1/chat/completions", json=body).status_code == 401 + response = requests.post( + f"{aimock.base_url}/v1/chat/completions", + json=body, + headers={"Authorization": "Bearer subprocess-key"}, + ) + assert response.status_code == 200 + assert response.json()["choices"][0]["message"]["content"] == "keyed fixture" +""", + encoding="utf-8", + ) + child_env = os.environ.copy() + # The child loads the plugin explicitly so source-tree and installed-package + # runs exercise the same fixture path without registering it twice. + child_env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "aimock_pytest.plugin", + "--aimock-api-key", + "subprocess-key", + str(test_file), + ], + env=child_env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_ordinary_plugin_fixture_ignores_inherited_api_key_in_a_subprocess(tmp_path): + """Ambient shell auth must not turn an unconfigured fixture into an auth server.""" + test_file = tmp_path / "test_inherited_key_fixture.py" + test_file.write_text( + """ +import requests + + +def test_unconfigured_fixture(aimock): + aimock.on_message("hello", {"content": "ambient key ignored"}) + response = requests.post( + f"{aimock.base_url}/v1/chat/completions", + json={"model": "gpt-4", "messages": [{"role": "user", "content": "hello"}]}, + ) + assert response.status_code == 200 + assert response.json()["choices"][0]["message"]["content"] == "ambient key ignored" +""", + encoding="utf-8", + ) + child_env = os.environ.copy() + child_env["AIMOCK_API_KEYS"] = "ambient-shell-key" + child_env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "aimock_pytest.plugin", + str(test_file), + ], + env=child_env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + + def test_add_fixture_and_match(aimock): """Add a fixture via control API, then hit it.""" aimock.on_message("hello", {"content": "Hi there!"}) diff --git a/src/__tests__/aimock-cli.test.ts b/src/__tests__/aimock-cli.test.ts index 05068f2c..15b7a5f1 100644 --- a/src/__tests__/aimock-cli.test.ts +++ b/src/__tests__/aimock-cli.test.ts @@ -9,12 +9,17 @@ import type { AimockConfig } from "../config-loader.js"; const CLI_PATH = resolve(__dirname, "../../dist/aimock-cli.js"); const CLI_AVAILABLE = existsSync(CLI_PATH); +// These integration tests spawn an additional Node process. Under the full +// parallel suite the default 5s runner deadline can expire before a healthy +// child gets CPU time to print its immediate startup/error result. +vi.setConfig({ testTimeout: 15_000 }); + /** Spawn the CLI and collect stdout/stderr/exit code. */ function runCli( args: string[], opts: { timeout?: number } = {}, ): Promise<{ stdout: string; stderr: string; code: number | null }> { - const timeout = opts.timeout ?? 5000; + const timeout = opts.timeout ?? 10_000; return new Promise((res) => { const cp = execFile("node", [CLI_PATH, ...args], { timeout }, (err, stdout, stderr) => { const code = cp.exitCode ?? (err && "code" in err ? (err as { code: number }).code : null); @@ -27,7 +32,10 @@ function runCli( * Spawn the CLI expecting a long-running server. Returns the child * process plus helpers to read accumulated output and send signals. */ -function spawnCli(args: string[]): { +function spawnCli( + args: string[], + envOverrides: NodeJS.ProcessEnv = {}, +): { cp: ChildProcess; stdout: () => string; stderr: () => string; @@ -36,7 +44,9 @@ function spawnCli(args: string[]): { } { let out = ""; let err = ""; - const cp = execFile("node", [CLI_PATH, ...args]); + const env = { ...process.env, ...envOverrides }; + if (!("AIMOCK_API_KEYS" in envOverrides)) delete env.AIMOCK_API_KEYS; + const cp = execFile("node", [CLI_PATH, ...args], { env }); cp.stdout?.on("data", (d) => { out += d; }); @@ -166,6 +176,48 @@ describe.skipIf(!CLI_AVAILABLE)("aimock CLI: server lifecycle", () => { }); }); + it("applies config auth and lets AIMOCK_API_KEYS override it", async () => { + const fixturePath = writeFixtureFile(tmpDir); + const configPath = writeConfig(tmpDir, { + llm: { fixtures: fixturePath }, + auth: { apiKeys: ["config-key"] }, + }); + const body = JSON.stringify({ + model: "gpt-4", + messages: [{ role: "user", content: "hello" }], + }); + const requestWith = (url: string, key: string): Promise => + fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` }, + body, + }); + + const configured = spawnCli(["--config", configPath]); + await configured.waitForOutput(/listening on/i); + const configuredUrl = configured.stdout().match(/listening on (http:\/\/\S+)/)?.[1]; + expect(configuredUrl).toBeTruthy(); + try { + expect((await requestWith(configuredUrl!, "wrong-key")).status).toBe(401); + expect((await requestWith(configuredUrl!, "config-key")).status).toBe(200); + } finally { + configured.kill(); + await new Promise((resolve) => configured.cp.on("close", () => resolve())); + } + + const overridden = spawnCli(["--config", configPath], { AIMOCK_API_KEYS: "environment-key" }); + await overridden.waitForOutput(/listening on/i); + const overriddenUrl = overridden.stdout().match(/listening on (http:\/\/\S+)/)?.[1]; + expect(overriddenUrl).toBeTruthy(); + try { + expect((await requestWith(overriddenUrl!, "config-key")).status).toBe(401); + expect((await requestWith(overriddenUrl!, "environment-key")).status).toBe(200); + } finally { + overridden.kill(); + await new Promise((resolve) => overridden.cp.on("close", () => resolve())); + } + }); + it("applies port override from --port flag", async () => { const configPath = writeConfig(tmpDir, {}); const child = spawnCli(["--config", configPath, "--port", "0"]); diff --git a/src/__tests__/api-key-auth.test.ts b/src/__tests__/api-key-auth.test.ts new file mode 100644 index 00000000..3a69f435 --- /dev/null +++ b/src/__tests__/api-key-auth.test.ts @@ -0,0 +1,375 @@ +import { afterEach, describe, expect, it } from "vitest"; +import * as http from "node:http"; +import * as net from "node:net"; +import { createServer, type ServerInstance } from "../server.js"; +import { isRecognizedApiKeyHeader, resolveInboundAuth } from "../api-key-auth.js"; +import { flattenHeaders } from "../helpers.js"; +import type { RecordConfig } from "../types.js"; +import { LLMock } from "../llmock.js"; +import { AGUIMock } from "../agui-mock.js"; + +let instance: ServerInstance | undefined; + +afterEach(async () => { + if (instance) await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; +}); + +function request( + url: string, + headers: Record = {}, + method = "POST", +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + const target = new URL(url); + return new Promise((resolve, reject) => { + const req = http.request( + { hostname: target.hostname, port: target.port, path: target.pathname, method, headers }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }), + ); + }, + ); + req.on("error", reject); + req.end( + method === "POST" + ? JSON.stringify({ model: "gpt-4", messages: [{ role: "user", content: "hello" }] }) + : undefined, + ); + }); +} + +function rawUpgrade(url: string, headers: string[]): Promise { + const target = new URL(url); + return new Promise((resolve, reject) => { + const socket = net.connect(Number(target.port), target.hostname); + let wire = ""; + socket.on("error", reject); + socket.on("data", (chunk) => { + wire += chunk.toString(); + if (wire.includes("\r\n\r\n")) socket.destroy(); + }); + socket.on("close", () => resolve(wire)); + socket.write( + [ + "GET /v1/realtime HTTP/1.1", + `Host: ${target.host}`, + "Upgrade: websocket", + "Connection: Upgrade", + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==", + "Sec-WebSocket-Version: 13", + ...headers, + "", + "", + ].join("\r\n"), + ); + }); +} + +describe("API key configuration and policy", () => { + it("rejects unsafe runtime shapes without exposing key values", () => { + for (const value of [ + null, + "key", + [], + { apiKeys: ["ok", 1] }, + { apiKeys: [" secret ", "secret"] }, + ]) { + expect(() => resolveInboundAuth({ value, label: "options.auth" })).toThrow(); + } + expect(() => + resolveInboundAuth({ value: { apiKeys: ["super-secret", 1] }, label: "options.auth" }), + ).toThrow("options.auth.apiKeys[1]"); + expect(() => + resolveInboundAuth({ value: { apiKeys: ["bad\nkey"] }, label: "options.auth" }), + ).toThrow("control"); + }); + + it("normalizes a valid public configuration", () => { + expect( + resolveInboundAuth({ value: { apiKeys: [" one ", "two"] }, label: "options.auth" }) + .publicConfig, + ).toEqual({ apiKeys: ["one", "two"] }); + }); +}); + +describe("API key HTTP boundary", () => { + it("rejects missing, wrong and duplicate-conflicting credentials before fixtures", async () => { + instance = await createServer( + [{ match: { userMessage: "hello" }, response: { content: "ok" } }], + { auth: { apiKeys: ["primary", "rotated"] } }, + ); + for (const headers of [ + {}, + { Authorization: "Bearer wrong" }, + { Authorization: "Bearer primary", "X-Api-Key": "rotated" }, + ]) { + const result = await request(`${instance.url}/v1/chat/completions`, headers); + expect(result.status).toBe(401); + expect(result.body).toBe( + '{"error":{"message":"Invalid API key","type":"authentication_error","code":"invalid_api_key"}}', + ); + expect(result.headers["www-authenticate"]).toBe('Bearer realm="aimock"'); + } + }); + + it("accepts every supported raw credential form and public probes", async () => { + instance = await createServer( + [{ match: { userMessage: "hello" }, response: { content: "ok" } }], + { auth: { apiKeys: ["primary"] } }, + ); + for (const headers of [ + { Authorization: "Bearer primary" }, + { Authorization: "Key primary" }, + { Authorization: "bearer primary" }, + { Authorization: "key primary" }, + { "X-Api-Key": "primary" }, + { "X-Goog-Api-Key": "primary" }, + { "Api-Key": "primary" }, + { "Xi-Api-Key": "primary" }, + ]) + expect((await request(`${instance.url}/v1/chat/completions`, headers)).status).toBe(200); + expect((await request(`${instance.url}/health`, {}, "GET")).status).toBe(200); + }); + + it("allows only genuine CORS preflights without a key", async () => { + instance = await createServer([], { auth: { apiKeys: ["primary"] } }); + expect( + ( + await request( + `${instance.url}/v1/chat/completions`, + { Origin: "https://example.test", "Access-Control-Request-Method": "POST" }, + "OPTIONS", + ) + ).status, + ).toBe(204); + expect((await request(`${instance.url}/v1/chat/completions`, {}, "OPTIONS")).status).toBe(401); + }); + + it("redacts every accepted inbound credential header through the shared registry", () => { + const acceptedHeaders = [ + "authorization", + "x-api-key", + "x-goog-api-key", + "api-key", + "xi-api-key", + ]; + const credentials = Object.fromEntries( + acceptedHeaders.map((name) => [name, `credential-for-${name}`]), + ); + + expect(acceptedHeaders.every(isRecognizedApiKeyHeader)).toBe(true); + expect(flattenHeaders(credentials)).toEqual( + Object.fromEntries(acceptedHeaders.map((name) => [name, "[REDACTED]"])), + ); + }); + + it("redacts accepted credential headers in a real journal entry", async () => { + instance = await createServer( + [{ match: { userMessage: "hello" }, response: { content: "ok" } }], + { auth: { apiKeys: ["primary"] } }, + ); + + const completion = await request(`${instance.url}/v1/chat/completions`, { + "X-Goog-Api-Key": "primary", + "Xi-Api-Key": "primary", + }); + expect(completion.status).toBe(200); + + const journal = await request( + `${instance.url}/__aimock/journal`, + { + Authorization: "Bearer primary", + }, + "GET", + ); + expect(journal.status).toBe(200); + const entry = JSON.parse(journal.body).at(-1) as { headers: Record }; + expect(entry.headers).toMatchObject({ + "x-goog-api-key": "[REDACTED]", + "xi-api-key": "[REDACTED]", + }); + expect(JSON.stringify(entry)).not.toContain("primary"); + }); +}); + +describe("API key WebSocket upgrade", () => { + it("uses a literal 401 rejection before websocket handshake", async () => { + instance = await createServer([], { auth: { apiKeys: ["primary", "rotated"] } }); + const rejected = await rawUpgrade(instance.url, ["Authorization: Bearer wrong"]); + expect(rejected).toContain("HTTP/1.1 401 Unauthorized"); + expect(rejected).toContain("Content-Type: application/json"); + expect(rejected).toContain("Connection: close"); + const accepted = await rawUpgrade(instance.url, ["Authorization: Bearer primary"]); + expect(accepted).toContain("HTTP/1.1 101 Switching Protocols"); + }); + + it("accepts lowercase schemes and duplicate raw credential lines", async () => { + instance = await createServer([], { auth: { apiKeys: ["primary"] } }); + const accepted = await rawUpgrade(instance.url, [ + "Authorization: bearer primary", + "X-Api-Key: primary", + "Authorization: key primary", + ]); + expect(accepted).toContain("HTTP/1.1 101 Switching Protocols"); + }); +}); + +describe("API key proxy isolation", () => { + it("scrubs the test key and injects the configured provider key before real egress", async () => { + let upstreamHeaders: http.IncomingHttpHeaders | undefined; + const upstream = http.createServer((req, res) => { + upstreamHeaders = req.headers; + req.resume(); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + id: "x", + object: "chat.completion", + created: 0, + model: "gpt-4", + choices: [ + { index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + ); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address() as net.AddressInfo; + instance = await createServer([], { + auth: { apiKeys: ["test-key"] }, + record: { + providers: { openai: `http://127.0.0.1:${address.port}` }, + providerKeys: { openai: "provider-key" }, + proxyOnly: true, + }, + }); + const result = await request(`${instance.url}/v1/chat/completions`, { + Authorization: "Bearer test-key", + "X-Api-Key": "test-key", + }); + expect(result.status).toBe(200); + expect(upstreamHeaders?.authorization).toBe("Bearer provider-key"); + expect(upstreamHeaders?.["x-api-key"]).toBeUndefined(); + await new Promise((resolve) => upstream.close(() => resolve())); + }); + + it("does not connect upstream when an auth-enabled proxy has no provider key", async () => { + let connections = 0; + const upstream = http.createServer((req, res) => { + connections += 1; + req.resume(); + res.end(); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address() as net.AddressInfo; + instance = await createServer([], { + auth: { apiKeys: ["test-key"] }, + record: { + providers: { openai: `http://127.0.0.1:${address.port}` }, + proxyOnly: true, + }, + }); + const result = await request(`${instance.url}/v1/chat/completions`, { + Authorization: "Bearer test-key", + }); + expect(result.status).toBe(502); + expect(connections).toBe(0); + await new Promise((resolve) => upstream.close(() => resolve())); + }); + + it("does not leak auth egress policy between servers that share a record config", async () => { + let upstreamAuthorization: string | undefined; + const upstream = http.createServer((req, res) => { + upstreamAuthorization = req.headers.authorization; + req.resume(); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + id: "x", + object: "chat.completion", + created: 0, + model: "gpt-4", + choices: [ + { index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + ); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address() as net.AddressInfo; + const record: RecordConfig = { + providers: { openai: `http://127.0.0.1:${address.port}` }, + proxyOnly: true, + }; + const authenticated = await createServer([], { auth: { apiKeys: ["primary"] }, record }); + const unauthenticated = await createServer([], { record }); + try { + const result = await request(`${unauthenticated.url}/v1/chat/completions`, { + Authorization: "Bearer external-client-key", + }); + expect(result.status).toBe(200); + expect(upstreamAuthorization).toBe("Bearer external-client-key"); + } finally { + await new Promise((resolve) => authenticated.server.close(() => resolve())); + await new Promise((resolve) => unauthenticated.server.close(() => resolve())); + await new Promise((resolve) => upstream.close(() => resolve())); + } + }); + + it("blocks authenticated AG-UI and fal direct egress without a provider key", async () => { + let upstreamRequests = 0; + const upstream = http.createServer((req, res) => { + upstreamRequests += 1; + req.resume(); + res.end(); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address() as net.AddressInfo; + const upstreamUrl = `http://127.0.0.1:${address.port}`; + const agui = new AGUIMock({ port: 0 }); + agui.enableRecording({ upstream: upstreamUrl, proxyOnly: true }); + const mock = new LLMock({ + port: 0, + auth: { apiKeys: ["test-key"] }, + record: { providers: { fal: upstreamUrl }, proxyOnly: true }, + }); + mock.mount("/agui", agui); + await mock.start(); + try { + const aguiResponse = await fetch(`${mock.url}/agui`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test-key" }, + body: JSON.stringify({ + threadId: "thread", + runId: "run", + messages: [{ id: "message", role: "user", content: "hello" }], + }), + }); + expect(aguiResponse.status).toBe(502); + + const falResponse = await fetch(`${mock.url}/fal/fal-ai/flux/dev`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer test-key", + "x-fal-target-host": "queue.fal.run", + }, + body: JSON.stringify({ input: { prompt: "hello" } }), + }); + expect(falResponse.status).toBe(502); + expect(upstreamRequests).toBe(0); + } finally { + await mock.stop(); + await new Promise((resolve) => upstream.close(() => resolve())); + } + }); +}); diff --git a/src/__tests__/cli.test.ts b/src/__tests__/cli.test.ts index 800a9350..ba708f83 100644 --- a/src/__tests__/cli.test.ts +++ b/src/__tests__/cli.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { execFile, type ChildProcess } from "node:child_process"; import { createServer as createHttpServer, type Server } from "node:http"; import { existsSync, mkdtempSync, writeFileSync, rmSync, mkdirSync } from "node:fs"; @@ -10,12 +10,16 @@ import { createHash } from "node:crypto"; const CLI_PATH = resolve(__dirname, "../../dist/cli.js"); const CLI_AVAILABLE = existsSync(CLI_PATH); +// CLI integration tests use child processes and file watchers. Give healthy +// children time to be scheduled while the full parallel suite is busy. +vi.setConfig({ testTimeout: 15_000 }); + /** Spawn the CLI and collect stdout/stderr/exit code. */ function runCli( args: string[], opts: { timeout?: number } = {}, ): Promise<{ stdout: string; stderr: string; code: number | null }> { - const timeout = opts.timeout ?? 5000; + const timeout = opts.timeout ?? 10_000; return new Promise((res) => { const cp = execFile("node", [CLI_PATH, ...args], { timeout }, (err, stdout, stderr) => { const code = cp.exitCode ?? (err && "code" in err ? (err as { code: number }).code : null); @@ -45,7 +49,7 @@ function spawnCli(args: string[]): { err += d; }); - const waitForOutput = (match: RegExp, timeoutMs = 5000): Promise => + const waitForOutput = (match: RegExp, timeoutMs = 10_000): Promise => new Promise((resolve, reject) => { const deadline = setTimeout(() => { reject(new Error(`Timed out waiting for ${match} — stdout: ${out}, stderr: ${err}`)); diff --git a/src/__tests__/multimedia-record.test.ts b/src/__tests__/multimedia-record.test.ts index 4d915ba1..0762dbfc 100644 --- a/src/__tests__/multimedia-record.test.ts +++ b/src/__tests__/multimedia-record.test.ts @@ -17,6 +17,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { proxyAndRecord } from "../recorder.js"; +import { createServer } from "../server.js"; import type { Fixture, RecordConfig, ChatCompletionRequest } from "../types.js"; import { Logger } from "../logger.js"; @@ -77,6 +78,21 @@ function makeTmpDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), "aimock-mm-record-")); } +async function requestStreamingTranscription(url: string, signal?: AbortSignal): Promise { + const formData = new FormData(); + formData.append("file", new Blob(["audio"], { type: "audio/wav" }), "audio.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + return fetch(`${url}/v1/audio/transcriptions`, { method: "POST", body: formData, signal }); +} + +async function requestJsonTranscription(url: string): Promise { + const formData = new FormData(); + formData.append("file", new Blob(["audio"], { type: "audio/wav" }), "audio.wav"); + formData.append("model", "gpt-transcribe"); + return fetch(`${url}/v1/audio/transcriptions`, { method: "POST", body: formData }); +} + // --------------------------------------------------------------------------- // Tests: buildFixtureResponse detection via proxyAndRecord // --------------------------------------------------------------------------- @@ -207,6 +223,220 @@ describe("multimedia record: image response detection", () => { }); describe("multimedia record: transcription response detection", () => { + it("records upstream gpt-transcribe SSE and replays it through the local handler", async () => { + const fixturePath = makeTmpDir(); + const upstreamBody = + 'data: {"type":"transcript.text.delta","delta":"Recorded "}\n\n' + + 'data: {"type":"transcript.text.delta","delta":"stream"}\n\n' + + 'data: {"type":"transcript.text.done","text":"Recorded stream","languages":[{"code":"en"}],"usage":{"type":"tokens","input_tokens":4,"output_tokens":5,"total_tokens":9}}\n\n'; + const { server: upstream, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.end(upstreamBody); + }); + const fixtures: Fixture[] = []; + const recorder = await createServer(fixtures, { + port: 0, + record: { providers: { openai: url }, fixturePath }, + }); + + try { + const recorded = await requestStreamingTranscription(recorder.url); + expect(recorded.status).toBe(200); + expect(await recorded.text()).toBe(upstreamBody); + expect(fixtures).toHaveLength(1); + expect(fixtures[0].response).toEqual({ + transcription: { + text: "Recorded stream", + languages: [{ code: "en" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }, + }); + + const savedFixture = JSON.parse( + fs.readFileSync( + path.join( + fixturePath, + fs.readdirSync(fixturePath).find((file) => file.endsWith(".json"))!, + ), + "utf8", + ), + ); + expect(savedFixture.fixtures[0].response).toEqual({ + transcription: { + text: "Recorded stream", + languages: [{ code: "en" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }, + }); + + // Replay goes through the local handler, which terminates the stream with + // the `[DONE]` sentinel the live API sends. The proxied passthrough above + // relays the upstream bytes verbatim, so only replay gains the sentinel. + const replay = await requestStreamingTranscription(recorder.url); + expect(replay.status).toBe(200); + expect(await replay.text()).toBe( + 'data: {"type":"transcript.text.delta","delta":"Recorded stream"}\n\n' + + 'data: {"type":"transcript.text.done","text":"Recorded stream","languages":[{"code":"en"}],"usage":{"type":"tokens","input_tokens":4,"output_tokens":5,"total_tokens":9}}\n\n' + + "data: [DONE]\n\n", + ); + } finally { + await closeServer(recorder.server); + await closeServer(upstream); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + + it("persists a typed terminal stream when the client closes before upstream end", async () => { + const fixturePath = makeTmpDir(); + const terminal = + 'data: {"type":"transcript.text.done","text":"Typed terminal","languages":[{"code":"en"}],"usage":{"input_tokens":4,"output_tokens":5}}\n\n'; + const { server: upstream, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.write(terminal); + setTimeout(() => res.end(), 150); + }); + const fixtures: Fixture[] = []; + const recorder = await createServer(fixtures, { + port: 0, + record: { providers: { openai: url }, fixturePath }, + }); + + try { + const controller = new AbortController(); + const recorded = await requestStreamingTranscription(recorder.url, controller.signal); + const reader = recorded.body!.getReader(); + const first = new TextDecoder().decode((await reader.read()).value); + expect(first).toContain('"type":"transcript.text.done"'); + controller.abort(); + await new Promise((resolve) => setTimeout(resolve, 250)); + + expect(fixtures).toHaveLength(1); + expect(fixtures[0].response).toEqual({ + transcription: { + text: "Typed terminal", + languages: [{ code: "en" }], + usage: { input_tokens: 4, output_tokens: 5 }, + }, + }); + const savedFixture = JSON.parse( + fs.readFileSync( + path.join( + fixturePath, + fs.readdirSync(fixturePath).find((file) => file.endsWith(".json"))!, + ), + "utf8", + ), + ); + expect(savedFixture.fixtures[0].response).toEqual(fixtures[0].response); + await closeServer(upstream); + + const replay = await requestStreamingTranscription(recorder.url); + expect(replay.status).toBe(200); + expect(await replay.text()).toContain( + '"text":"Typed terminal","languages":[{"code":"en"}],"usage":{"input_tokens":4,"output_tokens":5}', + ); + } finally { + await closeServer(recorder.server); + if (upstream.listening) await closeServer(upstream); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + + it("records and replays the modern gpt-transcribe text languages and usage response", async () => { + const fixturePath = makeTmpDir(); + const { server, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + text: "Modern transcript", + languages: [{ code: "en" }, { code: "es" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }), + ); + }); + + try { + const fixtures: Fixture[] = []; + const record: RecordConfig = { providers: { openai: url }, fixturePath }; + const logger = new Logger("silent"); + const request: ChatCompletionRequest = { + model: "gpt-transcribe", + messages: [], + _endpointType: "transcription", + }; + + const { req, res } = createMockReqRes("/v1/audio/transcriptions"); + await proxyAndRecord(req, res, request, "openai", "/v1/audio/transcriptions", fixtures, { + record, + logger, + }); + + const response = fixtures[0].response as { + transcription?: { + text: string; + languages?: Array<{ code: string }>; + usage?: Record; + }; + }; + expect(response.transcription).toEqual({ + text: "Modern transcript", + languages: [{ code: "en" }, { code: "es" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }); + } finally { + await closeServer(server); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + + it("records and locally replays JSON transcription languages and usage", async () => { + const fixturePath = makeTmpDir(); + const { server: upstream, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }), + ); + }); + const fixtures: Fixture[] = []; + const recorder = await createServer(fixtures, { + port: 0, + record: { providers: { openai: url }, fixturePath }, + }); + + try { + const recorded = await requestJsonTranscription(recorder.url); + expect(recorded.status).toBe(200); + expect(await recorded.json()).toEqual({ + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }); + expect(fixtures[0].response).toEqual({ + transcription: { + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }, + }); + + const replay = await requestJsonTranscription(recorder.url); + expect(replay.status).toBe(200); + expect(await replay.json()).toEqual({ + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }); + } finally { + await closeServer(recorder.server); + await closeServer(upstream); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + it("detects OpenAI transcription response", async () => { const fixturePath = makeTmpDir(); const { server, url } = await createUpstream((_req, res) => { diff --git a/src/__tests__/multimedia.test.ts b/src/__tests__/multimedia.test.ts index 1ee8686f..2c625e43 100644 --- a/src/__tests__/multimedia.test.ts +++ b/src/__tests__/multimedia.test.ts @@ -1,6 +1,32 @@ import { describe, test, expect } from "vitest"; import { LLMock } from "../llmock.js"; +async function readSSEFrameTimes( + response: Response, +): Promise<{ body: string; frameTimes: number[] }> { + const reader = response.body?.getReader(); + if (!reader) throw new Error("Expected an SSE response body"); + + const decoder = new TextDecoder(); + let body = ""; + let pending = ""; + const frameTimes: number[] = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + body += chunk; + pending += chunk; + while (pending.includes("\n\n")) { + const boundary = pending.indexOf("\n\n"); + pending = pending.slice(boundary + 2); + frameTimes.push(Date.now()); + } + } + body += decoder.decode(); + return { body, frameTimes }; +} + describe("image generation", () => { test("image generation returns fixture (OpenAI format)", async () => { const mock = new LLMock({ port: 0 }); @@ -87,7 +113,7 @@ describe("image generation", () => { }); describe("audio transcription", () => { - test("transcription returns text", async () => { + test("gpt-transcribe returns a nonstreaming transcription", async () => { const mock = new LLMock({ port: 0 }); mock.addFixture({ match: { endpoint: "transcription" }, @@ -97,7 +123,7 @@ describe("audio transcription", () => { const formData = new FormData(); formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); - formData.append("model", "whisper-1"); + formData.append("model", "gpt-transcribe"); const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { method: "POST", @@ -110,6 +136,194 @@ describe("audio transcription", () => { await mock.stop(); }); + test("gpt-transcribe streams transcript delta and completion events", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "transcription" }, + response: { transcription: { text: "Welcome" } }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + // Live `gpt-transcribe&stream=true` capture ends with the `[DONE]` + // sentinel after `transcript.text.done`; clients that loop until it + // otherwise never terminate. + expect(await res.text()).toBe( + 'data: {"type":"transcript.text.delta","delta":"Welcome"}\n\n' + + 'data: {"type":"transcript.text.done","text":"Welcome"}\n\n' + + "data: [DONE]\n\n", + ); + await mock.stop(); + }); + + test("gpt-transcribe stream schedules fixture chunks across separate SSE frames", async () => { + const mock = new LLMock({ port: 0, chunkSize: 2, latency: 40 }); + mock.addFixture({ + match: { endpoint: "transcription" }, + response: { transcription: { text: "abcdef" } }, + chunkSize: 2, + latency: 40, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + const { body, frameTimes } = await readSSEFrameTimes(res); + const deltas = [...body.matchAll(/"type":"transcript\.text\.delta","delta":"([^"]+)"/g)].map( + (match) => match[1], + ); + expect(deltas).toEqual(["ab", "cd", "ef"]); + // 3 delta frames + transcript.text.done + the terminal [DONE] sentinel. + expect(frameTimes).toHaveLength(5); + expect(body.endsWith("data: [DONE]\n\n")).toBe(true); + expect(frameTimes[1] - frameTimes[0]).toBeGreaterThanOrEqual(20); + expect(frameTimes[2] - frameTimes[1]).toBeGreaterThanOrEqual(20); + await mock.stop(); + }); + + test("gpt-transcribe stream records fixture truncation before its done event", async () => { + const mock = new LLMock({ port: 0, chunkSize: 2, latency: 1 }); + mock.addFixture({ + match: { endpoint: "transcription", model: "gpt-transcribe" }, + response: { transcription: { text: "abcdefgh" } }, + chunkSize: 2, + truncateAfterChunks: 2, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + body: formData, + }); + + await expect(res.text()).rejects.toThrow(); + expect(mock.journal.getLast()).toMatchObject({ + response: { interrupted: true, interruptReason: "truncateAfterChunks" }, + }); + await mock.stop(); + }); + + test("whisper-1 keeps its JSON response when stream is requested", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "transcription" }, + response: { transcription: { text: "Legacy transcript" } }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "whisper-1"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toEqual({ text: "Legacy transcript" }); + await mock.stop(); + }); + + test("gpt-transcribe stream errors remain JSON in strict mode", async () => { + const mock = new LLMock({ port: 0, strict: true }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.status).toBe(503); + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toMatchObject({ + error: { code: "no_fixture_match", type: "invalid_request_error" }, + }); + await mock.stop(); + }); + + test("gpt-transcribe stream no-match remains a non-strict 404 JSON error", async () => { + const mock = new LLMock({ port: 0 }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + body: formData, + }); + + expect(res.status).toBe(404); + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toMatchObject({ + error: { code: "no_fixture_match", type: "invalid_request_error" }, + }); + await mock.stop(); + }); + + test("gpt-transcribe stream ErrorResponse remains JSON", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "transcription", model: "gpt-transcribe" }, + response: { + error: { message: "Audio quota exhausted", type: "rate_limit_error", code: "quota" }, + status: 429, + }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + body: formData, + }); + + expect(res.status).toBe(429); + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toMatchObject({ + error: { message: "Audio quota exhausted", type: "rate_limit_error", code: "quota" }, + }); + await mock.stop(); + }); + test("verbose transcription includes words and segments", async () => { const mock = new LLMock({ port: 0 }); mock.addFixture({ @@ -168,6 +382,30 @@ describe("audio translation", () => { await mock.stop(); }); + test("translation keeps its JSON response when gpt-transcribe requests stream", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "translation" }, + response: { transcription: { text: "Translated" } }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/translations`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toEqual({ text: "Translated" }); + await mock.stop(); + }); + test("verbose translation includes task=translate", async () => { const mock = new LLMock({ port: 0 }); mock.addFixture({ diff --git a/src/__tests__/openrouter-video-record.test.ts b/src/__tests__/openrouter-video-record.test.ts index 02159be2..c83ba17a 100644 --- a/src/__tests__/openrouter-video-record.test.ts +++ b/src/__tests__/openrouter-video-record.test.ts @@ -709,10 +709,11 @@ describe("OpenRouter video record — poll proxy and eager capture", () => { async function submitRecordJob( m: LLMock, prompt: string, + authorization = "Bearer sk-test", ): Promise<{ id: string; polling_url: string }> { const res = await fetch(`${m.url}/api/v1/videos`, { method: "POST", - headers: { "Content-Type": "application/json", Authorization: "Bearer sk-test" }, + headers: { "Content-Type": "application/json", Authorization: authorization }, body: JSON.stringify({ model: "bytedance/seedance-2.0", prompt }), }); expect(res.status).toBe(200); @@ -908,7 +909,7 @@ describe("OpenRouter video record — poll proxy and eager capture", () => { expect(upstream.counts.status).toBe(1); }); - test("off-origin unsigned_urls are fetched WITHOUT the client's auth, with a warn", async () => { + test("off-origin unsigned_urls are fetched WITHOUT a configured Authorization credential, with a warn", async () => { const bytes = Buffer.from("cdn-hosted video"); contentHost = await startPlainContentHost(bytes); upstream = await startOpenRouterVideoUpstream({ unsignedUrlOrigin: contentHost.url }); @@ -916,14 +917,19 @@ describe("OpenRouter video record — poll proxy and eager capture", () => { mock = new LLMock({ port: 0, logLevel: "warn", - record: { providers: { openrouter: upstream.url }, fixturePath: tmpDir }, + auth: { apiKeys: ["test-key"] }, + record: { + providers: { openrouter: upstream.url }, + providerKeys: { openrouter: "provider-key" }, + fixturePath: tmpDir, + }, }); await mock.start(); - const envelope = await submitRecordJob(mock, "cdn capture"); + const envelope = await submitRecordJob(mock, "cdn capture", "Bearer test-key"); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); const poll = await ( - await fetch(envelope.polling_url, { headers: { Authorization: "Bearer poll-key" } }) + await fetch(envelope.polling_url, { headers: { Authorization: "Bearer test-key" } }) ).json(); expect(poll.status).toBe("completed"); await waitUntil(() => readRecordedFixtureFiles(tmpDir!).length === 1); diff --git a/src/__tests__/release-workflows.test.ts b/src/__tests__/release-workflows.test.ts new file mode 100644 index 00000000..9449f17a --- /dev/null +++ b/src/__tests__/release-workflows.test.ts @@ -0,0 +1,27 @@ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +const release = readFileSync( + resolve(__dirname, "../../.github/workflows/publish-release.yml"), + "utf8", +); +const standalonePytestPublish = resolve(__dirname, "../../.github/workflows/publish-pytest.yml"); +const unitTests = readFileSync(resolve(__dirname, "../../.github/workflows/test-unit.yml"), "utf8"); + +describe("release workflow sequencing", () => { + it("publishes pytest only from the Release workflow after npm", () => { + const pytestJob = release.slice(release.indexOf(" publish-pytest:")); + expect(pytestJob).toContain("needs: [build, publish]"); + expect(pytestJob).toContain("environment: pypi"); + expect(pytestJob).toContain('npm view "@copilotkit/aimock@${VERSION}" version'); + expect(existsSync(standalonePytestPublish)).toBe(false); + }); +}); + +describe("unit-test CLI coverage", () => { + it("builds the CLI before running the unit suite", () => { + expect(unitTests.indexOf("pnpm build")).toBeGreaterThan(-1); + expect(unitTests.indexOf("pnpm build")).toBeLessThan(unitTests.indexOf("pnpm test")); + }); +}); diff --git a/src/__tests__/stream-collapse.test.ts b/src/__tests__/stream-collapse.test.ts index d8789f19..0cc50b8e 100644 --- a/src/__tests__/stream-collapse.test.ts +++ b/src/__tests__/stream-collapse.test.ts @@ -19,6 +19,16 @@ import type { Fixture } from "../types.js"; // --------------------------------------------------------------------------- describe("collapseOpenAISSE", () => { + it("preserves typed transcription metadata, including an empty transcript", () => { + const languageResult = collapseOpenAISSE( + 'data: {"type":"transcript.text.done","text":"hello","languages":[{"code":"en"}]}\n\n', + ); + const emptyResult = collapseOpenAISSE('data: {"type":"transcript.text.done","text":""}\n\n'); + + expect(languageResult.transcription).toEqual({ text: "hello", languages: [{ code: "en" }] }); + expect(emptyResult).toEqual({ transcription: { text: "" }, content: "" }); + }); + it("collapses text content from SSE chunks", () => { const body = [ `data: ${JSON.stringify({ id: "chatcmpl-123", choices: [{ delta: { role: "assistant" } }] })}`, diff --git a/src/__tests__/streaming-physics.test.ts b/src/__tests__/streaming-physics.test.ts index 8cbb1324..b380a1de 100644 --- a/src/__tests__/streaming-physics.test.ts +++ b/src/__tests__/streaming-physics.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { PassThrough } from "node:stream"; import type * as http from "node:http"; import { writeSSEStream, calculateDelay } from "../sse-writer.js"; -import type { SSEChunk, StreamingProfile } from "../types.js"; +import type { SSEChunk, StreamingProfile, RecordedTimings } from "../types.js"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -198,6 +198,40 @@ describe("writeSSEStream with streamingProfile", () => { expect(output()).toContain(JSON.stringify(chunks[0])); }); + it("scales every recorded replay delay by replaySpeed", async () => { + vi.useFakeTimers(); + const { res, output, ended } = makeMockResponse(); + const chunks = Array.from({ length: 8 }, (_, index) => makeChunk(String(index), "chunk")); + const recordedTimings: RecordedTimings = { + ttftMs: 80, + interChunkDelaysMs: [40, 40, 40, 40], + totalDurationMs: 240, + }; + + const promise = writeSSEStream(res, chunks, { recordedTimings, replaySpeed: 2 }); + + // 8 emitted frames: 80ms / 2 for the first, then 40ms / 2 for each + // remaining frame (the writer uses the recorded average after the fourth gap). + for (const [index, delayMs] of [40, 20, 20, 20, 20, 20, 20, 20].entries()) { + const frame = JSON.stringify(chunks[index]); + await vi.advanceTimersByTimeAsync(delayMs - 1); + expect(output()).not.toContain(frame); + expect(ended()).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + expect(output()).toContain(frame); + for (let previous = 0; previous < index; previous++) { + expect(output().indexOf(JSON.stringify(chunks[previous]))).toBeLessThan( + output().indexOf(frame), + ); + } + expect(ended()).toBe(index === chunks.length - 1); + } + await promise; + + expect(output()).toContain("[DONE]"); + }); + it("jitter causes variable delays (not all identical)", async () => { // Use real timers for this test since we're measuring variance const delays: number[] = []; diff --git a/src/__tests__/timing-replay.test.ts b/src/__tests__/timing-replay.test.ts index ca034ddc..43ba191c 100644 --- a/src/__tests__/timing-replay.test.ts +++ b/src/__tests__/timing-replay.test.ts @@ -1,7 +1,8 @@ -import { describe, it, expect, afterEach } from "vitest"; +import { describe, it, expect, afterEach, vi } from "vitest"; import http from "node:http"; import { createServer, type ServerInstance } from "../server.js"; import type { Fixture, SSEChunk, ChatCompletionRequest, RecordedTimings } from "../types.js"; +import * as sseWriter from "../sse-writer.js"; // --------------------------------------------------------------------------- // Helpers @@ -102,7 +103,7 @@ describe("timing-aware replay through handleCompletions", () => { expect(elapsed).toBeGreaterThanOrEqual(40); // at least ~TTFT minus jitter }); - it("replaySpeed 2.0 halves the replay duration", async () => { + it("forwards fixture replaySpeed and recordedTimings through the completions route", async () => { // 80ms TTFT + 4 x 40ms inter-chunk = ~240ms at 1x speed const timings: RecordedTimings = { ttftMs: 80, @@ -124,19 +125,18 @@ describe("timing-aware replay through handleCompletions", () => { chunkSize: 5, }); - const start = Date.now(); + const writeSSEStream = vi.spyOn(sseWriter, "writeSSEStream"); const res = await httpPost(`${instance.url}/v1/chat/completions`, chatRequest("speed-test")); - const elapsed = Date.now() - start; expect(res.status).toBe(200); const chunks = parseSSEResponse(res.body); expect(chunks.length).toBeGreaterThan(1); - // At 2x speed, effective delays are halved. The full 1x duration would be - // ~240ms, so at 2x it should be ~120ms. We verify it's well below 1x - // but still non-trivial (delays are being applied, just faster). - expect(elapsed).toBeGreaterThanOrEqual(50); // still has meaningful delay - expect(elapsed).toBeLessThan(200); // well below 1x baseline of ~240ms + expect(writeSSEStream).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.objectContaining({ recordedTimings: timings, replaySpeed: 2 }), + ); }); it("recordedTimings alone impose real delays (positive control)", async () => { diff --git a/src/__tests__/ws-api-conformance.test.ts b/src/__tests__/ws-api-conformance.test.ts index edd8a787..8fb03f5f 100644 --- a/src/__tests__/ws-api-conformance.test.ts +++ b/src/__tests__/ws-api-conformance.test.ts @@ -615,14 +615,11 @@ describe("GA Realtime conformance", () => { expect(frame.type).toBe("session.created"); const session = frame.session; expect(session).toHaveProperty("audio"); - expect(session).toHaveProperty("type", "conversation"); + expect(session).toHaveProperty("type", "realtime"); expect(session).toHaveProperty("reasoning"); expect(session.audio).toMatchObject({ - voice: null, - input_audio_format: null, - output_audio_format: null, - input_audio_noise_reduction: null, - input_audio_transcription: null, + input: { format: null, noise_reduction: null, transcription: null }, + output: { format: null, voice: null }, }); }); @@ -686,6 +683,31 @@ describe("GA Realtime conformance", () => { expect(contentPartDone).toBeDefined(); expect(contentPartDone.part.type).toBe("output_text"); }); + + it("round-trips the documented PCM input format without dropping its rate", async () => { + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime-2"); + await ws.waitForMessages(1); + ws.send( + JSON.stringify({ + type: "session.update", + session: { + audio: { + input: { format: { type: "audio/pcm", rate: 24000 } }, + output: { voice: "alloy", format: { type: "audio/pcm", rate: 24000 } }, + }, + }, + }), + ); + const raw = await ws.waitForMessages(2); + ws.close(); + const frame = JSON.parse(raw[1]) as any; + expect(frame.type).toBe("session.updated"); + expect(frame.session.audio.input.format).toEqual({ type: "audio/pcm", rate: 24000 }); + expect(frame.session.audio.output).toEqual({ + voice: "alloy", + format: { type: "audio/pcm", rate: 24000 }, + }); + }); }); // --------------------------------------------------------------------------- @@ -709,6 +731,33 @@ describe("Beta Realtime conformance (OpenAI-Beta: realtime=v1)", () => { expect(session).not.toHaveProperty("reasoning"); }); + it("flattens turn_detection for default, nested GA, and legacy updates", async () => { + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime-2", { + "OpenAI-Beta": "realtime=v1", + }); + const created = JSON.parse((await ws.waitForMessages(1))[0]) as any; + expect(created.session.turn_detection).toBeNull(); + + ws.send( + JSON.stringify({ + type: "session.update", + session: { audio: { input: { turn_detection: { type: "server_vad", threshold: 0.7 } } } }, + }), + ); + const nested = JSON.parse((await ws.waitForMessages(2))[1]) as any; + expect(nested.session.turn_detection).toEqual({ type: "server_vad", threshold: 0.7 }); + + ws.send( + JSON.stringify({ + type: "session.update", + session: { turn_detection: { type: "semantic_vad" } }, + }), + ); + const legacy = JSON.parse((await ws.waitForMessages(3))[2]) as any; + expect(legacy.session.turn_detection).toEqual({ type: "semantic_vad" }); + ws.close(); + }); + it("emits Beta event names (response.text.delta, conversation.item.created)", async () => { const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime-2", { "OpenAI-Beta": "realtime=v1", diff --git a/src/__tests__/ws-realtime.test.ts b/src/__tests__/ws-realtime.test.ts index 518e1409..a56ebdab 100644 --- a/src/__tests__/ws-realtime.test.ts +++ b/src/__tests__/ws-realtime.test.ts @@ -72,6 +72,13 @@ function sessionUpdate(config: Record): string { return JSON.stringify({ type: "session.update", session: config }); } +function transcriptionSessionUpdate(model: string): string { + return JSON.stringify({ + type: "transcription_session.update", + session: { input_audio_transcription: { model } }, + }); +} + function functionCallOutputItem(callId: string, output: string): string { return JSON.stringify({ type: "conversation.item.create", @@ -146,16 +153,14 @@ describe("WebSocket /v1/realtime", () => { expect(typeof session.expires_at).toBe("number"); expect(session.max_response_output_tokens).toBe("inf"); expect(session.tool_choice).toBe("auto"); - expect(session.type).toBe("conversation"); + expect(session.type).toBe("realtime"); expect(session.reasoning).toBeNull(); // GA nested audio config const audio = session.audio as Record; - expect(audio).toBeDefined(); - expect(audio.voice).toBeNull(); - expect(audio.input_audio_format).toBeNull(); - expect(audio.output_audio_format).toBeNull(); - expect(audio.input_audio_noise_reduction).toBeNull(); - expect(audio.input_audio_transcription).toBeNull(); + expect(audio).toEqual({ + input: { format: null, noise_reduction: null, transcription: null, turn_detection: null }, + output: { format: null, voice: null }, + }); ws.close(); }); @@ -185,11 +190,11 @@ describe("WebSocket /v1/realtime", () => { expect(typeof session.expires_at).toBe("number"); expect(session.max_response_output_tokens).toBe("inf"); expect(session.tool_choice).toBe("auto"); - expect(session.type).toBe("conversation"); + expect(session.type).toBe("realtime"); // GA nested audio config const audio = session.audio as Record; expect(audio).toBeDefined(); - expect(audio.voice).toBeNull(); + expect((audio.output as Record).voice).toBeNull(); ws.close(); }); @@ -743,7 +748,7 @@ describe("WebSocket /v1/realtime", () => { ws.close(); }); - it("session.update updates modalities, model, and temperature", async () => { + it("session.update ignores an attempt to mutate the established connection model", async () => { instance = await createServer(allFixtures); const ws = await connectWebSocket(instance.url, "/v1/realtime"); @@ -757,21 +762,15 @@ describe("WebSocket /v1/realtime", () => { }), ); + // Live GA neither applies nor rejects the model: it returns a normal + // session.updated echoing the connection model, with the rest applied. const raw = await ws.waitForMessages(2); const event = JSON.parse(raw[1]) as WSEvent; expect(event.type).toBe("session.updated"); const session = event.session as Record; + expect(session.model).toBe("gpt-realtime-2"); expect(session.modalities).toEqual(["text", "audio"]); - expect(session.model).toBe("gpt-4o-mini-realtime"); expect(session.temperature).toBe(0.5); - expect(session.object).toBe("realtime.session"); - expect(typeof session.expires_at).toBe("number"); - expect(session.max_response_output_tokens).toBe("inf"); - expect(session.tool_choice).toBe("auto"); - expect(session.type).toBe("conversation"); - // GA nested audio config - const audio = session.audio as Record; - expect(audio).toBeDefined(); ws.close(); }); @@ -1035,7 +1034,9 @@ describe("WebSocket /v1/realtime", () => { expect(event.type).toBe("session.updated"); const session = event.session as Record; const audio = session.audio as Record; - expect(audio.input_audio_noise_reduction).toEqual({ type: "near_field" }); + expect((audio.input as Record).noise_reduction).toEqual({ + type: "near_field", + }); ws.close(); }); @@ -1061,7 +1062,7 @@ describe("WebSocket /v1/realtime", () => { expect(event.type).toBe("session.updated"); const session = event.session as Record; const audio = session.audio as Record; - expect(audio.input_audio_transcription).toEqual({ model: "whisper-1" }); + expect((audio.input as Record).transcription).toEqual({ model: "whisper-1" }); ws.close(); }); @@ -1092,9 +1093,15 @@ describe("WebSocket /v1/realtime", () => { expect(event.type).toBe("session.updated"); const session = event.session as Record; const audio = session.audio as Record; - expect(audio.voice).toBe("alloy"); - expect(audio.input_audio_format).toBe("pcm16"); - expect(audio.output_audio_format).toBe("pcm16"); + expect(audio).toEqual({ + input: { + format: { type: "pcm16" }, + noise_reduction: null, + transcription: null, + turn_detection: null, + }, + output: { format: { type: "pcm16" }, voice: "alloy" }, + }); expect(session.modalities).toEqual(["text", "audio"]); ws.close(); @@ -1290,19 +1297,23 @@ describe("WebSocket /v1/realtime", () => { // ── Translate/Whisper session types + audio buffer ───────────────────── it("accepts transcription session type and acknowledges audio buffer commit", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); // Skip session.created await ws.waitForMessages(1); // Update session to transcription mode with transcribe model - ws.send(sessionUpdate({ type: "transcription", model: "gpt-4o-transcribe" })); + ws.send(sessionUpdate({ type: "transcription" })); const updateRaw = await ws.waitForMessages(2); const updateEvent = parseEvents(updateRaw.slice(1))[0]; expect(updateEvent.type).toBe("session.updated"); - expect((updateEvent.session as Record).type).toBe("transcription"); - expect((updateEvent.session as Record).model).toBe("gpt-4o-transcribe"); + const updatedSession = updateEvent.session as Record; + expect(updatedSession.type).toBe("transcription"); + // A transcription session is serialized as its own resource and carries no + // session-level model — see the wire-fidelity tests for the full shape. + expect(updatedSession.object).toBe("realtime.transcription_session"); + expect(updatedSession).not.toHaveProperty("model"); // Send audio buffer messages ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: "base64data" })); @@ -1326,9 +1337,317 @@ describe("WebSocket /v1/realtime", () => { ws.close(); }); + it("streams live transcription after documented transcription_session.update", async () => { + const transcriptionFixture: Fixture = { + match: { endpoint: "realtime-transcription" }, + response: { transcription: { text: "Live caption" } }, + }; + instance = await createServer([transcriptionFixture]); + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?model=gpt-realtime&intent=transcription", + ); + + await ws.waitForMessages(1); // session.created + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + const update = parseEvents(await ws.waitForMessages(2))[1]; + expect(update.type).toBe("transcription_session.updated"); + expect(update.session).toMatchObject({ + audio: { + input: { transcription: { model: "gpt-live-transcribe-2026-07-01" } }, + }, + }); + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + const raw = await ws.waitForMessages(6); + const events = parseEvents(raw.slice(2)); + const delta = events.find( + (event) => event.type === "conversation.item.input_audio_transcription.delta", + ); + const completed = events.find( + (event) => event.type === "conversation.item.input_audio_transcription.completed", + ); + + expect(delta).toMatchObject({ content_index: 0, delta: "Live caption" }); + expect(completed).toMatchObject({ content_index: 0, transcript: "Live caption" }); + expect(completed!.item_id).toBe(delta!.item_id); + ws.close(); + }); + + it("schedules live transcription deltas across separate socket frames", async () => { + const transcriptionFixture: Fixture = { + match: { endpoint: "realtime-transcription" }, + response: { transcription: { text: "abcdef" } }, + chunkSize: 2, + latency: 40, + }; + instance = await createServer([transcriptionFixture], { chunkSize: 2, latency: 40 }); + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?model=gpt-realtime&intent=transcription", + ); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe")); + await ws.waitForMessages(2); + const startedAt = Date.now(); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const firstDelta = await ws.waitForMessages(5); + const firstDeltaAt = Date.now(); + await ws.waitForMessages(6); + const secondDeltaAt = Date.now(); + await ws.waitForMessages(7); + const thirdDeltaAt = Date.now(); + const events = parseEvents(await ws.waitForMessages(8)); + + expect(firstDeltaAt - startedAt).toBeGreaterThanOrEqual(20); + expect(secondDeltaAt - firstDeltaAt).toBeGreaterThanOrEqual(20); + expect(thirdDeltaAt - secondDeltaAt).toBeGreaterThanOrEqual(20); + expect(parseEvents(firstDelta.slice(4))[0]).toMatchObject({ delta: "ab" }); + expect(events[7]).toMatchObject({ + type: "conversation.item.input_audio_transcription.completed", + transcript: "abcdef", + }); + ws.close(); + }); + + it("uses the versioned model configured in session.audio.input.transcription", async () => { + const transcriptionFixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { transcription: { text: "Nested config caption" } }, + }; + instance = await createServer([transcriptionFixture]); + // The transcription session is established by intent; this test covers + // resolving its MODEL from the nested `audio.input.transcription` config. + // Configuring that field alone must not turn a conversation session into a + // transcription session — see the input_audio_transcription regressions. + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?intent=transcription&model=gpt-realtime", + ); + + await ws.waitForMessages(1); // session.created + ws.send( + sessionUpdate({ + audio: { + input: { transcription: { model: "gpt-live-transcribe-2026-07-01" } }, + }, + }), + ); + await ws.waitForMessages(2); // session.updated + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + const events = parseEvents(await ws.waitForMessages(7)); + const deltas = events + .filter((event) => event.type === "conversation.item.input_audio_transcription.delta") + .map((event) => event.delta) + .join(""); + expect(deltas).toBe("Nested config caption"); + expect(events[6]).toMatchObject({ + type: "conversation.item.input_audio_transcription.completed", + transcript: "Nested config caption", + }); + ws.close(); + }); + + it("abruptly destroys a documented transcription socket after truncateAfterChunks", async () => { + const fixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { transcription: { text: "abcdefgh" } }, + chunkSize: 2, + truncateAfterChunks: 2, + }; + instance = await createServer([fixture]); + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?intent=transcription&model=ignored-by-intent", + ); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + await ws.waitForClose(); + await expect(ws.waitForCloseFrame()).rejects.toThrow("without a close frame"); + const events = parseEvents(ws.getMessages()); + expect( + events.filter((event) => event.type === "conversation.item.input_audio_transcription.delta"), + ).toHaveLength(2); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "conversation.item.input_audio_transcription.completed" }), + ); + expect(instance.journal.getLast()).toMatchObject({ + body: { _endpointType: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { interrupted: true, interruptReason: "truncateAfterChunks" }, + }); + }); + + it("abruptly destroys a documented transcription socket after disconnectAfterMs", async () => { + const fixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { transcription: { text: "abcdefgh" } }, + chunkSize: 2, + latency: 60, + disconnectAfterMs: 15, + }; + instance = await createServer([fixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + await ws.waitForClose(); + await expect(ws.waitForCloseFrame()).rejects.toThrow("without a close frame"); + const events = parseEvents(ws.getMessages()); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "conversation.item.input_audio_transcription.delta" }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "conversation.item.input_audio_transcription.completed" }), + ); + expect(instance.journal.getLast()).toMatchObject({ + body: { _endpointType: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { interrupted: true, interruptReason: "disconnectAfterMs" }, + }); + }); + + it("closes documented transcription sockets in strict mode when a versioned fixture misses", async () => { + instance = await createServer([], { strict: true }); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); // session.created + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const close = await ws.waitForCloseFrame(); + expect(close.code).toBe(1008); + expect(instance.journal.getLast()).toMatchObject({ + response: { status: 503, fixture: null }, + body: { + _endpointType: "realtime-transcription", + model: "gpt-live-transcribe-2026-07-01", + }, + }); + }); + + it("emits a transcription failure event for a documented versioned ErrorResponse fixture", async () => { + const errorFixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { + error: { message: "Rate limited", type: "rate_limit_error", code: "rate_limit" }, + status: 429, + }, + }; + instance = await createServer([errorFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); // session.created + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(5); + const events = parseEvents(raw); + const item = events[3].item as Record; + expect(events[4]).toMatchObject({ + type: "conversation.item.input_audio_transcription.failed", + item_id: item.id, + content_index: 0, + error: { message: "Rate limited", type: "rate_limit_error", code: "rate_limit" }, + }); + expect(instance.journal.getLast()).toMatchObject({ + response: { status: 429, fixture: errorFixture }, + }); + ws.close(); + }); + + it("emits a transcription failure event for a non-strict live transcription no-match", async () => { + instance = await createServer([]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-live-transcribe"); + + await ws.waitForMessages(1); // session.created + ws.send(sessionUpdate({ type: "transcription" })); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(5); + const events = parseEvents(raw); + const item = events[3].item as Record; + expect(events[4]).toMatchObject({ + type: "conversation.item.input_audio_transcription.failed", + item_id: item.id, + content_index: 0, + error: { + message: "No fixture matched", + type: "invalid_request_error", + code: "no_fixture_match", + }, + }); + expect(instance.journal.getLast()).toMatchObject({ response: { status: 404, fixture: null } }); + ws.close(); + }); + + it("matches context-scoped live transcription fixtures", async () => { + const contextFixture: Fixture = { + match: { endpoint: "realtime-transcription", context: "call-42" }, + response: { transcription: { text: "Context transcript" } }, + }; + instance = await createServer([contextFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-live-transcribe", { + "x-aimock-context": "call-42", + }); + + await ws.waitForMessages(1); // session.created + ws.send(sessionUpdate({ type: "transcription" })); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(6); + const completed = parseEvents(raw)[5]; + expect(completed).toMatchObject({ + type: "conversation.item.input_audio_transcription.completed", + transcript: "Context transcript", + }); + expect(instance.journal.getLast()).toMatchObject({ body: { _context: "call-42" } }); + ws.close(); + }); + + it("emits a server failure for a non-transcription live transcription fixture", async () => { + const invalidFixture: Fixture = { + match: { endpoint: "realtime-transcription" }, + response: { content: "not a transcript" }, + }; + instance = await createServer([invalidFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-live-transcribe"); + + await ws.waitForMessages(1); // session.created + ws.send(sessionUpdate({ type: "transcription" })); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(5); + const events = parseEvents(raw); + const item = events[3].item as Record; + expect(events[4]).toMatchObject({ + type: "conversation.item.input_audio_transcription.failed", + item_id: item.id, + content_index: 0, + error: { message: "Fixture response is not a transcription type", type: "server_error" }, + }); + expect(instance.journal.getLast()).toMatchObject({ + response: { status: 500, fixture: invalidFixture }, + }); + ws.close(); + }); + it("input_audio_buffer.append is silently accepted", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); await ws.waitForMessages(1); // session.created @@ -1385,11 +1704,11 @@ describe("WebSocket /v1/realtime", () => { it("accepts translation session type", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); await ws.waitForMessages(1); // session.created - ws.send(sessionUpdate({ type: "translation", model: "gpt-4o-transcribe" })); + ws.send(sessionUpdate({ type: "translation" })); const raw = await ws.waitForMessages(2); const event = parseEvents(raw.slice(1))[0]; @@ -1425,7 +1744,7 @@ describe("WebSocket /v1/realtime", () => { await ws.waitForMessages(1); // session.created - ws.send(sessionUpdate({ type: "translation", model: "gpt-realtime-mini" })); + ws.send(sessionUpdate({ type: "translation" })); const raw = await ws.waitForMessages(2); const event = parseEvents(raw.slice(1))[0]; @@ -1440,11 +1759,11 @@ describe("WebSocket /v1/realtime", () => { it("audio buffer commit in translation mode adds placeholder conversation item", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); await ws.waitForMessages(1); // session.created - ws.send(sessionUpdate({ type: "translation", model: "gpt-4o-transcribe" })); + ws.send(sessionUpdate({ type: "translation" })); await ws.waitForMessages(2); // session.updated ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); @@ -1920,7 +2239,7 @@ describe("WebSocket /v1/realtime", () => { const session = event3.session as Record; // Model and type should still be the pre-rejection values expect(session.model).toBe("gpt-realtime-2"); - expect(session.type).toBe("conversation"); + expect(session.type).toBe("realtime"); expect(session.instructions).toBe("Updated instructions"); ws.close(); @@ -2112,3 +2431,296 @@ describe("realtimeItemsToMessages", () => { expect(messages[0].content).toBe(""); }); }); + +// ─── Regression: input transcription config must not hijack a conversation ── +// +// `input_audio_transcription` is the DOCUMENTED way to ask a normal realtime +// conversation session for input-transcription side events. It must never be +// read as "this is a transcription session": doing so routes +// `input_audio_buffer.commit` into the live-transcription path, which pushes a +// phantom `[audio]` turn and consumes a fixture match. Both corruptions are +// silent — the client still gets a well-formed response, just the WRONG one. +describe("realtime conversation with input_audio_transcription configured", () => { + // Two sequenced siblings: whoever asks first gets FIRST, the next gets SECOND. + // A stolen match in between is therefore observable as wrong-turn delivery. + const sequencedFixtures: Fixture[] = [ + { match: { sequenceIndex: 0 }, response: { content: "FIRST" } }, + { match: { sequenceIndex: 1 }, response: { content: "SECOND" } }, + ]; + + async function textOf(ws: Awaited>): Promise { + const deadline = Date.now() + 5000; + for (;;) { + const events = parseEvents(ws.getMessages()); + if (events.some((e) => e.type === "response.done")) { + return events + .filter((e) => e.type === "response.output_text.delta") + .map((e) => e.delta as string) + .join(""); + } + if (Date.now() > deadline) { + throw new Error(`timeout waiting for response.done; saw ${events.map((e) => e.type)}`); + } + await new Promise((r) => setTimeout(r, 20)); + } + } + + it("delivers the FIRST sequenced turn after a commit (no match is stolen)", async () => { + instance = await createServer(sequencedFixtures); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime"); + await ws.waitForMessages(1); // session.created + + // The canonical realtime setup: ask for input transcription on a plain + // conversation session. This is NOT a transcription session. + ws.send(sessionUpdate({ audio: { input: { transcription: { model: "whisper-1" } } } })); + await ws.waitForMessages(2); // session.updated + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + ws.send(conversationItemCreate("user", "hello")); + ws.send(responseCreate()); + + expect(await textOf(ws)).toBe("FIRST"); + ws.close(); + }); + + it("delivers the FIRST sequenced turn after a commit without input transcription", async () => { + // Control: identical flow with no input_audio_transcription configured. + instance = await createServer(sequencedFixtures); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime"); + await ws.waitForMessages(1); + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + ws.send(conversationItemCreate("user", "hello")); + ws.send(responseCreate()); + + expect(await textOf(ws)).toBe("FIRST"); + ws.close(); + }); + + it("does not burn a fixture match on commit", async () => { + instance = await createServer(sequencedFixtures); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime"); + await ws.waitForMessages(1); + + ws.send(sessionUpdate({ audio: { input: { transcription: { model: "whisper-1" } } } })); + await ws.waitForMessages(2); + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + // Give the (incorrect) transcription path time to run and consume a match. + await new Promise((r) => setTimeout(r, 200)); + + expect(instance.journal.getFixtureMatchCount(sequencedFixtures[0])).toBe(0); + expect(instance.journal.getFixtureMatchCount(sequencedFixtures[1])).toBe(0); + expect( + instance.journal + .getAll() + .filter( + (e) => + (e.body as Record | undefined)?._endpointType === + "realtime-transcription", + ), + ).toHaveLength(0); + ws.close(); + }); + + it("does not add a phantom audio item on commit", async () => { + instance = await createServer(sequencedFixtures); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime"); + await ws.waitForMessages(1); + + ws.send(sessionUpdate({ audio: { input: { transcription: { model: "whisper-1" } } } })); + await ws.waitForMessages(2); + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + await new Promise((r) => setTimeout(r, 200)); + + const types = parseEvents(ws.getMessages()).map((e) => e.type); + expect(types).toContain("input_audio_buffer.committed"); + expect(types).not.toContain("conversation.item.added"); + expect(types).not.toContain("conversation.item.input_audio_transcription.delta"); + ws.close(); + }); +}); + +// ─── Wire fidelity against live GA captures ───────────────────────────────── +describe("realtime wire fidelity", () => { + // Live GA capture: a session.update carrying a model DIFFERENT from the + // connection model returns a normal `session.updated` and silently IGNORES + // the model field (probe sent model:"gpt-realtime" on a "gpt-realtime-mini" + // connection; the reply echoed model:"gpt-realtime-mini" with the new + // instructions applied). Rejecting it strands clients that await + // session.updated -- they hang until their own timeout. + it("acknowledges session.update carrying a differing model, ignoring the field", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime-mini"); + await ws.waitForMessages(1); // session.created + + ws.send(sessionUpdate({ model: "gpt-realtime", instructions: "be brief" })); + + const event = parseEvents(await ws.waitForMessages(2))[1]; + expect(event.type).toBe("session.updated"); + const session = event.session as Record; + // The model field is ignored, not applied and not rejected. + expect(session.model).toBe("gpt-realtime-mini"); + expect(session.instructions).toBe("be brief"); + ws.close(); + }); + + // Live GA capture (transcription session): + // {"type":"transcription","object":"realtime.transcription_session", + // "id":"sess_…","expires_at":…,"audio":{"input":{…}},"include":null} + // No model, no modalities/tools/tool_choice/temperature/instructions/ + // max_response_output_tokens/reasoning, and no audio.output. + it("serializes a transcription session with the real transcription shape", async () => { + instance = await createServer([]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + const created = parseEvents(await ws.waitForMessages(1))[0]; + const session = created.session as Record; + + expect(session.object).toBe("realtime.transcription_session"); + expect(session.type).toBe("transcription"); + expect(session).not.toHaveProperty("model"); + expect(session).not.toHaveProperty("modalities"); + expect(session).not.toHaveProperty("tools"); + expect(session).not.toHaveProperty("tool_choice"); + expect(session).not.toHaveProperty("temperature"); + expect(session).not.toHaveProperty("instructions"); + expect(session).not.toHaveProperty("max_response_output_tokens"); + expect(session).not.toHaveProperty("reasoning"); + expect(session.include).toBeNull(); + expect(Object.keys(session.audio as Record)).toEqual(["input"]); + expect(typeof session.expires_at).toBe("number"); + expect((session.id as string).startsWith("sess_")).toBe(true); + ws.close(); + }); + + // A conversation session keeps the realtime.session shape untouched. + it("leaves the conversation session shape unchanged", async () => { + instance = await createServer([]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime"); + + const created = parseEvents(await ws.waitForMessages(1))[0]; + const session = created.session as Record; + + expect(session.object).toBe("realtime.session"); + expect(session.model).toBe("gpt-realtime"); + expect(session.type).toBe("realtime"); + expect(Object.keys(session.audio as Record).sort()).toEqual([ + "input", + "output", + ]); + ws.close(); + }); + + // Transcription usage is MODEL-DEPENDENT on the live API. Captured from + // api.openai.com with one 5s clip, identical on the realtime WS + // (conversation.item.input_audio_transcription.completed) and on HTTP + // (POST /v1/audio/transcriptions): + // whisper-1 → {"type":"duration","seconds":5} + // gpt-transcribe → {"type":"duration","seconds":5} + // gpt-live-transcribe → {"type":"duration","seconds":5} + // gpt-4o-transcribe → {"type":"tokens","total_tokens":66,…} + // gpt-4o-mini-transcribe → {"type":"tokens","total_tokens":66,…} + // A model-BLIND default fails silently in either direction, so each family is + // pinned separately — one family alone cannot catch a model-blind default. + async function completedTranscriptionUsage(model: string): Promise> { + instance = await createServer([ + { + match: { endpoint: "realtime-transcription" }, + response: { transcription: { text: "Live caption" } }, + }, + ]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate(model)); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const events = parseEvents(await ws.waitForMessages(6)); + const completed = events.find( + (e) => e.type === "conversation.item.input_audio_transcription.completed", + ); + ws.close(); + return completed!.usage as Record; + } + + it.each(["whisper-1", "gpt-transcribe", "gpt-live-transcribe", "gpt-live-transcribe-2026-07-01"])( + "reports duration-shaped usage for the duration family (%s)", + async (model) => { + const usage = await completedTranscriptionUsage(model); + expect(usage.type).toBe("duration"); + expect(usage).toHaveProperty("seconds"); + expect(usage).not.toHaveProperty("input_tokens"); + expect(usage).not.toHaveProperty("total_tokens"); + }, + ); + + it.each(["gpt-4o-transcribe", "gpt-4o-mini-transcribe", "gpt-4o-mini-transcribe-2024-12-17"])( + "reports token-shaped usage for the token family (%s)", + async (model) => { + const usage = await completedTranscriptionUsage(model); + expect(usage.type).toBe("tokens"); + expect(usage).toHaveProperty("total_tokens"); + expect(usage).toHaveProperty("input_tokens"); + expect(usage).toHaveProperty("output_tokens"); + expect(usage).not.toHaveProperty("seconds"); + }, + ); + + // A fixture-supplied usage still wins over the synthesized default. + it("passes through a fixture-supplied transcription usage", async () => { + instance = await createServer([ + { + match: { endpoint: "realtime-transcription" }, + response: { + transcription: { text: "Live caption", usage: { type: "duration", seconds: 4 } }, + }, + }, + ]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe")); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const events = parseEvents(await ws.waitForMessages(6)); + const completed = events.find( + (e) => e.type === "conversation.item.input_audio_transcription.completed", + ); + expect(completed!.usage).toEqual({ type: "duration", seconds: 4 }); + ws.close(); + }); +}); + +// A genuine transcription session must not consume a chat fixture's match +// count either. router.ts exempts `realtime*` requests from the response-shape +// gate, so a generic chat fixture still MATCHES a realtime-transcription +// lookup; burning its count on a response that is then rejected for having the +// wrong shape would silently advance a sequenced conversation. +describe("transcription session fixture accounting", () => { + it("does not burn a chat fixture's match count on a shape mismatch", async () => { + const chatFixture: Fixture = { match: { sequenceIndex: 0 }, response: { content: "FIRST" } }; + instance = await createServer([chatFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe")); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + // The lookup matches the chat fixture, then rejects it for shape. + const deadline = Date.now() + 3000; + for (;;) { + const types = parseEvents(ws.getMessages()).map((e) => e.type); + if (types.includes("conversation.item.input_audio_transcription.failed")) break; + if (Date.now() > deadline) throw new Error(`no failed event; saw ${types}`); + await new Promise((r) => setTimeout(r, 20)); + } + + expect(instance.journal.getFixtureMatchCount(chatFixture)).toBe(0); + ws.close(); + }); +}); diff --git a/src/agui-recorder.ts b/src/agui-recorder.ts index bb0720e1..87d6d8de 100644 --- a/src/agui-recorder.ts +++ b/src/agui-recorder.ts @@ -12,6 +12,7 @@ import type { } from "./agui-types.js"; import { extractLastUserMessage, getLastMessageIfToolResult } from "./agui-handler.js"; import type { Logger } from "./logger.js"; +import { isAuthenticatedRequest } from "./api-key-auth.js"; /** * Sentinel `match.message` value written to disk when the request had no @@ -85,6 +86,14 @@ export async function proxyAndRecordAGUI( config: AGUIRecordConfig, logger: Logger, ): Promise { + // AG-UI has no provider-specific static credential contract. An inbound + // test key must never become an implicit upstream credential, so fail before + // opening an upstream connection. + if (isAuthenticatedRequest(req)) { + res.writeHead(502, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "No configured provider credential" })); + return 502; + } if (!config.upstream) { logger.warn("No upstream URL configured for AG-UI recording — cannot proxy"); return false; diff --git a/src/api-key-auth.ts b/src/api-key-auth.ts new file mode 100644 index 00000000..a1a091f5 --- /dev/null +++ b/src/api-key-auth.ts @@ -0,0 +1,188 @@ +import { timingSafeEqual } from "node:crypto"; +import type * as http from "node:http"; +import type * as net from "node:net"; +import type { ApiKeyAuthConfig } from "./types.js"; + +export const INBOUND_API_KEY_HEADERS = [ + "authorization", + "x-api-key", + "x-goog-api-key", + "api-key", + "xi-api-key", +] as const; + +const RECOGNIZED_HEADERS = new Set(INBOUND_API_KEY_HEADERS); + +export const API_KEY_ERROR_BODY = JSON.stringify({ + error: { + message: "Invalid API key", + type: "authentication_error", + code: "invalid_api_key", + }, +}); + +/** @internal */ +export interface ApiKeyPolicy { + readonly enabled: boolean; + readonly keys: readonly Buffer[]; +} + +/** @internal */ +export interface ResolvedInboundAuth { + readonly publicConfig?: ApiKeyAuthConfig; + readonly policy: ApiKeyPolicy; +} + +export interface AuthResult { + readonly ok: boolean; + readonly reason?: "missing" | "invalid" | "conflict"; +} + +const DISABLED_POLICY: ApiKeyPolicy = { enabled: false, keys: [] }; +const authenticatedRequests = new WeakSet(); +const OWS_EDGES = /^[ \t]+|[ \t]+$/g; + +function fail(label: string, suffix: string): never { + throw new Error(`${label}${suffix}`); +} + +function hasControl(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 0x1f || code === 0x7f) return true; + } + return false; +} + +function normalizeKey(value: unknown, label: string): string { + if (typeof value !== "string") fail(label, " must be a string"); + const normalized = value.replace(OWS_EDGES, ""); + if (normalized.length === 0) fail(label, " must not be empty"); + if (hasControl(normalized)) fail(label, " must not contain control characters"); + return normalized; +} + +/** Parse a public auth source once and construct its private server policy. */ +export function resolveInboundAuth(source: { value: unknown; label: string }): ResolvedInboundAuth { + if (source.value === undefined) return { policy: DISABLED_POLICY }; + if (source.value === null || typeof source.value !== "object" || Array.isArray(source.value)) { + fail(source.label, " must be an object"); + } + const apiKeys = (source.value as { apiKeys?: unknown }).apiKeys; + if (!Array.isArray(apiKeys)) fail(`${source.label}.apiKeys`, " must be an array"); + if (apiKeys.length === 0) fail(`${source.label}.apiKeys`, " must not be empty"); + const keys = apiKeys.map((value, index) => + normalizeKey(value, `${source.label}.apiKeys[${index}]`), + ); + const seen = new Set(); + for (const key of keys) { + if (seen.has(key)) fail(source.label, " must not contain duplicate API keys"); + seen.add(key); + } + return { + publicConfig: { apiKeys: keys }, + policy: { enabled: true, keys: keys.map((key) => Buffer.from(key, "utf8")) }, + }; +} + +/** Select configuration without parsing it, so an environment override bypasses malformed JSON auth. */ +export function selectInboundAuthSource( + configAuth: unknown, + env: NodeJS.ProcessEnv = process.env, +): { value: unknown; label: string } { + if (Object.prototype.hasOwnProperty.call(env, "AIMOCK_API_KEYS")) { + return { value: { apiKeys: (env.AIMOCK_API_KEYS ?? "").split(",") }, label: "AIMOCK_API_KEYS" }; + } + return { value: configAuth, label: "aimock.json.auth" }; +} + +export function isRecognizedApiKeyHeader(name: string): boolean { + return RECOGNIZED_HEADERS.has(name.toLowerCase()); +} + +function rawCandidates(req: http.IncomingMessage): { + candidates?: string[]; + reason?: "missing" | "invalid"; +} { + const candidates: string[] = []; + for (let index = 0; index < req.rawHeaders.length; index += 2) { + const name = req.rawHeaders[index]; + const value = req.rawHeaders[index + 1] ?? ""; + const lower = name.toLowerCase(); + if (!isRecognizedApiKeyHeader(lower)) continue; + if (hasControl(value)) return { reason: "invalid" }; + if (lower === "authorization") { + const match = /^(?:Bearer|Key)[ \t]+(.+)$/i.exec(value.replace(OWS_EDGES, "")); + if (!match) return { reason: "invalid" }; + const candidate = match[1].replace(OWS_EDGES, ""); + if (candidate.length === 0 || hasControl(candidate)) return { reason: "invalid" }; + candidates.push(candidate); + } else { + const candidate = value.replace(OWS_EDGES, ""); + if (candidate.length === 0) return { reason: "invalid" }; + candidates.push(candidate); + } + } + return candidates.length > 0 ? { candidates } : { reason: "missing" }; +} + +function matchingIndex(candidate: string, keys: readonly Buffer[]): number | undefined { + const encoded = Buffer.from(candidate, "utf8"); + for (let index = 0; index < keys.length; index += 1) { + const key = keys[index]; + if (encoded.length === key.length && timingSafeEqual(encoded, key)) return index; + } + return undefined; +} + +export function validateRequestApiKey(req: http.IncomingMessage, policy: ApiKeyPolicy): AuthResult { + if (!policy.enabled) return { ok: true }; + const extracted = rawCandidates(req); + if (!extracted.candidates) return { ok: false, reason: extracted.reason }; + let resolved: number | undefined; + for (const candidate of extracted.candidates) { + const index = matchingIndex(candidate, policy.keys); + if (index === undefined) return { ok: false, reason: "invalid" }; + if (resolved !== undefined && resolved !== index) return { ok: false, reason: "conflict" }; + resolved = index; + } + return { ok: true }; +} + +/** @internal Opaque egress marker; it deliberately carries no key material. */ +export function markAuthenticatedRequest(req: http.IncomingMessage, policy: ApiKeyPolicy): void { + if (policy.enabled) authenticatedRequests.add(req); +} + +/** @internal Used by egress paths to select safe credential handling. */ +export function isAuthenticatedRequest(req: http.IncomingMessage): boolean { + return authenticatedRequests.has(req); +} + +export function writeApiKeyHttpRejection( + res: http.ServerResponse, + setCorsHeaders: (res: http.ServerResponse) => void, +): void { + setCorsHeaders(res); + res.writeHead(401, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(API_KEY_ERROR_BODY), + "WWW-Authenticate": 'Bearer realm="aimock"', + }); + res.end(API_KEY_ERROR_BODY); +} + +export function writeApiKeyUpgradeRejection(socket: net.Socket): void { + socket.write( + [ + "HTTP/1.1 401 Unauthorized", + "Content-Type: application/json", + `Content-Length: ${Buffer.byteLength(API_KEY_ERROR_BODY)}`, + 'WWW-Authenticate: Bearer realm="aimock"', + "Connection: close", + "", + API_KEY_ERROR_BODY, + ].join("\r\n"), + ); + socket.destroy(); +} diff --git a/src/cli.ts b/src/cli.ts index 7056472d..b3f6f732 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,6 +9,7 @@ import { watchFixtures } from "./watcher.js"; import { AGUIMock } from "./agui-mock.js"; import { resolveFixturesValue } from "./fixtures-remote.js"; import { readProviderKeysFromEnv } from "./provider-auth.js"; +import { resolveInboundAuth, selectInboundAuthSource } from "./api-key-auth.js"; import type { Fixture, ChaosConfig, RecordConfig } from "./types.js"; const HELP = ` @@ -52,6 +53,7 @@ Options: --chaos-drop Probability (0-1) of dropping requests with 500 --chaos-malformed Probability (0-1) of returning malformed JSON --chaos-disconnect Probability (0-1) of destroying connection + AIMOCK_API_KEYS Comma-separated inbound test API keys (environment only) --help Show this help message `.trim(); @@ -456,6 +458,7 @@ async function main() { strict: values.strict, journalMaxEntries: journalMax, fixtureCountsMaxTestIds: fixtureCountsMax, + auth: resolveInboundAuth(selectInboundAuthSource(undefined)).publicConfig, }, mounts, ); diff --git a/src/config-loader.ts b/src/config-loader.ts index 871904ca..0b3b71fa 100644 --- a/src/config-loader.ts +++ b/src/config-loader.ts @@ -1,10 +1,11 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { LLMock } from "./llmock.js"; +import { LLMock, createLLMockWithResolvedAuth } from "./llmock.js"; +import { resolveInboundAuth, selectInboundAuthSource } from "./api-key-auth.js"; import { MCPMock } from "./mcp-mock.js"; import { A2AMock } from "./a2a-mock.js"; import { AGUIMock } from "./agui-mock.js"; -import type { ChaosConfig, RecordConfig } from "./types.js"; +import type { ApiKeyAuthConfig, ChaosConfig, RecordConfig } from "./types.js"; import type { MCPToolDefinition, MCPPromptDefinition } from "./mcp-types.js"; import type { A2AAgentDefinition, A2APart, A2AArtifact, A2AStreamEvent } from "./a2a-types.js"; import type { AGUIEvent } from "./agui-types.js"; @@ -88,6 +89,7 @@ export interface VectorConfig { } export interface AimockConfig { + auth?: ApiKeyAuthConfig; llm?: { fixtures?: string; latency?: number; @@ -128,18 +130,22 @@ export async function startFromConfig( } // Load fixtures if specified - const llmock = new LLMock({ - port: overrides?.port ?? config.port ?? 0, - host: overrides?.host ?? config.host ?? "127.0.0.1", - latency: config.llm?.latency, - chunkSize: config.llm?.chunkSize, - replaySpeed, - logLevel: config.llm?.logLevel, - chaos: config.llm?.chaos, - record: config.llm?.record, - metrics: config.metrics, - strict: config.strict, - }); + const resolvedAuth = resolveInboundAuth(selectInboundAuthSource(config.auth)); + const llmock = createLLMockWithResolvedAuth( + { + port: overrides?.port ?? config.port ?? 0, + host: overrides?.host ?? config.host ?? "127.0.0.1", + latency: config.llm?.latency, + chunkSize: config.llm?.chunkSize, + replaySpeed, + logLevel: config.llm?.logLevel, + chaos: config.llm?.chaos, + record: config.llm?.record, + metrics: config.metrics, + strict: config.strict, + }, + resolvedAuth, + ); if (config.llm?.fixtures) { const fixturePath = path.resolve(config.llm.fixtures); diff --git a/src/fal-audio.ts b/src/fal-audio.ts index f79a8a64..9886f4f0 100644 --- a/src/fal-audio.ts +++ b/src/fal-audio.ts @@ -31,6 +31,7 @@ import { proxyAndRecord, sanitizeHeaderValue, } from "./recorder.js"; +import { isAuthenticatedRequest } from "./api-key-auth.js"; import { walkFalQueue } from "./fal.js"; import type { Journal } from "./journal.js"; import { applyChaos } from "./chaos.js"; @@ -580,6 +581,7 @@ async function tryRecordAudioQueueWalk(args: { // aimock's built-in fal key (Authorization: Key), injected by the walk on // a no/dummy caller credential; a real caller key overrides. builtinKey: record.providerKeys?.fal, + requireConfiguredKey: isAuthenticatedRequest(req), pollIntervalMs: record.fal?.pollIntervalMs, timeoutMs: record.fal?.timeoutMs, upstreamTimeoutMs: record.upstreamTimeoutMs, diff --git a/src/fal.ts b/src/fal.ts index 268aace6..15ca4ab0 100644 --- a/src/fal.ts +++ b/src/fal.ts @@ -35,6 +35,7 @@ import { proxyAndRecord, sanitizeHeaderValue, } from "./recorder.js"; +import { isAuthenticatedRequest } from "./api-key-auth.js"; import { resolveUpstreamUrl } from "./url.js"; import { applyProviderAuth } from "./provider-auth.js"; import type { Journal } from "./journal.js"; @@ -894,6 +895,7 @@ export async function walkFalQueue(args: { * route through `proxyAndRecord`, so own-key injection lives here. */ builtinKey?: string; + requireConfiguredKey?: boolean; }): Promise { const { upstreamBase, @@ -907,8 +909,13 @@ export async function walkFalQueue(args: { fallbackResultPath, logger, builtinKey, + requireConfiguredKey, } = args; + if (requireConfiguredKey && !builtinKey) { + throw new Error("No configured provider credential"); + } + // Inject aimock's own fal credential onto the walk headers up front so every // fetch below (submit + polls) carries it. fal's scheme ignores the target // URL, so any resolvable URL suffices. @@ -1083,6 +1090,7 @@ async function proxyAndRecordFalQueueSubmit(args: { // aimock's built-in fal key (Authorization: Key), injected by the walk on // a no/dummy caller credential; a real caller key overrides. builtinKey: record.providerKeys?.fal, + requireConfiguredKey: isAuthenticatedRequest(req), pollIntervalMs: record.fal?.pollIntervalMs, timeoutMs: record.fal?.timeoutMs, upstreamTimeoutMs: record.upstreamTimeoutMs, diff --git a/src/grok-video.ts b/src/grok-video.ts index 14949545..aa55332d 100644 --- a/src/grok-video.ts +++ b/src/grok-video.ts @@ -27,7 +27,7 @@ import { applyChaos } from "./chaos.js"; import { resolveProgression } from "./fal.js"; import { buildFixtureMatch, - buildForwardHeaders, + prepareEgressHeaders, persistFixture, sanitizeHeaderValue, } from "./recorder.js"; @@ -773,9 +773,11 @@ async function proxyGrokVideoSubmit(args: { let fetched: { status: number; contentType: string | null; text: string }; try { + const headers = prepareEgressHeaders(req, submitUrl, "grok", record.providerKeys?.grok); + if (!headers) return proxyError("No configured provider credential"); const upstreamRes = await fetch(submitUrl, { method: "POST", - headers: buildForwardHeaders(req), + headers, body: raw, signal: upstreamTimeoutSignal(record), }); @@ -946,8 +948,14 @@ async function proxyGrokVideoRecordPoll(args: { let fetched: { status: number; contentType: string | null; text: string }; try { + const target = new URL(job.upstreamPollingUrl); + const headers = prepareEgressHeaders(req, target, "grok", record.providerKeys?.grok); + if (!headers) { + proxyError("No configured provider credential"); + return; + } const upstreamRes = await fetch(job.upstreamPollingUrl, { - headers: buildForwardHeaders(req), + headers, signal: upstreamTimeoutSignal(record), }); fetched = { diff --git a/src/helpers.ts b/src/helpers.ts index eb18ebc0..748c475d 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -4,6 +4,7 @@ import type { IncomingHttpHeaders } from "node:http"; import { DEFAULT_TEST_ID } from "./constants.js"; import type { Logger } from "./logger.js"; import { isReasoningModel } from "./model-utils.js"; +import { isRecognizedApiKeyHeader } from "./api-key-auth.js"; import type { ChatCompletionRequest, Fixture, @@ -27,8 +28,6 @@ import type { ResponseOverrides, } from "./types.js"; -const REDACTED_HEADERS = new Set(["authorization", "x-api-key", "api-key"]); - /** * Resolve effective strict mode from per-request header and server default. * Header values override the server default — same precedence pattern as chaos @@ -196,7 +195,7 @@ export function flattenHeaders(headers: http.IncomingHttpHeaders): Record = {}; for (const [key, value] of Object.entries(headers)) { if (value === undefined) continue; - if (REDACTED_HEADERS.has(key.toLowerCase())) { + if (isRecognizedApiKeyHeader(key)) { flat[key] = "[REDACTED]"; } else { flat[key] = Array.isArray(value) ? value.join(", ") : value; diff --git a/src/index.ts b/src/index.ts index 8abe128d..8138f78b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -183,7 +183,7 @@ export { export type { CollapseResult } from "./stream-collapse.js"; // Mountable -export type { Mountable } from "./types.js"; +export type { Mountable, ApiKeyAuthConfig } from "./types.js"; // MCP export { MCPMock } from "./mcp-mock.js"; diff --git a/src/llmock.ts b/src/llmock.ts index a546f5fb..fe6b840d 100644 --- a/src/llmock.ts +++ b/src/llmock.ts @@ -17,7 +17,8 @@ import type { TranscriptionResponse, VideoResponse, } from "./types.js"; -import { createServer, type ServerInstance } from "./server.js"; +import { createServer, createServerWithResolvedAuth, type ServerInstance } from "./server.js"; +import type { ResolvedInboundAuth } from "./api-key-auth.js"; import { loadFixtureFile, loadFixturesFromDir, @@ -40,9 +41,11 @@ export class LLMock { private mounts: Array<{ path: string; handler: Mountable }> = []; private serverInstance: ServerInstance | null = null; private options: MockServerOptions; + private readonly resolvedInboundAuth?: ResolvedInboundAuth; - constructor(options?: MockServerOptions) { + constructor(options?: MockServerOptions, resolvedInboundAuth?: ResolvedInboundAuth) { this.options = options ?? {}; + this.resolvedInboundAuth = resolvedInboundAuth; } // ---- Fixture management ---- @@ -440,11 +443,23 @@ export class LLMock { if (this.serverInstance) { throw new Error("Server already started"); } - this.serverInstance = await createServer(this.fixtures, this.options, this.mounts, { - search: this.searchFixtures, - rerank: this.rerankFixtures, - moderation: this.moderationFixtures, - }); + this.serverInstance = await (this.resolvedInboundAuth + ? createServerWithResolvedAuth( + this.fixtures, + this.options, + this.resolvedInboundAuth, + this.mounts, + { + search: this.searchFixtures, + rerank: this.rerankFixtures, + moderation: this.moderationFixtures, + }, + ) + : createServer(this.fixtures, this.options, this.mounts, { + search: this.searchFixtures, + rerank: this.rerankFixtures, + moderation: this.moderationFixtures, + })); return this.serverInstance.url; } @@ -495,3 +510,11 @@ export class LLMock { return instance; } } + +/** @internal Configuration startup preserves a policy resolved from the selected source. */ +export function createLLMockWithResolvedAuth( + options: MockServerOptions, + resolvedAuth: ResolvedInboundAuth, +): LLMock { + return new LLMock(options, resolvedAuth); +} diff --git a/src/openrouter-video.ts b/src/openrouter-video.ts index 1f984522..a24c612f 100644 --- a/src/openrouter-video.ts +++ b/src/openrouter-video.ts @@ -28,7 +28,8 @@ import { applyChaos } from "./chaos.js"; import { resolveProgression } from "./fal.js"; import { buildFixtureMatch, - buildForwardHeaders, + prepareEgressHeaders, + removeForwardHeader, clampTimeout, persistFixture, sanitizeHeaderValue, @@ -928,9 +929,18 @@ async function proxyOpenRouterVideoRecordContent(args: { return; } } - const headers = buildForwardHeaders(req); + const headers = prepareEgressHeaders( + req, + contentUrl, + "openrouter", + record.providerKeys?.openrouter, + ); + if (!headers) { + proxyError("No configured provider credential"); + return; + } if (contentUrl.origin !== providerOrigin) { - delete headers.authorization; + removeForwardHeader(headers, "authorization"); logger.warn( `Upstream unsigned_urls[${index}] origin ${contentUrl.origin} differs from the provider origin ${providerOrigin} — fetching content WITHOUT the client's Authorization header`, ); @@ -1205,8 +1215,15 @@ export async function handleOpenRouterVideoModels( let relay: { text: string; contentType: string } | undefined; try { const target = resolveUpstreamUrl(upstreamBase, "/api/v1/videos/models"); + const headers = prepareEgressHeaders( + req, + target, + "openrouter", + record.providerKeys?.openrouter, + ); + if (!headers) throw new Error("No configured provider credential"); const upstreamRes = await fetch(target, { - headers: buildForwardHeaders(req), + headers, signal: upstreamTimeoutSignal(record), }); const text = await readEnvelopeText(upstreamRes, record); @@ -1750,9 +1767,16 @@ async function proxyOpenRouterVideoSubmit(args: { let fetched: { status: number; contentType: string | null; text: string }; try { + const headers = prepareEgressHeaders( + req, + submitUrl, + "openrouter", + record.providerKeys?.openrouter, + ); + if (!headers) return proxyError("No configured provider credential"); const upstreamRes = await fetch(submitUrl, { method: "POST", - headers: buildForwardHeaders(req), + headers, body: raw, signal: upstreamTimeoutSignal(record), }); @@ -1998,8 +2022,19 @@ async function proxyOpenRouterVideoRecordPoll(args: { let fetched: { status: number; contentType: string | null; text: string }; try { + const target = new URL(job.upstreamPollingUrl); + const headers = prepareEgressHeaders( + req, + target, + "openrouter", + record.providerKeys?.openrouter, + ); + if (!headers) { + proxyError("No configured provider credential"); + return; + } const upstreamRes = await fetch(job.upstreamPollingUrl, { - headers: buildForwardHeaders(req), + headers, // The locally captured `record` — not defaults.record, which is the // same object today but would silently diverge if the defaults were // ever swapped between the gate above and this fetch. @@ -2384,9 +2419,20 @@ async function captureOpenRouterVideoRecordFixture(args: { return; } } - const headers = buildForwardHeaders(req); + const headers = prepareEgressHeaders( + req, + contentUrl, + "openrouter", + record.providerKeys?.openrouter, + ); + if (!headers) { + logger.error( + `OpenRouter video capture for job ${job.upstreamJobId} aborted: no configured provider credential`, + ); + return; + } if (contentUrl.origin !== providerOrigin) { - delete headers.authorization; + removeForwardHeader(headers, "authorization"); logger.warn( `Upstream unsigned_urls[0] origin ${contentUrl.origin} differs from the provider origin ${providerOrigin} — fetching content WITHOUT the client's Authorization header`, ); diff --git a/src/provider-auth.ts b/src/provider-auth.ts index b00f436e..af3015d4 100644 --- a/src/provider-auth.ts +++ b/src/provider-auth.ts @@ -40,7 +40,7 @@ export function getDummyKeyMarker(): string { * and a real Entra bearer token from the caller is never dummy-prefixed, so it * is always forwarded verbatim (the caller overrides aimock). */ -type AuthScheme = +export type AuthScheme = | { kind: "bearer" } // Authorization: Bearer | { kind: "fal-key" } // Authorization: Key | { kind: "x-api-key" } // x-api-key: @@ -69,6 +69,36 @@ const PROVIDER_AUTH_SCHEMES: Partial> = { fal: { kind: "fal-key" }, }; +export function hasStaticProviderAuthScheme(providerKey: RecordProviderKey): boolean { + return PROVIDER_AUTH_SCHEMES[providerKey] !== undefined; +} + +/** Force a known provider credential onto an already-scrubbed outbound map. */ +export function applyConfiguredProviderAuth( + forwardHeaders: Record, + target: URL, + providerKey: RecordProviderKey, + providerCredential: string, +): boolean { + void target; + const scheme = PROVIDER_AUTH_SCHEMES[providerKey]; + if (!scheme) return false; + switch (scheme.kind) { + case "bearer": + takeHeader(forwardHeaders, "authorization"); + forwardHeaders.Authorization = `Bearer ${providerCredential}`; + break; + case "fal-key": + takeHeader(forwardHeaders, "authorization"); + forwardHeaders.Authorization = `Key ${providerCredential}`; + break; + default: + takeHeader(forwardHeaders, authHeaderName(scheme)); + forwardHeaders[authHeaderName(scheme)] = providerCredential; + } + return true; +} + /** * Case-insensitively delete a header from a forward-header map, returning the * value that was present (if any). Header maps preserve the original casing of diff --git a/src/recorder.ts b/src/recorder.ts index f98b0881..05ed6c52 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -20,10 +20,29 @@ import type { Logger } from "./logger.js"; import { collapseStreamingResponse, capturedRedactedData } from "./stream-collapse.js"; import { writeErrorResponse } from "./sse-writer.js"; import { resolveUpstreamUrl } from "./url.js"; -import { applyProviderAuth } from "./provider-auth.js"; +import { applyConfiguredProviderAuth, applyProviderAuth } from "./provider-auth.js"; +import { isAuthenticatedRequest, isRecognizedApiKeyHeader } from "./api-key-auth.js"; import { getTestId, slugifyTestId, slugifyContext } from "./helpers.js"; import { DEFAULT_TEST_ID } from "./constants.js"; +/** True when an SSE frame completes an OpenAI stream. */ +function isTerminalSSEFrame(frame: string): boolean { + const data = frame + .split(/\r?\n/) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice("data:".length).trimStart()) + .join("\n") + .trim(); + + if (data === "[DONE]") return true; + + try { + return (JSON.parse(data) as { type?: unknown }).type === "transcript.text.done"; + } catch { + return false; + } +} + /** Headers to strip when proxying — hop-by-hop (RFC 2616 §13.5.1) + client-set. */ /** * Default ceiling (bytes) for the in-memory proxy-path buffer. Chosen well @@ -124,7 +143,12 @@ export function buildForwardHeaders(req: http.IncomingMessage): Record = {}; for (const [name, val] of Object.entries(req.headers)) { const lower = name.toLowerCase(); - if (val === undefined || STRIP_HEADERS.has(lower) || lower.startsWith("x-aimock-chaos-")) { + if ( + val === undefined || + STRIP_HEADERS.has(lower) || + lower.startsWith("x-aimock-chaos-") || + (isAuthenticatedRequest(req) && isRecognizedApiKeyHeader(lower)) + ) { continue; } out[name] = Array.isArray(val) ? val.join(", ") : val; @@ -132,6 +156,38 @@ export function buildForwardHeaders(req: http.IncomingMessage): Record, name: string): void { + const lowerName = name.toLowerCase(); + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === lowerName) delete headers[key]; + } +} + +/** + * Construct the only safe egress map when inbound access control is enabled. + * Test credentials are always removed; a static configured provider credential + * is mandatory so an accepted test credential can never become an upstream one. + */ +export function prepareEgressHeaders( + req: http.IncomingMessage, + target: URL, + providerKey: RecordProviderKey, + configuredKey: string | undefined, +): Record | undefined { + const headers = buildForwardHeaders(req); + if (!isAuthenticatedRequest(req)) { + applyProviderAuth(headers, target, providerKey, configuredKey); + return headers; + } + for (const name of Object.keys(headers)) { + if (isRecognizedApiKeyHeader(name)) delete headers[name]; + } + return configuredKey && applyConfiguredProviderAuth(headers, target, providerKey, configuredKey) + ? headers + : undefined; +} + /** * Captured upstream response, exposed to the `beforeWriteResponse` hook so * callers can decide whether to relay it or mutate it (e.g. chaos injection). @@ -438,13 +494,22 @@ export async function proxyAndRecord( defaults.logger.warn(`NO FIXTURE MATCH — proxying to ${upstreamUrl}${pathname}`); // Forward all request headers except hop-by-hop and client-set ones. - const forwardHeaders = buildForwardHeaders(req); - - // If aimock owns a built-in upstream key for this provider, inject it now - // (opt-in, backward-compatible). A real caller credential overrides it; an - // absent or dummy-prefixed caller credential is replaced. gemini-interactions - // reuses the Gemini key, mirroring the upstream-URL remap above. - applyProviderAuth(forwardHeaders, target, providerKey, record.providerKeys?.[lookupKey]); + const forwardHeaders = prepareEgressHeaders( + req, + target, + providerKey, + record.providerKeys?.[lookupKey], + ); + if (!forwardHeaders) { + writeErrorResponse( + res, + 502, + JSON.stringify({ + error: { message: "No configured provider credential", type: "proxy_error" }, + }), + ); + return "relayed"; + } const requestBody = rawBody ?? JSON.stringify(request); @@ -630,7 +695,9 @@ export async function proxyAndRecord( // A single Gemini turn can interleave audio with a functionCall and/or // text/thought parts; preserve those companion modalities so the tool call // / content / reasoning are not silently dropped when audio is present. - if (collapsed.audioB64) { + if (collapsed.transcription) { + fixtureResponse = { transcription: collapsed.transcription }; + } else if (collapsed.audioB64) { const audioToolCallsSpread = collapsed.toolCalls && collapsed.toolCalls.length > 0 ? { @@ -792,14 +859,19 @@ export async function proxyAndRecord( // NOTE: base64 embeddings are decoded unconditionally inside // buildFixtureResponse regardless of the request's `encoding_format`, so // there is no need to re-parse it here — it was a dead param. - fixtureResponse = buildFixtureResponse(parsedResponse, upstreamStatus, defaults.logger); + fixtureResponse = buildFixtureResponse( + parsedResponse, + upstreamStatus, + defaults.logger, + request, + ); } } // Client may have closed its socket before upstream fired `end`. - // Distinguish two cases based on whether the SSE `[DONE]` marker was seen: - // - sawDone=true: client closed after consuming `data: [DONE]` (e.g. the - // OpenAI Python SDK closes the socket the moment it reads `[DONE]`). + // Distinguish two cases based on whether an SSE terminal frame was seen: + // - sawDone=true: client closed after consuming `data: [DONE]` or a typed + // `transcript.text.done` event. // Upstream ran to completion; the buffered body is intact. Log and // proceed to persist the full fixture. // - sawDone=false: genuine mid-stream abort. The buffered body is partial; @@ -1029,12 +1101,13 @@ function makeUpstreamRequest( */ hardCeilingExceeded: boolean; /** - * True when the SSE terminal marker `data: [DONE]` was observed in the - * upstream stream before the client disconnected. When `clientDisconnected` - * is true AND `sawDone` is true, the client closed after consuming `[DONE]` - * (e.g. the OpenAI Python SDK) — the stream was logically complete, so the - * fixture SHOULD be persisted. When `sawDone` is false the disconnect was a - * genuine mid-stream abort and the fixture MUST NOT be persisted. + * True when a terminal SSE frame was observed in the upstream stream before + * the client disconnected. The terminal may be legacy `data: [DONE]` or the + * typed OpenAI transcription event `transcript.text.done`. When + * `clientDisconnected` is true AND `sawDone` is true, the stream was + * logically complete, so the fixture SHOULD be persisted. When `sawDone` is + * false the disconnect was a genuine mid-stream abort and the fixture MUST + * NOT be persisted. */ sawDone: boolean; }> { @@ -1090,15 +1163,8 @@ function makeUpstreamRequest( let streamedToClient = false; let clientDisconnected = false; - // True once the SSE terminal marker `data: [DONE]` has been seen in the - // upstream stream. Used to distinguish two client-close scenarios: - // 1. Close AT/AFTER `[DONE]`: the stream is logically complete (e.g. - // the OpenAI Python SDK closes the socket the moment it reads - // `[DONE]`, before upstream fires `res.end`). The upstream should - // NOT be torn down — let it finish and persist the full fixture. - // 2. Close BEFORE `[DONE]`: genuine mid-stream abort. Restore the - // original teardown so upstream is destroyed and no partial fixture - // is persisted. + // A terminal can be legacy `[DONE]` or typed `transcript.text.done`. + // Either means a client close before upstream `res.end` is complete. let sawDone = false; if (isProgressiveStream && clientRes && !clientRes.headersSent) { const relayHeaders: Record = { @@ -1132,9 +1198,9 @@ function makeUpstreamRequest( frameBuffer = ""; binaryFrameBuffer = Buffer.alloc(0); } - // If sawDone is true: client closed after consuming `[DONE]` - // (e.g. the OpenAI Python SDK). The upstream is logically - // complete; let it run to `res.end` so the full body is + // If sawDone is true: client closed after consuming a terminal + // frame. The upstream is logically complete; let it run to + // `res.end` so the full body is // buffered and the fixture can be persisted. The // `!clientDisconnected` guard inside `onUpstreamData` already // prevents further writes to the now-closed client socket. @@ -1272,9 +1338,9 @@ function makeUpstreamRequest( const frame = parts[fi].trim(); if (frame.length > 0) { frameTimestamps.push(Date.now()); - // Track the SSE terminal marker so the client-close handler - // can distinguish a post-[DONE] close from a mid-stream abort. - if (frame === "data: [DONE]") sawDone = true; + // Track typed and legacy terminal frames so the client-close + // handler can distinguish a complete stream from an abort. + if (isTerminalSSEFrame(frame)) sawDone = true; } } // Last part stays in buffer (may be incomplete). Skip when the @@ -1450,7 +1516,12 @@ function toToolCallArguments(raw: unknown): string { * Detect the response format from the parsed upstream JSON and convert * it into an aimock FixtureResponse. */ -function buildFixtureResponse(parsed: unknown, status: number, logger?: Logger): FixtureResponse { +function buildFixtureResponse( + parsed: unknown, + status: number, + logger?: Logger, + request?: ChatCompletionRequest, +): FixtureResponse { if (parsed === null || parsed === undefined) { // Raw / unparseable response — save as error return { @@ -1551,16 +1622,17 @@ function buildFixtureResponse(parsed: unknown, status: number, logger?: Logger): return { images }; } - // OpenAI transcription: { text: "...", ... } - // Tightened: a bare `text` string alongside a single incidental `language` or - // `duration` field is too weak — many non-transcription payloads carry a - // `text` plus a `duration`-like number. Require an explicit - // `task: "transcribe"`, OR BOTH `language` and `duration` (the verbose - // transcription shape). Also reject anything carrying clear non-transcription - // markers (chat completions, events, etc.) so they route to their own branch. + // OpenAI transcription: { text: "...", ... }. Modern gpt-transcribe + // responses may be the minimal { text, languages?, usage? } shape, so trust + // that shape only on an audio transcription/translation request. Other + // endpoints still need the legacy markers to avoid misclassifying text APIs. + const isTranscriptionRequest = + request?._endpointType === "transcription" || request?._endpointType === "translation"; const looksLikeTranscription = typeof obj.text === "string" && - (obj.task === "transcribe" || (obj.language !== undefined && obj.duration !== undefined)) && + (isTranscriptionRequest || + obj.task === "transcribe" || + (obj.language !== undefined && obj.duration !== undefined)) && !("choices" in obj) && !("candidates" in obj) && !("object" in obj) && @@ -1571,7 +1643,22 @@ function buildFixtureResponse(parsed: unknown, status: number, logger?: Logger): transcription: { text: obj.text as string, ...(obj.language ? { language: String(obj.language) } : {}), + ...(Array.isArray(obj.languages) + ? { + languages: obj.languages + .filter( + (language): language is Record => + typeof language === "object" && + language !== null && + typeof language.code === "string", + ) + .map((language) => ({ code: language.code as string })), + } + : {}), ...(obj.duration !== undefined ? { duration: Number(obj.duration) } : {}), + ...(obj.usage && typeof obj.usage === "object" + ? { usage: obj.usage as Record } + : {}), ...(Array.isArray(obj.words) ? { words: obj.words } : {}), ...(Array.isArray(obj.segments) ? { segments: obj.segments } : {}), }, diff --git a/src/server.ts b/src/server.ts index 2947b9cc..f746ae21 100644 --- a/src/server.ts +++ b/src/server.ts @@ -110,6 +110,14 @@ import { GROK_VIDEO_STATUS_RE, } from "./metrics.js"; import { proxyAndRecord } from "./recorder.js"; +import { + resolveInboundAuth, + markAuthenticatedRequest, + validateRequestApiKey, + writeApiKeyHttpRejection, + writeApiKeyUpgradeRejection, + type ResolvedInboundAuth, +} from "./api-key-auth.js"; export interface ServerInstance { server: http.Server; @@ -1326,6 +1334,18 @@ export async function createServer( options?: MockServerOptions, mounts?: Array<{ path: string; handler: Mountable }>, serviceFixtures?: ServiceFixtures, +): Promise { + const resolvedAuth = resolveInboundAuth({ value: options?.auth, label: "options.auth" }); + return createServerWithResolvedAuth(fixtures, options, resolvedAuth, mounts, serviceFixtures); +} + +/** @internal Config startup uses this to avoid parsing a selected auth source twice. */ +export async function createServerWithResolvedAuth( + fixtures: Fixture[], + options: MockServerOptions | undefined, + resolvedAuth: ResolvedInboundAuth, + mounts?: Array<{ path: string; handler: Mountable }>, + serviceFixtures?: ServiceFixtures, ): Promise { const host = options?.host ?? "127.0.0.1"; const port = options?.port ?? 0; @@ -1458,12 +1478,6 @@ export async function createServer( req: http.IncomingMessage, res: http.ServerResponse, ): Promise { - // OPTIONS preflight - if (req.method === "OPTIONS") { - handleOptions(res); - return; - } - // Record start time for metrics const startTime = registry ? process.hrtime.bigint() : 0n; @@ -1502,6 +1516,29 @@ export async function createServer( }); } + // Browser CORS preflights do not carry application credentials. A bare + // OPTIONS is still a route request and must pass the normal auth boundary. + if ( + req.method === "OPTIONS" && + (!resolvedAuth.policy.enabled || + (req.headers.origin !== undefined && + req.headers["access-control-request-method"] !== undefined)) + ) { + handleOptions(res); + return; + } + + const isPublicProbe = + req.method === "GET" && + (pathname === HEALTH_PATH || pathname === READY_PATH || pathname === "/metrics"); + if (!isPublicProbe) { + if (!validateRequestApiKey(req, resolvedAuth.policy).ok) { + writeApiKeyHttpRejection(res, setCorsHeaders); + return; + } + markAuthenticatedRequest(req, resolvedAuth.policy); + } + // Control API — must be checked before mounts and path rewrites if (pathname.startsWith(CONTROL_PREFIX)) { await handleControlAPI( @@ -3101,6 +3138,11 @@ export async function createServer( const parsedUrl = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`); let pathname = parsedUrl.pathname; + if (!validateRequestApiKey(req, resolvedAuth.policy).ok) { + writeApiKeyUpgradeRejection(socket); + return; + } + // Dispatch to mounted services before any path rewrites if (mounts) { for (const { path: mountPath, handler } of mounts) { @@ -3166,10 +3208,14 @@ export async function createServer( upgradeHeaders: req.headers, }); } else if (pathname === REALTIME_PATH) { - const model = parsedUrl.searchParams.get("model") ?? "gpt-realtime-2"; + const transcriptionIntent = parsedUrl.searchParams.get("intent") === "transcription"; + const model = transcriptionIntent + ? "gpt-transcribe" + : (parsedUrl.searchParams.get("model") ?? "gpt-realtime-2"); handleWebSocketRealtime(ws, fixtures, journal, { ...defaults, model, + transcriptionIntent, testId: wsTestId, upgradeHeaders: req.headers, }); diff --git a/src/stream-collapse.ts b/src/stream-collapse.ts index 5061450f..88918d1e 100644 --- a/src/stream-collapse.ts +++ b/src/stream-collapse.ts @@ -97,6 +97,11 @@ function isCollapseInputTruncated(body: string): boolean { export interface CollapseResult { content?: string; + transcription?: { + text: string; + languages?: Array<{ code: string }>; + usage?: Record; + }; reasoning?: string; /** * The real cryptographic `signature` value captured from an Anthropic @@ -361,6 +366,10 @@ export function collapseOpenAISSE(rawBody: string): CollapseResult { const body = guardCollapseBody(rawBody); const lines = splitSSEEvents(body); let content = ""; + let transcript = ""; + let transcriptSeen = false; + let transcriptLanguages: Array<{ code: string }> | undefined; + let transcriptUsage: Record | undefined; let reasoning = ""; const webSearchQueries: string[] = []; let droppedChunks = 0; @@ -402,6 +411,29 @@ export function collapseOpenAISSE(rawBody: string): CollapseResult { } // Responses API reasoning events + if (parsed.type === "transcript.text.delta" && typeof parsed.delta === "string") { + transcript += parsed.delta; + transcriptSeen = true; + continue; + } + if (parsed.type === "transcript.text.done" && typeof parsed.text === "string") { + transcript = parsed.text; + transcriptSeen = true; + if (Array.isArray(parsed.languages)) { + transcriptLanguages = parsed.languages + .filter( + (language): language is Record => + typeof language === "object" && + language !== null && + typeof language.code === "string", + ) + .map((language) => ({ code: language.code as string })); + } + if (parsed.usage && typeof parsed.usage === "object") { + transcriptUsage = parsed.usage as Record; + } + continue; + } if ( parsed.type === "response.reasoning_summary_text.delta" && typeof parsed.delta === "string" @@ -553,6 +585,15 @@ export function collapseOpenAISSE(rawBody: string): CollapseResult { ...(tc.id ? { id: tc.id } : {}), })); return { + ...(transcriptSeen + ? { + transcription: { + text: transcript, + ...(transcriptLanguages ? { languages: transcriptLanguages } : {}), + ...(transcriptUsage ? { usage: transcriptUsage } : {}), + }, + } + : {}), ...(blocks ? { blocks } : {}), ...(content ? { content } : {}), // Fallback-only: harmonyToolCalls are populated ONLY in the @@ -573,6 +614,15 @@ export function collapseOpenAISSE(rawBody: string): CollapseResult { } return { + ...(transcriptSeen + ? { + transcription: { + text: transcript, + ...(transcriptLanguages ? { languages: transcriptLanguages } : {}), + ...(transcriptUsage ? { usage: transcriptUsage } : {}), + }, + } + : {}), content, ...(reasoning ? { reasoning } : {}), ...(webSearchQueries.length > 0 ? { webSearches: webSearchQueries } : {}), diff --git a/src/transcription.ts b/src/transcription.ts index 5426511a..d7c0915d 100644 --- a/src/transcription.ts +++ b/src/transcription.ts @@ -14,10 +14,11 @@ import { strictNoMatchLogLine, } from "./helpers.js"; import { matchFixtureDiagnostic } from "./router.js"; -import { writeErrorResponse } from "./sse-writer.js"; +import { calculateDelay, delay, writeErrorResponse } from "./sse-writer.js"; import type { Journal } from "./journal.js"; import { applyChaos } from "./chaos.js"; import { proxyAndRecord } from "./recorder.js"; +import { createInterruptionSignal } from "./interruption.js"; /** * Extract the multipart boundary string from a Content-Type header. @@ -100,6 +101,7 @@ export async function handleTranscription( const model = extractFormField(raw, "model", boundary) ?? "whisper-1"; const responseFormat = extractFormField(raw, "response_format", boundary) ?? "json"; + const stream = extractFormField(raw, "stream", boundary) === "true"; const syntheticReq: ChatCompletionRequest = { model, @@ -254,7 +256,7 @@ export async function handleTranscription( return; } - journal.add({ + const journalEntry = journal.add({ method, path, headers: flattenHeaders(req.headers), @@ -263,6 +265,59 @@ export async function handleTranscription( }); const t = response.transcription; + + // Only the transcription endpoint streams modern models. Whisper-1 ignores + // `stream=true`, and translations continue to return their JSON response. + if (endpointType === "transcription" && stream && model !== "whisper-1") { + const done = { + type: "transcript.text.done", + text: t.text, + ...(t.languages !== undefined ? { languages: t.languages } : {}), + ...(t.usage !== undefined ? { usage: t.usage } : {}), + }; + const latency = fixture.latency ?? defaults.latency; + const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); + const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + let chunkIndex = 0; + const interruption = createInterruptionSignal(fixture); + for (let index = 0; index < t.text.length; index += chunkSize) { + const chunkDelay = calculateDelay( + chunkIndex, + fixture.streamingProfile, + latency, + fixture.recordedTimings, + replaySpeed, + ); + if (chunkDelay > 0) await delay(chunkDelay, interruption?.signal); + if (interruption?.signal.aborted) break; + res.write( + `data: ${JSON.stringify({ type: "transcript.text.delta", delta: t.text.slice(index, index + chunkSize) })}\n\n`, + ); + interruption?.tick(); + chunkIndex++; + } + if (interruption?.signal.aborted) { + journalEntry.response.interrupted = true; + journalEntry.response.interruptReason = interruption.reason(); + interruption.cleanup(); + res.destroy(); + return; + } + res.write(`data: ${JSON.stringify(done)}\n\n`); + // The live `gpt-transcribe&stream=true` stream ends with the `[DONE]` + // sentinel after `transcript.text.done`. Omitting it leaves clients that + // loop until the sentinel waiting on a stream that never terminates. + res.write("data: [DONE]\n\n"); + res.end(); + interruption?.cleanup(); + return; + } + const useVerbose = responseFormat === "verbose_json" || t.words != null || t.segments != null; if (useVerbose) { @@ -282,6 +337,12 @@ export async function handleTranscription( res.end(JSON.stringify(verboseBody)); } else { res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ text: t.text })); + res.end( + JSON.stringify({ + text: t.text, + ...(t.languages !== undefined ? { languages: t.languages } : {}), + ...(t.usage !== undefined ? { usage: t.usage } : {}), + }), + ); } } diff --git a/src/types.ts b/src/types.ts index 020a136b..eb049746 100644 --- a/src/types.ts +++ b/src/types.ts @@ -410,7 +410,9 @@ export interface TranscriptionResponse { transcription: { text: string; language?: string; + languages?: Array<{ code: string }>; duration?: number; + usage?: Record; words?: Array<{ word: string; start: number; end: number }>; segments?: Array<{ id: number; text: string; start: number; end: number }>; }; @@ -470,8 +472,8 @@ export type RealtimePhase = "final_answer" | "commentary"; export interface GASessionAudioConfig { voice: string | null; - input_audio_format: string | null; - output_audio_format: string | null; + input_audio_format: { type: string; rate?: number; [key: string]: unknown } | null; + output_audio_format: { type: string; rate?: number; [key: string]: unknown } | null; input_audio_noise_reduction: { type: string } | null; input_audio_transcription: { model: string } | null; } @@ -1014,6 +1016,8 @@ export interface FalRecordConfig { } export interface MockServerOptions { + /** Optional inbound test-client access keys. Omit to preserve permissive behavior. */ + auth?: ApiKeyAuthConfig; port?: number; host?: string; latency?: number; @@ -1098,6 +1102,10 @@ export interface MockServerOptions { grokVideo?: FalQueueConfig; } +export interface ApiKeyAuthConfig { + apiKeys: readonly string[]; +} + /** * Poll-progression thresholds, documented below in fal queue terms; when used * as `openRouterVideo` the states map to `pending` / `in_progress` / diff --git a/src/veo-video.ts b/src/veo-video.ts index 2555e558..972481c3 100644 --- a/src/veo-video.ts +++ b/src/veo-video.ts @@ -27,7 +27,7 @@ import { applyChaos } from "./chaos.js"; import { resolveProgression } from "./fal.js"; import { buildFixtureMatch, - buildForwardHeaders, + prepareEgressHeaders, persistFixture, sanitizeHeaderValue, } from "./recorder.js"; @@ -687,9 +687,11 @@ async function proxyVeoVideoSubmit(args: { let fetched: { status: number; contentType: string | null; text: string }; try { + const headers = prepareEgressHeaders(req, submitUrl, "veo", record.providerKeys?.veo); + if (!headers) return proxyError("No configured provider credential"); const upstreamRes = await fetch(submitUrl, { method: "POST", - headers: buildForwardHeaders(req), + headers, body: raw, signal: upstreamTimeoutSignal(record), }); @@ -880,8 +882,14 @@ async function proxyVeoVideoRecordPoll(args: { let fetched: { status: number; contentType: string | null; text: string }; try { + const target = new URL(job.upstreamPollingUrl); + const headers = prepareEgressHeaders(req, target, "veo", record.providerKeys?.veo); + if (!headers) { + proxyError("No configured provider credential"); + return; + } const upstreamRes = await fetch(job.upstreamPollingUrl, { - headers: buildForwardHeaders(req), + headers, signal: upstreamTimeoutSignal(record), }); fetched = { diff --git a/src/ws-realtime.ts b/src/ws-realtime.ts index 4b1ad1e3..178ddeb2 100644 --- a/src/ws-realtime.ts +++ b/src/ws-realtime.ts @@ -21,6 +21,7 @@ import { isTextResponse, isToolCallResponse, isContentWithToolCallsResponse, + isTranscriptionResponse, isErrorResponse, resolveFixtureBlocks, resolveResponse, @@ -59,13 +60,13 @@ interface SessionConfig { instructions: string; tools: unknown[]; voice: string | null; - input_audio_format: string | null; - output_audio_format: string | null; + input_audio_format: Record | null; + output_audio_format: Record | null; input_audio_noise_reduction: { type: string } | null; - input_audio_transcription: { model: string } | null; + input_audio_transcription: { model: string; language?: string; prompt?: string } | null; turn_detection: unknown | null; temperature: number; - type: "conversation" | "transcription" | "translation"; + type: "conversation" | "realtime" | "transcription" | "translation"; reasoning: { effort: string } | null; } @@ -81,6 +82,150 @@ interface RealtimeMessage { }; } +/** + * Transcription `usage` is MODEL-DEPENDENT on the live API, so the admitted + * models are split by the usage shape they actually report rather than kept in + * one flat list. Verified against api.openai.com with a single 5s clip on BOTH + * the realtime WS (`conversation.item.input_audio_transcription.completed`) and + * HTTP (`POST /v1/audio/transcriptions`) surfaces: + * + * whisper-1 → {"type":"duration","seconds":5} + * gpt-transcribe → {"type":"duration","seconds":5} + * gpt-live-transcribe → {"type":"duration","seconds":5} (realtime only) + * gpt-4o-transcribe → {"type":"tokens","total_tokens":66,…} + * gpt-4o-mini-transcribe → {"type":"tokens","total_tokens":66,…} + * + * Keeping the split here — instead of a second list beside the admission + * predicate — means a newly admitted model has to declare its family, and the + * two can never disagree about which models exist. + */ +function isDurationUsageTranscriptionModel(model: string): boolean { + return ( + model === "whisper-1" || + model === "gpt-transcribe" || + model === "gpt-live-transcribe" || + model.startsWith("gpt-live-transcribe-") + ); +} + +function isTokenUsageTranscriptionModel(model: string): boolean { + return model === "gpt-4o-transcribe" || model.startsWith("gpt-4o-mini-transcribe"); +} + +function isLiveTranscriptionModel(model: string): boolean { + return isDurationUsageTranscriptionModel(model) || isTokenUsageTranscriptionModel(model); +} + +/** + * The synthesized `usage` for a transcription with no fixture-supplied one. + * aimock has no real audio to measure, so the magnitudes are zero — but the + * DISCRIMINATOR must match the model's live family, otherwise a consumer + * branching on `usage.type` reads `undefined` counts and fails silently. + */ +function defaultTranscriptionUsage(model: string): Record { + if (isTokenUsageTranscriptionModel(model)) { + return { + type: "tokens", + total_tokens: 0, + input_tokens: 0, + input_token_details: { text_tokens: 0, audio_tokens: 0 }, + output_tokens: 0, + }; + } + return { type: "duration", seconds: 0 }; +} + +function normalizeAudioFormat(value: unknown): Record | null { + if (value === null) return null; + if (typeof value === "string") return { type: value }; + if ( + value && + typeof value === "object" && + typeof (value as Record).type === "string" + ) { + return { ...(value as Record) }; + } + return null; +} + +function transcriptionModel(session: SessionConfig): string { + return session.input_audio_transcription?.model ?? session.model; +} + +/** + * A live-transcription session is discriminated by the session TYPE, never by + * the mere presence of `input_audio_transcription`. + * + * `input_audio_transcription` is the documented way to configure input + * transcription on an ordinary realtime CONVERSATION session, so treating it as + * the discriminator made the canonical realtime setup (`{model:"whisper-1"}`) + * route `input_audio_buffer.commit` into the transcription path. That silently + * corrupted the conversation: it appended a phantom `[audio]` turn and consumed + * a fixture match, so the next `response.create` replayed the FOLLOWING turn's + * content. The client saw a well-formed — but wrong — response. + * + * A transcription session is established explicitly, either by connecting with + * `?intent=transcription` or via `transcription_session.update` (both set + * `session.type`), which is what the OpenAI realtime API requires too. + */ +function isLiveTranscriptionSession(session: SessionConfig): boolean { + return isLiveTranscriptionModel(transcriptionModel(session)) && session.type === "transcription"; +} + +function serializeSession(session: SessionConfig, sessionId?: string): Record { + // A transcription session is a DIFFERENT resource on the wire. Live GA sends: + // {"type":"transcription","object":"realtime.transcription_session", + // "id":"sess_…","expires_at":…,"audio":{"input":{…}},"include":null} + // No model (there is no session-level model for a transcription session — the + // model lives in audio.input.transcription), and none of the conversation-only + // fields: modalities, instructions, tools, tool_choice, temperature, + // max_response_output_tokens, reasoning, or audio.output. + if (session.type === "transcription") { + return { + ...(sessionId ? { id: sessionId } : {}), + object: "realtime.transcription_session", + type: "transcription", + expires_at: Math.floor(Date.now() / 1000) + 3600, + audio: { + input: { + format: session.input_audio_format, + noise_reduction: session.input_audio_noise_reduction, + transcription: session.input_audio_transcription, + turn_detection: session.turn_detection, + }, + }, + include: null, + }; + } + return { + ...(sessionId ? { id: sessionId } : {}), + object: "realtime.session", + model: session.model, + expires_at: Math.floor(Date.now() / 1000) + 3600, + modalities: session.modalities, + instructions: session.instructions, + tools: session.tools, + tool_choice: "auto", + temperature: session.temperature, + max_response_output_tokens: "inf", + audio: { + input: { + format: session.input_audio_format, + noise_reduction: session.input_audio_noise_reduction, + transcription: session.input_audio_transcription, + turn_detection: session.turn_detection, + }, + output: { + format: session.output_audio_format, + voice: session.voice, + }, + }, + type: + session.type === "conversation" || session.type === "realtime" ? "realtime" : session.type, + reasoning: session.reasoning, + }; +} + // ─── Conversion helpers ───────────────────────────────────────────────────── export function realtimeItemsToMessages( @@ -259,10 +404,14 @@ function translateGAToBeta(event: Record): Record) }; if (session.audio && typeof session.audio === "object") { const audio = session.audio as Record; - session.voice = audio.voice; - session.input_audio_format = audio.input_audio_format; - session.output_audio_format = audio.output_audio_format; - session.input_audio_transcription = audio.input_audio_transcription; + const input = audio.input as Record | undefined; + const output = audio.output as Record | undefined; + session.voice = output?.voice; + session.input_audio_format = (input?.format as Record | undefined)?.type; + session.output_audio_format = (output?.format as Record | undefined)?.type; + session.input_audio_transcription = input?.transcription; + session.input_audio_noise_reduction = input?.noise_reduction; + session.turn_detection = input?.turn_detection ?? null; delete session.audio; } delete session.type; @@ -313,6 +462,7 @@ export function handleWebSocketRealtime( requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest; testId?: string; upgradeHeaders?: import("node:http").IncomingHttpHeaders; + transcriptionIntent?: boolean; }, ): void { const { logger } = defaults; @@ -334,7 +484,7 @@ export function handleWebSocketRealtime( input_audio_transcription: null, turn_detection: null, temperature: 0.8, - type: "conversation", + type: defaults.transcriptionIntent ? "transcription" : "conversation", reasoning: null, }; @@ -345,28 +495,7 @@ export function handleWebSocketRealtime( ws, { type: "session.created", - session: { - id: sessionId, - object: "realtime.session", - model: session.model, - expires_at: Math.floor(Date.now() / 1000) + 3600, - modalities: session.modalities, - instructions: session.instructions, - tools: session.tools, - tool_choice: "auto", - temperature: session.temperature, - max_response_output_tokens: "inf", - audio: { - voice: session.voice, - input_audio_format: session.input_audio_format, - output_audio_format: session.output_audio_format, - input_audio_noise_reduction: session.input_audio_noise_reduction, - input_audio_transcription: session.input_audio_transcription, - }, - turn_detection: session.turn_detection, - type: session.type, - reasoning: session.reasoning, - }, + session: serializeSession(session, sessionId), }, isBeta, ); @@ -414,6 +543,7 @@ async function processMessage( requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest; testId?: string; upgradeHeaders?: import("node:http").IncomingHttpHeaders; + transcriptionIntent?: boolean; }, session: SessionConfig, conversationItems: RealtimeItem[], @@ -436,13 +566,24 @@ async function processMessage( const msgType = parsed.type; - // ── session.update ──────────────────────────────────────────────────── - if (msgType === "session.update") { + // ── session.update / transcription_session.update ───────────────────── + const isTranscriptionSessionUpdate = msgType === "transcription_session.update"; + if (msgType === "session.update" || isTranscriptionSessionUpdate) { + if (isTranscriptionSessionUpdate && !defaults.transcriptionIntent) { + buildErrorRealtimeEvent( + ws, + "transcription_session.update requires a realtime connection with intent=transcription", + isBeta, + "invalid_request_error", + "invalid_session_config", + ); + return; + } if (parsed.session) { const s = parsed.session; // Validate session.type value before applying any mutations - const validTypes = new Set(["conversation", "transcription", "translation"]); + const validTypes = new Set(["conversation", "realtime", "transcription", "translation"]); if ((s as Record).type !== undefined) { if (!validTypes.has((s as Record).type as string)) { sendEvent( @@ -464,10 +605,20 @@ async function processMessage( // Capture full pre-mutation snapshot for rollback on validation failure const prevSession = { ...session }; + // `model` is deliberately NOT applied and NOT rejected. The session model + // is fixed when the connection is established, and live GA silently + // ignores the field: a probe that sent model:"gpt-realtime" on a + // "gpt-realtime-mini" connection got a normal `session.updated` echoing + // the ORIGINAL model with the rest of the update applied. Erroring here + // stranded the very common client that echoes back the session object it + // received — no `session.updated` ever arrived, so clients awaiting it + // hung until their own timeout. + + if (isTranscriptionSessionUpdate) session.type = "transcription"; + if (s.instructions !== undefined) session.instructions = s.instructions; if (s.tools !== undefined) session.tools = s.tools; if (s.modalities !== undefined) session.modalities = s.modalities; - if (s.model !== undefined) session.model = s.model; if (s.temperature !== undefined) session.temperature = s.temperature; if ((s as Record).type !== undefined) session.type = (s as Record).type as SessionConfig["type"]; @@ -476,9 +627,9 @@ async function processMessage( const audio = (s as Record).audio as Record; if (audio.voice !== undefined) session.voice = audio.voice as string | null; if (audio.input_audio_format !== undefined) - session.input_audio_format = audio.input_audio_format as string | null; + session.input_audio_format = normalizeAudioFormat(audio.input_audio_format); if (audio.output_audio_format !== undefined) - session.output_audio_format = audio.output_audio_format as string | null; + session.output_audio_format = normalizeAudioFormat(audio.output_audio_format); if (audio.input_audio_noise_reduction !== undefined) session.input_audio_noise_reduction = audio.input_audio_noise_reduction as { type: string; @@ -486,12 +637,38 @@ async function processMessage( if (audio.input_audio_transcription !== undefined) session.input_audio_transcription = audio.input_audio_transcription as { model: string; + language?: string; + prompt?: string; } | null; + // Current Realtime session shape nests input transcription under + // session.audio.input.transcription. + if (audio.input && typeof audio.input === "object") { + const input = audio.input as Record; + if (input.transcription !== undefined) + session.input_audio_transcription = input.transcription as { + model: string; + language?: string; + prompt?: string; + } | null; + if (input.noise_reduction !== undefined) + session.input_audio_noise_reduction = input.noise_reduction as { type: string } | null; + if (input.turn_detection !== undefined) session.turn_detection = input.turn_detection; + if (input.format !== undefined) + session.input_audio_format = normalizeAudioFormat(input.format); + } + if (audio.output && typeof audio.output === "object") { + const output = audio.output as Record; + if (output.voice !== undefined) session.voice = output.voice as string | null; + if (output.format !== undefined) + session.output_audio_format = normalizeAudioFormat(output.format); + } } // Beta flat fields (backward compat) if (s.voice !== undefined) session.voice = s.voice; - if (s.input_audio_format !== undefined) session.input_audio_format = s.input_audio_format; - if (s.output_audio_format !== undefined) session.output_audio_format = s.output_audio_format; + if (s.input_audio_format !== undefined) + session.input_audio_format = normalizeAudioFormat(s.input_audio_format); + if (s.output_audio_format !== undefined) + session.output_audio_format = normalizeAudioFormat(s.output_audio_format); if (s.input_audio_noise_reduction !== undefined) session.input_audio_noise_reduction = s.input_audio_noise_reduction; if (s.input_audio_transcription !== undefined) @@ -517,14 +694,19 @@ async function processMessage( "gpt-realtime-translate", ]); - if (session.type === "transcription" && !transcriptionModels.has(session.model)) { + const candidateTranscriptionModel = transcriptionModel(session); + if ( + session.type === "transcription" && + !transcriptionModels.has(candidateTranscriptionModel) && + !isLiveTranscriptionSession(session) + ) { Object.assign(session, prevSession); sendEvent( ws, { type: "error", error: { - message: `Model ${s.model ?? prevSession.model} does not support session type transcription`, + message: `Model ${candidateTranscriptionModel} does not support session type transcription`, type: "invalid_request_error", code: "invalid_session_config", }, @@ -554,28 +736,8 @@ async function processMessage( sendEvent( ws, { - type: "session.updated", - session: { - object: "realtime.session", - model: session.model, - expires_at: Math.floor(Date.now() / 1000) + 3600, - modalities: session.modalities, - instructions: session.instructions, - tools: session.tools, - tool_choice: "auto", - temperature: session.temperature, - max_response_output_tokens: "inf", - audio: { - voice: session.voice, - input_audio_format: session.input_audio_format, - output_audio_format: session.output_audio_format, - input_audio_noise_reduction: session.input_audio_noise_reduction, - input_audio_transcription: session.input_audio_transcription, - }, - turn_detection: session.turn_detection, - type: session.type, - reasoning: session.reasoning, - }, + type: isTranscriptionSessionUpdate ? "transcription_session.updated" : "session.updated", + session: serializeSession(session), }, isBeta, ); @@ -630,7 +792,9 @@ async function processMessage( // ── input_audio_buffer.commit ────────────────────────────────────── if (msgType === "input_audio_buffer.commit") { sendEvent(ws, { type: "input_audio_buffer.committed" }, isBeta); - // In transcription/translation mode, add a placeholder user item + // In transcription/translation mode, add a placeholder user item. A plain + // conversation session gets only the `committed` ack, even when it + // configures `input_audio_transcription` — see isLiveTranscriptionSession. if (session.type === "transcription" || session.type === "translation") { const audioItem: RealtimeItem = { type: "message", @@ -647,6 +811,17 @@ async function processMessage( }, isBeta, ); + if (isLiveTranscriptionSession(session)) { + await emitLiveTranscriptionEvents( + ws, + fixtures, + journal, + defaults, + session, + audioItem, + isBeta, + ); + } } return; } @@ -666,6 +841,222 @@ async function processMessage( // Unknown message type — ignore silently (matches OpenAI behavior) } +async function emitLiveTranscriptionEvents( + ws: WebSocketConnection, + fixtures: Fixture[], + journal: Journal, + defaults: { + latency: number; + chunkSize: number; + replaySpeed?: number; + model: string; + logger: Logger; + strict?: boolean; + requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest; + testId?: string; + upgradeHeaders?: import("node:http").IncomingHttpHeaders; + }, + session: SessionConfig, + audioItem: RealtimeItem, + isBeta: boolean, +): Promise { + const realtimeContextHeader = defaults.upgradeHeaders?.["x-aimock-context"]; + const realtimeContext = + typeof realtimeContextHeader === "string" + ? realtimeContextHeader + : Array.isArray(realtimeContextHeader) && realtimeContextHeader.length > 0 + ? realtimeContextHeader[0] + : undefined; + const request: ChatCompletionRequest = { + model: transcriptionModel(session), + messages: realtimeItemsToMessages([audioItem], undefined, defaults.logger), + _endpointType: "realtime-transcription", + _context: realtimeContext, + }; + const testId = defaults.testId ?? DEFAULT_TEST_ID; + const { fixture, skippedBySequenceOrTurn } = matchFixtureDiagnostic( + fixtures, + request, + journal.getFixtureMatchCountsForTest(testId), + defaults.requestTransform, + ); + const itemId = audioItem.id ?? realtimeId("item"); + + if (!fixture) { + if (resolveStrictMode(defaults.strict, defaults.upgradeHeaders)) { + const strictMessage = strictNoMatchMessage(skippedBySequenceOrTurn); + defaults.logger.error(strictNoMatchLogLine("WS", "/v1/realtime", skippedBySequenceOrTurn)); + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { + status: 503, + fixture: null, + ...strictOverrideField(defaults.strict, defaults.upgradeHeaders), + }, + }); + ws.close(1008, strictMessage); + return; + } + + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { + status: 404, + fixture: null, + ...strictOverrideField(defaults.strict, defaults.upgradeHeaders), + }, + }); + sendLiveTranscriptionFailure( + ws, + itemId, + { message: "No fixture matched", type: "invalid_request_error", code: "no_fixture_match" }, + isBeta, + ); + return; + } + // The match count is burned only once the response is confirmed usable for + // this endpoint. router.ts exempts `realtime*` requests from the + // response-shape gate, so a generic chat fixture still matches this lookup; + // counting it here — before the shape check below — silently advanced a + // sequenced conversation by one turn. An error fixture IS a deliberate match + // and still counts, matching the chat path. + const response = await resolveResponse(fixture, request); + if (isErrorResponse(response)) { + journal.incrementFixtureMatchCount(fixture, fixtures, testId); + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { status: response.status ?? 500, fixture }, + }); + sendLiveTranscriptionFailure( + ws, + itemId, + { + message: response.error.message, + type: response.error.type ?? "server_error", + ...(response.error.code !== undefined && { code: response.error.code }), + }, + isBeta, + ); + return; + } + if (!isTranscriptionResponse(response)) { + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { status: 500, fixture }, + }); + sendLiveTranscriptionFailure( + ws, + itemId, + { message: "Fixture response is not a transcription type", type: "server_error" }, + isBeta, + ); + return; + } + journal.incrementFixtureMatchCount(fixture, fixtures, testId); + + const transcript = response.transcription.text; + // The synthesized usage shape is MODEL-AWARE — duration for whisper-1 / + // gpt-transcribe / gpt-live-transcribe, a token breakdown for the + // gpt-4o-transcribe families (see defaultTranscriptionUsage). A + // fixture-supplied usage always wins. + const usage = response.transcription.usage ?? defaultTranscriptionUsage(request.model); + const journalEntry = journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { status: 200, fixture }, + }); + const latency = fixture.latency ?? defaults.latency; + const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); + const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; + const interruption = createInterruptionSignal(fixture); + let chunkIndex = 0; + for (let index = 0; index < transcript.length; index += chunkSize) { + const chunkDelay = calculateDelay( + chunkIndex, + fixture.streamingProfile, + latency, + fixture.recordedTimings, + replaySpeed, + ); + if (chunkDelay > 0) await delay(chunkDelay, interruption?.signal); + if (interruption?.signal.aborted || ws.isClosed) { + if (interruption?.signal.aborted) { + journalEntry.response.interrupted = true; + journalEntry.response.interruptReason = interruption.reason(); + ws.destroy(); + } + interruption?.cleanup(); + return; + } + sendEvent( + ws, + { + type: "conversation.item.input_audio_transcription.delta", + item_id: itemId, + content_index: 0, + delta: transcript.slice(index, index + chunkSize), + }, + isBeta, + ); + interruption?.tick(); + chunkIndex++; + } + if (interruption?.signal.aborted) { + journalEntry.response.interrupted = true; + journalEntry.response.interruptReason = interruption.reason(); + ws.destroy(); + interruption.cleanup(); + return; + } + sendEvent( + ws, + { + type: "conversation.item.input_audio_transcription.completed", + item_id: itemId, + content_index: 0, + transcript, + usage, + ...(response.transcription.languages !== undefined + ? { languages: response.transcription.languages } + : {}), + }, + isBeta, + ); + interruption?.cleanup(); +} + +function sendLiveTranscriptionFailure( + ws: WebSocketConnection, + itemId: string, + error: { message: string; type: string; code?: string }, + isBeta: boolean, +): void { + sendEvent( + ws, + { + type: "conversation.item.input_audio_transcription.failed", + item_id: itemId, + content_index: 0, + error, + }, + isBeta, + ); +} + async function handleResponseCreate( ws: WebSocketConnection, fixtures: Fixture[],