From fd32cda621327b00918b3184112ab3badcb64591 Mon Sep 17 00:00:00 2001 From: habeck Date: Fri, 31 Jul 2026 02:09:29 -0400 Subject: [PATCH 1/5] enh: add cpex-host-python crate Signed-off-by: habeck --- Cargo.lock | 18 + Cargo.toml | 1 + crates/README.md | 3 + crates/cpex-hosts-python/Cargo.toml | 63 + crates/cpex-hosts-python/README.md | 203 +++ crates/cpex-hosts-python/src/conversion.rs | 733 ++++++++++ crates/cpex-hosts-python/src/credentials.rs | 948 +++++++++++++ crates/cpex-hosts-python/src/error.rs | 133 ++ crates/cpex-hosts-python/src/factory.rs | 268 ++++ crates/cpex-hosts-python/src/legacy/mod.rs | 19 + .../cpex-hosts-python/src/legacy/payloads.rs | 445 ++++++ crates/cpex-hosts-python/src/lib.rs | 74 + crates/cpex-hosts-python/src/plugin.rs | 1225 +++++++++++++++++ crates/cpex-hosts-python/src/testing.rs | 283 ++++ crates/cpex-hosts-python/src/venv.rs | 1110 +++++++++++++++ crates/cpex-hosts-python/src/worker.rs | 1011 ++++++++++++++ crates/cpex-hosts-python/tests/config_e2e.rs | 439 ++++++ .../cpex-hosts-python/tests/credential_e2e.rs | 386 ++++++ .../tests/isolated_venv_e2e.rs | 382 +++++ 19 files changed, 7744 insertions(+) create mode 100644 crates/cpex-hosts-python/Cargo.toml create mode 100644 crates/cpex-hosts-python/README.md create mode 100644 crates/cpex-hosts-python/src/conversion.rs create mode 100644 crates/cpex-hosts-python/src/credentials.rs create mode 100644 crates/cpex-hosts-python/src/error.rs create mode 100644 crates/cpex-hosts-python/src/factory.rs create mode 100644 crates/cpex-hosts-python/src/legacy/mod.rs create mode 100644 crates/cpex-hosts-python/src/legacy/payloads.rs create mode 100644 crates/cpex-hosts-python/src/lib.rs create mode 100644 crates/cpex-hosts-python/src/plugin.rs create mode 100644 crates/cpex-hosts-python/src/testing.rs create mode 100644 crates/cpex-hosts-python/src/venv.rs create mode 100644 crates/cpex-hosts-python/src/worker.rs create mode 100644 crates/cpex-hosts-python/tests/config_e2e.rs create mode 100644 crates/cpex-hosts-python/tests/credential_e2e.rs create mode 100644 crates/cpex-hosts-python/tests/isolated_venv_e2e.rs diff --git a/Cargo.lock b/Cargo.lock index 9a49b7d6..885b8d54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -796,6 +796,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "cpex-hosts-python" +version = "0.2.2" +dependencies = [ + "async-trait", + "chrono", + "cpex-core", + "cpex-hosts-python", + "serde", + "serde_json", + "serde_yaml", + "sha2 0.10.9", + "tokio", + "tracing", + "uuid", + "zeroize", +] + [[package]] name = "cpex-orchestration" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 11f2bd51..3696bda0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ resolver = "2" members = [ "crates/cpex", "crates/cpex-core", + "crates/cpex-hosts-python", "crates/cpex-orchestration", "crates/cpex-sdk", "crates/cpex-builtins", diff --git a/crates/README.md b/crates/README.md index 62ace2ba..820cd9ec 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,6 +8,9 @@ Phase 1a — core runtime functional, no language bindings yet. - `cpex-core`: Plugin trait, typed hooks, 5-phase executor, plugin manager - `cpex-sdk`: Lean re-exports for plugin authors +- `cpex-hosts-python`: Runs existing Python plugins out-of-process, each in its + own cached virtualenv, under the `isolated_venv` kind + ([README](cpex-hosts-python/README.md)) ## Prerequisites diff --git a/crates/cpex-hosts-python/Cargo.toml b/crates/cpex-hosts-python/Cargo.toml new file mode 100644 index 00000000..04871a25 --- /dev/null +++ b/crates/cpex-hosts-python/Cargo.toml @@ -0,0 +1,63 @@ +# Location: ./crates/cpex-hosts-python/Cargo.toml +# Copyright 2026 +# SPDX-License-Identifier: Apache-2.0 +# Authors: Ted Habeck +# +# cpex-hosts-python — out-of-process host for existing Python CPEX +# plugins. Each plugin runs in its own cached virtualenv, driven by the +# Python framework's `worker.py` over a newline-delimited JSON stdio +# protocol. Registers under the `isolated_venv` plugin kind. +# +# Pure Rust plus a subprocess — no libpython link, so this crate stays +# in `default-members` (unlike `bindings/python`). + +[package] +name = "cpex-hosts-python" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "CPEX host — runs Python plugins out-of-process in isolated virtualenvs." +repository.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +rust-version.workspace = true + +[dependencies] +cpex-core = { workspace = true } + +async-trait = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +# Version-pinned rather than workspace-inherited, matching how the rest of +# the tree declares these two (crates/apl-cpex, builtins/session/valkey for +# sha2; crates/cpex-core, builtins/plugins/delegator-oauth for zeroize). +sha2 = "0.10" +zeroize = { version = "1.8", features = ["zeroize_derive"] } + +# `process` and `io-util` are absent from the workspace tokio floor (the +# embedded library never spawns children). This host's whole job is a +# long-lived child spoken to over piped stdio, so it opts both in. +tokio = { workspace = true, features = ["process", "io-util"] } + +[features] +# Exposes the `testing` module (temp dirs, python3 detection) to the +# integration tests under `tests/`, which cannot see a `cfg(test)` module. +# Not enabled by default — production builds carry none of it. +testing = [] + +[dev-dependencies] +chrono = { workspace = true } +serde_yaml = { workspace = true } +tokio = { workspace = true, features = ["process", "io-util", "macros", "rt", "rt-multi-thread"] } + +# Self-dependency with the test-only feature on, so `tests/*.rs` see +# `cpex_hosts_python::testing`. Standard trick for cfg(test)-adjacent helpers. +cpex-hosts-python = { path = ".", features = ["testing"] } + +[lints] +workspace = true diff --git a/crates/cpex-hosts-python/README.md b/crates/cpex-hosts-python/README.md new file mode 100644 index 00000000..dd433399 --- /dev/null +++ b/crates/cpex-hosts-python/README.md @@ -0,0 +1,203 @@ +# cpex-hosts-python + +Runs existing Python CPEX plugins out-of-process from the Rust `PluginManager`. +Each plugin gets its own cached virtualenv and a long-lived `worker.py` +subprocess, spoken to over newline-delimited JSON on stdio — the same protocol +the Python CLI already uses, so a plugin that works there works here unchanged. + +Registers under the plugin kind **`isolated_venv`**. + +## Why out-of-process + +The runtime never links libpython, so a plugin crash, a C-extension segfault, or +a dependency conflict cannot take the gateway down. Each plugin's dependencies +live in its own venv, so two plugins can pin incompatible versions of the same +package. This supersedes issue #20's original `python://` in-process (PyO3) +framing. + +## Usage + +Register the factory before loading config: + +```rust +use cpex_core::factory::PluginFactoryRegistry; +use cpex_hosts_python::{IsolatedVenvFactory, KIND}; + +let mut factories = PluginFactoryRegistry::new(); +factories.register(KIND, Box::new(IsolatedVenvFactory)); + +let manager = PluginManager::from_config(config, &factories)?; +``` + +## Config schema + +Host-specific settings live in the plugin entry's opaque `config:` map, because +`PluginConfig` has no fields for them: + +```yaml +plugins: + - name: pii-filter + kind: isolated_venv + hooks: [tool_pre_invoke] + config: + # Required — the fully-qualified Python class. + class_name: my_pkg.filters.PiiFilter + + # Optional. Installed into the venv on a cache miss. + requirements_file: requirements.txt + + # Optional, with defaults matching the Python reference implementation. + script_path: cpex/framework/isolated/worker.py + max_content_size: 10000000 # bytes, per outbound task + timeout_secs: 30 # per invocation +``` + +| Key | Required | Default | Notes | +|---|---|---|---| +| `class_name` | yes | — | Fully-qualified Python class. Also keys the venv cache. | +| `requirements_file` | no | none | Relative to the resolved plugin path. | +| `script_path` | no | `cpex/framework/isolated/worker.py` | Resolved inside the venv. | +| `max_content_size` | no | `10000000` | Oversized tasks are rejected before the write. | +| `timeout_secs` | no | `30` | The executor's global timeout also applies. | + +### Plugin directories are not configurable + +The host always uses **`plugins/` at the project root** — the process working +directory — as the worker's `sys.path` entry and as the directory the venv is +built under. There is no config key for it: + +- a `plugin_dirs:` key inside a plugin's `config:` block is **ignored** (the + factory logs a warning naming the plugin), and +- the top-level `plugin_dirs:` YAML key is **ignored** too — `cpex-core` parses + it, warns, and discards it. + +Both are ignored rather than rejected, so a config carrying either still loads. + +This is deliberate. `cpex plugin install` writes the plugin into +`/plugins/` and builds its venv there, so both sides already agree +on the location; a config key only creates a way for them to disagree, and the +failure mode is an `ImportError` inside the worker at invoke time — far from the +config that caused it. The worker independently restricts plugin dirs to its own +`ALLOWED_PLUGIN_DIRS`, which includes its working directory (inherited from this +host), so the resolved directory is importable by construction. + +## Hooks and payloads + +Both hook families work. The `cmf.` prefix selects the CMF message payload; +each legacy name maps to its own typed payload; anything unrecognized falls +back to a generic JSON payload. + +| Hook | Payload | +|---|---| +| `cmf.*` | `MessagePayload` | +| `tool_pre_invoke` / `tool_post_invoke` | `ToolPreInvokePayload` / `ToolPostInvokePayload` | +| `prompt_pre_fetch` / `prompt_post_fetch` | `PromptPreFetchPayload` / `PromptPostFetchPayload` | +| `resource_pre_fetch` / `resource_post_fetch` | `ResourcePreFetchPayload` / `ResourcePostFetchPayload` | +| `identity_resolve` | `IdentityResolvePayload` | +| `token_delegate` | `TokenDelegatePayload` | +| anything else | `GenericPayload` | + +A plugin error is returned to the executor, which applies the plugin's +configured `on_error` policy (fail / ignore / disable). This host does not +interpret that policy itself. + +### Extensions + +Only the `custom` slot of a returned `modified_extensions` is applied. The +`http`, `security`, and `delegation` slots are gated by `WriteToken`s the +executor mints, and this host has no authority to mint one — an attempt to +write them returns an error rather than being silently dropped. Full extensions +delivery is owned by a separate plan +(`docs/plans/2026-07-29-003-feat-out-of-process-extensions-delivery-plan.md`). + +## Credentials + +The framework strips raw tokens at every process boundary (the token fields are +`#[serde(skip)]`), which would leave identity resolvers and token delegators +unable to work out-of-process. This host adds a narrow, capability-gated +exception. + +A plugin declares what it needs: + +```yaml + capabilities: [read_inbound_credentials] # or read_delegated_tokens +``` + +Only `identity_resolve` and `token_delegate` are eligible — they are the only +hooks whose Python payload models a raw token (`IdentityPayload.raw_token`, +`DelegationPayload.bearer_token`). For a declaring plugin on one of those hooks, +the host attaches a dedicated `credential` object to the task JSON, built by +reading the in-memory token directly. Production credential types keep their +serde guard and are never serialized; the FFI, Python-bindings, and audit paths +are untouched. + +**Fail closed.** A plugin that declared nothing gets no `credential` field and +no token material. A plugin that declared a capability the request cannot honor +gets an error — not a silent no-token dispatch, which a resolver could read as +"no authentication required". + +**Worker requirement.** The `credential` field is only consumed by a `worker.py` +that implements `reconstruct_credential_payload`. An older worker silently drops +it. Assert a minimum `cpex` version for credential-bearing plugins. + +### Residual exposure + +The capability gate controls *which plugin* receives a token. It does not +constrain what happens next: once the plaintext is resident in the worker +process, **every transitively installed dependency in that venv can read it**. +That is a materially larger and less audited trust boundary than the in-process +host, and neither the gate nor the transport closes it. Sending raw credentials +to an out-of-process plugin means accepting that venv's whole dependency tree +into the credential trust boundary. + +The transport itself is sound by construction: the `credential` object rides on +the worker's stdin, a private pipe inherited only by the child. There is no +listening socket, and no other local process can read it. + +## Venv cache + +Reuse is keyed on a SHA256 of the requirements file plus the persisted +manifest's version and content hash. Layout mirrors the Python CLI's: `.venv` +and `.cpex/venv_cache` under `/`. + +Two deliberate behaviors: + +- **A missing manifest signal means "no signal", not "mismatch."** Metadata + written by an older CLI has no `manifest_version` / `manifest_hash` key. + Treating an absent key as a mismatch would wipe and rebuild every existing + venv on the first run after an upgrade. +- **Both the manifest and the metadata filename are keyed on the full class + name.** Plugins sharing a package share one venv directory by design. The + Python CLI keys only the manifest per class and derives the metadata filename + from the venv directory name, so those plugins share one metadata file and + each install invalidates its neighbour's cache — a rebuild loop. Keying both + closes it, at the cost of one extra rebuild the first time this host meets a + venv the Python CLI created. + +## Testing + +Unit tests run against a stub worker and need only `python3`: + +```bash +cargo test -p cpex-hosts-python +``` + +The end-to-end tests need a `cpex` Python source tree, and **skip with a printed +reason** when one is absent rather than failing: + +```bash +CPEX_PYTHON_SOURCE=/path/to/cpex-python \ + cargo test -p cpex-hosts-python --features testing --test isolated_venv_e2e --test credential_e2e +``` + +Two environmental caveats, both upstream of this crate: + +- The Python framework's declared dependencies currently have no satisfiable + resolution — `pyproject.toml` requires `mcp>=1.26`, but the framework imports + `McpError`, which mcp renamed to `MCPError` in 1.26. The e2e tests therefore + pre-build the venv in two pip passes (install as declared, then downgrade + `mcp`) and let the host reuse it via its own cache. +- The Python hook registry registers no `cmf.*` hooks, so a CMF round-trip + cannot be proven end to end yet. The host's CMF routing is covered by unit + tests, and the e2e suite asserts that a CMF hook is rejected cleanly rather + than silently allowed. diff --git a/crates/cpex-hosts-python/src/conversion.rs b/crates/cpex-hosts-python/src/conversion.rs new file mode 100644 index 00000000..a826887d --- /dev/null +++ b/crates/cpex-hosts-python/src/conversion.rs @@ -0,0 +1,733 @@ +// Location: ./crates/cpex-hosts-python/src/conversion.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Payload serialization, hook-name resolution, and the response-to-result +// conversion. +// +// Two directions, both keyed on the hook name: +// +// outbound: `&dyn PluginPayload` ──> JSON the worker's `json_to_payload` +// reconstructs into the matching Pydantic model +// inbound: the worker's response JSON ──> `ErasedResultFields` the +// executor consumes +// +// # Why a registry +// +// The host forwards arbitrary payloads, so nothing here can be generic over a +// single `P: PluginPayload`. Outbound, serialization is a downcast chain +// (mirroring `cpex-ffi`'s ordering) because the concrete type arrives erased. +// Inbound, the hook *name* selects which typed payload a `modified_payload` +// deserializes into — a `PayloadKind` per hook name, with `cmf.` routing to +// `MessagePayload` and unrecognized names to the generic payload. + +use cpex_core::cmf::MessagePayload; +use cpex_core::error::PluginViolation; +use cpex_core::executor::ErasedResultFields; +use cpex_core::hooks::payload::PluginPayload; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::error::HostError; +use crate::legacy::payloads::{ + IdentityResolvePayload, PromptPostFetchPayload, PromptPreFetchPayload, + ResourcePostFetchPayload, ResourcePreFetchPayload, TokenDelegatePayload, ToolPostInvokePayload, + ToolPreInvokePayload, +}; + +/// Prefix that marks a CMF hook name. +pub const CMF_PREFIX: &str = "cmf."; + +/// Wraps any JSON value for hooks with no typed payload. +/// +/// The third copy of this type in the workspace — `cpex-ffi` and the Python +/// bindings each define their own, because cpex-core exports the +/// `impl_plugin_payload!` macro but not a shared struct. Consolidating all +/// three into cpex-core is deferred follow-up work; it would touch the FFI and +/// the bindings, and is not needed to ship this host. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenericPayload { + pub value: Value, +} + +cpex_core::impl_plugin_payload!(GenericPayload); + +/// Which payload type a hook name maps to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PayloadKind { + /// Any `cmf.*` hook — the CMF message payload. + CmfMessage, + ToolPreInvoke, + ToolPostInvoke, + PromptPreFetch, + PromptPostFetch, + ResourcePreFetch, + ResourcePostFetch, + IdentityResolve, + TokenDelegate, + /// No typed payload for this name — carry the JSON as-is. + Generic, +} + +/// Resolve a hook name to its payload kind. +/// +/// The `cmf.` prefix is checked first: `cmf.tool_pre_invoke` must route to the +/// CMF message payload, not to the legacy tool payload whose name it contains. +pub fn payload_kind_for_hook(hook_name: &str) -> PayloadKind { + use cpex_core::hooks::types::hook_names; + + if hook_name.starts_with(CMF_PREFIX) { + return PayloadKind::CmfMessage; + } + + match hook_name { + hook_names::TOOL_PRE_INVOKE => PayloadKind::ToolPreInvoke, + hook_names::TOOL_POST_INVOKE => PayloadKind::ToolPostInvoke, + hook_names::PROMPT_PRE_FETCH => PayloadKind::PromptPreFetch, + hook_names::PROMPT_POST_FETCH => PayloadKind::PromptPostFetch, + hook_names::RESOURCE_PRE_FETCH => PayloadKind::ResourcePreFetch, + hook_names::RESOURCE_POST_FETCH => PayloadKind::ResourcePostFetch, + hook_names::IDENTITY_RESOLVE => PayloadKind::IdentityResolve, + hook_names::TOKEN_DELEGATE => PayloadKind::TokenDelegate, + _ => PayloadKind::Generic, + } +} + +/// Whether a hook carries raw credentials. +/// +/// Only `identity_resolve` and `token_delegate` do: they are the only two +/// hooks whose Python payload models a raw token at all +/// (`IdentityPayload.raw_token`, `DelegationPayload.bearer_token`). The Python +/// `Extensions` model has no raw-credential slot, so no other hook has +/// anywhere to receive one. +pub fn is_credential_hook(hook_name: &str) -> bool { + matches!( + payload_kind_for_hook(hook_name), + PayloadKind::IdentityResolve | PayloadKind::TokenDelegate + ) +} + +/// Serialize an erased payload to the JSON shape the worker reconstructs. +/// +/// The downcast chain covers every type this host can hand out, in rough +/// frequency order. `MessagePayload` comes first, mirroring `cpex-ffi`'s +/// `serialize_payload`. +/// +/// An unrecognized concrete type is an error rather than a silent `null`: it +/// means some other host handed this adapter a payload it cannot forward, and +/// sending `null` would surface inside the worker as a confusing Pydantic +/// validation failure instead. +pub fn serialize_payload(payload: &dyn PluginPayload) -> Result { + let any = payload.as_any(); + + // Macro rather than a chain of near-identical `if let` blocks — ten arms + // written out drift, and the pattern is mechanical. + macro_rules! try_downcast { + ($($ty:ty),+ $(,)?) => { + $( + if let Some(concrete) = any.downcast_ref::<$ty>() { + return serde_json::to_value(concrete).map_err(|e| HostError::Protocol { + message: format!( + "could not serialize a {} payload: {e}", + stringify!($ty) + ), + }); + } + )+ + }; + } + + try_downcast!( + MessagePayload, + ToolPreInvokePayload, + ToolPostInvokePayload, + PromptPreFetchPayload, + PromptPostFetchPayload, + ResourcePreFetchPayload, + ResourcePostFetchPayload, + IdentityResolvePayload, + TokenDelegatePayload, + ); + + // The generic payload unwraps to its inner value rather than serializing + // the wrapper, so the worker sees the payload itself. + if let Some(generic) = any.downcast_ref::() { + return Ok(generic.value.clone()); + } + + Err(HostError::Protocol { + message: "payload type is not one this host can forward to a Python worker \ + (expected a CMF message payload, a legacy typed payload, or the generic payload)" + .into(), + }) +} + +/// Rebuild a typed payload from JSON, per the hook's payload kind. +/// +/// Used for a `modified_payload` coming back from the worker: it must be +/// re-erased as the *same* concrete type the executor expects for the hook, or +/// a downstream downcast fails and the modification is dropped. +pub fn deserialize_payload( + hook_name: &str, + value: Value, +) -> Result, HostError> { + fn parse(value: Value, hook_name: &str) -> Result, HostError> + where + T: PluginPayload + serde::de::DeserializeOwned, + { + serde_json::from_value::(value) + .map(|p| Box::new(p) as Box) + .map_err(|e| HostError::Protocol { + message: format!("worker returned a modified payload that does not match hook '{hook_name}': {e}"), + }) + } + + match payload_kind_for_hook(hook_name) { + PayloadKind::CmfMessage => parse::(value, hook_name), + PayloadKind::ToolPreInvoke => parse::(value, hook_name), + PayloadKind::ToolPostInvoke => parse::(value, hook_name), + PayloadKind::PromptPreFetch => parse::(value, hook_name), + PayloadKind::PromptPostFetch => parse::(value, hook_name), + PayloadKind::ResourcePreFetch => parse::(value, hook_name), + PayloadKind::ResourcePostFetch => parse::(value, hook_name), + PayloadKind::IdentityResolve => parse::(value, hook_name), + PayloadKind::TokenDelegate => parse::(value, hook_name), + PayloadKind::Generic => Ok(Box::new(GenericPayload { value })), + } +} + +/// Convert a worker response into the erased result the executor consumes. +/// +/// Preserves all four decision-bearing fields: whether processing continues, +/// any violation, a modified payload, and modified extensions. A response +/// missing `continue_processing` defaults to `true`, matching +/// `PluginResult`'s Pydantic default — a plugin that returns nothing means +/// "allow", not "deny". +pub fn response_to_result( + hook_name: &str, + response: Value, +) -> Result { + let continue_processing = response + .get("continue_processing") + .and_then(Value::as_bool) + .unwrap_or(true); + + let violation = match response.get("violation") { + Some(Value::Null) | None => None, + Some(raw) => Some(parse_violation(raw.clone())?), + }; + + let modified_payload = match response.get("modified_payload") { + Some(Value::Null) | None => None, + Some(raw) => Some(deserialize_payload(hook_name, raw.clone())?), + }; + + let modified_extensions = match response.get("modified_extensions") { + Some(Value::Null) | None => None, + Some(raw) => owned_extensions_from_value(raw)?, + }; + + Ok(ErasedResultFields { + continue_processing, + modified_payload, + modified_extensions, + violation, + }) +} + +/// Rebuild an `OwnedExtensions` from a worker's `modified_extensions`. +/// +/// `OwnedExtensions` is deliberately not `Deserialize`: its `http`, `security`, +/// and `delegation` slots are gated by `WriteToken`s that the *executor* mints +/// when it grants a plugin write access. A token parsed out of worker JSON +/// would be a forgery, letting an out-of-process plugin write slots it was +/// never granted — so those slots cannot be reconstructed here, by design. +/// +/// What is safe is `custom`: a plain `HashMap` with no write +/// token, which is how an out-of-process plugin passes structured data back. +/// That slot is honored. +/// +/// A token-gated slot is reported as an error rather than dropped, so a plugin +/// attempting one learns its write did not land instead of silently losing it. +/// Full extensions delivery — including the returned-extensions merge for the +/// gated slots — is owned by +/// `docs/plans/2026-07-29-003-feat-out-of-process-extensions-delivery-plan.md`. +fn owned_extensions_from_value( + raw: &Value, +) -> Result, HostError> { + let obj = raw.as_object().ok_or_else(|| HostError::Protocol { + message: "worker returned modified_extensions that is not a JSON object".into(), + })?; + + // Slots whose writes are token-gated. Reconstructing one would require + // minting a WriteToken this host has no authority to mint. + const GATED_SLOTS: [&str; 3] = ["http", "security", "delegation"]; + for slot in GATED_SLOTS { + if obj.get(slot).is_some_and(|v| !v.is_null()) { + return Err(HostError::Protocol { + message: format!( + "worker returned modified_extensions.{slot}, which is write-token gated and \ + cannot be applied from an out-of-process plugin — only `custom` is supported \ + on this path" + ), + }); + } + } + + let custom = match obj.get("custom") { + Some(Value::Null) | None => None, + Some(value) => Some( + serde_json::from_value::>(value.clone()) + .map_err(|e| HostError::Protocol { + message: format!( + "worker returned a modified_extensions.custom this host cannot read: {e}" + ), + })?, + ), + }; + + // Nothing applicable — treat as no modification rather than an empty write. + if custom.is_none() { + return Ok(None); + } + + Ok(Some(cpex_core::hooks::payload::OwnedExtensions { + request: None, + agent: None, + mcp: None, + completion: None, + provenance: None, + llm: None, + framework: None, + meta: None, + raw_credentials: None, + http: None, + security: None, + delegation: None, + custom, + http_write_token: None, + labels_write_token: None, + delegation_write_token: None, + })) +} + +/// Parse a violation, tolerating the Python model's extra fields. +/// +/// The Python `PluginViolation` carries `mcp_error_code` and +/// `http_status_code`, which the Rust type does not model — so a strict +/// `from_value` into the Rust struct is not used. `reason` and `code` are +/// pulled out explicitly and defaulted, because a deny with an empty reason is +/// still a deny and must not be downgraded into an error. +fn parse_violation(raw: Value) -> Result { + let obj = raw.as_object().ok_or_else(|| HostError::Protocol { + message: "worker returned a violation that is not a JSON object".into(), + })?; + + let code = obj.get("code").and_then(Value::as_str).unwrap_or_default(); + let reason = obj + .get("reason") + .and_then(Value::as_str) + .unwrap_or_default(); + + let mut violation = PluginViolation::new(code, reason); + violation.description = obj + .get("description") + .and_then(Value::as_str) + .map(str::to_string); + + if let Some(details) = obj.get("details").and_then(Value::as_object) { + violation.details = details.clone().into_iter().collect(); + } + + // The Python side names this `mcp_error_code`; the Rust side calls the same + // concept `proto_error_code`. + violation.proto_error_code = obj + .get("mcp_error_code") + .or_else(|| obj.get("proto_error_code")) + .and_then(Value::as_i64); + + Ok(violation) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + // --- hook name resolution ----------------------------------------------- + + #[test] + fn cmf_hooks_route_to_the_message_payload() { + // The CMF-parity acceptance example. `cmf.tool_pre_invoke` contains the + // legacy name as a substring, so a match that checked the suffix or + // used `contains` would mis-route it to the tool payload. + assert_eq!( + payload_kind_for_hook("cmf.tool_pre_invoke"), + PayloadKind::CmfMessage + ); + assert_eq!( + payload_kind_for_hook("cmf.llm_input"), + PayloadKind::CmfMessage + ); + assert_eq!( + payload_kind_for_hook("cmf.some_future_hook"), + PayloadKind::CmfMessage, + "any cmf.* name is a message payload, even one this host predates" + ); + } + + #[test] + fn every_legacy_hook_resolves_to_its_own_typed_payload() { + let expected = [ + ("tool_pre_invoke", PayloadKind::ToolPreInvoke), + ("tool_post_invoke", PayloadKind::ToolPostInvoke), + ("prompt_pre_fetch", PayloadKind::PromptPreFetch), + ("prompt_post_fetch", PayloadKind::PromptPostFetch), + ("resource_pre_fetch", PayloadKind::ResourcePreFetch), + ("resource_post_fetch", PayloadKind::ResourcePostFetch), + ("identity_resolve", PayloadKind::IdentityResolve), + ("token_delegate", PayloadKind::TokenDelegate), + ]; + + for (hook, kind) in expected { + assert_eq!( + payload_kind_for_hook(hook), + kind, + "hook '{hook}' resolved wrongly" + ); + } + + // All eight are distinct — a copy-paste in the match arms would show up + // here rather than as a wrong payload at runtime. + let kinds: std::collections::HashSet<_> = + expected.iter().map(|(_, k)| format!("{k:?}")).collect(); + assert_eq!( + kinds.len(), + 8, + "each legacy hook needs its own payload kind" + ); + } + + #[test] + fn an_unknown_hook_name_falls_back_to_the_generic_payload() { + assert_eq!( + payload_kind_for_hook("some_custom_hook"), + PayloadKind::Generic + ); + assert_eq!(payload_kind_for_hook(""), PayloadKind::Generic); + } + + #[test] + fn only_identity_and_delegation_are_credential_hooks() { + assert!(is_credential_hook("identity_resolve")); + assert!(is_credential_hook("token_delegate")); + + for hook in [ + "tool_pre_invoke", + "tool_post_invoke", + "prompt_pre_fetch", + "resource_pre_fetch", + "cmf.tool_pre_invoke", + "unknown", + ] { + assert!( + !is_credential_hook(hook), + "'{hook}' must not receive credentials" + ); + } + } + + // --- outbound serialization --------------------------------------------- + + #[test] + fn a_tool_pre_invoke_payload_serializes_to_the_worker_field_shape() { + let payload = ToolPreInvokePayload { + name: "search".into(), + args: Some(HashMap::from([("q".into(), serde_json::json!("rust"))])), + headers: Some(HashMap::from([("X-Tenant".into(), "acme".into())])), + }; + + let json = serialize_payload(&payload).expect("serializes"); + assert_eq!(json["name"], "search"); + assert_eq!(json["args"]["q"], "rust"); + assert_eq!(json["headers"]["X-Tenant"], "acme"); + } + + #[test] + fn the_generic_payload_serializes_to_its_inner_value() { + // Not to `{"value": ...}` — the worker expects the payload itself. + let payload = GenericPayload { + value: serde_json::json!({"anything": [1, 2, 3]}), + }; + let json = serialize_payload(&payload).expect("serializes"); + assert_eq!(json["anything"][2], 3); + assert!( + json.get("value").is_none(), + "the wrapper must not appear on the wire" + ); + } + + #[test] + fn a_cmf_message_payload_serializes_as_itself() { + use cpex_core::cmf::{Message, Role}; + + let payload = MessagePayload { + message: Message::text(Role::User, "what is the weather?"), + }; + let json = serialize_payload(&payload).expect("serializes"); + assert_eq!(json["message"]["role"], "user"); + } + + #[test] + fn an_unforwardable_payload_type_errors_rather_than_sending_null() { + #[derive(Debug, Clone)] + struct ForeignPayload; + cpex_core::impl_plugin_payload!(ForeignPayload); + + let err = + serialize_payload(&ForeignPayload).expect_err("an unknown type cannot be forwarded"); + assert!(matches!(err, HostError::Protocol { .. }), "got {err:?}"); + } + + // --- inbound deserialization ------------------------------------------- + + #[test] + fn a_modified_payload_deserializes_into_the_hooks_typed_payload() { + let boxed = + deserialize_payload("tool_pre_invoke", serde_json::json!({ "name": "redacted" })) + .expect("parses"); + let typed = boxed + .as_any() + .downcast_ref::() + .expect("must come back as the tool payload, or the executor's downcast fails"); + assert_eq!(typed.name, "redacted"); + } + + #[test] + fn a_cmf_modified_payload_deserializes_as_a_message_payload() { + use cpex_core::cmf::{Message, Role}; + + // Round-tripped through the real type so the JSON matches the CMF + // schema rather than a hand-guessed shape. + let original = MessagePayload { + message: Message::text(Role::Assistant, "redacted"), + }; + let json = serde_json::to_value(&original).unwrap(); + + let boxed = deserialize_payload("cmf.tool_pre_invoke", json).expect("parses"); + let typed = boxed + .as_any() + .downcast_ref::() + .expect("a cmf hook's modified payload must come back as a MessagePayload"); + assert_eq!(typed.message.role, Role::Assistant); + } + + #[test] + fn an_unknown_hooks_modified_payload_becomes_a_generic_payload() { + let boxed = + deserialize_payload("mystery_hook", serde_json::json!({"x": 1})).expect("parses"); + let generic = boxed + .as_any() + .downcast_ref::() + .expect("generic"); + assert_eq!(generic.value["x"], 1); + } + + #[test] + fn a_mismatched_modified_payload_errors_and_names_the_hook() { + // `name` is required on the tool payload; omitting it must not silently + // yield a default-constructed payload. + let err = deserialize_payload("tool_pre_invoke", serde_json::json!({ "unexpected": true })) + .expect_err("a payload missing required fields must be rejected"); + assert!(err.to_string().contains("tool_pre_invoke"), "{err}"); + } + + // --- response to result -------------------------------------------------- + + #[test] + fn an_allow_response_becomes_an_allow_result() { + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ "continue_processing": true }), + ) + .expect("converts"); + + assert!(fields.continue_processing); + assert!(fields.violation.is_none()); + assert!(fields.modified_payload.is_none()); + assert!(fields.modified_extensions.is_none()); + } + + #[test] + fn a_response_without_continue_processing_defaults_to_allow() { + // Matches PluginResult's Pydantic default. Defaulting to deny would + // block traffic on any plugin that returns a bare result. + let fields = + response_to_result("tool_pre_invoke", serde_json::json!({})).expect("converts"); + assert!(fields.continue_processing); + } + + #[test] + fn a_deny_response_carries_its_violation() { + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ + "continue_processing": false, + "violation": { + "code": "PII_DETECTED", + "reason": "email address in args", + "description": "matched the email pattern", + "details": {"field": "q"}, + "mcp_error_code": -32603 + } + }), + ) + .expect("converts"); + + assert!(!fields.continue_processing); + let violation = fields.violation.expect("a deny must carry its violation"); + assert_eq!(violation.code, "PII_DETECTED"); + assert_eq!(violation.reason, "email address in args"); + assert_eq!( + violation.description.as_deref(), + Some("matched the email pattern") + ); + assert_eq!(violation.details["field"], "q"); + assert_eq!( + violation.proto_error_code, + Some(-32603), + "the Python mcp_error_code maps onto the Rust proto_error_code" + ); + } + + #[test] + fn a_violation_with_only_a_reason_still_denies() { + // Tolerance matters here: a partially-populated violation must not be + // converted into a host error, which would turn a deny into a + // policy-dependent failure. + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ "continue_processing": false, "violation": { "reason": "nope" } }), + ) + .expect("a sparse violation is still a violation"); + + assert!(!fields.continue_processing); + let violation = fields.violation.unwrap(); + assert_eq!(violation.reason, "nope"); + assert_eq!(violation.code, ""); + } + + #[test] + fn a_modified_payload_survives_the_round_trip() { + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ + "continue_processing": true, + "modified_payload": { "name": "search", "args": {"q": "[REDACTED]"} } + }), + ) + .expect("converts"); + + let payload = fields + .modified_payload + .expect("the modification must survive"); + let typed = payload + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(typed.args.as_ref().unwrap()["q"], "[REDACTED]"); + } + + #[test] + fn modified_extensions_survive_the_round_trip() { + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ + "continue_processing": true, + "modified_extensions": { "custom": { "seen_by": "py-plugin" } } + }), + ) + .expect("converts"); + + let extensions = fields.modified_extensions.expect("extensions must survive"); + assert_eq!( + extensions.custom.expect("custom slot")["seen_by"], + "py-plugin" + ); + } + + #[test] + fn a_write_token_gated_extension_slot_is_rejected_not_dropped() { + // `security` writes are gated by a WriteToken the executor mints. This + // host cannot mint one, so honoring the write would mean forging + // authorization — and dropping it silently would lose the plugin's + // intent. Erroring is the only honest option. + for slot in ["http", "security", "delegation"] { + // `ErasedResultFields` is not Debug, so `expect_err` is unavailable. + let Err(err) = response_to_result( + "tool_pre_invoke", + serde_json::json!({ + "continue_processing": true, + "modified_extensions": { slot: {"labels": ["PII"]} } + }), + ) else { + panic!("a token-gated slot ('{slot}') must not be silently dropped"); + }; + + assert!(matches!(err, HostError::Protocol { .. })); + assert!( + err.to_string().contains(slot), + "the error should name the slot the plugin tried to write: {err}" + ); + } + } + + #[test] + fn an_extensions_object_with_only_gated_nulls_is_no_modification() { + // Pydantic emits unset Optionals as null, so a plugin that touched + // nothing still sends every key. That must not read as an attempted + // gated write. + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ + "continue_processing": true, + "modified_extensions": { "http": null, "security": null, "custom": null } + }), + ) + .expect("all-null slots are not a write"); + assert!(fields.modified_extensions.is_none()); + } + + #[test] + fn explicit_nulls_are_treated_as_absent() { + // Pydantic serializes unset Optionals as null, so null and absent must + // behave identically or every allow response would try to parse a + // null payload. + let fields = response_to_result( + "tool_pre_invoke", + serde_json::json!({ + "continue_processing": true, + "modified_payload": null, + "modified_extensions": null, + "violation": null + }), + ) + .expect("nulls are absent, not malformed"); + + assert!(fields.modified_payload.is_none()); + assert!(fields.modified_extensions.is_none()); + assert!(fields.violation.is_none()); + } + + #[test] + fn a_malformed_modified_payload_surfaces_as_a_protocol_error() { + let Err(err) = response_to_result( + "tool_pre_invoke", + serde_json::json!({ "continue_processing": true, "modified_payload": "not an object" }), + ) else { + panic!("a payload that cannot be rebuilt must not be silently dropped"); + }; + assert!(matches!(err, HostError::Protocol { .. }), "got {err:?}"); + } +} diff --git a/crates/cpex-hosts-python/src/credentials.rs b/crates/cpex-hosts-python/src/credentials.rs new file mode 100644 index 00000000..4b43ee05 --- /dev/null +++ b/crates/cpex-hosts-python/src/credentials.rs @@ -0,0 +1,948 @@ +// Location: ./crates/cpex-hosts-python/src/credentials.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Capability-gated credential wire DTO. +// +// # Why this exists +// +// `RawInboundToken.token` and `RawDelegatedToken.token` are `#[serde(skip)]`, +// so serializing an `Extensions` yields no token bytes. That is the documented +// "raw credentials never leave the host process" invariant — and it means an +// out-of-process identity resolver or token delegator cannot do its job. +// +// This module deliberately reverses that invariant, narrowly: a dedicated DTO +// carries the token as a plain string, built only on the capability-gated +// dispatch path for the two hooks whose Python payload models a raw token at +// all. The production types keep their serde guard and are never serialized; +// the FFI, Python-bindings, and audit paths are untouched. The new leak surface +// is one constructor and one serialize site, which is the point. +// +// The DTO is built by reading the in-memory `Zeroizing` token field *directly*. +// A serialize-then-reparse would yield an empty token, because the serde guard +// strips the bytes on the way out — the guard is doing its job, so the only way +// to read the value is to read the field. +// +// # Fail closed +// +// Two rules, both load-bearing: +// +// 1. No declared capability → no DTO, no token material. Not an error; the +// plugin simply never sees credentials. +// 2. Declared capability that cannot be honored → error, not a silent +// no-token send. A plugin that asked for a token and got none would +// otherwise validate an empty bearer, or fall through to a code path that +// treats "no credential" as "no authentication required". +// +// # Residual exposure +// +// The capability gate controls *which* plugin receives a token. It does not +// constrain what happens next: once the plaintext is resident in the worker +// process, every transitively-installed dependency in that venv can read it. +// That is a materially larger and less audited trust boundary than the +// in-process host, and neither this gate nor the transport can close it. See +// the `worker` module for the transport side. + +use std::collections::{HashMap, HashSet}; + +use cpex_core::extensions::raw_credentials::{ + DelegationMode, RawDelegatedToken, RawInboundToken, TokenKind, TokenRole, +}; +use cpex_core::extensions::Extensions; +use serde::Serialize; +use serde_json::Value; + +use crate::conversion::payload_kind_for_hook; +use crate::error::HostError; +use crate::PayloadKind; + +/// Task field the DTO rides on. Contract with `worker.py`'s +/// `CREDENTIAL_FIELD`; both sides must agree verbatim. +pub const CREDENTIAL_FIELD: &str = "credential"; + +/// Capability a plugin declares to receive the inbound credential. +pub const CAP_READ_INBOUND: &str = "read_inbound_credentials"; + +/// Capability a plugin declares to receive a delegated token. +pub const CAP_READ_DELEGATED: &str = "read_delegated_tokens"; + +/// Placeholder printed instead of token bytes in `Debug`. +const REDACTED: &str = ""; + +/// The `credential` object attached to a task. +/// +/// Exactly one sub-object is populated, chosen by hook. `Debug` is +/// hand-written so neither ends up in a log line or panic message. +#[derive(Clone, Default, Serialize)] +pub struct CredentialDto { + #[serde(skip_serializing_if = "Option::is_none")] + pub inbound: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub delegated: Option, +} + +impl std::fmt::Debug for CredentialDto { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Which sub-object is present is safe and useful; its contents are not. + f.debug_struct("CredentialDto") + .field("inbound", &self.inbound.as_ref().map(|_| REDACTED)) + .field("delegated", &self.delegated.as_ref().map(|_| REDACTED)) + .finish() + } +} + +/// `credential.inbound` — built from `RawInboundToken`, for `identity_resolve`. +/// +/// The worker maps these onto `IdentityPayload`: `token` → `raw_token`, `kind` +/// → `source`, `headers` → `headers` (with the plaintext scrubbed out of the +/// values, since `IdentityPayload.headers` does not redact on serialization). +#[derive(Clone, Serialize)] +pub struct InboundCredential { + /// The plaintext, read straight from the in-memory `Zeroizing` field. + pub token: String, + + /// Header the credential arrived in. + pub source_header: String, + + /// Wire-format family — `jwt`, `opaque`, `spiffe_jwt`, `ucan`, `txn_token`. + pub kind: String, + + /// Headers to forward. Synthesized as `{source_header: token}` when the + /// extension carries none, so a header-driven extractor still learns which + /// header carried the credential. + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} + +impl std::fmt::Debug for InboundCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InboundCredential") + .field("token", &REDACTED) + .field("source_header", &self.source_header) + .field("kind", &self.kind) + // Values can embed the token ("Bearer "), so names only. + .field( + "header_names", + &self.headers.as_ref().map(|h| h.keys().collect::>()), + ) + .finish() + } +} + +/// `credential.delegated` — built from `RawDelegatedToken`, for +/// `token_delegate`. +/// +/// The worker maps `token` → `DelegationPayload.bearer_token`; the rest is +/// audit and validation context. +#[derive(Clone, Serialize)] +pub struct DelegatedCredential { + /// The plaintext, read straight from the in-memory `Zeroizing` field. + pub token: String, + + /// Where the consumer should attach the token upstream. + pub outbound_header: String, + + /// Audience the token was minted for. + pub audience: String, + + /// Effective scopes on the minted token. + pub scopes: Vec, +} + +impl std::fmt::Debug for DelegatedCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DelegatedCredential") + .field("token", &REDACTED) + .field("outbound_header", &self.outbound_header) + .field("audience", &self.audience) + .field("scopes", &self.scopes) + .finish() + } +} + +/// The wire name for a `TokenKind`. +/// +/// Matches the `snake_case` serde rename on the enum, which is what +/// `worker.py`'s `_BEARER_TOKEN_KINDS` matches against. Written out rather +/// than round-tripped through serde so the mapping is reviewable next to the +/// contract it implements. +fn token_kind_wire_name(kind: &TokenKind) -> &'static str { + match kind { + TokenKind::Jwt => "jwt", + TokenKind::Opaque => "opaque", + TokenKind::SpiffeJwt => "spiffe_jwt", + TokenKind::Ucan => "ucan", + TokenKind::TxnToken => "txn_token", + // `TokenKind` is `#[non_exhaustive]`, so a kind added upstream lands + // here. "opaque" is the safe default: the worker maps unknown kinds to + // `source: "custom"`, which makes a validator inspect the headers + // itself rather than trust a source it does not recognize. + _ => "opaque", + } +} + +/// Attach the `credential` object to a task, when the hook and the plugin's +/// declared capabilities both call for it. +/// +/// A no-op for every hook other than `identity_resolve` and `token_delegate`, +/// and for any plugin that declared neither capability. +pub fn attach_credential( + task: &mut Value, + hook_name: &str, + extensions: &Extensions, + capabilities: &HashSet, +) -> Result<(), HostError> { + let kind = payload_kind_for_hook(hook_name); + + let dto = match kind { + PayloadKind::IdentityResolve => { + if !capabilities.contains(CAP_READ_INBOUND) { + return Ok(()); + } + CredentialDto { + inbound: Some(build_inbound(extensions, hook_name)?), + delegated: None, + } + }, + PayloadKind::TokenDelegate => { + if !capabilities.contains(CAP_READ_DELEGATED) { + return Ok(()); + } + CredentialDto { + inbound: None, + delegated: Some(build_delegated(extensions, hook_name)?), + } + }, + // No other hook has a payload field to receive a token. + _ => return Ok(()), + }; + + let object = task.as_object_mut().ok_or_else(|| HostError::Protocol { + message: "task must be a JSON object to carry a credential".into(), + })?; + + let serialized = serde_json::to_value(&dto).map_err(|e| HostError::Credential { + // The error names the failure, never the value. + message: format!("could not serialize the credential DTO: {e}"), + })?; + object.insert(CREDENTIAL_FIELD.to_string(), serialized); + + Ok(()) +} + +/// Build `credential.inbound` from the filtered extensions view. +/// +/// The plugin declared `read_inbound_credentials`, so a missing or empty token +/// is a fail-closed error rather than an omitted field: sending no token to a +/// plugin that asked for one invites it to treat the request as unauthenticated. +fn build_inbound(extensions: &Extensions, hook_name: &str) -> Result { + let raw = extensions + .raw_credentials + .as_ref() + .ok_or_else(|| HostError::Credential { + message: format!( + "plugin declared '{CAP_READ_INBOUND}' for hook '{hook_name}' but the request \ + carries no raw-credentials extension — refusing to dispatch without the token \ + it asked for" + ), + })?; + + // Prefer the user token, then the client token, then a workload JWT-SVID. + // Ordering matters: an identity resolver wants the subject's credential, + // and falling straight to the client token would resolve the gateway's own + // identity as if it were the user's. + let token = [TokenRole::User, TokenRole::Client, TokenRole::Workload] + .iter() + .find_map(|role| raw.inbound_tokens.get(role)) + .or_else(|| raw.inbound_tokens.values().next()) + .ok_or_else(|| HostError::Credential { + message: format!( + "plugin declared '{CAP_READ_INBOUND}' for hook '{hook_name}' but no inbound token \ + is present — refusing to dispatch without it" + ), + })?; + + inbound_from_token(token, hook_name) +} + +/// Convert one `RawInboundToken` into its wire form. +fn inbound_from_token( + token: &RawInboundToken, + hook_name: &str, +) -> Result { + // Reading `*token.token` is the whole point: `serde` would give back an + // empty string here, because the field is `#[serde(skip)]`. + let plaintext = token.token.as_str(); + + // Whitespace-only is rejected as well as empty: "Authorization: Bearer " + // is not meaningfully different from an empty bearer downstream, and + // `worker.py` rejects it on its side too. + if plaintext.trim().is_empty() { + return Err(HostError::Credential { + message: format!( + "the inbound token for hook '{hook_name}' is empty — refusing to send an empty \ + credential (an empty bearer authenticates nowhere, and a plugin may read it as \ + 'no authentication required')" + ), + }); + } + + Ok(InboundCredential { + token: plaintext.to_string(), + source_header: token.source_header.clone(), + kind: token_kind_wire_name(&token.kind).to_string(), + // Synthesized so a header-driven extractor learns which header carried + // the credential. The worker scrubs the plaintext back out of these + // values before a plugin can echo them onto a serialized payload. + headers: Some(HashMap::from([( + token.source_header.clone(), + plaintext.to_string(), + )])), + }) +} + +/// Build `credential.delegated` from the filtered extensions view. +fn build_delegated( + extensions: &Extensions, + hook_name: &str, +) -> Result { + let raw = extensions + .raw_credentials + .as_ref() + .ok_or_else(|| HostError::Credential { + message: format!( + "plugin declared '{CAP_READ_DELEGATED}' for hook '{hook_name}' but the request \ + carries no raw-credentials extension — refusing to dispatch without the token \ + it asked for" + ), + })?; + + // Prefer an on-behalf-of-user token: a delegator asked to act for the user + // should not silently fall back to the gateway's own identity. + let token = raw + .delegated_tokens + .iter() + .find(|(key, _)| key.mode == DelegationMode::OnBehalfOfUser) + .or_else(|| raw.delegated_tokens.iter().next()) + .map(|(_, token)| token) + .ok_or_else(|| HostError::Credential { + message: format!( + "plugin declared '{CAP_READ_DELEGATED}' for hook '{hook_name}' but no delegated \ + token is present — refusing to dispatch without it" + ), + })?; + + delegated_from_token(token, hook_name) +} + +/// Convert one `RawDelegatedToken` into its wire form. +fn delegated_from_token( + token: &RawDelegatedToken, + hook_name: &str, +) -> Result { + let plaintext = token.token.as_str(); + + if plaintext.trim().is_empty() { + return Err(HostError::Credential { + message: format!( + "the delegated token for hook '{hook_name}' is empty — refusing to send an empty \ + credential" + ), + }); + } + + Ok(DelegatedCredential { + token: plaintext.to_string(), + outbound_header: token.outbound_header.clone(), + audience: token.audience.clone(), + scopes: token.scopes.clone(), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use cpex_core::extensions::raw_credentials::{DelegationKey, RawCredentialsExtension}; + + use super::*; + + const INBOUND_SECRET: &str = "eyJhbGciOiJSUzI1NiJ9.INBOUND-SECRET-BYTES.sig"; + const DELEGATED_SECRET: &str = "MINTED-DELEGATED-SECRET-BYTES"; + + fn caps(names: &[&str]) -> HashSet { + names.iter().map(|s| (*s).to_string()).collect() + } + + /// Extensions carrying both an inbound and a delegated token. + fn extensions_with_credentials() -> Extensions { + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new(INBOUND_SECRET, "Authorization", TokenKind::Jwt), + ); + raw.delegated_tokens.insert( + DelegationKey { + subject_id: "alice".into(), + audience: "https://billing.example.com".into(), + scopes: vec!["read".into()], + mode: DelegationMode::OnBehalfOfUser, + }, + RawDelegatedToken::new( + DELEGATED_SECRET, + "Authorization", + "https://billing.example.com", + vec!["read".into()], + chrono::Utc::now(), + ), + ); + + Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + } + } + + fn task() -> Value { + serde_json::json!({ "task_type": "load_and_run_hook" }) + } + + // --- the capability gate ------------------------------------------------- + + #[test] + fn a_declaring_plugin_receives_the_inbound_token() { + let mut task = task(); + attach_credential( + &mut task, + "identity_resolve", + &extensions_with_credentials(), + &caps(&[CAP_READ_INBOUND]), + ) + .expect("a declared capability is honored"); + + let credential = &task[CREDENTIAL_FIELD]; + assert_eq!(credential["inbound"]["token"], INBOUND_SECRET); + assert_eq!(credential["inbound"]["source_header"], "Authorization"); + assert_eq!(credential["inbound"]["kind"], "jwt"); + assert!( + credential.get("delegated").is_none(), + "an identity hook must not receive delegated material" + ); + } + + #[test] + fn a_non_declaring_plugin_receives_no_token_material() { + let mut task = task(); + attach_credential( + &mut task, + "identity_resolve", + &extensions_with_credentials(), + &caps(&[]), + ) + .expect("declaring nothing is not an error"); + + assert!( + task.get(CREDENTIAL_FIELD).is_none(), + "no credential field at all" + ); + assert!( + !serde_json::to_string(&task) + .unwrap() + .contains("INBOUND-SECRET-BYTES"), + "no token bytes anywhere in the task" + ); + } + + #[test] + fn two_plugins_on_one_hook_are_gated_independently() { + // The capability-gated acceptance example: same hook, same request, one + // plugin declared the capability and the other did not. + let extensions = extensions_with_credentials(); + + let mut declaring = task(); + attach_credential( + &mut declaring, + "identity_resolve", + &extensions, + &caps(&[CAP_READ_INBOUND]), + ) + .unwrap(); + + let mut non_declaring = task(); + attach_credential( + &mut non_declaring, + "identity_resolve", + &extensions, + &caps(&[]), + ) + .unwrap(); + + assert!(serde_json::to_string(&declaring) + .unwrap() + .contains(INBOUND_SECRET)); + assert!(!serde_json::to_string(&non_declaring) + .unwrap() + .contains("INBOUND-SECRET-BYTES")); + } + + #[test] + fn the_wrong_capability_does_not_unlock_a_hook() { + // `read_delegated_tokens` must not grant inbound material, and vice + // versa — the two are independently scoped. + let extensions = extensions_with_credentials(); + + let mut identity = task(); + attach_credential( + &mut identity, + "identity_resolve", + &extensions, + &caps(&[CAP_READ_DELEGATED]), + ) + .unwrap(); + assert!(identity.get(CREDENTIAL_FIELD).is_none()); + + let mut delegate = task(); + attach_credential( + &mut delegate, + "token_delegate", + &extensions, + &caps(&[CAP_READ_INBOUND]), + ) + .unwrap(); + assert!(delegate.get(CREDENTIAL_FIELD).is_none()); + } + + #[test] + fn a_declaring_plugin_receives_the_delegated_token() { + let mut task = task(); + attach_credential( + &mut task, + "token_delegate", + &extensions_with_credentials(), + &caps(&[CAP_READ_DELEGATED]), + ) + .expect("a declared capability is honored"); + + let delegated = &task[CREDENTIAL_FIELD]["delegated"]; + assert_eq!(delegated["token"], DELEGATED_SECRET); + assert_eq!(delegated["outbound_header"], "Authorization"); + assert_eq!(delegated["audience"], "https://billing.example.com"); + assert_eq!(delegated["scopes"][0], "read"); + assert!(task[CREDENTIAL_FIELD].get("inbound").is_none()); + } + + #[test] + fn non_credential_hooks_never_receive_a_credential() { + // Even a plugin that declared both capabilities gets nothing on a hook + // whose payload has nowhere to put a token. + for hook in [ + "tool_pre_invoke", + "tool_post_invoke", + "prompt_pre_fetch", + "resource_post_fetch", + "cmf.tool_pre_invoke", + "some_custom_hook", + ] { + let mut task = task(); + attach_credential( + &mut task, + hook, + &extensions_with_credentials(), + &caps(&[CAP_READ_INBOUND, CAP_READ_DELEGATED]), + ) + .unwrap(); + + assert!( + task.get(CREDENTIAL_FIELD).is_none(), + "hook '{hook}' must not receive credentials" + ); + } + } + + // --- fail closed --------------------------------------------------------- + + #[test] + fn a_declared_capability_with_no_extension_errors() { + // Fail closed rather than dispatching without the token: a plugin that + // asked for a credential and silently got none may treat the request + // as unauthenticated. + let err = attach_credential( + &mut task(), + "identity_resolve", + &Extensions::default(), + &caps(&[CAP_READ_INBOUND]), + ) + .expect_err("a declared capability that cannot be honored must error"); + + assert!(matches!(err, HostError::Credential { .. }), "got {err:?}"); + } + + #[test] + fn a_declared_capability_with_no_matching_token_errors() { + // The extension exists but carries only delegated tokens, so the + // inbound capability cannot be honored. + let mut raw = RawCredentialsExtension::default(); + raw.delegated_tokens.insert( + DelegationKey { + subject_id: "alice".into(), + audience: "aud".into(), + scopes: vec![], + mode: DelegationMode::AsGateway, + }, + RawDelegatedToken::new("t", "Authorization", "aud", vec![], chrono::Utc::now()), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let err = attach_credential( + &mut task(), + "identity_resolve", + &extensions, + &caps(&[CAP_READ_INBOUND]), + ) + .expect_err("no inbound token means the capability cannot be honored"); + assert!(matches!(err, HostError::Credential { .. })); + } + + #[test] + fn an_empty_token_is_rejected_rather_than_sent() { + // A `#[serde(skip)]`-stripped token deserializes to "" — exactly the + // shape that reaches here if anyone ever reintroduces a + // serialize-then-reparse. Sending it would authenticate as an empty + // bearer. + for token in ["", " ", "\t\n"] { + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new(token, "Authorization", TokenKind::Jwt), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let err = attach_credential( + &mut task(), + "identity_resolve", + &extensions, + &caps(&[CAP_READ_INBOUND]), + ) + .expect_err("an empty or whitespace-only token must be refused"); + assert!(matches!(err, HostError::Credential { .. })); + } + } + + #[test] + fn an_empty_delegated_token_is_rejected() { + let mut raw = RawCredentialsExtension::default(); + raw.delegated_tokens.insert( + DelegationKey { + subject_id: "alice".into(), + audience: "aud".into(), + scopes: vec![], + mode: DelegationMode::OnBehalfOfUser, + }, + RawDelegatedToken::new("", "Authorization", "aud", vec![], chrono::Utc::now()), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let err = attach_credential( + &mut task(), + "token_delegate", + &extensions, + &caps(&[CAP_READ_DELEGATED]), + ) + .expect_err("an empty delegated token must be refused"); + assert!(matches!(err, HostError::Credential { .. })); + } + + #[test] + fn a_fail_closed_error_never_names_the_token() { + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new(" ", "Authorization", TokenKind::Jwt), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let err = attach_credential( + &mut task(), + "identity_resolve", + &extensions, + &caps(&[CAP_READ_INBOUND]), + ) + .expect_err("errors"); + let message = err.to_string(); + assert!( + message.contains("identity_resolve"), + "the message should name the hook: {message}" + ); + assert!(message.contains("empty")); + } + + // --- the DTO is the only carrier ---------------------------------------- + + #[test] + fn production_credential_types_still_refuse_to_serialize_tokens() { + // The guard this module works around must remain intact: nothing here + // loosened it, so a direct serialize of the production types still + // strips the bytes. If this ever fails, the DTO stopped being the only + // path token material can take. + let inbound = RawInboundToken::new(INBOUND_SECRET, "Authorization", TokenKind::Jwt); + let json = serde_json::to_string(&inbound).unwrap(); + assert!( + !json.contains("INBOUND-SECRET-BYTES"), + "the serde guard regressed: {json}" + ); + + let delegated = RawDelegatedToken::new( + DELEGATED_SECRET, + "Authorization", + "aud", + vec![], + chrono::Utc::now(), + ); + let json = serde_json::to_string(&delegated).unwrap(); + assert!( + !json.contains("MINTED-DELEGATED"), + "the serde guard regressed: {json}" + ); + + // And the inbound sub-map, which is what an audit or trace path dumps. + // (`delegated_tokens` is keyed by a struct, so the extension as a whole + // is not JSON-serializable — a pre-existing property of the type, not + // something this host changed.) + let extensions = extensions_with_credentials(); + let raw = extensions.raw_credentials.as_ref().unwrap(); + let json = serde_json::to_string(&raw.inbound_tokens).unwrap(); + assert!( + !json.contains("INBOUND-SECRET-BYTES"), + "the serde guard regressed: {json}" + ); + + for token in raw.delegated_tokens.values() { + let json = serde_json::to_string(token).unwrap(); + assert!( + !json.contains("MINTED-DELEGATED"), + "the serde guard regressed: {json}" + ); + } + } + + #[test] + fn a_serialize_then_reparse_would_have_yielded_an_empty_token() { + // Documents *why* the DTO reads the in-memory field directly. This is + // the tempting shortcut, and it silently produces an empty credential. + let inbound = RawInboundToken::new(INBOUND_SECRET, "Authorization", TokenKind::Jwt); + let round_tripped: RawInboundToken = + serde_json::from_str(&serde_json::to_string(&inbound).unwrap()).unwrap(); + + assert_eq!( + &*round_tripped.token, "", + "the guard strips the bytes on the way out" + ); + assert_eq!(round_tripped.source_header, "Authorization"); + } + + // --- redaction ----------------------------------------------------------- + + #[test] + fn the_dto_debug_output_hides_the_token() { + let dto = CredentialDto { + inbound: Some(InboundCredential { + token: INBOUND_SECRET.into(), + source_header: "Authorization".into(), + kind: "jwt".into(), + headers: Some(HashMap::from([( + "Authorization".into(), + format!("Bearer {INBOUND_SECRET}"), + )])), + }), + delegated: None, + }; + + // Both the outer DTO and the inner sub-object are printed, since either + // could reach a log line on its own. + let outer = format!("{dto:?}"); + let inner = format!("{:?}", dto.inbound.as_ref().unwrap()); + + for debug in [&outer, &inner] { + assert!( + !debug.contains("INBOUND-SECRET-BYTES"), + "token leaked into Debug: {debug}" + ); + } + // Non-secret context stays diagnosable. + assert!(inner.contains("Authorization")); + assert!(inner.contains("jwt")); + } + + #[test] + fn the_delegated_dto_debug_output_hides_the_token() { + let credential = DelegatedCredential { + token: DELEGATED_SECRET.into(), + outbound_header: "Authorization".into(), + audience: "https://billing.example.com".into(), + scopes: vec!["read".into()], + }; + + let debug = format!("{credential:?}"); + assert!( + !debug.contains("MINTED-DELEGATED"), + "token leaked into Debug: {debug}" + ); + assert!(debug.contains("billing.example.com")); + assert!(debug.contains("read")); + } + + // --- wire contract details ---------------------------------------------- + + #[test] + fn token_kinds_use_the_wire_names_the_worker_matches_on() { + // worker.py's _BEARER_TOKEN_KINDS is {jwt, opaque, spiffe_jwt, ucan}; + // a mismatch here would silently map a bearer to source "custom". + assert_eq!(token_kind_wire_name(&TokenKind::Jwt), "jwt"); + assert_eq!(token_kind_wire_name(&TokenKind::Opaque), "opaque"); + assert_eq!(token_kind_wire_name(&TokenKind::SpiffeJwt), "spiffe_jwt"); + assert_eq!(token_kind_wire_name(&TokenKind::Ucan), "ucan"); + assert_eq!(token_kind_wire_name(&TokenKind::TxnToken), "txn_token"); + } + + #[test] + fn wire_names_match_the_serde_representation() { + // Guards the hand-written mapping against the enum's own serde rename, + // so adding a variant upstream cannot silently desync the two. + for kind in [ + TokenKind::Jwt, + TokenKind::Opaque, + TokenKind::SpiffeJwt, + TokenKind::Ucan, + TokenKind::TxnToken, + ] { + let via_serde = serde_json::to_value(&kind).unwrap(); + assert_eq!( + via_serde.as_str().unwrap(), + token_kind_wire_name(&kind), + "hand-written wire name disagrees with serde for {kind:?}" + ); + } + } + + #[test] + fn the_user_token_is_preferred_over_the_client_token() { + // An identity resolver wants the subject's credential; resolving the + // gateway's own token as the user's would be an authorization bug. + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::Client, + RawInboundToken::new("CLIENT-TOKEN", "X-Client", TokenKind::Opaque), + ); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new("USER-TOKEN", "Authorization", TokenKind::Jwt), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let mut task = task(); + attach_credential( + &mut task, + "identity_resolve", + &extensions, + &caps(&[CAP_READ_INBOUND]), + ) + .unwrap(); + assert_eq!(task[CREDENTIAL_FIELD]["inbound"]["token"], "USER-TOKEN"); + } + + #[test] + fn headers_are_synthesized_from_the_source_header() { + let mut task = task(); + attach_credential( + &mut task, + "identity_resolve", + &extensions_with_credentials(), + &caps(&[CAP_READ_INBOUND]), + ) + .unwrap(); + + // The worker scrubs the plaintext out of these values before a plugin + // can echo them; the host's job is only to name the carrying header. + assert_eq!( + task[CREDENTIAL_FIELD]["inbound"]["headers"]["Authorization"], + INBOUND_SECRET + ); + } + + #[test] + fn an_on_behalf_of_user_token_is_preferred_over_a_gateway_token() { + let mut raw = RawCredentialsExtension::default(); + raw.delegated_tokens.insert( + DelegationKey { + subject_id: "gw".into(), + audience: "aud".into(), + scopes: vec![], + mode: DelegationMode::AsGateway, + }, + RawDelegatedToken::new( + "GATEWAY-TOKEN", + "Authorization", + "aud", + vec![], + chrono::Utc::now(), + ), + ); + raw.delegated_tokens.insert( + DelegationKey { + subject_id: "alice".into(), + audience: "aud".into(), + scopes: vec![], + mode: DelegationMode::OnBehalfOfUser, + }, + RawDelegatedToken::new( + "USER-DELEGATED-TOKEN", + "Authorization", + "aud", + vec![], + chrono::Utc::now(), + ), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let mut task = task(); + attach_credential( + &mut task, + "token_delegate", + &extensions, + &caps(&[CAP_READ_DELEGATED]), + ) + .unwrap(); + assert_eq!( + task[CREDENTIAL_FIELD]["delegated"]["token"], "USER-DELEGATED-TOKEN", + "a delegator acting for the user must not fall back to the gateway's identity" + ); + } + + #[test] + fn the_credential_field_name_matches_the_worker_contract() { + assert_eq!( + CREDENTIAL_FIELD, "credential", + "the worker reads task_data['credential']" + ); + } +} diff --git a/crates/cpex-hosts-python/src/error.rs b/crates/cpex-hosts-python/src/error.rs new file mode 100644 index 00000000..a9318691 --- /dev/null +++ b/crates/cpex-hosts-python/src/error.rs @@ -0,0 +1,133 @@ +// Location: ./crates/cpex-hosts-python/src/error.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Host-internal errors and their mapping to `cpex_core::error::PluginError`. +// +// The host distinguishes failure modes the executor cannot see — a venv that +// would not build, a worker that died mid-flight, a task over the size cap — +// so operators get an actionable reason. At the trait boundary each maps to a +// `PluginError`, and the *executor* applies the configured error policy +// (fail / ignore / disable). The host never implements that policy itself. + +use std::fmt; + +use cpex_core::error::PluginError; + +/// A failure inside the host, before or around the plugin's own logic. +#[derive(Debug)] +pub enum HostError { + /// The plugin's configuration cannot support a venv (no plugin dirs, an + /// unusable class name). + Config { message: String }, + + /// Building the virtualenv or installing its requirements failed. + VenvBuild { message: String }, + + /// The worker subprocess could not be launched. + WorkerStart { message: String }, + + /// The worker process is gone — reader EOF, or a non-zero exit — while a + /// request was outstanding. Distinct from a timeout: there is nothing + /// left to wait for. + WorkerDied { message: String }, + + /// No response arrived within the per-invocation timeout. + Timeout { timeout_secs: u64 }, + + /// The serialized task exceeded the configured `max_content_size`. Caught + /// before the write, so nothing was sent. + TaskTooLarge { size: usize, limit: usize }, + + /// The worker returned a structured error response for this request. + WorkerError { message: String }, + + /// A payload or response could not be serialized or parsed. + /// + /// The message must never carry payload *values* — see the note on + /// `Self::redacted_detail`. + Protocol { message: String }, + + /// A credential-bearing hook could not be served safely, so the host + /// failed closed rather than dispatching without the token. + /// + /// The message never contains token material. + Credential { message: String }, +} + +impl fmt::Display for HostError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Config { message } => write!(f, "configuration error: {message}"), + Self::VenvBuild { message } => write!(f, "venv build failed: {message}"), + Self::WorkerStart { message } => write!(f, "worker failed to start: {message}"), + Self::WorkerDied { message } => write!(f, "worker process died: {message}"), + Self::Timeout { timeout_secs } => { + write!(f, "worker did not respond within {timeout_secs}s") + }, + Self::TaskTooLarge { size, limit } => write!( + f, + "serialized task is {size} bytes, over the {limit}-byte max_content_size" + ), + Self::WorkerError { message } => write!(f, "worker returned an error: {message}"), + Self::Protocol { message } => write!(f, "protocol error: {message}"), + Self::Credential { message } => write!(f, "credential error: {message}"), + } + } +} + +impl std::error::Error for HostError {} + +impl HostError { + /// Short, stable code for the `PluginError::Execution.code` field, so a + /// host can branch on the failure mode without string-matching messages. + pub fn code(&self) -> &'static str { + match self { + Self::Config { .. } => "config", + Self::VenvBuild { .. } => "venv_build_failed", + Self::WorkerStart { .. } => "worker_start_failed", + Self::WorkerDied { .. } => "worker_died", + Self::Timeout { .. } => "timeout", + Self::TaskTooLarge { .. } => "task_too_large", + Self::WorkerError { .. } => "worker_error", + Self::Protocol { .. } => "protocol_error", + Self::Credential { .. } => "credential_error", + } + } + + /// Convert to the framework error type for a named plugin. + /// + /// A timeout becomes `PluginError::Timeout` so the executor's existing + /// timeout accounting applies; a config fault becomes + /// `PluginError::Config`; everything else is an `Execution` error the + /// executor routes through the plugin's `on_error` policy. + pub fn into_plugin_error(self, plugin_name: &str) -> Box { + match self { + Self::Timeout { timeout_secs } => PluginError::Timeout { + plugin_name: plugin_name.to_string(), + timeout_ms: timeout_secs.saturating_mul(1000), + proto_error_code: None, + } + .boxed(), + + Self::Config { ref message } => PluginError::Config { + message: format!("plugin '{plugin_name}' (isolated_venv): {message}"), + } + .boxed(), + + other => { + let code = other.code(); + PluginError::Execution { + plugin_name: plugin_name.to_string(), + message: other.to_string(), + source: None, + code: Some(code.to_string()), + details: Default::default(), + proto_error_code: None, + } + .boxed() + }, + } + } +} diff --git a/crates/cpex-hosts-python/src/factory.rs b/crates/cpex-hosts-python/src/factory.rs new file mode 100644 index 00000000..72f7c7b3 --- /dev/null +++ b/crates/cpex-hosts-python/src/factory.rs @@ -0,0 +1,268 @@ +// Location: ./crates/cpex-hosts-python/src/factory.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// The `isolated_venv` plugin factory. +// +// Hosts register this before `PluginManager::from_config()`. For each config +// entry whose `kind:` is `isolated_venv`, the manager calls `create()`, which +// parses the venv settings out of the entry's opaque `config` map and returns +// the plugin plus one handler per declared hook. + +use std::sync::Arc; + +use cpex_core::{ + error::PluginError, + factory::{PluginFactory, PluginInstance}, + plugin::PluginConfig, +}; + +use crate::plugin::{IsolatedPythonPlugin, PythonHookAdapter}; + +/// `kind:` string operators write in CPEX YAML to declare a Python plugin +/// that runs out-of-process in its own virtualenv. +/// +/// This supersedes issue #20's `python://` in-process framing: the host runs +/// the plugin in a subprocess, not via PyO3, so the runtime never links +/// libpython and a plugin crash cannot take the gateway down. +pub const KIND: &str = "isolated_venv"; + +/// Creates out-of-process Python plugins from `isolated_venv` config entries. +pub struct IsolatedVenvFactory; + +/// Whether an entry sets the ignored `plugin_dirs` key in its `config:` block. +/// +/// Split out from the warning call so the *decision* is directly assertable — +/// the `tracing::warn!` itself is a no-op without a subscriber, and the +/// workspace carries none in its test profile. +fn declares_ignored_plugin_dirs(config: &PluginConfig) -> bool { + config + .config + .as_ref() + .and_then(serde_json::Value::as_object) + .is_some_and(|block| block.contains_key("plugin_dirs")) +} + +impl PluginFactory for IsolatedVenvFactory { + fn create(&self, config: &PluginConfig) -> Result> { + if config.hooks.is_empty() { + return Err(PluginError::Config { + message: format!( + "plugin '{}' (isolated_venv): `hooks:` must list at least one hook \ + for the Python plugin to handle (e.g. tool_pre_invoke)", + config.name + ), + } + .boxed()); + } + + // `plugin_dirs` is no longer configurable — the host always uses + // `/plugins`. An entry that still sets it would otherwise + // appear to work while pointing somewhere else entirely, so say so once, + // naming the plugin. + if declares_ignored_plugin_dirs(config) { + tracing::warn!( + plugin = %config.name, + "plugin '{}' (isolated_venv) sets `plugin_dirs` in its config block, but the \ + host always uses '{}' at the project root. Setting ignored — remove it.", + config.name, + crate::plugin::DEFAULT_PLUGIN_DIR, + ); + } + + // Parsing happens here rather than in `initialize()` so a malformed + // entry fails at config load — while the manager can still report + // which plugin was bad — instead of midway through a rollback. + let plugin = Arc::new(IsolatedPythonPlugin::from_config(config)?); + + let handlers: Vec<_> = config + .hooks + .iter() + .map( + |hook| -> (&'static str, Arc) { + // The registry keys handlers by `&'static str`, and hook names + // arrive as owned Strings from YAML. Leaking is what + // audit-logger does: the set is bounded by config size and + // lives as long as the process holds the plugin anyway. + let leaked: &'static str = Box::leak(hook.clone().into_boxed_str()); + ( + leaked, + Arc::new(PythonHookAdapter::new(Arc::clone(&plugin), leaked)), + ) + }, + ) + .collect(); + + Ok(PluginInstance { plugin, handlers }) + } +} + +#[cfg(test)] +mod tests { + use cpex_core::config::CpexConfig; + use cpex_core::factory::PluginFactoryRegistry; + use cpex_core::manager::PluginManager; + + use super::*; + + /// A config with one `isolated_venv` plugin, mirroring the YAML shape an + /// operator writes. No `plugin_dirs`: the host always resolves + /// `/plugins` — see `plugin::DEFAULT_PLUGIN_DIR`. + fn minimal_config_yaml() -> &'static str { + r#" +plugins: + - name: pii-filter + kind: isolated_venv + hooks: [tool_pre_invoke] + config: + class_name: my_pkg.filters.PiiFilter +"# + } + + fn registry() -> PluginFactoryRegistry { + let mut factories = PluginFactoryRegistry::new(); + factories.register(KIND, Box::new(IsolatedVenvFactory)); + factories + } + + #[test] + fn factory_registers_under_isolated_venv_kind() { + let factories = registry(); + assert!(factories.has(KIND)); + assert_eq!(KIND, "isolated_venv"); + } + + #[test] + fn loading_an_isolated_venv_config_succeeds() { + let config: CpexConfig = serde_yaml::from_str(minimal_config_yaml()).expect("valid YAML"); + let factories = registry(); + + // The load path is what would raise "no factory registered for plugin + // kind 'isolated_venv'" if registration did not take. + PluginManager::from_config(config, &factories) + .expect("config loads with the factory registered"); + } + + #[test] + fn missing_class_name_is_a_config_error_not_a_panic() { + // The block carries only the now-ignored `plugin_dirs`, so this also + // covers that an ignored key does not mask the missing required one. + let yaml = r#" +plugins: + - name: broken + kind: isolated_venv + hooks: [tool_pre_invoke] + config: + plugin_dirs: ["./plugins"] +"#; + let config: CpexConfig = serde_yaml::from_str(yaml).expect("valid YAML"); + // `PluginManager` is not Debug, so `expect_err` is unavailable here. + let Err(err) = PluginManager::from_config(config, ®istry()) else { + panic!("a config block without class_name must be rejected"); + }; + + assert!( + matches!(*err, PluginError::Config { .. }), + "expected a Config error, got: {err:?}" + ); + let msg = err.to_string(); + assert!( + msg.contains("class_name"), + "the error should name the missing field so an operator can fix it: {msg}" + ); + } + + #[test] + fn empty_hooks_list_is_rejected() { + let yaml = r#" +plugins: + - name: no-hooks + kind: isolated_venv + hooks: [] + config: + class_name: my_pkg.filters.PiiFilter +"#; + let config: CpexConfig = serde_yaml::from_str(yaml).expect("valid YAML"); + let Err(err) = PluginManager::from_config(config, ®istry()) else { + panic!("a plugin declaring no hooks can never be invoked"); + }; + assert!(matches!(*err, PluginError::Config { .. })); + } + + #[test] + fn the_ignored_plugin_dirs_key_is_detected_for_the_warning() { + // Guards the warning's trigger condition. Without this an operator + // upgrading gets silence: the key stops working and nothing says so. + let with_key = PluginConfig { + name: "legacy".into(), + kind: KIND.into(), + hooks: vec!["tool_pre_invoke".into()], + config: Some(serde_json::json!({ + "class_name": "my_pkg.P", + "plugin_dirs": ["/somewhere"], + })), + ..Default::default() + }; + assert!(declares_ignored_plugin_dirs(&with_key)); + + let without_key = PluginConfig { + config: Some(serde_json::json!({ "class_name": "my_pkg.P" })), + ..with_key.clone() + }; + assert!( + !declares_ignored_plugin_dirs(&without_key), + "a clean config must not warn" + ); + + // An absent block must not panic or warn. + let no_block = PluginConfig { + config: None, + ..with_key + }; + assert!(!declares_ignored_plugin_dirs(&no_block)); + } + + #[test] + fn an_ignored_plugin_dirs_key_still_loads() { + // An operator upgrading from a config that set `plugin_dirs` must not + // hit a load failure — the key is ignored (with a warning), not + // rejected. A hard error here would break every existing config. + let yaml = r#" +plugins: + - name: legacy-dirs + kind: isolated_venv + hooks: [tool_pre_invoke] + config: + class_name: my_pkg.filters.PiiFilter + plugin_dirs: ["/somewhere/else"] +"#; + let config: CpexConfig = serde_yaml::from_str(yaml).expect("valid YAML"); + PluginManager::from_config(config, ®istry()) + .expect("an ignored plugin_dirs key must not fail the load"); + } + + #[test] + fn one_handler_is_produced_per_declared_hook() { + let config = PluginConfig { + name: "multi".into(), + kind: KIND.into(), + hooks: vec![ + "tool_pre_invoke".into(), + "tool_post_invoke".into(), + "cmf.tool_pre_invoke".into(), + ], + config: Some(serde_json::json!({ "class_name": "my_pkg.Multi" })), + ..Default::default() + }; + + let instance = IsolatedVenvFactory + .create(&config) + .expect("a valid multi-hook entry creates an instance"); + + assert_eq!(instance.handlers.len(), 3); + let names: Vec<&str> = instance.handlers.iter().map(|(n, _)| *n).collect(); + assert!(names.contains(&"tool_pre_invoke")); + assert!(names.contains(&"cmf.tool_pre_invoke")); + } +} diff --git a/crates/cpex-hosts-python/src/legacy/mod.rs b/crates/cpex-hosts-python/src/legacy/mod.rs new file mode 100644 index 00000000..4fa718e6 --- /dev/null +++ b/crates/cpex-hosts-python/src/legacy/mod.rs @@ -0,0 +1,19 @@ +// Location: ./crates/cpex-hosts-python/src/legacy/mod.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Legacy (non-CMF) hook payloads. +// +// "Legacy" is the framework's own term for the pre-CMF hook family — +// `tool_pre_invoke` and friends, as opposed to `cmf.tool_pre_invoke`. Both +// families are supported; they differ only in which payload type a hook name +// resolves to, which is what `crate::conversion` decides. + +pub mod payloads; + +pub use payloads::{ + AttenuationConfig, IdentityResolvePayload, PromptPostFetchPayload, PromptPreFetchPayload, + ResourcePostFetchPayload, ResourcePreFetchPayload, TokenDelegatePayload, ToolPostInvokePayload, + ToolPreInvokePayload, +}; diff --git a/crates/cpex-hosts-python/src/legacy/payloads.rs b/crates/cpex-hosts-python/src/legacy/payloads.rs new file mode 100644 index 00000000..f6669aa4 --- /dev/null +++ b/crates/cpex-hosts-python/src/legacy/payloads.rs @@ -0,0 +1,445 @@ +// Location: ./crates/cpex-hosts-python/src/legacy/payloads.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Typed payloads for the eight legacy (non-CMF) hooks. +// +// Each struct's serialized JSON must match the Pydantic model `worker.py` +// reconstructs via `json_to_payload`, field for field — a name mismatch +// surfaces as a Pydantic validation error inside the worker, at invoke time, +// far from its cause. The Python model each one mirrors is named in its doc +// comment; the field-shape tests in this module are the guard. +// +// # Redaction +// +// `IdentityResolvePayload` and `TokenDelegatePayload` carry token-adjacent +// fields, so both get a hand-written `Debug` that prints a placeholder instead +// of the value. A derive would put credential material into any `{:?}` — a +// tracing call, an `unwrap()` panic message, an assertion failure. The +// equivalent gap on the pre-existing cpex-core types is deferred follow-up +// work; these new types do not reintroduce it. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Placeholder printed in place of any token-adjacent value. +const REDACTED: &str = ""; + +/// `tool_pre_invoke` — mirrors `ToolPreInvokePayload`. +/// +/// `headers` mirrors `HttpHeaderPayload`, which serializes as a plain string +/// map. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ToolPreInvokePayload { + pub name: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub args: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} +cpex_core::impl_plugin_payload!(ToolPreInvokePayload); + +/// `tool_post_invoke` — mirrors `ToolPostInvokePayload`. +/// +/// `result` is `Any` on the Python side, so it stays an untyped JSON value. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ToolPostInvokePayload { + pub name: String, + + #[serde(default)] + pub result: serde_json::Value, +} +cpex_core::impl_plugin_payload!(ToolPostInvokePayload); + +/// `prompt_pre_fetch` — mirrors `PromptPrehookPayload`. +/// +/// Note `args` is `dict[str, str]` here, not `dict[str, Any]` as on the tool +/// payload. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct PromptPreFetchPayload { + pub prompt_id: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub args: Option>, +} +cpex_core::impl_plugin_payload!(PromptPreFetchPayload); + +/// `prompt_post_fetch` — mirrors `PromptPosthookPayload`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct PromptPostFetchPayload { + pub prompt_id: String, + + #[serde(default)] + pub result: serde_json::Value, +} +cpex_core::impl_plugin_payload!(PromptPostFetchPayload); + +/// `resource_pre_fetch` — mirrors `ResourcePreFetchPayload`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ResourcePreFetchPayload { + pub uri: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option>, +} +cpex_core::impl_plugin_payload!(ResourcePreFetchPayload); + +/// `resource_post_fetch` — mirrors `ResourcePostFetchPayload`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ResourcePostFetchPayload { + pub uri: String, + + #[serde(default)] + pub content: serde_json::Value, +} +cpex_core::impl_plugin_payload!(ResourcePostFetchPayload); + +/// `identity_resolve` — mirrors `IdentityPayload`. +/// +/// `raw_token` is `SecretStr` on the Python side and redacts on serialization, +/// so the value that travels in *this* field is a placeholder, never the real +/// token. The plaintext rides the separate capability-gated `credential` +/// object on the task (see the `credentials` module) and the worker folds it +/// back onto the payload before calling the plugin. +/// +/// `Debug` is hand-written — see the module docs. +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct IdentityResolvePayload { + /// Redacted on the wire. Populated from the `credential` object by the + /// worker, not from this field. + #[serde(default)] + pub raw_token: String, + + /// How the credential was presented — `bearer`, `custom`, and so on. + #[serde(default)] + pub source: String, + + /// Headers the credential arrived in. Not a `SecretStr` on the Python + /// side, so the worker scrubs the plaintext out of these values before a + /// plugin can echo them back. + #[serde(default)] + pub headers: HashMap, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_host: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_port: Option, +} +cpex_core::impl_plugin_payload!(IdentityResolvePayload); + +impl std::fmt::Debug for IdentityResolvePayload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // `raw_token` and every header value are redacted: a header can carry + // the credential ("Authorization: Bearer "), so printing keys + // alone is the safe maximum. + f.debug_struct("IdentityResolvePayload") + .field("raw_token", &REDACTED) + .field("source", &self.source) + .field("header_names", &self.headers.keys().collect::>()) + .field("client_host", &self.client_host) + .field("client_port", &self.client_port) + .finish() + } +} + +/// Scope-attenuation config — mirrors `AttenuationConfig`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AttenuationConfig { + #[serde(default)] + pub capabilities: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_template: Option, + + #[serde(default)] + pub actions: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl_seconds: Option, +} + +/// `token_delegate` — mirrors `DelegationPayload`. +/// +/// `bearer_token` is `SecretStr | None` on the Python side and redacts on +/// serialization, so this field carries a placeholder. The plaintext travels +/// via the `credential` object. +/// +/// `Debug` is hand-written — see the module docs. +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct TokenDelegatePayload { + pub target_name: String, + + #[serde(default = "default_target_type")] + pub target_type: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_audience: Option, + + #[serde(default)] + pub required_permissions: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trust_domain: Option, + + #[serde(default = "default_auth_enforced_by")] + pub auth_enforced_by: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub route_attenuation: Option, + + /// Redacted on the wire; the plaintext comes from the `credential` object. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bearer_token: Option, +} +cpex_core::impl_plugin_payload!(TokenDelegatePayload); + +fn default_target_type() -> String { + "tool".to_string() +} + +fn default_auth_enforced_by() -> String { + "caller".to_string() +} + +impl std::fmt::Debug for TokenDelegatePayload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TokenDelegatePayload") + .field("target_name", &self.target_name) + .field("target_type", &self.target_type) + .field("target_audience", &self.target_audience) + .field("required_permissions", &self.required_permissions) + .field("trust_domain", &self.trust_domain) + .field("auth_enforced_by", &self.auth_enforced_by) + .field("route_attenuation", &self.route_attenuation) + // Presence is useful for diagnosis; the value never is. + .field( + "bearer_token", + &self.bearer_token.as_ref().map(|_| REDACTED), + ) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Serialize and return the JSON object, so field names can be asserted + /// against the Pydantic models `worker.py` reconstructs. + fn json_of(value: &T) -> serde_json::Value { + serde_json::to_value(value).expect("payload serializes") + } + + #[test] + fn tool_pre_invoke_matches_the_python_field_shape() { + let payload = ToolPreInvokePayload { + name: "search".into(), + args: Some(HashMap::from([("q".into(), serde_json::json!("rust"))])), + headers: Some(HashMap::from([("X-Tenant".into(), "acme".into())])), + }; + let json = json_of(&payload); + + assert_eq!(json["name"], "search"); + assert_eq!(json["args"]["q"], "rust"); + assert_eq!(json["headers"]["X-Tenant"], "acme"); + } + + #[test] + fn tool_pre_invoke_omits_absent_optionals() { + // Pydantic defaults `args` to {} and `headers` to None. Emitting + // explicit nulls would override those defaults on the Python side, so + // absent fields must stay absent. + let json = json_of(&ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }); + assert_eq!( + json.as_object().unwrap().len(), + 1, + "only `name` should be present: {json}" + ); + } + + #[test] + fn tool_pre_invoke_round_trips() { + let payload = ToolPreInvokePayload { + name: "search".into(), + args: Some(HashMap::from([("q".into(), serde_json::json!(7))])), + headers: None, + }; + let restored: ToolPreInvokePayload = serde_json::from_value(json_of(&payload)).unwrap(); + assert_eq!(restored.name, "search"); + assert_eq!(restored.args.unwrap()["q"], 7); + } + + #[test] + fn tool_post_invoke_matches_the_python_field_shape() { + let json = json_of(&ToolPostInvokePayload { + name: "search".into(), + result: serde_json::json!({"hits": 3}), + }); + assert_eq!(json["name"], "search"); + assert_eq!(json["result"]["hits"], 3); + } + + #[test] + fn prompt_payloads_use_prompt_id_not_name() { + // The prompt hooks key on `prompt_id`; using `name` here would fail + // Pydantic validation inside the worker. + let pre = json_of(&PromptPreFetchPayload { + prompt_id: "greeting".into(), + args: Some(HashMap::from([("lang".into(), "en".into())])), + }); + assert_eq!(pre["prompt_id"], "greeting"); + assert_eq!(pre["args"]["lang"], "en"); + assert!(pre.get("name").is_none()); + + let post = json_of(&PromptPostFetchPayload { + prompt_id: "greeting".into(), + result: serde_json::json!({"messages": []}), + }); + assert_eq!(post["prompt_id"], "greeting"); + assert!(post["result"]["messages"].is_array()); + } + + #[test] + fn prompt_pre_fetch_args_are_strings() { + // dict[str, str] on the Python side, so a non-string value must not + // deserialize into it. + let parsed: Result = + serde_json::from_value(serde_json::json!({ "prompt_id": "p", "args": {"n": 5} })); + assert!( + parsed.is_err(), + "prompt args are dict[str, str], not dict[str, Any]" + ); + } + + #[test] + fn resource_payloads_key_on_uri() { + let pre = json_of(&ResourcePreFetchPayload { + uri: "file:///tmp/x".into(), + metadata: Some(HashMap::from([("etag".into(), serde_json::json!("abc"))])), + }); + assert_eq!(pre["uri"], "file:///tmp/x"); + assert_eq!(pre["metadata"]["etag"], "abc"); + + let post = json_of(&ResourcePostFetchPayload { + uri: "file:///tmp/x".into(), + content: serde_json::json!("hello"), + }); + assert_eq!(post["content"], "hello"); + } + + #[test] + fn identity_resolve_matches_the_python_field_shape() { + let json = json_of(&IdentityResolvePayload { + raw_token: "**********".into(), + source: "bearer".into(), + headers: HashMap::from([("Authorization".into(), "**********".into())]), + client_host: Some("10.0.0.1".into()), + client_port: Some(443), + }); + + assert_eq!(json["source"], "bearer"); + assert_eq!(json["client_host"], "10.0.0.1"); + assert_eq!(json["client_port"], 443); + assert!( + json.get("raw_token").is_some(), + "the field exists; its value is a placeholder" + ); + } + + #[test] + fn token_delegate_matches_the_python_field_shape_and_defaults() { + let parsed: TokenDelegatePayload = + serde_json::from_value(serde_json::json!({ "target_name": "billing-api" })).unwrap(); + + // Pydantic defaults these two; the host must agree or a minimal + // payload would deserialize with empty strings. + assert_eq!(parsed.target_type, "tool"); + assert_eq!(parsed.auth_enforced_by, "caller"); + assert!(parsed.required_permissions.is_empty()); + assert!(parsed.bearer_token.is_none()); + } + + #[test] + fn token_delegate_carries_attenuation_config() { + let json = json_of(&TokenDelegatePayload { + target_name: "billing-api".into(), + route_attenuation: Some(AttenuationConfig { + capabilities: vec!["read".into()], + resource_template: Some("/v1/*".into()), + actions: vec!["GET".into()], + ttl_seconds: Some(300), + }), + ..Default::default() + }); + + assert_eq!(json["route_attenuation"]["capabilities"][0], "read"); + assert_eq!(json["route_attenuation"]["ttl_seconds"], 300); + } + + // --- redaction ---------------------------------------------------------- + + #[test] + fn identity_resolve_debug_hides_token_and_header_values() { + let payload = IdentityResolvePayload { + raw_token: "eyJhbGciOiJSUzI1NiJ9.SECRET-TOKEN-BYTES".into(), + source: "bearer".into(), + headers: HashMap::from([("Authorization".into(), "Bearer SECRET-TOKEN-BYTES".into())]), + client_host: None, + client_port: None, + }; + + let debug = format!("{payload:?}"); + assert!( + !debug.contains("SECRET-TOKEN-BYTES"), + "token material leaked into Debug: {debug}" + ); + // The header *name* is still useful for diagnosis. + assert!(debug.contains("Authorization")); + assert!( + debug.contains("bearer"), + "non-secret fields stay readable: {debug}" + ); + } + + #[test] + fn token_delegate_debug_hides_the_bearer_token() { + let payload = TokenDelegatePayload { + target_name: "billing-api".into(), + bearer_token: Some("MINTED-SECRET-BYTES".into()), + ..Default::default() + }; + + let debug = format!("{payload:?}"); + assert!( + !debug.contains("MINTED-SECRET-BYTES"), + "delegated token leaked into Debug: {debug}" + ); + // Presence is diagnosable without exposing the value. + assert!(debug.contains("billing-api")); + assert!(debug.contains(REDACTED)); + } + + #[test] + fn token_delegate_debug_distinguishes_absent_from_present() { + let absent = format!( + "{:?}", + TokenDelegatePayload { + target_name: "x".into(), + ..Default::default() + } + ); + assert!( + absent.contains("None"), + "an absent token should read as None: {absent}" + ); + assert!(!absent.contains(REDACTED)); + } +} diff --git a/crates/cpex-hosts-python/src/lib.rs b/crates/cpex-hosts-python/src/lib.rs new file mode 100644 index 00000000..c9cf3b92 --- /dev/null +++ b/crates/cpex-hosts-python/src/lib.rs @@ -0,0 +1,74 @@ +// Location: ./crates/cpex-hosts-python/src/lib.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// cpex-hosts-python — out-of-process host for existing Python CPEX plugins. +// +// The Rust `PluginManager` cannot run a Python plugin on its own. This crate +// is the Rust counterpart to the Python CLI's `client.py` + `venv_comm.py`: +// it builds a per-plugin virtualenv, launches the framework's `worker.py` in +// that venv as a long-lived subprocess, and speaks the same +// newline-delimited-JSON stdio protocol the Python CLI already uses. Existing +// PII filters, identity resolvers, and token delegators keep working while +// the gateway migrates to the Rust manager. +// +// # Topology +// +// The factory produces one plugin object per config entry. That object owns +// the venv manager (build + cache) and the worker client (subprocess + +// protocol), and supplies one hook adapter per declared hook: +// +// ```text +// PluginManager +// └── isolated_venv factory ──> IsolatedPythonPlugin ──┬── venv manager +// (initialize/shutdown) └── worker client +// └── hook adapter (serialize, send, deserialize) │ +// ▼ +// worker.py subprocess in the venv +// ``` +// +// The three concerns are split deliberately rather than ported as one +// one-to-one translation of `client.py`: each tests in isolation, and +// shared-package venv handling falls naturally to the venv manager. +// +// # Lifecycle +// +// Venv construction and worker launch happen in `initialize()`, which the +// manager awaits once per plugin (with rollback on failure), and teardown in +// `shutdown()`, which it calls in reverse order. Neither belongs on the +// invoke path — a cold pip install is measured in minutes. +// +// # Credentials +// +// The framework strips raw tokens at every process boundary (the token +// fields are `#[serde(skip)]`). Identity and delegation plugins genuinely +// need them, so this host adds a capability-gated wire DTO: a plugin that +// declared `read_inbound_credentials` or `read_delegated_tokens` gets a +// dedicated `credential` object on the task JSON, built by reading the +// in-memory token directly. Production credential types keep their serde +// guard and are never serialized. See the `credentials` module for the +// fail-closed rules and the residual exposure this does not close. + +pub mod conversion; +pub mod credentials; +pub mod error; +pub mod factory; +pub mod legacy; +pub mod plugin; +pub mod venv; +pub mod worker; + +// Test-only helpers. Also exposed behind the `testing` feature so the +// integration tests under `tests/` can use them — a `cfg(test)` module is +// invisible to a separate test binary. +#[cfg(any(test, feature = "testing"))] +pub mod testing; + +pub use conversion::{GenericPayload, PayloadKind}; +pub use credentials::CredentialDto; +pub use error::HostError; +pub use factory::{IsolatedVenvFactory, KIND}; +pub use plugin::{IsolatedPythonPlugin, VenvConfig, DEFAULT_PLUGIN_DIR}; +pub use venv::{CacheVerdict, EnsureOutcome, VenvManager}; +pub use worker::WorkerClient; diff --git a/crates/cpex-hosts-python/src/plugin.rs b/crates/cpex-hosts-python/src/plugin.rs new file mode 100644 index 00000000..36b2d7a4 --- /dev/null +++ b/crates/cpex-hosts-python/src/plugin.rs @@ -0,0 +1,1225 @@ +// Location: ./crates/cpex-hosts-python/src/plugin.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// `IsolatedPythonPlugin` — the plugin object the factory produces, and +// `PythonHookAdapter` — the per-hook handler that drives one invocation. +// +// The plugin owns the venv manager and the worker client and implements the +// lifecycle half of the contract (initialize / shutdown). The adapter +// implements `AnyHookHandler` directly rather than going through +// `TypedHandlerAdapter`: that adapter downcasts to one fixed payload type, +// but this host forwards whatever payload arrives, serialized, to a worker +// that reconstructs it on the Python side. The concrete type is not known at +// Rust compile time, so there is nothing to downcast to. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use cpex_core::{ + context::PluginContext, + error::PluginError, + extensions::Extensions, + hooks::payload::PluginPayload, + plugin::{Plugin, PluginConfig}, + registry::AnyHookHandler, +}; +use serde::Deserialize; + +use crate::error::HostError; +use crate::venv::VenvManager; +use crate::worker::WorkerClient; +use crate::{conversion, credentials}; + +/// Default cap on the serialized size of one outbound task, in bytes. +/// +/// Mirrors `client.py`'s `max_content_size` default. The bound exists so a +/// pathological payload fails fast on this side rather than after the worker +/// has already buffered it. +const DEFAULT_MAX_CONTENT_SIZE: usize = 10_000_000; + +/// Default per-invocation timeout, in seconds. +/// +/// Mirrors `venv_comm.py`'s `send_task` default. A plugin entry may override +/// it; absent both, the executor's global `plugin_settings.plugin_timeout` +/// still bounds the call from outside. +const DEFAULT_TIMEOUT_SECS: u64 = 30; + +/// Path to the worker script, relative to the venv's installed `cpex`. +/// +/// The worker ships inside the venv via the plugin's `cpex` dependency, so +/// the host does not carry a copy of it. +const DEFAULT_SCRIPT_PATH: &str = "cpex/framework/isolated/worker.py"; + +/// Directory, relative to the project root, holding installed Python plugins. +/// +/// This is the sole source of the worker's `sys.path` entries and of the +/// directory the venv is built under. It is *not* configurable per plugin: a +/// `plugin_dirs:` key inside a plugin's `config:` block is ignored (see +/// `VenvConfig`), as is the top-level `plugin_dirs:` YAML key, which +/// `cpex_core` already parses and discards. +/// +/// # Why a fixed default rather than a config key +/// +/// `cpex plugin install` writes the plugin into `/plugins/` and +/// builds its venv there. Both the Python CLI and this host therefore already +/// agree on the location; making it configurable only creates a way for the +/// two to disagree, and the failure mode is an import error inside the worker +/// at invoke time, far from the config that caused it. +pub const DEFAULT_PLUGIN_DIR: &str = "plugins"; + +/// The project root the `plugins` directory is resolved against. +/// +/// The process working directory. That is not an arbitrary choice: the +/// worker's own `ALLOWED_PLUGIN_DIRS` allowlist accepts `os.getcwd()`, and the +/// worker inherits this host's cwd, so a directory resolved this way is +/// importable by construction. Anchoring anywhere else would produce paths the +/// worker rejects. +fn project_root() -> PathBuf { + // A cwd that cannot be read is not recoverable here, and returning a + // relative path keeps the failure legible: the venv layout error names + // `plugins`, which is what an operator would look for. + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) +} + +/// The plugin directory list the worker and venv layout both use. +/// +/// One entry: `/plugins`. Absolute, so the value does not shift +/// if something later changes the process cwd between `initialize()` and an +/// invocation. +fn default_plugin_dirs() -> Vec { + vec![project_root() + .join(DEFAULT_PLUGIN_DIR) + .display() + .to_string()] +} + +/// Global-state key a host can set to propagate its own request id into the +/// worker's `GlobalContext.request_id`. +/// +/// The Rust `PluginContext` has no dedicated request-id field, but the Python +/// `GlobalContext` requires one. Reading it from global state lets a gateway +/// that tracks request ids keep them consistent across the boundary. +pub const GLOBAL_REQUEST_ID_KEY: &str = "request_id"; + +/// Venv-relevant settings, parsed from a plugin entry's opaque `config` map. +/// +/// `PluginConfig` has no fields for a class name, a content-size cap, or a +/// per-plugin timeout — those are host-specific, so they live in the +/// free-form `config` value and are parsed here. +/// +/// # `plugin_dirs` is deliberately absent +/// +/// A `plugin_dirs:` key in this block is **ignored**, as is the top-level +/// `plugin_dirs:` YAML key. Plugin directories are not configurable: the host +/// always uses `/plugins`, which is where `cpex plugin install` +/// puts them. See `DEFAULT_PLUGIN_DIR`. Serde ignores unknown fields, so an +/// existing config that still carries the key parses without error — the value +/// simply has no effect. +#[derive(Debug, Clone, Deserialize)] +pub struct VenvConfig { + /// Fully-qualified Python class implementing the plugin, e.g. + /// `my_pkg.filters.PiiFilter`. Required: the worker needs it to import + /// and instantiate the plugin, and the venv cache is keyed on it. + pub class_name: String, + + /// Requirements file to install into the venv, relative to the package + /// root. Optional — a plugin installed by FQN conversion gets its + /// dependencies from the install channel, not a requirements file, and + /// then the manifest version plus hash is the sole cache signal. + #[serde(default)] + pub requirements_file: Option, + + /// Worker script path inside the venv. Defaults to the framework's + /// `worker.py`; overridable for tests and for a host that vendors its own. + #[serde(default = "default_script_path")] + pub script_path: String, + + /// Cap on one serialized outbound task, in bytes. + #[serde(default = "default_max_content_size")] + pub max_content_size: usize, + + /// Per-invocation timeout, in seconds. Falls back to the framework + /// default when absent. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +fn default_script_path() -> String { + DEFAULT_SCRIPT_PATH.to_string() +} + +fn default_max_content_size() -> usize { + DEFAULT_MAX_CONTENT_SIZE +} + +fn default_timeout_secs() -> u64 { + DEFAULT_TIMEOUT_SECS +} + +impl VenvConfig { + /// Parse the venv settings out of a plugin entry's `config` map. + /// + /// A missing or malformed `class_name` is a config error, not a panic — + /// the manager surfaces it against the plugin name at load time. + pub fn from_plugin_config(config: &PluginConfig) -> Result> { + let raw = config + .config + .clone() + .unwrap_or_else(|| serde_json::json!({})); + + serde_json::from_value(raw).map_err(|e| { + PluginError::Config { + message: format!( + "plugin '{}' (isolated_venv): invalid `config:` block — {}. \ + `class_name` is required (the fully-qualified Python class, \ + e.g. my_pkg.filters.PiiFilter)", + config.name, e + ), + } + .boxed() + }) + } +} + +/// A Python plugin running out-of-process in its own virtualenv. +pub struct IsolatedPythonPlugin { + /// The authoritative config entry, kept for `Plugin::config()`. + config: PluginConfig, + + /// Venv settings parsed from the entry's `config` map. + venv_config: VenvConfig, + + /// Directories the worker prepends to `sys.path`, and under whose first + /// entry the venv is built. + /// + /// Host-owned rather than config-derived: `/plugins`, per + /// `DEFAULT_PLUGIN_DIR`. Held here rather than recomputed per invocation so + /// the value stays fixed for the plugin's lifetime even if the process cwd + /// changes after construction. + plugin_dirs: Vec, + + /// Builds and caches the venv. `None` only when the layout cannot be + /// resolved (an empty dir list, or a `class_name` with no package + /// segment), which is valid solely for a `worker_override` test setup. + venv: Option, + + /// The live worker, populated by `initialize()`. + worker: tokio::sync::RwLock>>, + + /// Absolute worker-script path that bypasses venv resolution. Tests point + /// this at a stub; production leaves it unset and resolves from the venv. + worker_override: Option, + + /// Working directory for the worker process. + /// + /// Defaults to the host's own cwd, matching `venv_comm.py`'s + /// `cwd=os.getcwd()`. This is load-bearing twice over: the worker's + /// `ALLOWED_PLUGIN_DIRS` allowlist accepts `os.getcwd()`, so a plugin dir + /// outside it is rejected outright; and a plugin's relative-path side + /// effects land here. + worker_cwd: Option, +} + +impl IsolatedPythonPlugin { + /// Build a plugin from its config entry, parsing the venv settings. + pub fn from_config(config: &PluginConfig) -> Result> { + let venv_config = VenvConfig::from_plugin_config(config)?; + + // Not read from the config block — see `DEFAULT_PLUGIN_DIR`. A + // `plugin_dirs:` key there (or at the YAML top level) is ignored; + // warning about one is the factory's job, since it can name the plugin. + let plugin_dirs = default_plugin_dirs(); + + // A layout that will not resolve leaves the venv absent. That is a + // config error in production, but the failure belongs in + // `initialize()` rather than here: the factory should still hand back a + // named plugin the manager can report against. + let venv = VenvManager::new( + &plugin_dirs, + &venv_config.class_name, + venv_config.requirements_file.as_deref(), + config.version.as_deref(), + ) + .ok(); + + Ok(Self { + config: config.clone(), + venv_config, + plugin_dirs, + venv, + worker: tokio::sync::RwLock::new(None), + worker_override: None, + worker_cwd: None, + }) + } + + /// The resolved plugin directories — `/plugins`. + pub fn plugin_dirs(&self) -> &[String] { + &self.plugin_dirs + } + + /// Run the worker with an explicit working directory. + /// + /// Production leaves this unset and inherits the host's cwd. Tests set it + /// so their scratch plugin dir falls inside the worker's + /// `ALLOWED_PLUGIN_DIRS` (which accepts `os.getcwd()`), and so a plugin's + /// marker files land somewhere the test can find and clean up. + #[cfg(any(test, feature = "testing"))] + pub fn with_worker_cwd(mut self, cwd: PathBuf) -> Self { + self.worker_cwd = Some(cwd); + self + } + + /// Point the plugin at explicit plugin directories, replacing the + /// `/plugins` default. + /// + /// Test-only seam. Production has no override by design — the directory is + /// fixed so the host and `cpex plugin install` cannot disagree. The e2e + /// tests scaffold a plugin into a temp dir and need the venv built there + /// rather than in the developer's real `plugins/`, which a test must not + /// touch. + /// + /// Rebuilds the venv manager, since the layout is derived from these dirs. + #[cfg(any(test, feature = "testing"))] + pub fn with_plugin_dirs(mut self, dirs: Vec) -> Self { + self.venv = VenvManager::new( + &dirs, + &self.venv_config.class_name, + self.venv_config.requirements_file.as_deref(), + self.config.version.as_deref(), + ) + .ok(); + self.plugin_dirs = dirs; + self + } + + /// Point the plugin at an explicit worker script, skipping venv build and + /// script resolution. + /// + /// Test-only seam: the adapter tests need to exercise dispatch against a + /// stub worker without paying for a real venv, and the credential tests + /// need a worker that reports exactly what it received. + #[cfg(any(test, feature = "testing"))] + pub fn with_worker_override(mut self, script: PathBuf) -> Self { + self.worker_override = Some(script); + self + } + + /// The parsed venv settings. + pub fn venv_config(&self) -> &VenvConfig { + &self.venv_config + } + + /// The live worker's retained stderr, for leak assertions in tests. + /// + /// Returns an empty vector when no worker is running. + #[cfg(any(test, feature = "testing"))] + pub async fn worker_stderr(&self) -> Vec { + match self.worker.read().await.as_ref() { + Some(worker) => worker.stderr_lines().await, + None => Vec::new(), + } + } + + /// The live worker, or a plugin error when `initialize()` has not run. + async fn worker(&self) -> Result, Box> { + self.worker.read().await.clone().ok_or_else(|| { + HostError::WorkerStart { + message: "plugin has no worker — initialize() did not run or it failed".into(), + } + .into_plugin_error(&self.config.name) + }) + } + + /// Build the `load_and_run_hook` task for one invocation. + /// + /// The field names and encodings are the protocol contract with + /// `worker.py`: `config` is a JSON *string* the worker `json.loads`es into + /// `PluginConfig(**config_raw)`, and `context` is the worker's own + /// `{state, global_context, metadata}` shape rather than the Rust + /// `PluginContext` layout. + fn build_task( + &self, + hook_name: &str, + payload: serde_json::Value, + ctx: &PluginContext, + ) -> Result { + // The worker constructs a Python `PluginConfig` from this, so it must + // carry that model's field names — not this host's venv settings. + let config_json = serde_json::json!({ + "name": self.config.name, + "kind": self.config.kind, + "hooks": self.config.hooks, + "config": self.config.config.clone().unwrap_or(serde_json::Value::Null), + }); + let config_string = + serde_json::to_string(&config_json).map_err(|e| HostError::Protocol { + message: format!("could not serialize the plugin config for the worker: {e}"), + })?; + + // Prefer a request id the pipeline already put in global state, so a + // plugin's logs correlate with the gateway's. `PluginContext` has no + // dedicated field for one, so fall back to a fresh UUID rather than + // sending a placeholder that would collide across requests. + let request_id = ctx + .get_global(GLOBAL_REQUEST_ID_KEY) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + + Ok(serde_json::json!({ + "task_type": crate::worker::TASK_LOAD_AND_RUN_HOOK, + "plugin_dirs": self.plugin_dirs, + "class_name": self.venv_config.class_name, + "config": config_string, + "hook_type": hook_name, + "plugin_name": self.config.name, + "payload": payload, + "context": { + // `local_state` is this plugin's private slice; `global_state` + // is the pipeline-wide map the worker exposes as + // `global_context`. + "state": ctx.local_state, + "global_context": { + // `request_id` is the one *required* field on the Python + // `GlobalContext`; omitting it fails Pydantic validation + // inside the worker before the plugin is ever called. The + // Rust `PluginContext` carries no request id, so one is + // synthesized per invocation — a plugin that correlates + // logs by request id still gets a stable, unique value for + // the call. + "request_id": request_id, + "state": ctx.global_state, + }, + "metadata": {}, + }, + })) + } +} + +#[async_trait] +impl Plugin for IsolatedPythonPlugin { + fn config(&self) -> &PluginConfig { + &self.config + } + + /// Build the venv (or reuse a cached one) and launch the worker. + /// + /// Heavy setup lives here rather than on the invoke path: the manager + /// awaits this once per plugin, with rollback on failure. + async fn initialize(&self) -> Result<(), Box> { + let to_plugin_error = |e: HostError| e.into_plugin_error(&self.config.name); + + // With an override the worker script is given outright, so neither a + // venv nor script resolution is needed — but the interpreter still is. + let (python, script) = match self.worker_override.as_ref() { + Some(script) => (which_python3().map_err(to_plugin_error)?, script.clone()), + None => { + let venv = self.venv.as_ref().ok_or_else(|| { + to_plugin_error(HostError::Config { + message: format!( + "could not resolve a venv layout under {} — check that `class_name` \ + ('{}') begins with a package segment", + self.plugin_dirs.join(", "), + self.venv_config.class_name, + ), + }) + })?; + + venv.ensure().await.map_err(to_plugin_error)?; + let script = crate::worker::resolve_worker_script( + &venv.layout().venv_path, + &self.venv_config.script_path, + ) + .map_err(to_plugin_error)?; + + (venv.python_executable(), script) + }, + }; + + // Matching `venv_comm.py`'s `cwd=os.getcwd()`. The worker's + // ALLOWED_PLUGIN_DIRS allowlist accepts its own cwd, so inheriting the + // host's is what lets a plugin dir declared relative to the gateway be + // importable at all. + let worker = WorkerClient::spawn( + &python, + &script, + self.worker_cwd.as_deref(), + self.venv_config.max_content_size, + self.venv_config.timeout_secs, + ) + .await + .map_err(to_plugin_error)?; + + *self.worker.write().await = Some(Arc::new(worker)); + Ok(()) + } + + /// Stop the worker, killing it if it will not exit on its own. + async fn shutdown(&self) -> Result<(), Box> { + let worker = self.worker.write().await.take(); + if let Some(worker) = worker { + worker + .shutdown() + .await + .map_err(|e| e.into_plugin_error(&self.config.name))?; + } + Ok(()) + } +} + +/// Resolve an absolute `python3` from PATH. +/// +/// `WorkerClient::spawn` takes an interpreter path rather than doing a PATH +/// lookup, so the override path needs one. Production goes through the venv's +/// own interpreter and never calls this. +fn which_python3() -> Result { + let candidates = [ + "/usr/bin/python3", + "/usr/local/bin/python3", + "/opt/homebrew/bin/python3", + ]; + if let Some(found) = candidates.iter().map(Path::new).find(|p| p.exists()) { + return Ok(found.to_path_buf()); + } + + // Fall back to asking the shell, which honors PATH (pyenv, conda, nix). + let output = std::process::Command::new("sh") + .args(["-c", "command -v python3"]) + .output() + .map_err(|e| HostError::WorkerStart { + message: format!("could not look up python3: {e}"), + })?; + + let path = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if path.is_empty() { + return Err(HostError::WorkerStart { + message: "python3 was not found on PATH".into(), + }); + } + Ok(PathBuf::from(path)) +} + +/// Handler for one hook of one Python plugin. +/// +/// Holds the hook name because a single plugin object serves every hook it +/// declared, and the worker needs to be told which one to run. +pub struct PythonHookAdapter { + plugin: Arc, + hook_name: &'static str, +} + +impl PythonHookAdapter { + pub fn new(plugin: Arc, hook_name: &'static str) -> Self { + Self { plugin, hook_name } + } +} + +#[async_trait] +impl AnyHookHandler for PythonHookAdapter { + /// Serialize, send, and convert one hook invocation. + /// + /// Every failure returns a `PluginError` and stops there — the *executor* + /// applies the plugin's configured `on_error` policy (fail / ignore / + /// disable). This host deliberately does not interpret that policy itself; + /// doing so would double-apply it. + async fn invoke( + &self, + payload: &dyn PluginPayload, + extensions: &Extensions, + ctx: &mut PluginContext, + ) -> Result, Box> { + let plugin_name = &self.plugin.config.name; + let to_plugin_error = |e: HostError| e.into_plugin_error(plugin_name); + + let payload_json = conversion::serialize_payload(payload).map_err(to_plugin_error)?; + let mut task = self + .plugin + .build_task(self.hook_name, payload_json, ctx) + .map_err(to_plugin_error)?; + + // Raw credentials, for the two hooks that carry them and only for a + // plugin that declared the matching capability. Fails closed. + credentials::attach_credential( + &mut task, + self.hook_name, + extensions, + &self.plugin.config.capabilities, + ) + .map_err(to_plugin_error)?; + + let worker = self.plugin.worker().await?; + let response = worker.send_task(task).await.map_err(to_plugin_error)?; + + // State the worker sends back is merged into the caller's context + // before the result is returned, so a plugin's context writes survive. + apply_context_deltas(&response, ctx); + + let fields = + conversion::response_to_result(self.hook_name, response).map_err(to_plugin_error)?; + Ok(Box::new(fields)) + } + + fn hook_type_name(&self) -> &'static str { + self.hook_name + } +} + +/// Merge worker-returned context state back into the mutable plugin context. +/// +/// The worker returns the context under the same `{state, global_context}` +/// shape it received. Merging rather than replacing is deliberate: a plugin +/// that touched one key must not blank out the rest of the pipeline's state. +fn apply_context_deltas(response: &serde_json::Value, ctx: &mut PluginContext) { + let Some(context) = response.get("context") else { + return; + }; + + if let Some(state) = context.get("state").and_then(|v| v.as_object()) { + for (key, value) in state { + ctx.local_state.insert(key.clone(), value.clone()); + } + } + + if let Some(global) = context + .get("global_context") + .and_then(|g| g.get("state")) + .and_then(|v| v.as_object()) + { + for (key, value) in global { + ctx.global_state.insert(key.clone(), value.clone()); + } + } +} + +#[cfg(test)] +mod tests { + use crate::testing::TempDir; + + use super::*; + + fn config_with(value: serde_json::Value) -> PluginConfig { + PluginConfig { + name: "py-plugin".into(), + kind: crate::factory::KIND.into(), + hooks: vec!["tool_pre_invoke".into()], + config: Some(value), + ..Default::default() + } + } + + #[test] + fn full_config_block_deserializes_every_field() { + let cfg = VenvConfig::from_plugin_config(&config_with(serde_json::json!({ + "class_name": "my_pkg.filters.PiiFilter", + "requirements_file": "requirements.txt", + "plugin_dirs": ["./plugins", "./vendor"], + "script_path": "custom/worker.py", + "max_content_size": 2048, + "timeout_secs": 5, + }))) + .expect("a fully populated block parses"); + + assert_eq!(cfg.class_name, "my_pkg.filters.PiiFilter"); + assert_eq!(cfg.requirements_file.as_deref(), Some("requirements.txt")); + assert_eq!(cfg.script_path, "custom/worker.py"); + assert_eq!(cfg.max_content_size, 2048); + assert_eq!(cfg.timeout_secs, 5); + // The block above also carries `plugin_dirs`, which is no longer a + // field — an unknown key must parse harmlessly rather than error, so + // an existing config keeps loading. + } + + #[test] + fn absent_optionals_fall_back_to_documented_defaults() { + let cfg = VenvConfig::from_plugin_config(&config_with(serde_json::json!({ + "class_name": "my_pkg.Minimal", + }))) + .expect("class_name alone is a valid block"); + + // These three defaults are the contract with `client.py` / + // `venv_comm.py` — a drift here changes behavior silently. + assert_eq!( + cfg.max_content_size, 10_000_000, + "matches client.py's max_content_size" + ); + assert_eq!( + cfg.timeout_secs, 30, + "matches venv_comm.py's send_task timeout" + ); + assert_eq!(cfg.script_path, "cpex/framework/isolated/worker.py"); + + assert!( + cfg.requirements_file.is_none(), + "no requirements file is valid" + ); + } + + #[test] + fn missing_class_name_errors() { + let err = VenvConfig::from_plugin_config(&config_with(serde_json::json!({ + "plugin_dirs": ["./plugins"], + }))) + .expect_err("class_name is required"); + + assert!(matches!(*err, PluginError::Config { .. })); + assert!(err.to_string().contains("class_name")); + } + + #[test] + fn absent_config_block_errors_rather_than_panicking() { + let cfg = PluginConfig { + name: "no-config".into(), + kind: crate::factory::KIND.into(), + hooks: vec!["tool_pre_invoke".into()], + config: None, + ..Default::default() + }; + + let err = VenvConfig::from_plugin_config(&cfg) + .expect_err("an absent block cannot supply class_name"); + assert!(matches!(*err, PluginError::Config { .. })); + } + + /// The plugin dir a config with no usable `plugin_dirs` anywhere resolves + /// to: `/plugins`. + fn expected_default_dir() -> String { + std::env::current_dir() + .unwrap() + .join("plugins") + .display() + .to_string() + } + + #[test] + fn plugin_dirs_default_to_plugins_at_the_project_root() { + // Neither key is set anywhere, which is the shape the host now expects. + let plugin = IsolatedPythonPlugin::from_config(&config_with(serde_json::json!({ + "class_name": "my_pkg.Minimal", + }))) + .expect("class_name alone is a valid block"); + + assert_eq!( + plugin.plugin_dirs(), + [expected_default_dir()], + "the host supplies the plugin dir; the config no longer can" + ); + // Absolute, so a later cwd change cannot move the venv out from under + // a running plugin. + assert!( + Path::new(&plugin.plugin_dirs()[0]).is_absolute(), + "the resolved dir must be absolute: {:?}", + plugin.plugin_dirs() + ); + } + + #[test] + fn a_plugin_dirs_key_in_the_config_block_is_ignored() { + // The key an existing config still carries. It must not steer the host, + // and it must not fail the parse either — an operator upgrading should + // see the warning (emitted by the factory) rather than a load error. + let plugin = IsolatedPythonPlugin::from_config(&config_with(serde_json::json!({ + "class_name": "my_pkg.Minimal", + "plugin_dirs": ["/somewhere/else", "/and/here"], + }))) + .expect("an ignored key must not fail the parse"); + + assert_eq!( + plugin.plugin_dirs(), + [expected_default_dir()], + "`plugin_dirs` in the config block must be ignored" + ); + } + + #[test] + fn the_top_level_plugin_dirs_key_is_also_ignored() { + // `cpex plugin install` writes the key here. cpex_core parses and + // discards it; this pins that it does not reach the host either. + let yaml = r#" +plugin_dirs: ["./top-level-dirs"] +plugins: + - name: py-plugin + kind: isolated_venv + hooks: [tool_pre_invoke] + config: + class_name: my_pkg.Minimal +"#; + let cpex_config: cpex_core::config::CpexConfig = + serde_yaml::from_str(yaml).expect("valid YAML"); + assert_eq!( + cpex_config.plugin_dirs, + vec!["./top-level-dirs"], + "the top-level key still parses — it is simply not the host's source" + ); + + let plugin = + IsolatedPythonPlugin::from_config(&cpex_config.plugins[0]).expect("config parses"); + assert_eq!( + plugin.plugin_dirs(), + [expected_default_dir()], + "the top-level key must not steer the host's plugin dir" + ); + } + + #[test] + fn the_resolved_plugin_dir_is_what_the_worker_receives() { + // The dir has to reach the worker's `sys.path`, not just the venv + // layout — otherwise the venv is built in the right place and the + // import still fails. + let plugin = IsolatedPythonPlugin::from_config(&config_with(serde_json::json!({ + "class_name": "my_pkg.Minimal", + "plugin_dirs": ["/ignored"], + }))) + .expect("parses"); + + let task = plugin + .build_task( + "tool_pre_invoke", + serde_json::json!({"name": "t"}), + &PluginContext::new(), + ) + .expect("task builds"); + + assert_eq!( + task["plugin_dirs"], + serde_json::json!([expected_default_dir()]), + "the worker must be told the resolved dir, not the ignored one" + ); + } + + // --- adapter dispatch --------------------------------------------------- + + /// A stub worker that answers `load_and_run_hook` with a canned result and + /// records the task it received. + /// + /// Behavior is driven by a marker file so a single script covers every + /// case: the test writes `mode` next to the script before invoking. + const ADAPTER_STUB: &str = r#" +import json, os, sys + +here = os.path.dirname(os.path.abspath(__file__)) + +def mode(): + try: + with open(os.path.join(here, "mode")) as f: + return f.read().strip() + except FileNotFoundError: + return "allow" + +while True: + line = sys.stdin.readline() + if not line: + break + line = line.strip() + if not line: + continue + task = json.loads(line) + rid = task.get("request_id", "unknown") + + if task.get("task_type") == "shutdown": + print(json.dumps({"status": "success", "request_id": rid}), flush=True) + break + + # Record the task so the test can assert on what the host actually sent. + with open(os.path.join(here, "last_task.json"), "w") as f: + json.dump(task, f) + + m = mode() + if m == "deny": + response = { + "continue_processing": False, + "violation": {"code": "PII_DETECTED", "reason": "email in args", "details": {"field": "q"}}, + } + elif m == "error": + response = {"status": "error", "message": "the plugin raised ValueError"} + elif m == "context": + response = { + "continue_processing": True, + "context": {"state": {"seen": 1}, "global_context": {"state": {"shared": "yes"}}}, + } + elif m == "modify": + response = { + "continue_processing": True, + "modified_payload": {"name": "search", "args": {"q": "[REDACTED]"}}, + } + else: + response = {"continue_processing": True} + + response["request_id"] = rid + print(json.dumps(response), flush=True) +"#; + + /// Build a plugin wired to the adapter stub, plus its scratch dir. + async fn adapter_plugin( + hooks: &[&str], + capabilities: &[&str], + ) -> (TempDir, Arc) { + let dir = TempDir::new(); + let script = dir.path().join("adapter_stub.py"); + std::fs::write(&script, ADAPTER_STUB).unwrap(); + + let config = PluginConfig { + name: "py-plugin".into(), + kind: crate::factory::KIND.into(), + hooks: hooks.iter().map(|h| (*h).to_string()).collect(), + capabilities: capabilities.iter().map(|c| (*c).to_string()).collect(), + config: Some(serde_json::json!({ "class_name": "my_pkg.Plugin" })), + ..Default::default() + }; + + let plugin = Arc::new( + IsolatedPythonPlugin::from_config(&config) + .expect("config parses") + .with_worker_override(script), + ); + plugin.initialize().await.expect("the stub worker launches"); + (dir, plugin) + } + + fn set_mode(dir: &TempDir, mode: &str) { + std::fs::write(dir.path().join("mode"), mode).unwrap(); + } + + fn last_task(dir: &TempDir) -> serde_json::Value { + let raw = std::fs::read_to_string(dir.path().join("last_task.json")) + .expect("the stub recorded a task"); + serde_json::from_str(&raw).expect("recorded task is JSON") + } + + /// Invoke a hook through the adapter and return the erased result fields. + async fn invoke( + plugin: &Arc, + hook: &'static str, + payload: &dyn PluginPayload, + ctx: &mut PluginContext, + ) -> Result> { + let adapter = PythonHookAdapter::new(Arc::clone(plugin), hook); + let erased = adapter.invoke(payload, &Extensions::default(), ctx).await?; + Ok(cpex_core::executor::extract_erased(erased) + .expect("the adapter returns ErasedResultFields")) + } + + #[tokio::test] + async fn an_invoke_round_trips_and_returns_allow() { + if crate::testing::skip_without_python3("an_invoke_round_trips_and_returns_allow") { + return; + } + let (dir, plugin) = adapter_plugin(&["tool_pre_invoke"], &[]).await; + set_mode(&dir, "allow"); + + let payload = crate::legacy::ToolPreInvokePayload { + name: "search".into(), + args: Some(std::collections::HashMap::from([( + "q".into(), + serde_json::json!("rust"), + )])), + headers: None, + }; + let mut ctx = PluginContext::new(); + let fields = invoke(&plugin, "tool_pre_invoke", &payload, &mut ctx) + .await + .expect("the round trip succeeds"); + + assert!(fields.continue_processing); + assert!(fields.violation.is_none()); + + // The task the worker received must carry the protocol contract. + let task = last_task(&dir); + assert_eq!(task["task_type"], "load_and_run_hook"); + assert_eq!(task["hook_type"], "tool_pre_invoke"); + assert_eq!(task["plugin_name"], "py-plugin"); + assert_eq!(task["class_name"], "my_pkg.Plugin"); + assert_eq!(task["payload"]["name"], "search"); + assert_eq!(task["payload"]["args"]["q"], "rust"); + + // `config` is a JSON *string* the worker json.loads()es — not an object. + let config_str = task["config"] + .as_str() + .expect("config must be a JSON string"); + let config: serde_json::Value = serde_json::from_str(config_str).unwrap(); + assert_eq!(config["name"], "py-plugin"); + + plugin.shutdown().await.expect("shuts down"); + } + + #[tokio::test] + async fn a_deny_response_becomes_a_deny_result_with_its_violation() { + if crate::testing::skip_without_python3( + "a_deny_response_becomes_a_deny_result_with_its_violation", + ) { + return; + } + let (dir, plugin) = adapter_plugin(&["tool_pre_invoke"], &[]).await; + set_mode(&dir, "deny"); + + let payload = crate::legacy::ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }; + let mut ctx = PluginContext::new(); + let fields = invoke(&plugin, "tool_pre_invoke", &payload, &mut ctx) + .await + .expect("a deny is a successful invocation that returns a deny"); + + assert!(!fields.continue_processing); + let violation = fields.violation.expect("a deny must carry its violation"); + assert_eq!(violation.code, "PII_DETECTED"); + assert_eq!(violation.reason, "email in args"); + assert_eq!(violation.details["field"], "q"); + + plugin.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn a_worker_error_surfaces_as_a_plugin_error_for_the_executor_to_police() { + // The host returns the error and stops. Applying the on_error policy is + // the executor's job — doing it here would double-apply it. + if crate::testing::skip_without_python3( + "a_worker_error_surfaces_as_a_plugin_error_for_the_executor_to_police", + ) { + return; + } + let (dir, plugin) = adapter_plugin(&["tool_pre_invoke"], &[]).await; + set_mode(&dir, "error"); + + let payload = crate::legacy::ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }; + let mut ctx = PluginContext::new(); + let Err(err) = invoke(&plugin, "tool_pre_invoke", &payload, &mut ctx).await else { + panic!("a worker error response must not read as success"); + }; + + match *err { + PluginError::Execution { + ref plugin_name, + ref code, + ref message, + .. + } => { + assert_eq!(plugin_name, "py-plugin"); + assert_eq!(code.as_deref(), Some("worker_error")); + assert!( + message.contains("ValueError"), + "the plugin's message should survive: {message}" + ); + }, + ref other => panic!("expected an Execution error, got {other:?}"), + } + + plugin.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn context_deltas_are_written_back_into_the_plugin_context() { + if crate::testing::skip_without_python3( + "context_deltas_are_written_back_into_the_plugin_context", + ) { + return; + } + let (dir, plugin) = adapter_plugin(&["tool_pre_invoke"], &[]).await; + set_mode(&dir, "context"); + + let payload = crate::legacy::ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }; + let mut ctx = PluginContext::new(); + ctx.set_local("preexisting", serde_json::json!("kept")); + + invoke(&plugin, "tool_pre_invoke", &payload, &mut ctx) + .await + .unwrap(); + + assert_eq!(ctx.get_local("seen"), Some(&serde_json::json!(1))); + assert_eq!(ctx.get_global("shared"), Some(&serde_json::json!("yes"))); + assert_eq!( + ctx.get_local("preexisting"), + Some(&serde_json::json!("kept")), + "deltas merge — a plugin touching one key must not blank the rest" + ); + + plugin.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn a_modified_payload_comes_back_as_the_hooks_typed_payload() { + if crate::testing::skip_without_python3( + "a_modified_payload_comes_back_as_the_hooks_typed_payload", + ) { + return; + } + let (dir, plugin) = adapter_plugin(&["tool_pre_invoke"], &[]).await; + set_mode(&dir, "modify"); + + let payload = crate::legacy::ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }; + let mut ctx = PluginContext::new(); + let fields = invoke(&plugin, "tool_pre_invoke", &payload, &mut ctx) + .await + .unwrap(); + + let modified = fields.modified_payload.expect("the modification survives"); + let typed = modified + .as_any() + .downcast_ref::() + .expect("must come back as the tool payload or the executor's downcast fails"); + assert_eq!(typed.args.as_ref().unwrap()["q"], "[REDACTED]"); + + plugin.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn a_cmf_hook_sends_the_message_payload_shape() { + if crate::testing::skip_without_python3("a_cmf_hook_sends_the_message_payload_shape") { + return; + } + let (dir, plugin) = adapter_plugin(&["cmf.tool_pre_invoke"], &[]).await; + set_mode(&dir, "allow"); + + let payload = cpex_core::cmf::MessagePayload { + message: cpex_core::cmf::Message::text(cpex_core::cmf::Role::User, "hello"), + }; + let mut ctx = PluginContext::new(); + invoke(&plugin, "cmf.tool_pre_invoke", &payload, &mut ctx) + .await + .unwrap(); + + let task = last_task(&dir); + assert_eq!(task["hook_type"], "cmf.tool_pre_invoke"); + assert_eq!( + task["payload"]["message"]["role"], "user", + "a cmf hook must send the CMF message payload, not the generic wrapper" + ); + + plugin.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn invoking_before_initialize_is_a_plugin_error_not_a_panic() { + let config = PluginConfig { + name: "uninitialized".into(), + kind: crate::factory::KIND.into(), + hooks: vec!["tool_pre_invoke".into()], + config: Some(serde_json::json!({ "class_name": "my_pkg.Plugin" })), + ..Default::default() + }; + let plugin = Arc::new(IsolatedPythonPlugin::from_config(&config).unwrap()); + + let payload = crate::legacy::ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }; + let mut ctx = PluginContext::new(); + let Err(err) = invoke(&plugin, "tool_pre_invoke", &payload, &mut ctx).await else { + panic!("dispatch without a worker cannot succeed"); + }; + assert!(matches!(*err, PluginError::Execution { .. })); + } + + #[tokio::test] + async fn a_credential_capability_attaches_the_dto_on_an_identity_hook() { + // Ties the capability gate to real dispatch: the DTO must reach the + // worker's task, on the contract's field name. + if crate::testing::skip_without_python3( + "a_credential_capability_attaches_the_dto_on_an_identity_hook", + ) { + return; + } + use cpex_core::extensions::raw_credentials::{ + RawCredentialsExtension, RawInboundToken, TokenKind, TokenRole, + }; + + let (dir, plugin) = + adapter_plugin(&["identity_resolve"], &["read_inbound_credentials"]).await; + set_mode(&dir, "allow"); + + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new("E2E-SECRET-TOKEN", "Authorization", TokenKind::Jwt), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let payload = crate::legacy::IdentityResolvePayload::default(); + let mut ctx = PluginContext::new(); + let adapter = PythonHookAdapter::new(Arc::clone(&plugin), "identity_resolve"); + adapter + .invoke(&payload, &extensions, &mut ctx) + .await + .expect("dispatch succeeds"); + + let task = last_task(&dir); + assert_eq!(task["credential"]["inbound"]["token"], "E2E-SECRET-TOKEN"); + assert_eq!(task["credential"]["inbound"]["kind"], "jwt"); + + plugin.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn no_capability_means_no_credential_reaches_the_worker() { + if crate::testing::skip_without_python3( + "no_capability_means_no_credential_reaches_the_worker", + ) { + return; + } + use cpex_core::extensions::raw_credentials::{ + RawCredentialsExtension, RawInboundToken, TokenKind, TokenRole, + }; + + // Same hook, same request — this plugin simply declared nothing. + let (dir, plugin) = adapter_plugin(&["identity_resolve"], &[]).await; + set_mode(&dir, "allow"); + + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new("E2E-SECRET-TOKEN", "Authorization", TokenKind::Jwt), + ); + let extensions = Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + }; + + let payload = crate::legacy::IdentityResolvePayload::default(); + let mut ctx = PluginContext::new(); + let adapter = PythonHookAdapter::new(Arc::clone(&plugin), "identity_resolve"); + adapter + .invoke(&payload, &extensions, &mut ctx) + .await + .unwrap(); + + let task = last_task(&dir); + assert!(task.get("credential").is_none(), "no credential field"); + assert!( + !serde_json::to_string(&task) + .unwrap() + .contains("E2E-SECRET-TOKEN"), + "no token bytes anywhere in the task the worker received" + ); + + plugin.shutdown().await.unwrap(); + } + + #[test] + fn unknown_config_keys_are_tolerated() { + // Plugin `config:` blocks are shared with the Python side, which reads + // keys this host does not model. Rejecting them would break configs + // that work under the Python CLI. + let cfg = VenvConfig::from_plugin_config(&config_with(serde_json::json!({ + "class_name": "my_pkg.Minimal", + "some_python_only_setting": true, + }))) + .expect("extra keys belong to the Python plugin, not this host"); + assert_eq!(cfg.class_name, "my_pkg.Minimal"); + } +} diff --git a/crates/cpex-hosts-python/src/testing.rs b/crates/cpex-hosts-python/src/testing.rs new file mode 100644 index 00000000..cc4731b3 --- /dev/null +++ b/crates/cpex-hosts-python/src/testing.rs @@ -0,0 +1,283 @@ +// Location: ./crates/cpex-hosts-python/src/testing.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Test-only helpers. Compiled under `cfg(test)` for unit tests and behind the +// `testing` feature for the integration tests in `tests/`, which cannot see a +// `cfg(test)` module. +// +// The venv and worker tests need a scratch directory that cleans itself up. +// The workspace carries no temp-dir dependency, and adding one for this is not +// worth the supply-chain surface for ~40 lines. + +use std::path::{Path, PathBuf}; + +/// A scratch directory removed when the value drops. +/// +/// Names combine the process id with a monotonic counter, so concurrent test +/// threads (and concurrent `cargo test` invocations) never collide. +#[derive(Debug)] +pub struct TempDir { + path: PathBuf, +} + +impl TempDir { + /// Create a uniquely named directory under the system temp dir. + pub fn new() -> Self { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + + let unique = COUNTER.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("cpex-hosts-python-{}-{unique}", std::process::id())); + + // A leftover directory from a killed run would otherwise leak state + // into this one. + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).expect("create temp dir"); + + Self { path } + } + + /// The directory path. + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Default for TempDir { + fn default() -> Self { + Self::new() + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + // Best-effort: a failed cleanup must not mask the test's own failure. + let _ = std::fs::remove_dir_all(&self.path); + } +} + +/// Whether a usable `python3` is on PATH. +/// +/// The venv and end-to-end tests need a real interpreter. Per the plan they +/// *skip* rather than fail when one is absent, so CI without python3 stays +/// green — while not pretending to have covered the path. +pub fn python3_available() -> bool { + std::process::Command::new("python3") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +/// Skip the calling test (with a printed reason) when python3 is missing. +/// +/// Rust has no first-class runtime skip, so this returns a bool the caller +/// early-returns on. The printed line is what distinguishes "skipped" from +/// "passed" when reading `cargo test -- --nocapture` output. +#[must_use] +pub fn skip_without_python3(test_name: &str) -> bool { + if python3_available() { + return false; + } + println!("SKIP {test_name}: python3 not found on PATH"); + true +} + +/// Env var naming a checkout of the `cpex` Python package for the e2e tests. +pub const SOURCE_ENV: &str = "CPEX_PYTHON_SOURCE"; + +/// Locate a `cpex` Python source tree, or explain why there is none. +/// +/// The Python framework lives on a different branch than this Rust host, and +/// the published PyPI package is behind it (its `worker.py` predates the +/// credential field). So the e2e tests install from a local checkout: set +/// `CPEX_PYTHON_SOURCE`, or keep a sibling checkout. +pub fn python_source() -> Result { + if let Ok(explicit) = std::env::var(SOURCE_ENV) { + let path = PathBuf::from(&explicit); + if path.join("pyproject.toml").is_file() { + return Ok(path); + } + return Err(format!("{SOURCE_ENV}={explicit} has no pyproject.toml")); + } + + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .ok_or("could not locate the repository root")?; + + [ + repo_root.join("cpex-python"), + repo_root.join("../cpex-python"), + repo_root.to_path_buf(), + ] + .iter() + .find(|p| { + p.join("pyproject.toml").is_file() && p.join("cpex/framework/isolated/worker.py").is_file() + }) + .cloned() + .ok_or_else(|| { + format!( + "no cpex Python source tree found — set {SOURCE_ENV} to a checkout of the Python side \ + (the branch carrying cpex/framework/isolated/worker.py)" + ) + }) +} + +/// Whether a framework checkout's worker consumes the `credential` field. +/// +/// The credential path needs a `worker.py` that reads the DTO and repopulates +/// the redacted `SecretStr`. An older worker silently drops the field, so a +/// credential test should skip with a precise reason rather than fail. +pub fn worker_consumes_credentials(source: &Path) -> bool { + std::fs::read_to_string(source.join("cpex/framework/isolated/worker.py")) + .map(|s| s.contains("reconstruct_credential_payload") && s.contains("CREDENTIAL_FIELD")) + .unwrap_or(false) +} + +/// Lay out a plugin package under `/plugins/`. +/// +/// Writes `__init__.py`, the plugin module, and a `requirements.txt` pointing +/// at the local framework checkout — so the venv gets *that* `worker.py` rather +/// than whatever version PyPI currently serves. Returns the plugin dir. +pub fn scaffold_plugin( + dir: &TempDir, + source: &Path, + package: &str, + module: &str, + plugin_source: &str, +) -> PathBuf { + let plugin_dir = dir.path().join("plugins"); + let package_dir = plugin_dir.join(package); + std::fs::create_dir_all(&package_dir).expect("create package dir"); + + std::fs::write(package_dir.join("__init__.py"), "").expect("write __init__.py"); + std::fs::write(package_dir.join(format!("{module}.py")), plugin_source) + .expect("write plugin module"); + std::fs::write( + package_dir.join("requirements.txt"), + format!("{}\n", source.display()), + ) + .expect("write requirements.txt"); + + plugin_dir +} + +/// Pre-build a plugin's venv and write the cache metadata that makes the host +/// reuse it. +/// +/// # Why the test builds the venv instead of letting the host do it +/// +/// The Python framework's declared dependencies currently have no satisfiable +/// resolution: `pyproject.toml` requires `mcp>=1.26`, but +/// `cpex.framework.__init__` imports its MCP client, which does +/// `from mcp import McpError` — a symbol mcp renamed to `MCPError` in 1.26. A +/// clean install therefore pulls an mcp whose API the framework cannot import, +/// and the worker dies with an `ImportError` before reading a task. Adding +/// `mcp<1.26` to the requirements file instead makes pip fail outright with +/// `ResolutionImpossible`, because that contradicts the framework's own floor. +/// +/// The only combination that runs is "install as declared, then downgrade mcp" +/// — two sequential pip passes. The venv manager issues one `pip install -r`, +/// which is correct for a working package; contorting production code around a +/// contradictory upstream manifest would be the wrong fix, and the Python +/// framework is out of scope here. So the test arranges the venv and the host +/// then finds it cached, which exercises a real host path +/// (`CacheVerdict::Valid` → reuse) rather than bypassing one. +/// +/// Returns `Err` with a printable reason when the venv cannot be built. +pub fn prebuild_venv( + plugin_dir: &Path, + source: &Path, + class_name: &str, + package: &str, +) -> Result<(), String> { + let venv = crate::venv::VenvManager::new( + &[plugin_dir.display().to_string()], + class_name, + Some("requirements.txt"), + Some("1.0.0"), + ) + .map_err(|e| format!("could not resolve the venv layout: {e}"))?; + + let layout = venv.layout(); + std::fs::create_dir_all(&layout.cache_dir).map_err(|e| e.to_string())?; + + let run = |program: &Path, args: &[&std::ffi::OsStr]| -> Result<(), String> { + let output = std::process::Command::new(program) + .args(args) + .output() + .map_err(|e| format!("could not run {}: {e}", program.display()))?; + if output.status.success() { + return Ok(()); + } + Err(format!( + "{} exited {}: {}", + program.display(), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )) + }; + + run( + Path::new("python3"), + &["-m".as_ref(), "venv".as_ref(), layout.venv_path.as_os_str()], + )?; + + let python = layout.python_executable(); + // Pass 1: the framework as declared, with its full transitive tree. + run( + &python, + &[ + "-m".as_ref(), + "pip".as_ref(), + "install".as_ref(), + "-q".as_ref(), + source.as_os_str(), + ], + )?; + // Pass 2: downgrade mcp to a version whose API the framework can import. + // pip warns about the deliberate conflict; that warning is expected. + run( + &python, + &[ + "-m".as_ref(), + "pip".as_ref(), + "install".as_ref(), + "-q".as_ref(), + "mcp<1.26".as_ref(), + ], + )?; + + // Metadata matching what the host computes, so its cache check says Valid + // and `initialize()` skips the (unsatisfiable) reinstall. + let requirements = plugin_dir.join(package).join("requirements.txt"); + let metadata = serde_json::json!({ + "venv_path": layout.venv_path.display().to_string(), + "requirements_file": requirements.display().to_string(), + "requirements_hash": crate::venv::hash_file_or_empty(Some(&requirements)), + "manifest_version": "1.0.0", + // No manifest file on disk — the empty-content digest, matching what + // the host computes for an absent manifest. + "manifest_hash": crate::venv::hash_file_or_empty(None), + }); + std::fs::write( + &layout.metadata_path, + serde_json::to_string_pretty(&metadata).unwrap(), + ) + .map_err(|e| e.to_string())?; + + if !venv.cache_verdict().is_valid() { + return Err(format!( + "the pre-built venv is not recognized as cached ({:?}) — the metadata written here \ + disagrees with what the host computes", + venv.cache_verdict() + )); + } + Ok(()) +} diff --git a/crates/cpex-hosts-python/src/venv.rs b/crates/cpex-hosts-python/src/venv.rs new file mode 100644 index 00000000..ec832b32 --- /dev/null +++ b/crates/cpex-hosts-python/src/venv.rs @@ -0,0 +1,1110 @@ +// Location: ./crates/cpex-hosts-python/src/venv.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Venv manager — builds, caches, and reuses a per-plugin virtualenv. +// +// Ported from `cpex/framework/isolated/client.py` (`create_venv`, +// `_is_venv_cache_valid`, `_save_cache_metadata`, `_manifest_path`). The +// layout is deliberately identical to the Python CLI's so a venv built by +// either side is reusable by the other: `.venv` under the plugin's class-root +// directory, cache metadata under `.cpex/venv_cache`. +// +// # Cache validity +// +// Reuse is keyed on a SHA256 of the requirements file plus the persisted +// manifest's version and content hash. Requirements alone is not enough: a +// plugin installed by FQN conversion has no requirements file, so its +// requirements hash is a constant and the manifest signals are the only way +// to notice it changed. +// +// The manifest signals are read as *optional*. Metadata written by an older +// CLI has no `manifest_version` / `manifest_hash` keys, and reading an absent +// key as `None` and comparing it against a real value would treat every +// pre-existing install as a mismatch — wiping and rebuilding every venv on +// the first run after an upgrade. An absent key therefore means "no signal" +// and is skipped; only a key that is present *and* differs invalidates. +// +// # Blocking work +// +// `initialize()` is awaited sequentially by the manager, so a multi-minute +// pip install on the runtime thread would stall every other plugin's init +// and the rollback path with it. Subprocesses go through `tokio::process` +// and synchronous filesystem work through `spawn_blocking`. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::error::HostError; + +/// Cache metadata persisted next to each venv. +/// +/// Field names match what `client.py`'s `_save_cache_metadata` writes, so +/// metadata is interchangeable between the two implementations. +/// +/// `manifest_version` and `manifest_hash` are `Option` to distinguish +/// "absent" (older metadata, no signal) from "present and different" +/// (a real change). That distinction is the whole point — see the module +/// docs. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheMetadata { + /// Absolute path of the venv this metadata describes. + pub venv_path: String, + + /// Absolute path of the requirements file, when there was one. + #[serde(default)] + pub requirements_file: Option, + + /// SHA256 of the requirements file content (empty-content digest when + /// there is no file). + pub requirements_hash: String, + + /// Manifest version at install time. Absent in metadata written before + /// this signal existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest_version: Option, + + /// SHA256 of the persisted manifest content. Absent in older metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest_hash: Option, + + /// Interpreter version the venv was built with, for diagnostics. + #[serde(default)] + pub python_version: Option, +} + +/// Why a cached venv was rejected. Carried rather than logged-and-dropped so +/// tests can pin the specific rule that fired and operators get a precise +/// reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CacheVerdict { + /// The cache is usable as-is. + Valid, + /// The venv directory does not exist. + VenvMissing, + /// No metadata file sits alongside the venv. + MetadataMissing, + /// The metadata file could not be parsed. + MetadataUnreadable, + /// The requirements file content changed. + RequirementsChanged, + /// The manifest version changed (signal present in metadata). + ManifestVersionChanged, + /// The manifest content changed (signal present in metadata). + ManifestHashChanged, +} + +impl CacheVerdict { + /// Whether the cached venv can be reused without a rebuild. + pub fn is_valid(&self) -> bool { + matches!(self, Self::Valid) + } +} + +/// SHA256 of a file's bytes, hex-encoded. +/// +/// An absent or unreadable path hashes to the empty-content digest, matching +/// `client.py`'s `_compute_requirements_hash`: a plugin with no requirements +/// file gets a stable constant rather than an error, so the manifest signals +/// carry the change detection instead. +pub fn hash_file_or_empty(path: Option<&Path>) -> String { + let mut hasher = Sha256::new(); + match path { + Some(p) => match std::fs::read(p) { + Ok(bytes) => hasher.update(&bytes), + Err(_) => hasher.update(b""), + }, + None => hasher.update(b""), + } + format!("{:x}", hasher.finalize()) +} + +/// Filesystem-safe stem for a fully-qualified class name. +/// +/// Sanitization must match the Python side's `manifest_filename_for_class` +/// byte for byte: non-alphanumerics (other than `-` and `_`) become `-`, +/// leading and trailing `-` are stripped, and an empty result degrades to +/// `plugin` rather than producing a dotfile. +fn manifest_stem_for_class(class_name: &str) -> String { + let safe: String = class_name + .trim() + .chars() + .map(|c| { + if c.is_alphanumeric() || c == '-' || c == '_' { + c + } else { + '-' + } + }) + .collect(); + let safe = safe.trim_matches('-'); + if safe.is_empty() { + "plugin".to_string() + } else { + safe.to_string() + } +} + +/// Per-plugin manifest filename for a fully-qualified class name. +/// +/// Plugins that share a package share one venv directory (it is keyed on the +/// class root), but each needs its own manifest file: if `pkg.a.PluginA` and +/// `pkg.b.PluginB` shared one, installing either would change the other's +/// manifest hash and both would rebuild forever. +/// +/// Must produce byte-identical output to the Python side's +/// `manifest_filename_for_class` — both implementations resolve the same path, +/// and a divergence means each sees the other's install as a change. +pub fn manifest_filename_for_class(class_name: &str) -> String { + format!( + "{}.plugin-manifest.yaml", + manifest_stem_for_class(class_name) + ) +} + +/// Decide whether a cached venv can be reused. +/// +/// `expected_manifest_version` and `expected_manifest_hash` are what the +/// plugin declares *now*; the metadata carries what was true at install +/// time. A metadata signal that is absent is skipped rather than compared — +/// see the module docs for why that asymmetry matters. +pub fn evaluate_cache( + venv_path: &Path, + metadata_path: &Path, + expected_requirements_hash: &str, + expected_manifest_version: Option<&str>, + expected_manifest_hash: Option<&str>, +) -> CacheVerdict { + if !venv_path.exists() { + return CacheVerdict::VenvMissing; + } + if !metadata_path.exists() { + return CacheVerdict::MetadataMissing; + } + + let Ok(raw) = std::fs::read_to_string(metadata_path) else { + return CacheVerdict::MetadataUnreadable; + }; + let Ok(metadata) = serde_json::from_str::(&raw) else { + return CacheVerdict::MetadataUnreadable; + }; + + if metadata.requirements_hash != expected_requirements_hash { + return CacheVerdict::RequirementsChanged; + } + + // Present-and-different invalidates; absent is "no signal" and skipped. + if let Some(cached) = metadata.manifest_version.as_deref() { + if Some(cached) != expected_manifest_version { + return CacheVerdict::ManifestVersionChanged; + } + } + if let Some(cached) = metadata.manifest_hash.as_deref() { + if Some(cached) != expected_manifest_hash { + return CacheVerdict::ManifestHashChanged; + } + } + + CacheVerdict::Valid +} + +/// Resolved on-disk layout for one plugin's venv. +/// +/// The class *root* (the first dotted segment of the class name) names the +/// directory, which is what makes plugins in one package share a venv. +#[derive(Debug, Clone)] +pub struct VenvLayout { + /// Directory holding the venv and its cache — `/`. + pub plugin_path: PathBuf, + /// The virtualenv itself. + pub venv_path: PathBuf, + /// Cache-metadata directory. + pub cache_dir: PathBuf, + /// This plugin's metadata file within `cache_dir`. + pub metadata_path: PathBuf, + /// This plugin's persisted manifest. + pub manifest_path: PathBuf, +} + +impl VenvLayout { + /// Resolve the layout for a class name under the first configured plugin + /// dir, mirroring `client.py`'s `__init__`. + pub fn resolve(plugin_dirs: &[String], class_name: &str) -> Result { + // The host supplies `/plugins`, so an empty list here is + // an internal error rather than a misconfiguration — the message avoids + // naming a config key an operator could set, because there isn't one. + let first = plugin_dirs.first().ok_or_else(|| HostError::Config { + message: "isolated_venv requires at least one plugin directory — \ + the venv is built under the first one" + .into(), + })?; + + let class_root = class_name.split('.').next().unwrap_or(class_name); + if class_root.is_empty() { + return Err(HostError::Config { + message: format!("`class_name` '{class_name}' has no leading package segment"), + }); + } + + let plugin_path = Path::new(first).join(class_root); + let venv_path = plugin_path.join(".venv"); + let cache_dir = plugin_path.join(".cpex").join("venv_cache"); + + // Both filenames are keyed on the full class name. + // + // This diverges from `client.py`, which keys the *metadata* filename on + // the venv directory name (`_get_cache_metadata_path` reads + // `Path(venv_path).name`, always `.venv`) while keying the *manifest* + // per class. Since plugins in one package deliberately share a venv + // directory, that gives them one shared `.venv_metadata.json`: each + // install overwrites the other's `requirements_hash` and + // `manifest_hash`, so each plugin's build invalidates its neighbour's + // cache and the two rebuild in a loop — the exact thrash the per-class + // manifest naming exists to prevent, left half-solved. + // + // Keying both on the class name closes it. The cost is that a venv + // built by the Python CLI is not recognized as cached by this host on + // first run (its metadata sits under the other filename), so the host + // rebuilds once and is a cache hit forever after. That one-time + // rebuild is strictly better than a permanent rebuild loop. + let metadata_path = cache_dir.join(format!( + "{}_metadata.json", + manifest_stem_for_class(class_name) + )); + let manifest_path = plugin_path.join(manifest_filename_for_class(class_name)); + + Ok(Self { + plugin_path, + venv_path, + cache_dir, + metadata_path, + manifest_path, + }) + } + + /// The venv's own Python interpreter. + pub fn python_executable(&self) -> PathBuf { + if cfg!(windows) { + self.venv_path.join("Scripts").join("python.exe") + } else { + self.venv_path.join("bin").join("python") + } + } +} + +/// Outcome of a `VenvManager::ensure` call. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EnsureOutcome { + /// The cached venv was reused — no rebuild, no reinstall. + Reused, + /// The venv was created (or recreated) and requirements installed. + Built { + /// Whether a requirements install actually ran. False when the plugin + /// has no requirements file — the venv is still fresh. + installed_requirements: bool, + }, +} + +/// Builds and caches one plugin's virtualenv. +/// +/// Construction is cheap and does no I/O; `ensure()` does the work and is +/// called from the plugin's `initialize()`. +#[derive(Debug, Clone)] +pub struct VenvManager { + layout: VenvLayout, + /// Absolute path of the requirements file, when the plugin has one and it + /// exists on disk. + requirements_file: Option, + /// Manifest version the plugin declares now, if any. + manifest_version: Option, +} + +impl VenvManager { + /// Resolve the layout for a plugin and record its cache inputs. + /// + /// `requirements_file` is interpreted relative to the resolved plugin + /// path when it is not already absolute, mirroring `client.py`'s + /// `package_path / requirements_file_input`. + pub fn new( + plugin_dirs: &[String], + class_name: &str, + requirements_file: Option<&str>, + manifest_version: Option<&str>, + ) -> Result { + let layout = VenvLayout::resolve(plugin_dirs, class_name)?; + + let requirements_file = requirements_file.map(|rel| { + let path = Path::new(rel); + if path.is_absolute() { + path.to_path_buf() + } else { + layout.plugin_path.join(path) + } + }); + + Ok(Self { + layout, + requirements_file, + manifest_version: manifest_version.map(str::to_string), + }) + } + + /// The resolved on-disk layout. + pub fn layout(&self) -> &VenvLayout { + &self.layout + } + + /// The venv's interpreter, for launching the worker. + pub fn python_executable(&self) -> PathBuf { + self.layout.python_executable() + } + + /// Whether the cached venv is currently reusable, and why not if it isn't. + pub fn cache_verdict(&self) -> CacheVerdict { + evaluate_cache( + &self.layout.venv_path, + &self.layout.metadata_path, + &hash_file_or_empty(self.requirements_file.as_deref()), + self.manifest_version.as_deref(), + Some(&hash_file_or_empty(Some(&self.layout.manifest_path))), + ) + } + + /// Ensure a usable venv exists, building and installing only when the + /// cache says it must. + /// + /// All blocking work is moved off the runtime thread: the manager awaits + /// each plugin's `initialize()` sequentially, so a cold pip install run + /// inline would stall every other plugin's init and the rollback path. + pub async fn ensure(&self) -> Result { + let verdict = self.cache_verdict(); + if verdict.is_valid() { + tracing::info!(venv = %self.layout.venv_path.display(), "reusing cached venv"); + return Ok(EnsureOutcome::Reused); + } + + tracing::info!( + venv = %self.layout.venv_path.display(), + reason = ?verdict, + "building venv" + ); + + self.prepare_directories().await?; + self.create_venv().await?; + + let installed_requirements = self.install_requirements().await?; + self.save_metadata().await?; + + Ok(EnsureOutcome::Built { + installed_requirements, + }) + } + + /// Create the cache directory and clear any stale venv. + async fn prepare_directories(&self) -> Result<(), HostError> { + let venv_path = self.layout.venv_path.clone(); + let cache_dir = self.layout.cache_dir.clone(); + let plugin_path = self.layout.plugin_path.clone(); + + tokio::task::spawn_blocking(move || { + std::fs::create_dir_all(&plugin_path).map_err(|e| HostError::VenvBuild { + message: format!("could not create plugin dir {}: {e}", plugin_path.display()), + })?; + std::fs::create_dir_all(&cache_dir).map_err(|e| HostError::VenvBuild { + message: format!("could not create cache dir {}: {e}", cache_dir.display()), + })?; + + // An invalid cache means the venv contents cannot be trusted; + // client.py rmtree's before rebuilding rather than upgrading in + // place, so a removed dependency actually disappears. + if venv_path.exists() { + std::fs::remove_dir_all(&venv_path).map_err(|e| HostError::VenvBuild { + message: format!("could not remove stale venv {}: {e}", venv_path.display()), + })?; + } + Ok(()) + }) + .await + .map_err(|e| HostError::VenvBuild { + message: format!("venv directory preparation panicked: {e}"), + })? + } + + /// Run `python3 -m venv --system-site-packages=false` on the venv path. + /// + /// `client.py` uses `venv.EnvBuilder(with_pip=True, symlinks=True)`; the + /// CLI equivalent is `python3 -m venv` with pip left enabled. + async fn create_venv(&self) -> Result<(), HostError> { + let output = tokio::process::Command::new("python3") + .arg("-m") + .arg("venv") + .arg(&self.layout.venv_path) + .output() + .await + .map_err(|e| HostError::VenvBuild { + message: format!("could not run `python3 -m venv`: {e} (is python3 on PATH?)"), + })?; + + if !output.status.success() { + return Err(HostError::VenvBuild { + message: format!( + "`python3 -m venv {}` exited {}: {}", + self.layout.venv_path.display(), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }); + } + Ok(()) + } + + /// Install the plugin's requirements into the venv. + /// + /// Returns whether an install ran. A plugin with no requirements file is + /// not an error: an FQN-converted plugin gets its package from the install + /// channel instead, and installing requirements transitively brings in the + /// `cpex` framework (and with it `worker.py`). + async fn install_requirements(&self) -> Result { + let Some(requirements) = self.requirements_file.as_ref().filter(|p| p.exists()) else { + tracing::info!("no requirements file; skipping install"); + return Ok(false); + }; + + let python = self.python_executable(); + let output = tokio::process::Command::new(&python) + .args(["-m", "pip", "install", "-r"]) + .arg(requirements) + .output() + .await + .map_err(|e| HostError::VenvBuild { + message: format!("could not run pip in {}: {e}", python.display()), + })?; + + if !output.status.success() { + return Err(HostError::VenvBuild { + message: format!( + "pip install -r {} exited {}: {}", + requirements.display(), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }); + } + Ok(true) + } + + /// Persist the cache metadata that makes the next run a cache hit. + async fn save_metadata(&self) -> Result<(), HostError> { + let metadata = CacheMetadata { + venv_path: self.layout.venv_path.display().to_string(), + requirements_file: self + .requirements_file + .as_ref() + .filter(|p| p.exists()) + .map(|p| p.display().to_string()), + requirements_hash: hash_file_or_empty(self.requirements_file.as_deref()), + manifest_version: self.manifest_version.clone(), + manifest_hash: Some(hash_file_or_empty(Some(&self.layout.manifest_path))), + python_version: None, + }; + + let path = self.layout.metadata_path.clone(); + let json = serde_json::to_string_pretty(&metadata).map_err(|e| HostError::VenvBuild { + message: format!("could not serialize cache metadata: {e}"), + })?; + + tokio::task::spawn_blocking(move || { + std::fs::write(&path, json).map_err(|e| HostError::VenvBuild { + message: format!("could not write cache metadata {}: {e}", path.display()), + }) + }) + .await + .map_err(|e| HostError::VenvBuild { + message: format!("cache metadata write panicked: {e}"), + })? + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::TempDir; + + /// Empty-content SHA256 — what a plugin with no requirements file hashes + /// to, so it appears in most of these fixtures. + const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + /// Lay down a venv dir plus a metadata file, and return both paths. + fn scaffold(dir: &TempDir, metadata: &str) -> (PathBuf, PathBuf) { + let venv = dir.path().join(".venv"); + std::fs::create_dir_all(&venv).unwrap(); + let metadata_path = dir.path().join("metadata.json"); + std::fs::write(&metadata_path, metadata).unwrap(); + (venv, metadata_path) + } + + fn metadata_json( + requirements_hash: &str, + version: Option<&str>, + manifest_hash: Option<&str>, + ) -> String { + let mut obj = serde_json::json!({ + "venv_path": "/tmp/.venv", + "requirements_file": null, + "requirements_hash": requirements_hash, + "python_version": "3.12.1", + }); + if let Some(v) = version { + obj["manifest_version"] = serde_json::json!(v); + } + if let Some(h) = manifest_hash { + obj["manifest_hash"] = serde_json::json!(h); + } + serde_json::to_string(&obj).unwrap() + } + + // --- hashing ------------------------------------------------------------ + + #[test] + fn absent_requirements_file_hashes_to_the_empty_digest() { + // client.py hashes b"" for both "no file configured" and "configured + // but not on disk", so both reuse a venv rather than erroring. + assert_eq!(hash_file_or_empty(None), EMPTY_SHA256); + assert_eq!( + hash_file_or_empty(Some(Path::new("/nonexistent/requirements.txt"))), + EMPTY_SHA256 + ); + } + + #[test] + fn requirements_hash_tracks_content_not_path() { + let dir = TempDir::new(); + let a = dir.path().join("a.txt"); + let b = dir.path().join("b.txt"); + std::fs::write(&a, "requests==2.31.0\n").unwrap(); + std::fs::write(&b, "requests==2.31.0\n").unwrap(); + assert_eq!(hash_file_or_empty(Some(&a)), hash_file_or_empty(Some(&b))); + + std::fs::write(&b, "requests==2.32.0\n").unwrap(); + assert_ne!(hash_file_or_empty(Some(&a)), hash_file_or_empty(Some(&b))); + } + + // --- manifest filename (contract with the Python side) ------------------ + + #[test] + fn manifest_filename_matches_the_python_sanitization_rule() { + // Byte-for-byte agreement with utils.manifest_filename_for_class: + // dots become dashes, alphanumerics and -_ survive. + assert_eq!( + manifest_filename_for_class("pkg.module.ClassName"), + "pkg-module-ClassName.plugin-manifest.yaml" + ); + assert_eq!( + manifest_filename_for_class("my_pkg.filters.PiiFilter"), + "my_pkg-filters-PiiFilter.plugin-manifest.yaml" + ); + // Leading/trailing separators are stripped, and an all-separator name + // degrades to "plugin" rather than producing a dotfile. + assert_eq!( + manifest_filename_for_class(" .A. "), + "A.plugin-manifest.yaml" + ); + assert_eq!( + manifest_filename_for_class("..."), + "plugin.plugin-manifest.yaml" + ); + } + + #[test] + fn plugins_sharing_a_package_get_distinct_manifest_filenames() { + // The anti-thrash guarantee: one shared venv dir, one manifest each. + assert_ne!( + manifest_filename_for_class("pkg.a.PluginA"), + manifest_filename_for_class("pkg.b.PluginB") + ); + } + + // --- cache validity ----------------------------------------------------- + + #[test] + fn valid_when_every_present_signal_agrees() { + let dir = TempDir::new(); + let (venv, meta) = scaffold( + &dir, + &metadata_json(EMPTY_SHA256, Some("1.0.0"), Some("abc123")), + ); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, Some("1.0.0"), Some("abc123")), + CacheVerdict::Valid + ); + } + + #[test] + fn invalid_when_the_venv_directory_is_gone() { + let dir = TempDir::new(); + let meta = dir.path().join("metadata.json"); + std::fs::write(&meta, metadata_json(EMPTY_SHA256, None, None)).unwrap(); + + assert_eq!( + evaluate_cache(&dir.path().join(".venv"), &meta, EMPTY_SHA256, None, None), + CacheVerdict::VenvMissing + ); + } + + #[test] + fn invalid_when_metadata_is_absent() { + let dir = TempDir::new(); + let venv = dir.path().join(".venv"); + std::fs::create_dir_all(&venv).unwrap(); + + assert_eq!( + evaluate_cache( + &venv, + &dir.path().join("missing.json"), + EMPTY_SHA256, + None, + None + ), + CacheVerdict::MetadataMissing + ); + } + + #[test] + fn invalid_when_metadata_will_not_parse() { + // client.py catches JSONDecodeError and returns False — a corrupt + // metadata file rebuilds rather than propagating. + let dir = TempDir::new(); + let (venv, meta) = scaffold(&dir, "{ this is not json"); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, None, None), + CacheVerdict::MetadataUnreadable + ); + } + + #[test] + fn changed_requirements_hash_invalidates() { + let dir = TempDir::new(); + let (venv, meta) = scaffold( + &dir, + &metadata_json("old-hash", Some("1.0.0"), Some("abc123")), + ); + + assert_eq!( + evaluate_cache(&venv, &meta, "new-hash", Some("1.0.0"), Some("abc123")), + CacheVerdict::RequirementsChanged + ); + } + + #[test] + fn changed_manifest_version_invalidates_when_the_signal_is_present() { + let dir = TempDir::new(); + let (venv, meta) = scaffold( + &dir, + &metadata_json(EMPTY_SHA256, Some("1.0.0"), Some("abc123")), + ); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, Some("2.0.0"), Some("abc123")), + CacheVerdict::ManifestVersionChanged + ); + } + + #[test] + fn changed_manifest_hash_invalidates_when_the_signal_is_present() { + let dir = TempDir::new(); + let (venv, meta) = scaffold( + &dir, + &metadata_json(EMPTY_SHA256, Some("1.0.0"), Some("abc123")), + ); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, Some("1.0.0"), Some("def456")), + CacheVerdict::ManifestHashChanged + ); + } + + #[test] + fn absent_manifest_signals_do_not_invalidate() { + // THE upgrade-safety rule. Metadata written by an older CLI carries + // neither manifest key. Reading absent-as-None and comparing against a + // live value would report a mismatch and wipe every existing venv on + // the first run after upgrading the host. + let dir = TempDir::new(); + let (venv, meta) = scaffold(&dir, &metadata_json(EMPTY_SHA256, None, None)); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, Some("1.0.0"), Some("abc123")), + CacheVerdict::Valid, + "absent manifest signals mean 'no signal', not 'mismatch'" + ); + } + + #[test] + fn each_manifest_signal_is_evaluated_independently() { + // Half-populated metadata is real: only the key that is present gets + // compared, so a version-only record still catches a version bump and + // still ignores the hash. + let dir = TempDir::new(); + let (venv, meta) = scaffold(&dir, &metadata_json(EMPTY_SHA256, Some("1.0.0"), None)); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, Some("1.0.0"), Some("any-hash")), + CacheVerdict::Valid, + "an absent manifest_hash is skipped even when manifest_version is present" + ); + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, Some("9.9.9"), Some("any-hash")), + CacheVerdict::ManifestVersionChanged, + "the present signal is still enforced" + ); + } + + #[test] + fn a_present_signal_against_an_unknown_expected_value_invalidates() { + // The converse of the upgrade rule: metadata that recorded a version + // while the plugin now declares none is a real change, not "no signal". + let dir = TempDir::new(); + let (venv, meta) = scaffold(&dir, &metadata_json(EMPTY_SHA256, Some("1.0.0"), None)); + + assert_eq!( + evaluate_cache(&venv, &meta, EMPTY_SHA256, None, None), + CacheVerdict::ManifestVersionChanged + ); + } + + #[test] + fn requirements_are_checked_before_manifest_signals() { + // Ordering matters for the reported reason: client.py compares the + // requirements hash first, so a config that changed both should + // attribute the rebuild to requirements. + let dir = TempDir::new(); + let (venv, meta) = scaffold(&dir, &metadata_json("old", Some("1.0.0"), Some("abc"))); + + assert_eq!( + evaluate_cache(&venv, &meta, "new", Some("2.0.0"), Some("def")), + CacheVerdict::RequirementsChanged + ); + } + + #[test] + fn metadata_round_trips_without_inventing_manifest_keys() { + // Serializing must not write `manifest_version: null` — the Python + // side reads key *presence* as the signal, so a null key would turn + // "no signal" into a permanent mismatch. + let metadata = CacheMetadata { + venv_path: "/tmp/x/.venv".into(), + requirements_file: None, + requirements_hash: EMPTY_SHA256.into(), + manifest_version: None, + manifest_hash: None, + python_version: Some("3.12.1".into()), + }; + let json = serde_json::to_string(&metadata).unwrap(); + assert!( + !json.contains("manifest_version"), + "absent signals must stay absent: {json}" + ); + assert!(!json.contains("manifest_hash")); + + let parsed: CacheMetadata = serde_json::from_str(&json).unwrap(); + assert!(parsed.manifest_version.is_none()); + } + + // --- layout ------------------------------------------------------------- + + #[test] + fn layout_is_keyed_on_the_class_root_so_a_package_shares_one_venv() { + let dirs = vec!["/plugins".to_string()]; + let a = VenvLayout::resolve(&dirs, "pkg.a.PluginA").unwrap(); + let b = VenvLayout::resolve(&dirs, "pkg.b.PluginB").unwrap(); + + assert_eq!(a.venv_path, PathBuf::from("/plugins/pkg/.venv")); + assert_eq!(a.venv_path, b.venv_path, "one venv per package root"); + assert_eq!(a.cache_dir, PathBuf::from("/plugins/pkg/.cpex/venv_cache")); + + // ...but distinct manifests, so neither install invalidates the other. + assert_ne!(a.manifest_path, b.manifest_path); + } + + #[test] + fn layout_uses_the_first_plugin_dir() { + let dirs = vec!["/first".to_string(), "/second".to_string()]; + let layout = VenvLayout::resolve(&dirs, "pkg.Plugin").unwrap(); + assert_eq!(layout.plugin_path, PathBuf::from("/first/pkg")); + } + + #[test] + fn layout_requires_at_least_one_plugin_dir() { + let err = VenvLayout::resolve(&[], "pkg.Plugin").unwrap_err(); + assert!(matches!(err, HostError::Config { .. })); + assert!( + err.to_string().contains("plugin directory"), + "the message should explain what is missing: {err}" + ); + } + + #[test] + fn metadata_filename_is_keyed_on_the_full_class_name() { + // Deliberately unlike client.py, which keys this on the venv directory + // name and so hands every plugin in a shared package the same file. + // See the comment in `VenvLayout::resolve`. + let layout = VenvLayout::resolve(&["/plugins".to_string()], "pkg.Plugin").unwrap(); + assert_eq!( + layout.metadata_path, + PathBuf::from("/plugins/pkg/.cpex/venv_cache/pkg-Plugin_metadata.json") + ); + } + + #[test] + fn plugins_sharing_a_package_get_distinct_metadata_files() { + // The other half of the anti-thrash guarantee. A shared metadata file + // would let each plugin's build overwrite the other's requirements and + // manifest hashes, invalidating it on the next run — forever. + let dirs = vec!["/plugins".to_string()]; + let a = VenvLayout::resolve(&dirs, "pkg.a.PluginA").unwrap(); + let b = VenvLayout::resolve(&dirs, "pkg.b.PluginB").unwrap(); + assert_eq!(a.venv_path, b.venv_path, "still one shared venv"); + assert_ne!(a.metadata_path, b.metadata_path); + } + + #[test] + fn python_executable_sits_inside_the_venv() { + let layout = VenvLayout::resolve(&["/plugins".to_string()], "pkg.Plugin").unwrap(); + let exe = layout.python_executable(); + assert!(exe.starts_with(&layout.venv_path)); + assert!(exe.ends_with(if cfg!(windows) { + "python.exe" + } else { + "python" + })); + } + + // --- manager ------------------------------------------------------------ + + fn manager_in(dir: &TempDir, class_name: &str, requirements: Option<&str>) -> VenvManager { + VenvManager::new( + &[dir.path().display().to_string()], + class_name, + requirements, + Some("1.0.0"), + ) + .expect("layout resolves") + } + + #[test] + fn relative_requirements_paths_resolve_under_the_plugin_path() { + let dir = TempDir::new(); + let mgr = manager_in(&dir, "pkg.Plugin", Some("requirements.txt")); + assert_eq!( + mgr.requirements_file.as_deref(), + Some(dir.path().join("pkg").join("requirements.txt").as_path()) + ); + } + + #[test] + fn absolute_requirements_paths_are_left_alone() { + let dir = TempDir::new(); + let mgr = manager_in(&dir, "pkg.Plugin", Some("/etc/shared/requirements.txt")); + assert_eq!( + mgr.requirements_file.as_deref(), + Some(Path::new("/etc/shared/requirements.txt")) + ); + } + + #[test] + fn a_missing_venv_reports_an_invalid_cache() { + let dir = TempDir::new(); + let mgr = manager_in(&dir, "pkg.Plugin", None); + assert_eq!(mgr.cache_verdict(), CacheVerdict::VenvMissing); + } + + #[tokio::test] + async fn ensure_builds_then_reuses_the_venv() { + // The cached-venv-reuse acceptance example: a second initialize with + // unchanged inputs must not rebuild or reinstall. + if crate::testing::skip_without_python3("ensure_builds_then_reuses_the_venv") { + return; + } + let dir = TempDir::new(); + let mgr = manager_in(&dir, "pkg.Plugin", None); + + let first = mgr.ensure().await.expect("venv builds"); + assert_eq!( + first, + EnsureOutcome::Built { + installed_requirements: false + }, + "no requirements file means a fresh venv with no install" + ); + assert!( + mgr.python_executable().exists(), + "the interpreter must exist after a build" + ); + + let second = mgr.ensure().await.expect("cached venv is reusable"); + assert_eq!( + second, + EnsureOutcome::Reused, + "unchanged inputs must not rebuild" + ); + } + + #[tokio::test] + async fn a_changed_requirements_hash_forces_a_rebuild() { + if crate::testing::skip_without_python3("a_changed_requirements_hash_forces_a_rebuild") { + return; + } + let dir = TempDir::new(); + std::fs::create_dir_all(dir.path().join("pkg")).unwrap(); + let requirements = dir.path().join("pkg").join("requirements.txt"); + // Empty requirements: pip succeeds without network access, which keeps + // this test hermetic while still exercising the install branch. + std::fs::write(&requirements, "").unwrap(); + + let mgr = manager_in(&dir, "pkg.Plugin", Some("requirements.txt")); + assert!(matches!( + mgr.ensure().await.unwrap(), + EnsureOutcome::Built { .. } + )); + assert_eq!(mgr.ensure().await.unwrap(), EnsureOutcome::Reused); + + // Editing the file changes its hash, which must invalidate. + std::fs::write(&requirements, "# a comment changes the hash\n").unwrap(); + assert_eq!(mgr.cache_verdict(), CacheVerdict::RequirementsChanged); + assert!(matches!( + mgr.ensure().await.unwrap(), + EnsureOutcome::Built { .. } + )); + } + + #[tokio::test] + async fn an_absent_requirements_file_builds_and_skips_install() { + if crate::testing::skip_without_python3( + "an_absent_requirements_file_builds_and_skips_install", + ) { + return; + } + let dir = TempDir::new(); + // Configured but not present on disk — must not error. + let mgr = manager_in(&dir, "pkg.Plugin", Some("does-not-exist.txt")); + + assert_eq!( + mgr.ensure() + .await + .expect("a missing requirements file is not fatal"), + EnsureOutcome::Built { + installed_requirements: false + } + ); + } + + #[tokio::test] + async fn two_plugins_sharing_a_package_do_not_thrash_each_others_cache() { + // Both resolve to one venv dir but distinct manifest + metadata files, + // so building either leaves the other's cache valid. + if crate::testing::skip_without_python3( + "two_plugins_sharing_a_package_do_not_thrash_each_others_cache", + ) { + return; + } + let dir = TempDir::new(); + let a = manager_in(&dir, "pkg.a.PluginA", None); + let b = manager_in(&dir, "pkg.b.PluginB", None); + + assert_eq!( + a.layout().venv_path, + b.layout().venv_path, + "one shared venv" + ); + assert_ne!(a.layout().metadata_path, b.layout().metadata_path); + + assert!(matches!( + a.ensure().await.unwrap(), + EnsureOutcome::Built { .. } + )); + assert_eq!(a.ensure().await.unwrap(), EnsureOutcome::Reused); + + // B has no metadata yet, so it builds once... + assert!(matches!( + b.ensure().await.unwrap(), + EnsureOutcome::Built { .. } + )); + // ...and crucially A is still a cache hit afterwards. A shared manifest + // filename here would have invalidated A and started a rebuild loop. + assert_eq!( + a.ensure().await.unwrap(), + EnsureOutcome::Reused, + "B's build must not invalidate A" + ); + assert_eq!(b.ensure().await.unwrap(), EnsureOutcome::Reused); + } + + #[tokio::test] + async fn a_manifest_edit_invalidates_the_cache() { + if crate::testing::skip_without_python3("a_manifest_edit_invalidates_the_cache") { + return; + } + let dir = TempDir::new(); + let mgr = manager_in(&dir, "pkg.Plugin", None); + assert!(matches!( + mgr.ensure().await.unwrap(), + EnsureOutcome::Built { .. } + )); + assert_eq!(mgr.ensure().await.unwrap(), EnsureOutcome::Reused); + + // Writing a manifest where there was none changes the manifest hash — + // the sole change signal for a plugin with no requirements file. + std::fs::write(&mgr.layout().manifest_path, "version: 1.0.1\n").unwrap(); + assert_eq!(mgr.cache_verdict(), CacheVerdict::ManifestHashChanged); + } + + #[tokio::test] + async fn a_declared_version_bump_invalidates_the_cache() { + if crate::testing::skip_without_python3("a_declared_version_bump_invalidates_the_cache") { + return; + } + let dir = TempDir::new(); + let v1 = manager_in(&dir, "pkg.Plugin", None); + assert!(matches!( + v1.ensure().await.unwrap(), + EnsureOutcome::Built { .. } + )); + + // Same plugin, newer declared version: the metadata recorded 1.0.0. + let v2 = VenvManager::new( + &[dir.path().display().to_string()], + "pkg.Plugin", + None, + Some("2.0.0"), + ) + .unwrap(); + assert_eq!(v2.cache_verdict(), CacheVerdict::ManifestVersionChanged); + } + + #[tokio::test] + async fn metadata_written_by_this_host_is_a_cache_hit_for_it() { + // Round-trip guard: whatever `save_metadata` writes must satisfy + // `evaluate_cache` on the next run. A field-name drift between the two + // would otherwise rebuild every venv on every single run. + if crate::testing::skip_without_python3( + "metadata_written_by_this_host_is_a_cache_hit_for_it", + ) { + return; + } + let dir = TempDir::new(); + let mgr = manager_in(&dir, "pkg.Plugin", None); + mgr.ensure().await.unwrap(); + + let raw = std::fs::read_to_string(&mgr.layout().metadata_path).unwrap(); + let parsed: CacheMetadata = + serde_json::from_str(&raw).expect("our own metadata must parse"); + assert_eq!(parsed.manifest_version.as_deref(), Some("1.0.0")); + assert!(parsed.manifest_hash.is_some()); + assert_eq!(mgr.cache_verdict(), CacheVerdict::Valid); + } +} diff --git a/crates/cpex-hosts-python/src/worker.rs b/crates/cpex-hosts-python/src/worker.rs new file mode 100644 index 00000000..8e64d13b --- /dev/null +++ b/crates/cpex-hosts-python/src/worker.rs @@ -0,0 +1,1011 @@ +// Location: ./crates/cpex-hosts-python/src/worker.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Worker client — launches `worker.py` in the plugin's venv and speaks the +// newline-delimited JSON protocol to it. +// +// The async analog of `cpex/framework/isolated/venv_comm.py`. Where that uses +// two threads and a `Queue` per request, this uses two tokio tasks and a +// one-shot channel per request: +// +// ```text +// send_task ──> outbound channel ──> writer task ──> child stdin +// registers a one-shot in `pending` +// child stdout ──> reader task ──> demux on request_id ──> that one-shot +// ``` +// +// Demuxing on `request_id` rather than assuming request/response ordering is +// what lets several hook invocations be in flight against one worker at once — +// the executor dispatches parallel-phase plugins concurrently, and a +// FIFO-coupled client would serialize them. +// +// # Failure modes, kept distinct +// +// A hung worker (`Timeout`) and a dead worker (`WorkerDied`) call for different +// operator responses, so they do not collapse into one error. Process death is +// detected by reader EOF, at which point every pending one-shot is dropped — +// each in-flight `send_task` then resolves to `WorkerDied` instead of waiting +// out a timeout that can never be satisfied. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; + +use serde_json::Value; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::process::{Child, Command}; +use tokio::sync::{mpsc, oneshot, Mutex}; + +use crate::error::HostError; + +/// How long to wait for a worker to exit after a `shutdown` task before +/// killing it. Matches `venv_comm.py`'s `stop_worker` grace window. +const SHUTDOWN_GRACE_SECS: u64 = 5; + +/// Task type for a hook invocation, as `worker.py` dispatches on it. +pub const TASK_LOAD_AND_RUN_HOOK: &str = "load_and_run_hook"; + +/// Task type that asks the worker to exit. +pub const TASK_SHUTDOWN: &str = "shutdown"; + +/// Map of in-flight request ids to the channel awaiting each response. +type Pending = Arc>>>; + +/// How many recent stderr lines to retain for diagnosis. +/// +/// A worker that dies at import time (a bad dependency pin, a missing module) +/// says so on stderr and nowhere else. Without this, the host reports only +/// "worker process died" and the actual cause is lost — the tail is small +/// enough to be cheap and long enough to hold a Python traceback. +const STDERR_RETAIN_LINES: usize = 40; + +/// Ring buffer of the worker's most recent stderr lines. +type StderrTail = Arc>>; + +/// Set once the reader task observes that the worker's stdout has closed. +/// +/// Needed because clearing `pending` only rescues requests that were *already* +/// registered when the worker died. A send issued afterwards would otherwise +/// register a fresh one-shot into a map nobody is reading, succeed at writing +/// into an mpsc channel whose consumer has exited, and then wait out the full +/// timeout for a response that can never arrive. Checking this flag turns that +/// into an immediate `WorkerDied`. +type Alive = Arc; + +/// A live `worker.py` subprocess and the plumbing that talks to it. +pub struct WorkerClient { + /// Outbound task lines, drained by the writer task. + outbound: mpsc::UnboundedSender, + + /// Response channels keyed by request id. + pending: Pending, + + /// False once the reader task has seen the worker's stdout close. + alive: Alive, + + /// Recent stderr, for explaining a death. + stderr_tail: StderrTail, + + /// The child handle, kept so shutdown can wait on and kill it. `None` + /// once shutdown has consumed it. + child: Mutex>, + + /// Cap on one serialized task, enforced before the write. + max_content_size: usize, + + /// Per-invocation response timeout. + timeout_secs: u64, +} + +/// Manual rather than derived: the outbound channel carries serialized task +/// lines, which on a credential-bearing hook include the plaintext token. A +/// derive would put those in any debug dump of the client. +impl std::fmt::Debug for WorkerClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkerClient") + .field("max_content_size", &self.max_content_size) + .field("timeout_secs", &self.timeout_secs) + .finish_non_exhaustive() + } +} + +impl WorkerClient { + /// Launch `worker.py` with the venv's interpreter and start the I/O tasks. + /// + /// `cwd` becomes the child's working directory, which is where a plugin's + /// relative-path side effects land. + pub async fn spawn( + python: &Path, + script_path: &Path, + cwd: Option<&Path>, + max_content_size: usize, + timeout_secs: u64, + ) -> Result { + if !python.exists() { + return Err(HostError::WorkerStart { + message: format!("venv interpreter not found at {}", python.display()), + }); + } + + let mut command = Command::new(python); + command + .arg(script_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + // Without this, a killed host leaves orphaned workers behind. + .kill_on_drop(true); + + if let Some(dir) = cwd { + command.current_dir(dir); + } + + let mut child = command.spawn().map_err(|e| HostError::WorkerStart { + message: format!( + "could not spawn {} {}: {e}", + python.display(), + script_path.display() + ), + })?; + + let stdin = child.stdin.take().ok_or_else(|| HostError::WorkerStart { + message: "worker stdin was not piped".into(), + })?; + let stdout = child.stdout.take().ok_or_else(|| HostError::WorkerStart { + message: "worker stdout was not piped".into(), + })?; + let stderr = child.stderr.take().ok_or_else(|| HostError::WorkerStart { + message: "worker stderr was not piped".into(), + })?; + + let pending: Pending = Arc::new(Mutex::new(HashMap::new())); + let alive: Alive = Arc::new(std::sync::atomic::AtomicBool::new(true)); + let stderr_tail: StderrTail = Arc::new(Mutex::new(std::collections::VecDeque::new())); + let (outbound, outbound_rx) = mpsc::unbounded_channel::(); + + tokio::spawn(writer_loop(stdin, outbound_rx)); + tokio::spawn(reader_loop( + stdout, + Arc::clone(&pending), + Arc::clone(&alive), + )); + tokio::spawn(stderr_loop(stderr, Arc::clone(&stderr_tail))); + + Ok(Self { + outbound, + pending, + alive, + stderr_tail, + child: Mutex::new(Some(child)), + max_content_size, + timeout_secs, + }) + } + + /// Build a `WorkerDied` error, appending recent stderr when there is any. + /// + /// The stderr tail is what turns "worker process died" into something + /// actionable — an import error, a missing dependency, a traceback. + async fn died(&self, context: &str) -> HostError { + let tail = self.stderr_tail.lock().await; + if tail.is_empty() { + return HostError::WorkerDied { + message: context.to_string(), + }; + } + + let recent: Vec<&str> = tail.iter().map(String::as_str).collect(); + HostError::WorkerDied { + message: format!("{context}; recent worker stderr: {}", recent.join(" | ")), + } + } + + /// The worker's retained stderr lines, oldest first. + /// + /// Exposed so a test can assert the credential channel never surfaces here: + /// this buffer is the one place the host copies worker stderr, and the one + /// place it is echoed onward (into a `WorkerDied` message). + pub async fn stderr_lines(&self) -> Vec { + self.stderr_tail.lock().await.iter().cloned().collect() + } + + /// Whether the worker's stdout is still open. + /// + /// A `false` here is definitive — the process has closed its output. A + /// `true` is only as fresh as the reader task's last observation, so a + /// send can still race a death; that path is covered by the dropped-sender + /// branch in `send_task`. + pub fn is_alive(&self) -> bool { + self.alive.load(std::sync::atomic::Ordering::Acquire) + } + + /// Send a task and await its response. + /// + /// The caller supplies the task body; this method injects the `request_id` + /// and handles registration, the size check, the write, and the timeout. + pub async fn send_task(&self, mut task: Value) -> Result { + // Checked up front: once the worker is known dead, waiting out the + // timeout tells the caller nothing it does not already know. + if !self.is_alive() { + return Err(self.died("worker process is no longer running").await); + } + + let request_id = uuid::Uuid::new_v4().to_string(); + + { + let obj = task.as_object_mut().ok_or_else(|| HostError::Protocol { + message: "task must be a JSON object".into(), + })?; + obj.insert("request_id".into(), Value::String(request_id.clone())); + } + + let line = serde_json::to_string(&task).map_err(|e| HostError::Protocol { + message: format!("could not serialize task: {e}"), + })?; + + // Checked before registering or writing, so an oversized task costs + // nothing and cannot desync the worker's line-oriented reader. + if line.len() > self.max_content_size { + return Err(HostError::TaskTooLarge { + size: line.len(), + limit: self.max_content_size, + }); + } + + let (tx, rx) = oneshot::channel(); + self.pending.lock().await.insert(request_id.clone(), tx); + + // Registration happens before the write so a fast response cannot + // arrive before there is anywhere to route it. + if self.outbound.send(format!("{line}\n")).is_err() { + self.pending.lock().await.remove(&request_id); + return Err(self + .died("writer task is gone — the worker is no longer accepting tasks") + .await); + } + + let response = + match tokio::time::timeout(std::time::Duration::from_secs(self.timeout_secs), rx).await + { + Ok(Ok(response)) => response, + // The sender was dropped without a value: the reader task saw EOF + // and cleared `pending`, so the worker is gone. + Ok(Err(_)) => { + return Err(self + .died("worker exited while the request was in flight") + .await) + }, + Err(_) => { + self.pending.lock().await.remove(&request_id); + return Err(HostError::Timeout { + timeout_secs: self.timeout_secs, + }); + }, + }; + + interpret_response(response) + } + + /// Ask the worker to exit, then wait out the grace window and kill it. + /// + /// Idempotent: a second call after the child has been reaped is a no-op, + /// which matters because `shutdown` may run from both the plugin lifecycle + /// and a drop path. + pub async fn shutdown(&self) -> Result<(), HostError> { + let Some(mut child) = self.child.lock().await.take() else { + return Ok(()); + }; + + // `venv_comm.py` sends a fixed "shutdown" request id rather than a + // UUID, and the worker echoes it back; nothing awaits the reply. + let task = serde_json::json!({ "task_type": TASK_SHUTDOWN, "request_id": TASK_SHUTDOWN }); + if let Ok(line) = serde_json::to_string(&task) { + let _ = self.outbound.send(format!("{line}\n")); + } + + match tokio::time::timeout( + std::time::Duration::from_secs(SHUTDOWN_GRACE_SECS), + child.wait(), + ) + .await + { + Ok(Ok(_status)) => Ok(()), + Ok(Err(e)) => Err(HostError::WorkerDied { + message: format!("could not wait on the worker process: {e}"), + }), + Err(_) => { + // A worker that ignores the shutdown task (or is wedged inside + // plugin code) must not hold teardown open. + tracing::warn!("worker did not exit within {SHUTDOWN_GRACE_SECS}s; killing it"); + child.kill().await.map_err(|e| HostError::WorkerDied { + message: format!("could not kill the unresponsive worker: {e}"), + })?; + Ok(()) + }, + } + } +} + +/// Turn a worker response envelope into a result. +/// +/// The worker signals failure with `{"status": "error", "message": ...}`; any +/// other envelope is the hook result. `request_id` is stripped — it is +/// transport bookkeeping, and `venv_comm.py` removes it too before returning. +fn interpret_response(mut response: Value) -> Result { + if let Some(obj) = response.as_object_mut() { + obj.remove("request_id"); + + if obj.get("status").and_then(Value::as_str) == Some("error") { + let message = obj + .get("message") + .and_then(Value::as_str) + .unwrap_or("worker reported an error with no message") + .to_string(); + return Err(HostError::WorkerError { message }); + } + } + Ok(response) +} + +/// Drain outbound task lines into the child's stdin. +/// +/// Ends when the channel closes (client dropped) or the pipe breaks (worker +/// gone). Either way the reader loop is what surfaces the failure to callers. +async fn writer_loop( + mut stdin: tokio::process::ChildStdin, + mut outbound: mpsc::UnboundedReceiver, +) { + while let Some(line) = outbound.recv().await { + if stdin.write_all(line.as_bytes()).await.is_err() { + tracing::debug!("worker stdin closed; writer stopping"); + break; + } + if stdin.flush().await.is_err() { + tracing::debug!("worker stdin flush failed; writer stopping"); + break; + } + } +} + +/// Read response lines and route each to the one-shot for its request id. +/// +/// On EOF, `pending` is drained and every sender dropped, which resolves each +/// in-flight `send_task` to `WorkerDied` rather than leaving it to time out. +async fn reader_loop(stdout: tokio::process::ChildStdout, pending: Pending, alive: Alive) { + let mut lines = BufReader::new(stdout).lines(); + + loop { + match lines.next_line().await { + Ok(Some(line)) => { + let line = line.trim(); + if line.is_empty() { + continue; + } + + let Ok(response) = serde_json::from_str::(line) else { + // A plugin that printed to stdout lands here. Skipping the + // line keeps the stream in sync; the request still times + // out or gets its real response later. + tracing::warn!("worker emitted a non-JSON stdout line; ignoring it"); + continue; + }; + + let Some(request_id) = response.get("request_id").and_then(Value::as_str) else { + tracing::warn!("worker response had no request_id; dropping it"); + continue; + }; + + match pending.lock().await.remove(request_id) { + Some(tx) => { + // Receiver gone means the caller already timed out. + let _ = tx.send(response); + }, + None => { + // The fixed "shutdown" id lands here by design, as does + // any late response whose caller has given up. + tracing::debug!(request_id, "response for an unknown request id"); + }, + } + }, + Ok(None) => { + tracing::debug!("worker stdout closed"); + break; + }, + Err(e) => { + tracing::warn!("error reading worker stdout: {e}"); + break; + }, + } + } + + // Order matters: mark dead *before* clearing, so a send that races this + // teardown either sees the flag and fails fast, or finds its sender + // dropped. Either way it does not wait out the timeout. + alive.store(false, std::sync::atomic::Ordering::Release); + + // Dropping the senders is what converts a dead worker into an immediate + // error for everyone still waiting. + pending.lock().await.clear(); +} + +/// Log the worker's stderr and retain a bounded tail for diagnosis. +/// +/// Logged at debug: the worker's own logging goes here, and a credential-bearing +/// hook must never surface token material through it. `worker.py` scrubs its +/// side (it holds the plaintext only to scrub it back out of anything the +/// plugin logs, raises, or returns); this side never parses these lines or +/// echoes them anywhere except into a `WorkerDied` message, which is the one +/// place an operator genuinely needs them. +async fn stderr_loop(stderr: tokio::process::ChildStderr, tail: StderrTail) { + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + tracing::debug!(target: "cpex_hosts_python::worker_stderr", "{line}"); + + let mut tail = tail.lock().await; + if tail.len() == STDERR_RETAIN_LINES { + tail.pop_front(); + } + tail.push_back(line); + } +} + +/// Resolve `worker.py` inside a venv's site-packages. +/// +/// The worker ships with the `cpex` framework, which the plugin's requirements +/// pull into the venv — so the host does not carry its own copy. Both +/// `lib/pythonX.Y/site-packages` (Unix) and `Lib/site-packages` (Windows) are +/// searched because the interpreter version is not known up front. +pub fn resolve_worker_script(venv_path: &Path, relative: &str) -> Result { + // An absolute override (used by tests and by hosts that vendor a worker) + // bypasses venv resolution entirely. + let relative_path = Path::new(relative); + if relative_path.is_absolute() { + return Ok(relative_path.to_path_buf()); + } + + let mut candidates = Vec::new(); + + let unix_lib = venv_path.join("lib"); + if let Ok(entries) = std::fs::read_dir(&unix_lib) { + for entry in entries.flatten() { + candidates.push(entry.path().join("site-packages").join(relative_path)); + } + } + candidates.push( + venv_path + .join("Lib") + .join("site-packages") + .join(relative_path), + ); + // Last resort: a worker sitting directly under the venv. + candidates.push(venv_path.join(relative_path)); + + candidates + .iter() + .find(|p| p.exists()) + .cloned() + .ok_or_else(|| HostError::WorkerStart { + message: format!( + "could not find '{relative}' in the venv at {} — is `cpex` installed in it? \ + (a plugin's requirements file must pull in the framework that ships worker.py)", + venv_path.display() + ), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::{skip_without_python3, TempDir}; + + /// A stub worker that speaks the same protocol as `worker.py`. + /// + /// Deterministic stand-in for the real thing: no venv, no framework + /// import, and each behavior selectable per task so the failure modes are + /// reachable without contriving a real plugin to misbehave. + /// + /// Recognized `task_type` values: + /// - `echo` — reply with `{"status":"success","echo":}` + /// - `slow` — sleep `delay` seconds, then echo (drives the timeout) + /// - `fail` — reply `{"status":"error","message":}` + /// - `die` — exit immediately without replying + /// - `noisy` — print a non-JSON line first, then reply + /// - `deaf` — ignore `shutdown` and keep running (drives the kill) + /// - `shutdown` — reply and exit, unless previously put in `deaf` mode + const STUB_WORKER: &str = r#" +import json, sys, time + +deaf = False +while True: + line = sys.stdin.readline() + if not line: + break + line = line.strip() + if not line: + continue + task = json.loads(line) + kind = task.get("task_type") + rid = task.get("request_id", "unknown") + + if kind == "shutdown": + if deaf: + # Acknowledge but refuse to exit, so the host must kill us. + print(json.dumps({"status": "success", "request_id": rid}), flush=True) + while True: + time.sleep(0.05) + print(json.dumps({"status": "success", "message": "Shutting down", "request_id": rid}), flush=True) + break + if kind == "deaf": + deaf = True + print(json.dumps({"status": "success", "request_id": rid}), flush=True) + continue + if kind == "die": + sys.exit(3) + if kind == "slow": + time.sleep(float(task.get("delay", 1.0))) + print(json.dumps({"status": "success", "echo": task.get("payload"), "request_id": rid}), flush=True) + continue + if kind == "fail": + print(json.dumps({"status": "error", "message": task.get("message", "stub failure"), "request_id": rid}), flush=True) + continue + if kind == "noisy": + print("this is not JSON and must not desync the stream", flush=True) + print(json.dumps({"status": "success", "echo": task.get("payload"), "request_id": rid}), flush=True) + continue + + print(json.dumps({"status": "success", "echo": task.get("payload"), "request_id": rid}), flush=True) +"#; + + /// Write the stub to disk and spawn a client against it. + async fn stub_client( + dir: &TempDir, + max_content_size: usize, + timeout_secs: u64, + ) -> WorkerClient { + let script = dir.path().join("stub_worker.py"); + std::fs::write(&script, STUB_WORKER).unwrap(); + + let python = which_python3(); + WorkerClient::spawn( + &python, + &script, + Some(dir.path()), + max_content_size, + timeout_secs, + ) + .await + .expect("stub worker spawns") + } + + /// Absolute path to python3, since `WorkerClient::spawn` requires an + /// existing interpreter path rather than a PATH lookup. + fn which_python3() -> PathBuf { + let out = std::process::Command::new("sh") + .args(["-c", "command -v python3"]) + .output() + .expect("run command -v"); + PathBuf::from(String::from_utf8_lossy(&out.stdout).trim()) + } + + #[tokio::test] + async fn a_task_round_trips_to_its_response() { + if skip_without_python3("a_task_round_trips_to_its_response") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 10).await; + + let response = client + .send_task(serde_json::json!({ "task_type": "echo", "payload": {"tool": "search"} })) + .await + .expect("the stub replies"); + + assert_eq!(response["echo"]["tool"], "search"); + assert!( + response.get("request_id").is_none(), + "request_id is transport bookkeeping and must be stripped" + ); + } + + #[tokio::test] + async fn concurrent_sends_each_receive_their_own_response() { + // Demux correctness. The stub answers the slower task second, so a + // client that assumed FIFO ordering would hand each caller the other's + // response — this test fails loudly on that. + if skip_without_python3("concurrent_sends_each_receive_their_own_response") { + return; + } + let dir = TempDir::new(); + let client = Arc::new(stub_client(&dir, 1_000_000, 10).await); + + let slow = { + let client = Arc::clone(&client); + tokio::spawn(async move { + client + .send_task(serde_json::json!({ + "task_type": "slow", "delay": 0.4, "payload": "slow-one" + })) + .await + }) + }; + + // Give the slow task a head start so it is genuinely outstanding. + tokio::time::sleep(std::time::Duration::from_millis(80)).await; + + let fast = client + .send_task(serde_json::json!({ "task_type": "echo", "payload": "fast-one" })) + .await + .expect("the fast task replies"); + + let slow = slow.await.unwrap().expect("the slow task replies"); + + assert_eq!(fast["echo"], "fast-one"); + assert_eq!(slow["echo"], "slow-one"); + } + + #[tokio::test] + async fn many_concurrent_sends_all_resolve_correctly() { + // Scales the demux check past two, where an off-by-one in the pending + // map would still look fine. + if skip_without_python3("many_concurrent_sends_all_resolve_correctly") { + return; + } + let dir = TempDir::new(); + let client = Arc::new(stub_client(&dir, 1_000_000, 20).await); + + let mut handles = Vec::with_capacity(16); + for i in 0..16 { + let client = Arc::clone(&client); + handles.push(tokio::spawn(async move { + let response = client + .send_task(serde_json::json!({ "task_type": "echo", "payload": i })) + .await + .expect("each task replies"); + (i, response["echo"].as_i64().unwrap()) + })); + } + + for handle in handles { + let (sent, echoed) = handle.await.unwrap(); + assert_eq!( + sent, echoed, + "response {echoed} was delivered to the caller that sent {sent}" + ); + } + } + + #[tokio::test] + async fn an_oversized_task_is_rejected_before_it_is_sent() { + if skip_without_python3("an_oversized_task_is_rejected_before_it_is_sent") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 256, 10).await; + + let err = client + .send_task(serde_json::json!({ "task_type": "echo", "payload": "x".repeat(1024) })) + .await + .expect_err("a task over the cap must be rejected"); + + match err { + HostError::TaskTooLarge { size, limit } => { + assert!(size > limit); + assert_eq!(limit, 256); + }, + other => panic!("expected TaskTooLarge, got {other:?}"), + } + + // The client must still be usable — rejecting a task locally neither + // wrote to the pipe nor desynced the worker. + let response = client + .send_task(serde_json::json!({ "task_type": "echo", "payload": "small" })) + .await + .expect("the client survives a rejected task"); + assert_eq!(response["echo"], "small"); + } + + #[tokio::test] + async fn no_response_within_the_timeout_yields_a_timeout_error() { + if skip_without_python3("no_response_within_the_timeout_yields_a_timeout_error") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 1).await; + + let err = client + .send_task(serde_json::json!({ "task_type": "slow", "delay": 5.0 })) + .await + .expect_err("a task that outruns the timeout must error"); + + assert!( + matches!(err, HostError::Timeout { timeout_secs: 1 }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn a_worker_error_response_becomes_a_worker_error() { + if skip_without_python3("a_worker_error_response_becomes_a_worker_error") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 10).await; + + let err = client + .send_task(serde_json::json!({ "task_type": "fail", "message": "plugin blew up" })) + .await + .expect_err("status=error must not be returned as success"); + + match err { + HostError::WorkerError { message } => assert!(message.contains("plugin blew up")), + other => panic!("expected WorkerError, got {other:?}"), + } + } + + #[tokio::test] + async fn worker_death_mid_flight_resolves_rather_than_hangs() { + // The important one: a dead worker must not leave a caller waiting out + // a timeout that can never be satisfied. The generous 30s client + // timeout means a hang here would show up as a test that runs for 30s + // and then reports Timeout instead of WorkerDied. + if skip_without_python3("worker_death_mid_flight_resolves_rather_than_hangs") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 30).await; + + let started = std::time::Instant::now(); + let err = client + .send_task(serde_json::json!({ "task_type": "die" })) + .await + .expect_err("a worker that exits without replying must error"); + + assert!( + matches!(err, HostError::WorkerDied { .. }), + "expected WorkerDied (not Timeout), got {err:?}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the error must arrive on worker death, not on timeout expiry" + ); + } + + #[tokio::test] + async fn sends_after_worker_death_fail_fast() { + if skip_without_python3("sends_after_worker_death_fail_fast") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 30).await; + + let _ = client + .send_task(serde_json::json!({ "task_type": "die" })) + .await; + + // A send issued *after* the death must also fail fast. Clearing + // `pending` on EOF only rescues already-registered requests; without an + // explicit liveness flag this send would register into an unwatched map + // and wait out the full 30s timeout. + let started = std::time::Instant::now(); + let err = client + .send_task(serde_json::json!({ "task_type": "echo", "payload": "anyone home?" })) + .await + .expect_err("a send to a dead worker cannot succeed"); + + assert!( + matches!(err, HostError::WorkerDied { .. }), + "expected WorkerDied, got {err:?}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "a send to a known-dead worker must fail immediately, not on timeout: {:?}", + started.elapsed() + ); + assert!(!client.is_alive()); + } + + #[tokio::test] + async fn a_non_json_stdout_line_does_not_desync_the_stream() { + // Plugins print. A stray line must be skipped, not consumed as if it + // were the response to the pending request. + if skip_without_python3("a_non_json_stdout_line_does_not_desync_the_stream") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 10).await; + + let response = client + .send_task(serde_json::json!({ "task_type": "noisy", "payload": "after-the-noise" })) + .await + .expect("the real response still arrives"); + assert_eq!(response["echo"], "after-the-noise"); + } + + #[tokio::test] + async fn shutdown_terminates_a_live_worker() { + if skip_without_python3("shutdown_terminates_a_live_worker") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 10).await; + + client + .send_task(serde_json::json!({ "task_type": "echo", "payload": "alive" })) + .await + .expect("worker is up"); + + client.shutdown().await.expect("a cooperative worker exits"); + // Idempotent: teardown can be reached more than once. + client + .shutdown() + .await + .expect("a second shutdown is a no-op"); + } + + #[tokio::test] + async fn a_worker_that_ignores_shutdown_is_killed_after_the_grace_window() { + if skip_without_python3("a_worker_that_ignores_shutdown_is_killed_after_the_grace_window") { + return; + } + let dir = TempDir::new(); + let client = stub_client(&dir, 1_000_000, 10).await; + + // Put the stub into a mode where it acknowledges shutdown and then + // spins forever. + client + .send_task(serde_json::json!({ "task_type": "deaf" })) + .await + .expect("stub accepts deaf mode"); + + let started = std::time::Instant::now(); + client + .shutdown() + .await + .expect("an unresponsive worker is killed, not waited on forever"); + + let elapsed = started.elapsed(); + assert!( + elapsed >= std::time::Duration::from_secs(SHUTDOWN_GRACE_SECS), + "the grace window should be honored before killing: {elapsed:?}" + ); + assert!( + elapsed < std::time::Duration::from_secs(SHUTDOWN_GRACE_SECS + 5), + "the kill must happen promptly once the window expires: {elapsed:?}" + ); + } + + #[tokio::test] + async fn a_death_error_carries_the_workers_stderr() { + // A worker that dies during startup — a bad dependency pin, a missing + // module — explains itself on stderr and nowhere else. Without the + // tail, the host reports a bare "worker process died" and the operator + // has nothing to act on. + if skip_without_python3("a_death_error_carries_the_workers_stderr") { + return; + } + let dir = TempDir::new(); + let script = dir.path().join("import_error.py"); + std::fs::write( + &script, + "import sys\nprint('ImportError: cannot import name McpError', file=sys.stderr, flush=True)\nsys.exit(1)\n", + ) + .unwrap(); + + let client = + WorkerClient::spawn(&which_python3(), &script, Some(dir.path()), 1_000_000, 30) + .await + .expect("the process starts, then exits"); + + let err = client + .send_task(serde_json::json!({ "task_type": "echo" })) + .await + .expect_err("a dead worker cannot answer"); + + let message = err.to_string(); + assert!(matches!(err, HostError::WorkerDied { .. }), "got {err:?}"); + assert!( + message.contains("McpError"), + "the death error should carry the stderr that explains it: {message}" + ); + } + + #[tokio::test] + async fn spawning_against_a_missing_interpreter_errors() { + let err = WorkerClient::spawn( + Path::new("/nonexistent/bin/python"), + Path::new("/tmp/worker.py"), + None, + 1024, + 5, + ) + .await + .expect_err("a missing interpreter cannot be launched"); + + assert!(matches!(err, HostError::WorkerStart { .. }), "got {err:?}"); + } + + // --- response interpretation (no subprocess needed) --------------------- + + #[test] + fn an_error_envelope_becomes_a_worker_error() { + let err = interpret_response(serde_json::json!({ + "status": "error", "message": "boom", "request_id": "abc" + })) + .expect_err("status=error is a failure"); + match err { + HostError::WorkerError { message } => assert_eq!(message, "boom"), + other => panic!("expected WorkerError, got {other:?}"), + } + } + + #[test] + fn an_error_envelope_without_a_message_still_errors() { + let err = interpret_response(serde_json::json!({ "status": "error" })) + .expect_err("a message-less error is still an error"); + assert!(matches!(err, HostError::WorkerError { .. })); + } + + #[test] + fn a_success_envelope_is_returned_without_its_request_id() { + let response = interpret_response(serde_json::json!({ + "status": "success", "continue_processing": true, "request_id": "abc" + })) + .expect("success passes through"); + assert!(response.get("request_id").is_none()); + assert_eq!(response["continue_processing"], true); + } + + #[test] + fn a_result_envelope_with_no_status_field_is_success() { + // The hook path returns `model_dump()` of a result model, which has no + // `status` key at all — treating "no status" as an error would fail + // every successful invocation. + let response = interpret_response(serde_json::json!({ + "continue_processing": false, "violation": {"code": "pii"} + })) + .expect("a bare result model is not an error"); + assert_eq!(response["violation"]["code"], "pii"); + } + + // --- worker script resolution ------------------------------------------ + + #[test] + fn an_absolute_script_path_bypasses_venv_resolution() { + let resolved = resolve_worker_script(Path::new("/venv"), "/opt/custom/worker.py").unwrap(); + assert_eq!(resolved, PathBuf::from("/opt/custom/worker.py")); + } + + #[test] + fn the_worker_script_is_found_in_unix_site_packages() { + let dir = TempDir::new(); + let site = dir + .path() + .join("lib") + .join("python3.12") + .join("site-packages"); + let worker = site.join("cpex").join("framework").join("isolated"); + std::fs::create_dir_all(&worker).unwrap(); + std::fs::write(worker.join("worker.py"), "# stub").unwrap(); + + let resolved = + resolve_worker_script(dir.path(), "cpex/framework/isolated/worker.py").unwrap(); + assert_eq!(resolved, worker.join("worker.py")); + } + + #[test] + fn a_missing_worker_script_names_the_venv_and_the_likely_cause() { + let dir = TempDir::new(); + let err = resolve_worker_script(dir.path(), "cpex/framework/isolated/worker.py") + .expect_err("an empty venv has no worker"); + + let message = err.to_string(); + assert!(message.contains("worker.py")); + assert!( + message.contains("cpex"), + "the message should point at the missing framework: {message}" + ); + } +} diff --git a/crates/cpex-hosts-python/tests/config_e2e.rs b/crates/cpex-hosts-python/tests/config_e2e.rs new file mode 100644 index 00000000..eaeb0b7c --- /dev/null +++ b/crates/cpex-hosts-python/tests/config_e2e.rs @@ -0,0 +1,439 @@ +// Location: ./crates/cpex-hosts-python/tests/config_e2e.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// End-to-end through the *manager*: load `plugins/config.yaml`, register the +// `isolated_venv` factory, and invoke hooks on cpex-test-plugin — +// `tool_pre_invoke`, plus the two credential-adjacent hooks `identity_resolve` +// and `token_delegate`. +// +// # What this covers that `isolated_venv_e2e.rs` does not +// +// The other e2e tests build an `IsolatedPythonPlugin` directly and call its +// adapter, which skips two layers a real gateway goes through: YAML → factory +// → registry, and `PluginManager::invoke_by_name` → executor → adapter. This +// one drives both, against a plugin installed the way an operator installs one +// (`cpex plugin install`, not a scaffolded fixture). So it is the test that +// catches a config-shape or dispatch-wiring break that a direct adapter call +// would not see. +// +// # Why it is `#[ignore]` +// +// It depends on state this repo does not create: an installed plugin with a +// built venv under `plugins/`. See the setup comment on the test itself. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use cpex_core::extensions::Extensions; +use cpex_core::hooks::types::hook_names; +use cpex_core::manager::PluginManager; +use cpex_hosts_python::factory::{IsolatedVenvFactory, KIND}; +use cpex_hosts_python::legacy::{ + IdentityResolvePayload, TokenDelegatePayload, ToolPreInvokePayload, +}; +use cpex_hosts_python::plugin::IsolatedPythonPlugin; +use cpex_hosts_python::testing::skip_without_python3; + +/// Plugin name in `plugins/config.yaml`. +const PLUGIN_NAME: &str = "cpex-test-plugin"; + +/// Marker `TestPlugin.identity_resolve` writes when `raw_token` arrives with a +/// non-empty secret value — evidence the field survived the wire as a populated +/// `SecretStr` rather than as `None` or an empty string. +const IDENTITY_RAW_TOKEN_MARKER: &str = "identity_resolve_raw_token"; + +/// Marker `TestPlugin.token_delegate` writes when `bearer_token` arrives with a +/// non-empty secret value. See [`IDENTITY_RAW_TOKEN_MARKER`]. +const TOKEN_DELEGATE_BEARER_MARKER: &str = "token_delegate_bearer_token"; + +/// The repository root — `plugins/config.yaml` and the installed plugin's +/// `plugin_dirs` are both relative to it. +fn repo_root() -> PathBuf { + // CARGO_MANIFEST_DIR = .../cpex/crates/cpex-hosts-python + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("the manifest dir has a crates/ ancestry") + .to_path_buf() +} + +/// Skip guard: `Some(config_path)` when the installed-plugin fixture is present. +/// +/// Every branch prints why it skipped — a silent no-op is indistinguishable +/// from a pass in `cargo test` output otherwise. +fn require_installed_plugin(test_name: &str) -> Option { + if skip_without_python3(test_name) { + return None; + } + + let root = repo_root(); + let config_path = root.join("plugins").join("config.yaml"); + if !config_path.is_file() { + println!( + "SKIP {test_name}: no {} — install the plugin first (see the setup comment)", + config_path.display() + ); + return None; + } + + // The venv the worker runs in. Its absence means the plugin was never + // installed, and the host would try a cold build from the plugin's + // requirements.txt (two git clones) inside the test. + let venv = root + .join("plugins") + .join("cpex_test_plugin") + .join(".venv") + .join("bin") + .join("python"); + if !venv.is_file() { + println!( + "SKIP {test_name}: no venv at {} — run `cpex plugin --type test-pypi install \ + \"cpex-test-plugin@>=0.2.0\"` first", + venv.display() + ); + return None; + } + + Some(config_path) +} + +/// Remove a stale `.` marker and return its path. +/// +/// The plugin's hooks each write `.` at the worker's cwd, and the +/// two credential hooks write a second `._` when their +/// redacting field arrives non-empty. Clearing first is what makes a marker +/// evidence of *this* run rather than a previous one. +fn clear_marker(root: &Path, name: &str) -> PathBuf { + let marker = root.join(format!(".{name}")); + let _ = std::fs::remove_file(&marker); + marker +} + +/// Assert a hook body ran, then remove the marker so the next run starts clean. +fn assert_marker_written(marker: &Path, description: &str) { + let touched = marker.is_file(); + let _ = std::fs::remove_file(marker); + assert!( + touched, + "expected TestPlugin.{description} to create {} — the hook body did not run", + marker.display() + ); +} + +/// Assert a result is the unconditional allow every `TestPlugin` hook returns. +/// +/// `errors` is checked first: a worker that failed to start or a hook the +/// framework rejected surfaces there rather than as a deny, so checking the +/// allow first would point the failure message at the wrong layer. +fn assert_allowed(result: &cpex_core::executor::PipelineResult, hook_name: &str) { + assert!( + result.errors.is_empty(), + "the pipeline recorded plugin errors on {hook_name}: {:?}", + result.errors + ); + assert!( + result.continue_processing, + "TestPlugin.{hook_name} returns continue_processing=True unconditionally: violation={:?}", + result.violation + ); + assert!( + result.violation.is_none(), + "an allow carries no violation on {hook_name}: {:?}", + result.violation + ); +} + +// --------------------------------------------------------------------------- +// Setup +// +// The plugin must be installed under the repo root, which builds +// `plugins/cpex_test_plugin/.venv` and writes `plugins/config.yaml`: +// +// cpex plugin --type test-pypi install "cpex-test-plugin@>=0.2.0" +// +// Then: +// +// cargo test -p cpex-hosts-python --test config_e2e -- --ignored --nocapture +// +// Note the first run may rebuild the venv even though one exists: the Python +// CLI writes its cache metadata as `.venv_metadata.json`, while this host keys +// that filename per class (see the comment in `venv.rs::VenvLayout::resolve`). +// It is a cache hit on every run after. +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires cpex-test-plugin installed under plugins/ with a built venv"] +async fn config_yaml_dispatches_tool_pre_invoke_to_the_installed_plugin() { + let test_name = "config_yaml_dispatches_tool_pre_invoke_to_the_installed_plugin"; + let Some(config_path) = require_installed_plugin(test_name) else { + return; + }; + let root = repo_root(); + + // The process cwd is the project root the host resolves `plugins` against, + // and it is also the worker's cwd — whose ALLOWED_PLUGIN_DIRS allowlist + // accepts it, and where the plugin's marker file lands. So this one call + // sets both, and they cannot disagree. + std::env::set_current_dir(&root).expect("cd to the repo root"); + + // The plugin's `tool_pre_invoke` writes `.tool_pre_invoke` at the worker's + // cwd. Remove any stale copy so its existence proves *this* run executed + // the hook body rather than a previous one. + let marker = clear_marker(&root, hook_names::TOOL_PRE_INVOKE); + + // Loaded straight from disk with no patching: the whole point is that a + // config.yaml as written by `cpex plugin install` works as-is. + let manager = Arc::new(PluginManager::default()); + manager.register_factory(KIND, Box::new(IsolatedVenvFactory)); + manager + .load_config_file(&config_path) + .expect("plugins/config.yaml loads with the isolated_venv factory registered"); + + // A config that loaded but registered nothing would make every assertion + // below vacuous: `invoke_by_name` short-circuits to an allow when no plugin + // is on the hook, so the "clean input is allowed" checks would pass with no + // Python involved at all. + assert!( + manager.has_hooks_for(hook_names::TOOL_PRE_INVOKE), + "no plugin registered on {} — check that config.yaml still declares \ + kind: {KIND} and that hook", + hook_names::TOOL_PRE_INVOKE + ); + assert!( + manager.plugin_names().iter().any(|n| n == PLUGIN_NAME), + "expected '{PLUGIN_NAME}' among the loaded plugins, got {:?}", + manager.plugin_names() + ); + + manager + .initialize() + .await + .expect("the venv resolves and the worker starts"); + + // `tool_pre_invoke` carries the native Pydantic shape the Python plugin + // expects — `name` plus `args` — not the generic wrapper. + let payload = ToolPreInvokePayload { + name: "test_tool".to_string(), + args: Some(HashMap::from([( + "query".to_string(), + serde_json::json!("hello world"), + )])), + headers: None, + }; + + let (result, _background) = manager + .invoke_by_name( + hook_names::TOOL_PRE_INVOKE, + Box::new(payload), + Extensions::default(), + None, + ) + .await; + + manager.shutdown().await; + + assert_allowed(&result, hook_names::TOOL_PRE_INVOKE); + + // The marker proves the plugin's hook body ran. Without it, a response + // shaped like an allow is indistinguishable from the manager's + // no-plugins-registered short circuit. + assert_marker_written(&marker, hook_names::TOOL_PRE_INVOKE); +} + +/// The two credential-adjacent hooks dispatch through the same config → factory +/// → registry → executor → worker path as `tool_pre_invoke`. +/// +/// # Why these two are worth their own test +/// +/// `identity_resolve` and `token_delegate` are the only hooks the host treats +/// specially: `is_credential_hook` singles them out, and their payloads carry +/// redacting fields (`raw_token`, `bearer_token`) that no other hook has. That +/// makes them the two most likely to break in a way `tool_pre_invoke` cannot +/// detect — a payload-kind mapping that falls through to `PayloadKind::Generic`, +/// for instance, still round-trips as an allow, so only a marker file +/// distinguishes a real dispatch from a silent no-op. The plugin writes two +/// markers per credential hook: one for the hook body, one for the redacting +/// field arriving non-empty. This test asserts both. +/// +/// No `capabilities` are declared in `plugins/config.yaml`, so the host attaches +/// no `credential` object here. This is the no-credential path: the hooks +/// dispatch on payload shape alone. `credential_e2e.rs` covers the gated +/// plaintext channel. +/// +/// Both hooks run in one test because they share the manager, the worker, and +/// the setup cost — a second `#[ignore]` test would double a venv-backed +/// startup to assert the same wiring. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires cpex-test-plugin installed under plugins/ with a built venv"] +async fn config_yaml_dispatches_identity_resolve_and_token_delegate_to_the_installed_plugin() { + let test_name = + "config_yaml_dispatches_identity_resolve_and_token_delegate_to_the_installed_plugin"; + let Some(config_path) = require_installed_plugin(test_name) else { + return; + }; + let root = repo_root(); + + // Sets the project root the host resolves `plugins` against and the worker's + // cwd at once — see the comment in the test above. + std::env::set_current_dir(&root).expect("cd to the repo root"); + + let identity_marker = clear_marker(&root, hook_names::IDENTITY_RESOLVE); + let delegate_marker = clear_marker(&root, hook_names::TOKEN_DELEGATE); + + // Each credential hook writes a *second* marker when its redacting field + // arrives non-empty (`.identity_resolve_raw_token`, + // `.token_delegate_bearer_token`). Both payloads below populate that field, + // so both land on every run — clear them here and assert them below, or the + // test leaves them behind in the working tree. + let identity_field_marker = clear_marker(&root, IDENTITY_RAW_TOKEN_MARKER); + let delegate_field_marker = clear_marker(&root, TOKEN_DELEGATE_BEARER_MARKER); + + let manager = Arc::new(PluginManager::default()); + manager.register_factory(KIND, Box::new(IsolatedVenvFactory)); + manager + .load_config_file(&config_path) + .expect("plugins/config.yaml loads with the isolated_venv factory registered"); + + // Without these, both invocations below short-circuit to an allow with no + // Python involved, and every assertion that follows would be vacuous. + for hook in [hook_names::IDENTITY_RESOLVE, hook_names::TOKEN_DELEGATE] { + assert!( + manager.has_hooks_for(hook), + "no plugin registered on {hook} — check that config.yaml still lists it under \ + `hooks:` for a kind: {KIND} plugin" + ); + } + assert!( + manager.plugin_names().iter().any(|n| n == PLUGIN_NAME), + "expected '{PLUGIN_NAME}' among the loaded plugins, got {:?}", + manager.plugin_names() + ); + + manager + .initialize() + .await + .expect("the venv resolves and the worker starts"); + + // `identity_resolve` mirrors Python's `IdentityPayload`. `raw_token` is a + // `SecretStr` there and carries a placeholder on the wire — the plaintext + // would ride the separate capability-gated `credential` object, which this + // config declares no capability for. + let identity_payload = IdentityResolvePayload { + raw_token: "".to_string(), + source: "bearer".to_string(), + headers: HashMap::from([("Authorization".to_string(), "".to_string())]), + client_host: Some("10.0.0.1".to_string()), + client_port: Some(8443), + }; + + let (identity_result, _background) = manager + .invoke_by_name( + hook_names::IDENTITY_RESOLVE, + Box::new(identity_payload), + Extensions::default(), + None, + ) + .await; + + // `token_delegate` mirrors `DelegationPayload`. `target_type` and + // `auth_enforced_by` are left to their defaults so the worker's Pydantic + // model has to supply them — an omitted-field mismatch surfaces as a + // validation error in `errors`, not as a deny. + // + // `bearer_token` carries a mock value rather than staying `None`: on the + // Python side it is a `SecretStr | None`, so a populated field exercises the + // `SecretStr` coercion in the worker's model reconstruction that a `None` + // leaves untested. The value is deliberately not a real credential — this + // config declares no capability, so the plaintext channel (`credential`) + // is absent and only this redacting field is in play. `credential_e2e.rs` + // covers the gated plaintext path. + let delegate_payload = TokenDelegatePayload { + target_name: "billing-api".to_string(), + target_audience: Some("https://billing.internal".to_string()), + required_permissions: vec!["invoices:read".to_string()], + bearer_token: Some("mock-bearer-token".to_string()), + ..Default::default() + }; + + let (delegate_result, _background) = manager + .invoke_by_name( + hook_names::TOKEN_DELEGATE, + Box::new(delegate_payload), + Extensions::default(), + None, + ) + .await; + + manager.shutdown().await; + + assert_allowed(&identity_result, hook_names::IDENTITY_RESOLVE); + assert_allowed(&delegate_result, hook_names::TOKEN_DELEGATE); + + // The markers are what separate a real dispatch from an allow-shaped no-op: + // a hook that never reached the plugin still comes back as an allow, so + // without these the assertions above would pass on a silent no-op. + assert_marker_written(&identity_marker, hook_names::IDENTITY_RESOLVE); + assert_marker_written(&delegate_marker, hook_names::TOKEN_DELEGATE); + + // And these prove the redacting fields survived the round trip as populated + // `SecretStr`s. A payload-kind mapping that fell through to + // `PayloadKind::Generic` would drop them to `None`, which the hook markers + // above cannot detect — the hook body still runs either way. + assert_marker_written( + &identity_field_marker, + "identity_resolve (non-empty raw_token)", + ); + assert_marker_written( + &delegate_field_marker, + "token_delegate (non-empty bearer_token)", + ); +} + +/// An installer-generated config resolves `plugins` at the project root with no +/// patching, and neither `plugin_dirs` key steers it. +/// +/// Needs no venv and no python3, so unlike the test above it runs on every +/// `cargo test`. This is the regression guard for the gap that used to require a +/// shim here: a config shaped exactly as `cpex plugin install` writes it must +/// load *and* resolve to the right directory. +#[test] +fn an_installer_generated_config_resolves_plugins_at_the_project_root() { + // The shape `cpex plugin install` generates: plugin_dirs at the top level, + // absent from the plugin's own config block. Plus a per-plugin + // `plugin_dirs` pointing somewhere bogus, to prove neither key wins. + let yaml = r#" +plugin_dirs: + - /top/level/ignored +plugins: + - name: cpex-test-plugin + kind: isolated_venv + hooks: [tool_pre_invoke] + version: 0.2.1 + config: + class_name: cpex_test_plugin.plugin.TestPlugin + requirements_file: requirements.txt + plugin_dirs: ["/per/plugin/ignored"] +"#; + + let config = cpex_core::config::parse_config(yaml).expect("valid config"); + + // Both keys still parse — they are simply not the host's source. + assert_eq!(config.plugin_dirs, vec!["/top/level/ignored".to_string()]); + + let plugin = IsolatedPythonPlugin::from_config(&config.plugins[0]) + .expect("an ignored plugin_dirs key must not fail the load"); + + let expected = std::env::current_dir() + .unwrap() + .join(cpex_hosts_python::DEFAULT_PLUGIN_DIR) + .display() + .to_string(); + assert_eq!( + plugin.plugin_dirs(), + [expected], + "the host must resolve /plugins, ignoring both config keys" + ); +} diff --git a/crates/cpex-hosts-python/tests/credential_e2e.rs b/crates/cpex-hosts-python/tests/credential_e2e.rs new file mode 100644 index 00000000..0fe18f11 --- /dev/null +++ b/crates/cpex-hosts-python/tests/credential_e2e.rs @@ -0,0 +1,386 @@ +// Location: ./crates/cpex-hosts-python/tests/credential_e2e.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// End-to-end: capability-gated raw credentials reaching a real Python plugin. +// +// The unit tests prove the gate and the DTO shape. These prove the whole +// channel: the host reads the in-memory token, attaches the DTO, `worker.py` +// folds it back onto the redacted `SecretStr` payload field, and the plugin +// reads the plaintext — while a plugin on the same hook that declared nothing +// gets no token at all. +// +// # Transport +// +// The `credential` object rides as a field in the existing task JSON on the +// worker's stdin: a private pipe inherited only by the child. There is no +// listening socket and no other local process can read it, so the +// "loopback-only, access-controlled" constraint holds by construction rather +// than by configuration. What still needs proving is that nothing *logs* it — +// hence the stderr assertions below. +// +// # Residual exposure this cannot close +// +// Once the plaintext is resident in the worker process, every transitively +// installed dependency in that venv can read it. That is a materially larger +// and less audited trust boundary than the in-process host, and neither the +// capability gate (which controls *which plugin* receives a token) nor the +// transport (which controls *how it travels*) constrains it. Shipping raw +// credentials to an out-of-process plugin means accepting that venv's whole +// dependency tree into the credential trust boundary. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use cpex_core::context::PluginContext; +use cpex_core::extensions::raw_credentials::{ + RawCredentialsExtension, RawInboundToken, TokenKind, TokenRole, +}; +use cpex_core::extensions::Extensions; +use cpex_core::plugin::{Plugin, PluginConfig}; +use cpex_core::registry::AnyHookHandler; +use cpex_hosts_python::factory::KIND; +use cpex_hosts_python::legacy::IdentityResolvePayload; +use cpex_hosts_python::plugin::{IsolatedPythonPlugin, PythonHookAdapter}; +use cpex_hosts_python::testing::{ + prebuild_venv, python_source, scaffold_plugin, skip_without_python3, + worker_consumes_credentials, TempDir, +}; + +/// The plaintext under test. Distinctive so a leak is unambiguous in any +/// captured output. +const SECRET: &str = "E2E-INBOUND-PLAINTEXT-9f3c1a"; + +/// Fully-qualified class name of the fixture plugin. +const FIXTURE_CLASS: &str = "cred_pkg.identity_plugin.TokenReadingPlugin"; + +/// An identity plugin that records whether it could read the plaintext. +/// +/// It writes what it saw to a file rather than returning it: a returned token +/// would ride the response channel, which is exactly what must not happen. +/// The recorded value is a *verdict*, not the secret. +const FIXTURE_PLUGIN: &str = r#" +import os + +from cpex.framework.base import Plugin +from cpex.framework.models import PluginResult + +EXPECTED = "E2E-INBOUND-PLAINTEXT-9f3c1a" + + +class TokenReadingPlugin(Plugin): + """Reports whether the raw inbound token arrived intact.""" + + def __init__(self, config): + super().__init__(config) + + async def identity_resolve(self, payload, context): + raw = payload.raw_token + # SecretStr: get_secret_value() is the only way to the plaintext. + actual = raw.get_secret_value() if hasattr(raw, "get_secret_value") else raw + + if actual == EXPECTED: + verdict = "GOT_TOKEN" + elif not actual: + verdict = "NO_TOKEN" + else: + # Never write the value itself, even on the unexpected path. + verdict = f"OTHER:len={len(actual)}" + + marker = os.path.join(os.getcwd(), "credential_verdict.txt") + with open(marker, "w") as f: + f.write(f"{verdict};source={payload.source}") + + return PluginResult(continue_processing=True) +"#; + +/// Skip guard: `Some(source)` when the credential path is exercisable. +/// +/// Beyond python3 and a framework checkout, the credential path needs a +/// `worker.py` that consumes the `credential` field — an older one drops it +/// silently, which would look like a host bug. +fn require_environment(test_name: &str) -> Option { + if skip_without_python3(test_name) { + return None; + } + let source = match python_source() { + Ok(source) => source, + Err(reason) => { + println!("SKIP {test_name}: {reason}"); + return None; + }, + }; + if !worker_consumes_credentials(&source) { + println!( + "SKIP {test_name}: the cpex source at {} has a worker.py that does not consume the \ + `credential` field — the credential path has no consumer there", + source.display() + ); + return None; + } + Some(source) +} + +/// Scaffold the credential fixture and pre-build its venv. +/// +/// Returns `None` (after printing why) when the venv cannot be built. +fn setup(dir: &TempDir, source: &Path, test_name: &str) -> Option { + let plugin_dir = scaffold_plugin(dir, source, "cred_pkg", "identity_plugin", FIXTURE_PLUGIN); + match prebuild_venv(&plugin_dir, source, FIXTURE_CLASS, "cred_pkg") { + Ok(()) => Some(plugin_dir), + Err(reason) => { + println!("SKIP {test_name}: could not prepare the plugin venv — {reason}"); + None + }, + } +} + +/// Extensions carrying the inbound token under test. +fn extensions_with_token() -> Extensions { + let mut raw = RawCredentialsExtension::default(); + raw.inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new(SECRET, "Authorization", TokenKind::Jwt), + ); + Extensions { + raw_credentials: Some(Arc::new(raw)), + ..Default::default() + } +} + +/// No `plugin_dirs` here: the host ignores that key and always resolves +/// `/plugins`. These tests scaffold into a temp dir and point the +/// plugin at it via `with_plugin_dirs`, so a run never touches the real one. +fn plugin_config(capabilities: &[&str]) -> PluginConfig { + PluginConfig { + name: "identity-reader".into(), + kind: KIND.into(), + hooks: vec!["identity_resolve".into()], + capabilities: capabilities.iter().map(|c| (*c).to_string()).collect(), + version: Some("1.0.0".into()), + config: Some(serde_json::json!({ + "class_name": FIXTURE_CLASS, + "requirements_file": "requirements.txt", + "timeout_secs": 300, + })), + ..Default::default() + } +} + +/// Build the plugin under test, pointing it at the scaffolded temp plugin dir. +fn plugin_for( + config: &PluginConfig, + plugin_dir: &Path, + worker_cwd: &Path, +) -> Arc { + Arc::new( + IsolatedPythonPlugin::from_config(config) + .expect("config parses") + .with_plugin_dirs(vec![plugin_dir.display().to_string()]) + .with_worker_cwd(worker_cwd.to_path_buf()), + ) +} + +/// Run one identity_resolve invocation. +/// +/// Returns the plugin's recorded verdict plus the worker's retained stderr, +/// captured before shutdown so the credential-leak assertions have something to +/// inspect. +async fn run_identity_hook( + dir: &TempDir, + plugin_dir: &Path, + capabilities: &[&str], +) -> (String, Vec) { + let config = plugin_config(capabilities); + let plugin = plugin_for(&config, plugin_dir, dir.path()); + plugin + .initialize() + .await + .expect("the cached venv is reused and the worker starts"); + + let payload = IdentityResolvePayload::default(); + let adapter = PythonHookAdapter::new(Arc::clone(&plugin), "identity_resolve"); + let mut ctx = PluginContext::new(); + adapter + .invoke(&payload, &extensions_with_token(), &mut ctx) + .await + .expect("the hook round-trips"); + + // Read the stderr the host retained while the worker was alive; after + // shutdown the client is gone. + let stderr = plugin.worker_stderr().await; + plugin.shutdown().await.expect("shuts down"); + + let marker = dir.path().join("credential_verdict.txt"); + let verdict = std::fs::read_to_string(&marker) + .unwrap_or_else(|e| panic!("the plugin recorded no verdict: {e}")); + (verdict, stderr) +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_capable_plugin_reads_the_raw_token_end_to_end() { + // The capability-gated acceptance example, all the way through: the host + // reads the in-memory Zeroizing token, the DTO carries it, worker.py folds + // it onto the redacted SecretStr field, and the plugin reads the plaintext. + let Some(source) = require_environment("a_capable_plugin_reads_the_raw_token_end_to_end") + else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "a_capable_plugin_reads_the_raw_token_end_to_end", + ) else { + return; + }; + + let (verdict, _stderr) = + run_identity_hook(&dir, &plugin_dir, &["read_inbound_credentials"]).await; + + assert!( + verdict.starts_with("GOT_TOKEN"), + "a plugin declaring read_inbound_credentials must receive the plaintext; got: {verdict}" + ); + // The worker maps kind `jwt` onto source `bearer`. + assert!( + verdict.contains("source=bearer"), + "the token kind should map onto the payload's source field: {verdict}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_non_capable_plugin_on_the_same_hook_receives_no_token() { + // Same hook, same request, same extensions — this plugin simply declared + // no capability. It must see an empty credential, not the plaintext. + let Some(source) = + require_environment("a_non_capable_plugin_on_the_same_hook_receives_no_token") + else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "a_non_capable_plugin_on_the_same_hook_receives_no_token", + ) else { + return; + }; + + let (verdict, _stderr) = run_identity_hook(&dir, &plugin_dir, &[]).await; + + assert!( + verdict.starts_with("NO_TOKEN"), + "a plugin that declared nothing must not receive token material; got: {verdict}" + ); + assert!( + !verdict.contains(SECRET), + "the plaintext must not appear in what the plugin observed: {verdict}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_credential_never_reaches_a_log_sink_the_host_controls() { + // The transport's own guarantee. The DTO travels on the child's stdin — a + // private inherited pipe — so the exposure worth checking is what the host + // *writes down*. The host copies worker stderr into exactly one buffer (the + // retained tail used to explain a `WorkerDied`), which is also the only + // place it echoes stderr onward. Asserting on that buffer tests the real + // sink rather than a log-formatting side effect. + let Some(source) = + require_environment("the_credential_never_reaches_a_log_sink_the_host_controls") + else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "the_credential_never_reaches_a_log_sink_the_host_controls", + ) else { + return; + }; + + let (verdict, stderr) = + run_identity_hook(&dir, &plugin_dir, &["read_inbound_credentials"]).await; + + // Precondition: the hook really did receive the token. Without this the + // test could pass trivially by never having a secret to leak. + assert!( + verdict.starts_with("GOT_TOKEN"), + "precondition: the plugin must have received the token for this test to mean anything; got {verdict}" + ); + + let captured = stderr.join("\n"); + assert!( + !captured.contains(SECRET), + "the plaintext leaked into the worker stderr the host retains:\n{captured}" + ); + // A fragment is as bad as the whole value. + assert!( + !captured.contains("E2E-INBOUND-PLAINTEXT"), + "a fragment of the plaintext leaked into retained worker stderr:\n{captured}" + ); + + // The plugin's own recorded output must carry a verdict, never the value — + // the fixture is written to record only a verdict, and this pins it. + assert!( + !verdict.contains(SECRET), + "the plugin's recorded output must not contain the plaintext: {verdict}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_declared_capability_that_cannot_be_honored_fails_closed_end_to_end() { + // Fail-closed, through the real stack: the plugin declared the capability + // but the request carries no credential extension. The host must refuse to + // dispatch rather than silently invoke the plugin with an empty token, + // which a resolver could read as "no authentication required". + let Some(source) = + require_environment("a_declared_capability_that_cannot_be_honored_fails_closed_end_to_end") + else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "a_declared_capability_that_cannot_be_honored_fails_closed_end_to_end", + ) else { + return; + }; + + let config = plugin_config(&["read_inbound_credentials"]); + let plugin = plugin_for(&config, &plugin_dir, dir.path()); + plugin.initialize().await.expect("initializes"); + + let payload = IdentityResolvePayload::default(); + let adapter = PythonHookAdapter::new(Arc::clone(&plugin), "identity_resolve"); + let mut ctx = PluginContext::new(); + // No raw-credentials extension at all. + let result = adapter + .invoke(&payload, &Extensions::default(), &mut ctx) + .await; + + let Err(err) = result else { + panic!("a declared capability that cannot be honored must not dispatch successfully"); + }; + let message = err.to_string(); + assert!( + !message.contains(SECRET), + "the fail-closed error must not name any credential: {message}" + ); + + // And the plugin was never invoked — no verdict file was written. + assert!( + !dir.path().join("credential_verdict.txt").exists(), + "the plugin must not run at all when its declared capability cannot be honored" + ); + + plugin.shutdown().await.expect("shuts down"); +} diff --git a/crates/cpex-hosts-python/tests/isolated_venv_e2e.rs b/crates/cpex-hosts-python/tests/isolated_venv_e2e.rs new file mode 100644 index 00000000..f9aba22b --- /dev/null +++ b/crates/cpex-hosts-python/tests/isolated_venv_e2e.rs @@ -0,0 +1,382 @@ +// Location: ./crates/cpex-hosts-python/tests/isolated_venv_e2e.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// End-to-end: Rust manager → venv → real `worker.py` → Python plugin → back. +// +// The unit tests cover each layer against a stub worker. These prove the whole +// path against the *real* framework worker, which is the only thing that can +// catch a field-name disagreement between this host's task JSON and what +// `worker.py` actually reads. +// +// # Requirements, and why these skip instead of failing +// +// Two things must be present: a `python3`, and a `cpex` Python source tree to +// install into the venv. Neither is guaranteed in CI, and the plan calls for a +// clean skip rather than a red build when they are absent — a machine without +// them has not covered this path, but it has not broken it either. Every skip +// prints its reason so a silent no-op is distinguishable from a pass. +// +// The framework source is discovered rather than assumed: `cpex` on PyPI is +// behind this branch (its `worker.py` predates the credential field), so the +// tests install from a local checkout of the Python side. Set +// `CPEX_PYTHON_SOURCE` to point at one; otherwise a sibling checkout is tried. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use cpex_core::context::PluginContext; +use cpex_core::extensions::Extensions; +use cpex_core::plugin::{Plugin, PluginConfig}; +use cpex_core::registry::AnyHookHandler; +use cpex_hosts_python::factory::KIND; +use cpex_hosts_python::legacy::ToolPreInvokePayload; +use cpex_hosts_python::plugin::{IsolatedPythonPlugin, PythonHookAdapter}; +use cpex_hosts_python::testing::{ + prebuild_venv, python_source, scaffold_plugin, skip_without_python3, TempDir, +}; + +/// A Python plugin that marks the filesystem when its hook body runs, so the +/// test can prove the plugin *executed* rather than merely that a response +/// came back shaped like a success. +const FIXTURE_PLUGIN: &str = r#" +import os + +from cpex.framework.base import Plugin +from cpex.framework.models import PluginResult, PluginViolation + + +class MarkerPlugin(Plugin): + """Writes a marker file, then allows — or denies when args say to.""" + + def __init__(self, config): + super().__init__(config) + + async def tool_pre_invoke(self, payload, context): + with open(os.path.join(os.getcwd(), "marker.txt"), "w") as f: + f.write(f"ran:{payload.name}") + + args = payload.args or {} + if args.get("q") == "DENY-ME": + return PluginResult( + continue_processing=False, + violation=PluginViolation( + reason="denied by fixture", + description="the fixture was asked to deny", + code="FIXTURE_DENY", + details={"arg": "q"}, + ), + ) + return PluginResult(continue_processing=True) +"#; + +/// Skip guard: returns `Some(source)` when the path is exercisable. +fn require_environment(test_name: &str) -> Option { + if skip_without_python3(test_name) { + return None; + } + match python_source() { + Ok(source) => Some(source), + Err(reason) => { + println!("SKIP {test_name}: {reason}"); + None + }, + } +} + +/// Full setup: scaffold the plugin, pre-build its venv, return the plugin dir. +/// +/// Returns `None` (after printing why) when the venv cannot be built, so the +/// caller skips rather than fails. +fn setup(dir: &TempDir, source: &Path, test_name: &str) -> Option { + let plugin_dir = scaffold_plugin(dir, source, "fixture_pkg", "marker_plugin", FIXTURE_PLUGIN); + match prebuild_venv(&plugin_dir, source, FIXTURE_CLASS, "fixture_pkg") { + Ok(()) => Some(plugin_dir), + Err(reason) => { + println!("SKIP {test_name}: could not prepare the plugin venv — {reason}"); + None + }, + } +} + +/// Fully-qualified class name of the fixture plugin. +const FIXTURE_CLASS: &str = "fixture_pkg.marker_plugin.MarkerPlugin"; + +/// Build the plugin config the manager would load from YAML. +/// +/// No `plugin_dirs` here: the host ignores that key and always resolves +/// `/plugins`. These tests scaffold into a temp dir instead and +/// point the plugin at it via `with_plugin_dirs`, so a test run never builds a +/// venv in the developer's real `plugins/`. +fn plugin_config(hooks: &[&str], capabilities: &[&str]) -> PluginConfig { + PluginConfig { + name: "fixture-marker".into(), + kind: KIND.into(), + hooks: hooks.iter().map(|h| (*h).to_string()).collect(), + capabilities: capabilities.iter().map(|c| (*c).to_string()).collect(), + version: Some("1.0.0".into()), + config: Some(serde_json::json!({ + "class_name": FIXTURE_CLASS, + "requirements_file": "requirements.txt", + // Generous: a cold venv build plus a pip install of the framework + // is minutes, and the hook call itself follows in the same window. + "timeout_secs": 300, + })), + ..Default::default() + } +} + +/// Build the plugin under test: config + the scaffolded temp plugin dir + the +/// worker cwd both e2e assertions depend on. +fn plugin_for( + config: &PluginConfig, + plugin_dir: &Path, + worker_cwd: &Path, +) -> Arc { + Arc::new( + IsolatedPythonPlugin::from_config(config) + .expect("config parses") + .with_plugin_dirs(vec![plugin_dir.display().to_string()]) + .with_worker_cwd(worker_cwd.to_path_buf()), + ) +} + +/// Invoke a hook through the adapter and return the erased result. +async fn invoke( + plugin: &Arc, + hook: &'static str, + payload: &ToolPreInvokePayload, +) -> Result { + let adapter = PythonHookAdapter::new(Arc::clone(plugin), hook); + let mut ctx = PluginContext::new(); + let erased = adapter + .invoke(payload, &Extensions::default(), &mut ctx) + .await + .map_err(|e| e.to_string())?; + Ok(cpex_core::executor::extract_erased(erased).expect("adapter returns ErasedResultFields")) +} + +#[tokio::test(flavor = "multi_thread")] +async fn legacy_tool_pre_invoke_round_trips_through_the_real_worker() { + // The plan's primary acceptance example: clean args return + // continue_processing = true with no violation, and the plugin's hook body + // demonstrably ran (its marker file exists at the worker's cwd). + let Some(source) = + require_environment("legacy_tool_pre_invoke_round_trips_through_the_real_worker") + else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "legacy_tool_pre_invoke_round_trips_through_the_real_worker", + ) else { + return; + }; + let config = plugin_config(&["tool_pre_invoke"], &[]); + + let plugin = plugin_for(&config, &plugin_dir, dir.path()); + plugin + .initialize() + .await + .expect("the cached venv is reused and the worker starts"); + + let payload = ToolPreInvokePayload { + name: "search".into(), + args: Some(std::collections::HashMap::from([( + "q".into(), + serde_json::json!("clean query"), + )])), + headers: None, + }; + + let fields = invoke(&plugin, "tool_pre_invoke", &payload) + .await + .expect("the hook round-trips"); + + assert!(fields.continue_processing, "clean args must be allowed"); + assert!(fields.violation.is_none(), "an allow carries no violation"); + + // The marker proves the plugin's own hook body executed — a response can + // be shaped like an allow without the plugin ever having run. It lands in + // the worker's cwd, which is this test's scratch dir. + let marker = dir.path().join("marker.txt"); + assert!( + marker.is_file(), + "the plugin hook body did not run — no marker at {}", + marker.display() + ); + let contents = std::fs::read_to_string(&marker).unwrap(); + assert_eq!( + contents, "ran:search", + "the marker should record the payload the plugin saw" + ); + + plugin.shutdown().await.expect("the worker shuts down"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_real_plugin_deny_surfaces_as_a_deny_result() { + // Proves the deny path through the real framework, not just a stub that + // emits a hand-written violation envelope. + let Some(source) = require_environment("a_real_plugin_deny_surfaces_as_a_deny_result") else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "a_real_plugin_deny_surfaces_as_a_deny_result", + ) else { + return; + }; + let config = plugin_config(&["tool_pre_invoke"], &[]); + + let plugin = plugin_for(&config, &plugin_dir, dir.path()); + plugin.initialize().await.expect("initializes"); + + let payload = ToolPreInvokePayload { + name: "search".into(), + args: Some(std::collections::HashMap::from([( + "q".into(), + serde_json::json!("DENY-ME"), + )])), + headers: None, + }; + + let fields = invoke(&plugin, "tool_pre_invoke", &payload) + .await + .expect("a deny is a successful round trip"); + + assert!(!fields.continue_processing); + let violation = fields.violation.expect("a deny carries its violation"); + assert_eq!(violation.code, "FIXTURE_DENY"); + assert_eq!(violation.reason, "denied by fixture"); + assert_eq!(violation.details["arg"], "q"); + + plugin.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_cmf_hook_is_rejected_cleanly_by_a_framework_that_has_no_cmf_hooks() { + // The plan asks for a CMF round-trip here. It is not achievable against + // this framework version, and the reason is worth pinning rather than + // omitting: the Python hook registry knows only the legacy names — + // `get_result_type("cmf.tool_pre_invoke")` returns None, and + // `json_to_payload` raises "No payload defined for hook cmf.tool_pre_invoke". + // No `cmf.*` hook is registered anywhere in `cpex/framework/hooks/`. + // + // So the CMF path cannot be proven end to end until the Python side + // registers those hooks. What *is* provable, and asserted here, is that + // this host does its half correctly: it serializes the CMF message payload + // (not the generic wrapper — verified in the conversion and adapter unit + // tests), and the worker's rejection surfaces as a clean plugin error + // rather than a hang, a crash, or a silent allow. The last of those is the + // one that would matter in production: a CMF hook that quietly returned + // "allow" would bypass policy. + let Some(source) = + require_environment("a_cmf_hook_is_rejected_cleanly_by_a_framework_that_has_no_cmf_hooks") + else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup( + &dir, + &source, + "a_cmf_hook_is_rejected_cleanly_by_a_framework_that_has_no_cmf_hooks", + ) else { + return; + }; + let config = plugin_config(&["cmf.tool_pre_invoke"], &[]); + + let plugin = plugin_for(&config, &plugin_dir, dir.path()); + plugin.initialize().await.expect("initializes"); + + let payload = cpex_core::cmf::MessagePayload { + message: cpex_core::cmf::Message::text(cpex_core::cmf::Role::User, "what is the weather?"), + }; + let adapter = PythonHookAdapter::new(Arc::clone(&plugin), "cmf.tool_pre_invoke"); + let mut ctx = PluginContext::new(); + let result = adapter + .invoke(&payload, &Extensions::default(), &mut ctx) + .await; + + let Err(err) = result else { + panic!( + "a framework with no cmf hooks must not report success — a silent allow on a CMF hook \ + would bypass policy" + ); + }; + let message = err.to_string(); + assert!( + message.contains("cmf.tool_pre_invoke") || message.to_lowercase().contains("payload"), + "the error should explain that the hook is unknown to the framework: {message}" + ); + + // Still healthy: one rejected hook must not poison the worker. + assert!( + plugin.shutdown().await.is_ok(), + "the worker should survive a rejected hook and shut down cleanly" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_second_run_reuses_the_cached_venv() { + // The cached-venv-reuse acceptance example, end to end: the second + // initialize must not rebuild or reinstall. Observed through the venv + // manager's own verdict, which is what gates the reinstall. + let Some(source) = require_environment("a_second_run_reuses_the_cached_venv") else { + return; + }; + + let dir = TempDir::new(); + let Some(plugin_dir) = setup(&dir, &source, "a_second_run_reuses_the_cached_venv") else { + return; + }; + let config = plugin_config(&["tool_pre_invoke"], &[]); + + let first = plugin_for(&config, &plugin_dir, dir.path()); + first + .initialize() + .await + .expect("first initialize builds the venv"); + first.shutdown().await.unwrap(); + + // A fresh plugin object against the same directories: the venv on disk is + // the only thing carried over. + let second = plugin_for(&config, &plugin_dir, dir.path()); + + let venv = cpex_hosts_python::VenvManager::new( + &[plugin_dir.display().to_string()], + FIXTURE_CLASS, + Some("requirements.txt"), + Some("1.0.0"), + ) + .expect("layout resolves"); + assert_eq!( + venv.cache_verdict(), + cpex_hosts_python::CacheVerdict::Valid, + "the venv built by the first run must be a cache hit for the second" + ); + + // And the second run still works off that cached venv. + second + .initialize() + .await + .expect("second initialize reuses the venv"); + let payload = ToolPreInvokePayload { + name: "search".into(), + ..Default::default() + }; + let fields = invoke(&second, "tool_pre_invoke", &payload) + .await + .expect("still works"); + assert!(fields.continue_processing); + + second.shutdown().await.unwrap(); +} From 1ed160fcac22fc5777834fcc8c0641fd92616dc0 Mon Sep 17 00:00:00 2001 From: habeck Date: Fri, 31 Jul 2026 11:08:16 -0400 Subject: [PATCH 2/5] feat: out of process extensions delivery Signed-off-by: habeck --- crates/cpex-hosts-python/README.md | 46 +- crates/cpex-hosts-python/src/conversion.rs | 139 ++-- crates/cpex-hosts-python/src/extensions.rs | 746 ++++++++++++++++++ crates/cpex-hosts-python/src/lib.rs | 11 + crates/cpex-hosts-python/src/plugin.rs | 13 +- crates/cpex-hosts-python/src/testing.rs | 15 + .../tests/extensions_merge_e2e.rs | 718 +++++++++++++++++ 7 files changed, 1587 insertions(+), 101 deletions(-) create mode 100644 crates/cpex-hosts-python/src/extensions.rs create mode 100644 crates/cpex-hosts-python/tests/extensions_merge_e2e.rs diff --git a/crates/cpex-hosts-python/README.md b/crates/cpex-hosts-python/README.md index dd433399..8641e412 100644 --- a/crates/cpex-hosts-python/README.md +++ b/crates/cpex-hosts-python/README.md @@ -103,12 +103,46 @@ interpret that policy itself. ### Extensions -Only the `custom` slot of a returned `modified_extensions` is applied. The -`http`, `security`, and `delegation` slots are gated by `WriteToken`s the -executor mints, and this host has no authority to mint one — an attempt to -write them returns an error rather than being silently dropped. Full extensions -delivery is owned by a separate plan -(`docs/plans/2026-07-29-003-feat-out-of-process-extensions-delivery-plan.md`). +The capability-filtered `Extensions` the executor produced for a plugin is +serialized onto the task under an `extensions` field, so a 3-arg +`(payload, context, extensions)` hook sees out-of-process what its in-process +equivalent would. Slot visibility comes from the filtered view — this host does +not re-derive capability filtering. + +Two rules apply in **both** directions: + +- `raw_credentials` never rides this channel. Raw tokens travel the + capability-gated `credential` DTO instead (see below). +- `Authorization`, `Cookie`, and `X-API-Key` are stripped from + `http.request_headers` and `http.response_headers` (spec §3.5), matched + case-insensitively. + +A plugin's returned extensions come back on the response's +`modified_extensions` field — the field the Python `PluginResult` model already +carries, so returning extensions works the same way in and out of process. The +host rebuilds an `OwnedExtensions` and hands it to the executor's existing +copy-on-write merge, which enforces the mutability tiers. The host adds no tier +logic of its own; what it does add is two structural guarantees the merge relies +on: + +| Slot | Tier | Out-of-process behavior | +|---|---|---| +| `request`, `agent`, `mcp`, `completion`, `provenance`, `llm`, `framework`, `meta` | Immutable | The inbound `Arc` is reused, so a forged edit is dropped before the merge sees it | +| `security` (labels) | Monotonic | Applied only with `append_labels`; the executor then enforces `before ⊆ after` | +| `delegation` | Monotonic | Applied only with `append_delegation` | +| `http` | Guarded | Applied only with `write_headers` | +| `custom` | Mutable | Applied as-is | + +Reusing the inbound `Arc`s is load-bearing: the executor validates the immutable +tier with `Arc::ptr_eq`, and a JSON round trip allocates a fresh `Arc` for every +slot. Deserializing the whole object from the wire would fail that check on every +immutable slot and get the entire return rejected, even for a plugin that only +touched `custom`. + +Write authority comes from the `WriteToken` the executor mints per plugin from +its declared capabilities and carries on the inbound view. `WriteToken::new()` is +`pub(crate)` to `cpex-core`, so this host cannot mint one — a gated write with no +token behind it is dropped rather than honored. ## Credentials diff --git a/crates/cpex-hosts-python/src/conversion.rs b/crates/cpex-hosts-python/src/conversion.rs index a826887d..530a88f6 100644 --- a/crates/cpex-hosts-python/src/conversion.rs +++ b/crates/cpex-hosts-python/src/conversion.rs @@ -204,9 +204,15 @@ pub fn deserialize_payload( /// missing `continue_processing` defaults to `true`, matching /// `PluginResult`'s Pydantic default — a plugin that returns nothing means /// "allow", not "deny". +/// +/// `inbound` is the capability-filtered view this plugin was dispatched with. +/// The returned-extensions path needs it to reuse the original `Arc`s for +/// immutable slots and to read the write tokens the executor issued — see the +/// `extensions` module for why both are load-bearing. pub fn response_to_result( hook_name: &str, response: Value, + inbound: &cpex_core::extensions::Extensions, ) -> Result { let continue_processing = response .get("continue_processing") @@ -223,9 +229,9 @@ pub fn response_to_result( Some(raw) => Some(deserialize_payload(hook_name, raw.clone())?), }; - let modified_extensions = match response.get("modified_extensions") { + let modified_extensions = match response.get(crate::extensions::MODIFIED_EXTENSIONS_FIELD) { Some(Value::Null) | None => None, - Some(raw) => owned_extensions_from_value(raw)?, + Some(raw) => crate::extensions::owned_from_returned_slot(raw, inbound)?, }; Ok(ErasedResultFields { @@ -236,82 +242,6 @@ pub fn response_to_result( }) } -/// Rebuild an `OwnedExtensions` from a worker's `modified_extensions`. -/// -/// `OwnedExtensions` is deliberately not `Deserialize`: its `http`, `security`, -/// and `delegation` slots are gated by `WriteToken`s that the *executor* mints -/// when it grants a plugin write access. A token parsed out of worker JSON -/// would be a forgery, letting an out-of-process plugin write slots it was -/// never granted — so those slots cannot be reconstructed here, by design. -/// -/// What is safe is `custom`: a plain `HashMap` with no write -/// token, which is how an out-of-process plugin passes structured data back. -/// That slot is honored. -/// -/// A token-gated slot is reported as an error rather than dropped, so a plugin -/// attempting one learns its write did not land instead of silently losing it. -/// Full extensions delivery — including the returned-extensions merge for the -/// gated slots — is owned by -/// `docs/plans/2026-07-29-003-feat-out-of-process-extensions-delivery-plan.md`. -fn owned_extensions_from_value( - raw: &Value, -) -> Result, HostError> { - let obj = raw.as_object().ok_or_else(|| HostError::Protocol { - message: "worker returned modified_extensions that is not a JSON object".into(), - })?; - - // Slots whose writes are token-gated. Reconstructing one would require - // minting a WriteToken this host has no authority to mint. - const GATED_SLOTS: [&str; 3] = ["http", "security", "delegation"]; - for slot in GATED_SLOTS { - if obj.get(slot).is_some_and(|v| !v.is_null()) { - return Err(HostError::Protocol { - message: format!( - "worker returned modified_extensions.{slot}, which is write-token gated and \ - cannot be applied from an out-of-process plugin — only `custom` is supported \ - on this path" - ), - }); - } - } - - let custom = match obj.get("custom") { - Some(Value::Null) | None => None, - Some(value) => Some( - serde_json::from_value::>(value.clone()) - .map_err(|e| HostError::Protocol { - message: format!( - "worker returned a modified_extensions.custom this host cannot read: {e}" - ), - })?, - ), - }; - - // Nothing applicable — treat as no modification rather than an empty write. - if custom.is_none() { - return Ok(None); - } - - Ok(Some(cpex_core::hooks::payload::OwnedExtensions { - request: None, - agent: None, - mcp: None, - completion: None, - provenance: None, - llm: None, - framework: None, - meta: None, - raw_credentials: None, - http: None, - security: None, - delegation: None, - custom, - http_write_token: None, - labels_write_token: None, - delegation_write_token: None, - })) -} - /// Parse a violation, tolerating the Python model's extra fields. /// /// The Python `PluginViolation` carries `mcp_error_code` and @@ -355,6 +285,17 @@ mod tests { use super::*; use std::collections::HashMap; + /// Inbound view for the response-conversion tests. + /// + /// Empty by default: these tests exercise response parsing, not the + /// capability-filtered delivery path, and an empty view carries no write + /// tokens — so a gated slot in a response is dropped rather than applied. + /// `extensions::tests` and `tests/extensions_merge_e2e.rs` cover the + /// token-bearing cases. + fn no_inbound() -> cpex_core::extensions::Extensions { + cpex_core::extensions::Extensions::default() + } + // --- hook name resolution ----------------------------------------------- #[test] @@ -550,6 +491,7 @@ mod tests { let fields = response_to_result( "tool_pre_invoke", serde_json::json!({ "continue_processing": true }), + &no_inbound(), ) .expect("converts"); @@ -563,8 +505,8 @@ mod tests { fn a_response_without_continue_processing_defaults_to_allow() { // Matches PluginResult's Pydantic default. Defaulting to deny would // block traffic on any plugin that returns a bare result. - let fields = - response_to_result("tool_pre_invoke", serde_json::json!({})).expect("converts"); + let fields = response_to_result("tool_pre_invoke", serde_json::json!({}), &no_inbound()) + .expect("converts"); assert!(fields.continue_processing); } @@ -582,6 +524,7 @@ mod tests { "mcp_error_code": -32603 } }), + &no_inbound(), ) .expect("converts"); @@ -609,6 +552,7 @@ mod tests { let fields = response_to_result( "tool_pre_invoke", serde_json::json!({ "continue_processing": false, "violation": { "reason": "nope" } }), + &no_inbound(), ) .expect("a sparse violation is still a violation"); @@ -626,6 +570,7 @@ mod tests { "continue_processing": true, "modified_payload": { "name": "search", "args": {"q": "[REDACTED]"} } }), + &no_inbound(), ) .expect("converts"); @@ -647,6 +592,7 @@ mod tests { "continue_processing": true, "modified_extensions": { "custom": { "seen_by": "py-plugin" } } }), + &no_inbound(), ) .expect("converts"); @@ -658,27 +604,31 @@ mod tests { } #[test] - fn a_write_token_gated_extension_slot_is_rejected_not_dropped() { - // `security` writes are gated by a WriteToken the executor mints. This - // host cannot mint one, so honoring the write would mean forging - // authorization — and dropping it silently would lose the plugin's - // intent. Erroring is the only honest option. + fn a_gated_slot_write_without_the_capability_is_dropped() { + // `http`, `security`, and `delegation` writes are gated by a WriteToken + // the *executor* mints from the plugin's declared capabilities and + // carries on the inbound view. An empty inbound view means no token was + // issued, so the write is dropped rather than applied — the host cannot + // mint a token and must not honor an unauthorized write. + // + // This used to be a hard error, back when the host had no inbound view + // to consult and so could not tell an authorized write from an + // unauthorized one. It can now, so the tier is enforced instead of + // refused. `extensions::tests` covers the drop at slot granularity. for slot in ["http", "security", "delegation"] { - // `ErasedResultFields` is not Debug, so `expect_err` is unavailable. - let Err(err) = response_to_result( + let fields = response_to_result( "tool_pre_invoke", serde_json::json!({ "continue_processing": true, "modified_extensions": { slot: {"labels": ["PII"]} } }), - ) else { - panic!("a token-gated slot ('{slot}') must not be silently dropped"); - }; + &no_inbound(), + ) + .expect("an unauthorized gated write is dropped, not a protocol error"); - assert!(matches!(err, HostError::Protocol { .. })); assert!( - err.to_string().contains(slot), - "the error should name the slot the plugin tried to write: {err}" + fields.modified_extensions.is_none(), + "the '{slot}' write had no token behind it, so nothing should merge" ); } } @@ -694,6 +644,7 @@ mod tests { "continue_processing": true, "modified_extensions": { "http": null, "security": null, "custom": null } }), + &no_inbound(), ) .expect("all-null slots are not a write"); assert!(fields.modified_extensions.is_none()); @@ -712,6 +663,7 @@ mod tests { "modified_extensions": null, "violation": null }), + &no_inbound(), ) .expect("nulls are absent, not malformed"); @@ -725,6 +677,7 @@ mod tests { let Err(err) = response_to_result( "tool_pre_invoke", serde_json::json!({ "continue_processing": true, "modified_payload": "not an object" }), + &no_inbound(), ) else { panic!("a payload that cannot be rebuilt must not be silently dropped"); }; diff --git a/crates/cpex-hosts-python/src/extensions.rs b/crates/cpex-hosts-python/src/extensions.rs new file mode 100644 index 00000000..ca10c605 --- /dev/null +++ b/crates/cpex-hosts-python/src/extensions.rs @@ -0,0 +1,746 @@ +// Location: ./crates/cpex-hosts-python/src/extensions.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// Extensions wire contract for out-of-process Python plugins. +// +// # Why this exists +// +// `worker.py` calls `execute_plugin` without an `extensions` argument, so every +// out-of-process hook sees `extensions=None`. A plugin using the 3-arg +// `(payload, context, extensions)` signature — the form `_accepts_extensions` +// detects in `cpex/framework/base.py` — silently loses all extension context +// when it runs out-of-process: security labels, agent lineage, HTTP headers, +// MCP metadata. In-process plugins see all of it. That gap is what this module +// closes, in both directions. +// +// # Two directions, one contract +// +// Outbound (`attach_extensions`): serialize the capability-filtered +// `&Extensions` the executor handed the adapter and attach it to the task. The +// filtered view is used deliberately rather than the full extensions, so a +// plugin sees only the slots its declared capabilities permit and this host +// never re-derives filtering the executor already did. +// +// Inbound (`parse_returned_extensions`): read the worker's returned extensions +// and rebuild an `OwnedExtensions` for the executor's existing copy-on-write +// merge, which enforces the mutability tiers. This module adds no tier logic. +// +// # Why the return path rebuilds instead of deserializing +// +// `Extensions::validate_immutable` enforces the immutable tier with +// `Arc::ptr_eq` — pointer identity, not value equality. A JSON round trip +// allocates a fresh `Arc` for every slot, so an `OwnedExtensions` deserialized +// wholesale from the wire would fail that check on *every* immutable slot and +// the executor would reject the whole return with a spurious "violated +// immutable tier" warning, even for a plugin that only touched `custom`. +// +// So the return path reuses the inbound `Arc`s for immutable slots — exactly +// what `cow_copy()` does — and takes only the mutable, monotonic, and guarded +// slots (`http`, `security`, `delegation`, `custom`) from the wire. Two +// consequences, both intended: +// +// 1. Honest plugins pass `validate_immutable`, because the immutable slots +// are the same pointers the executor issued. +// 2. A plugin that tries to forge an immutable slot has that edit dropped +// here rather than rejected downstream. The spec outcome ("immutable slots +// do not change") holds structurally, without trusting the wire. +// +// # Credentials are not on this channel +// +// This channel carries non-secret context only. `raw_credentials` is never +// serialized here — raw tokens travel the capability-gated DTO in the +// `credentials` module instead. Sensitive headers are stripped in both +// directions per `docs/specs/cmf-message-spec.md` §3.5; a plugin that needs a +// bearer token uses the credential path, not `http.request_headers`. + +use std::collections::HashMap; + +use cpex_core::extensions::container::OwnedExtensions; +use cpex_core::extensions::delegation::DelegationExtension; +use cpex_core::extensions::guarded::Guarded; +use cpex_core::extensions::http::HttpExtension; +use cpex_core::extensions::security::SecurityExtension; +use cpex_core::extensions::Extensions; +use serde_json::Value; + +use crate::error::HostError; + +/// Task field the inbound extensions ride on. Contract with `worker.py`, which +/// reads it and passes the reconstructed object as `extensions=` to +/// `execute_plugin`; both sides must agree verbatim. +pub const EXTENSIONS_FIELD: &str = "extensions"; + +/// Response field the returned extensions ride on. +/// +/// Deliberately *not* the same name as [`EXTENSIONS_FIELD`]. The worker's +/// response is a serialized Python `PluginResult`, and that model already +/// carries `modified_extensions: Optional[Extensions]` (`cpex/framework/ +/// models.py`) — the same field the in-process Python manager accumulates. This +/// host reads the field the model already produces rather than asking the worker +/// to invent a second one, so an out-of-process plugin returns extensions the +/// exact same way its in-process equivalent does. +pub const MODIFIED_EXTENSIONS_FIELD: &str = "modified_extensions"; + +/// Headers stripped in both directions (spec §3.5). +/// +/// Matched case-insensitively: HTTP header names are case-insensitive, and +/// `HttpExtension`'s own accessors look up that way, so a plugin sending +/// `authorization` must not slip past a case-sensitive compare. +const SENSITIVE_HEADERS: &[&str] = &["authorization", "cookie", "x-api-key"]; + +/// True when `name` is a header this channel must not carry. +fn is_sensitive(name: &str) -> bool { + SENSITIVE_HEADERS + .iter() + .any(|s| name.eq_ignore_ascii_case(s)) +} + +/// Drop sensitive entries from a header map, leaving the rest untouched. +fn strip_sensitive(headers: &HashMap) -> HashMap { + headers + .iter() + .filter(|(name, _)| !is_sensitive(name)) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +/// Copy an `HttpExtension` with sensitive headers removed from both maps. +/// +/// Both maps are scrubbed: `response_headers` can carry a `Set-Cookie` or an +/// upstream `Authorization` echo just as `request_headers` carries the inbound +/// credential. +fn sanitize_http(http: &HttpExtension) -> HttpExtension { + HttpExtension { + request_headers: strip_sensitive(&http.request_headers), + response_headers: strip_sensitive(&http.response_headers), + method: http.method.clone(), + path: http.path.clone(), + host: http.host.clone(), + scheme: http.scheme.clone(), + } +} + +/// Serialize the filtered extensions and attach them to `task`. +/// +/// `extensions` is the capability-filtered view the executor produced for this +/// plugin, so slot visibility is already correct — an absent slot means the +/// plugin's capabilities excluded it, and it stays absent on the wire. +/// +/// `raw_credentials` is excluded outright. Its token fields are `#[serde(skip)]` +/// so they would serialize empty anyway, but sending a hollow slot invites a +/// plugin to read it as "no credential present" rather than "not on this +/// channel". Omitting it says the latter unambiguously. +/// +/// Attaching nothing is valid: extensions with no visible slots produce no +/// field, and the worker passes `None` to `execute_plugin` exactly as today. +pub fn attach_extensions(task: &mut Value, extensions: &Extensions) -> Result<(), HostError> { + let wire = to_wire(extensions)?; + + // An empty object carries no information the worker can act on; omitting + // the field keeps the "no extensions" path byte-identical to a task built + // before this feature existed. + if wire.as_object().is_none_or(serde_json::Map::is_empty) { + return Ok(()); + } + + let object = task.as_object_mut().ok_or_else(|| HostError::Protocol { + message: "task must be a JSON object to carry extensions".into(), + })?; + object.insert(EXTENSIONS_FIELD.to_string(), wire); + + Ok(()) +} + +/// Build the wire JSON for a filtered extensions view. +/// +/// Slots serialize through their own serde impls — those are the source of +/// truth for sub-field shape, so this function stays correct as the extension +/// structs evolve. Only `http` is rewritten on the way out, to strip sensitive +/// headers. +fn to_wire(extensions: &Extensions) -> Result { + let mut map = serde_json::Map::new(); + + let mut put = |key: &str, value: Result| -> Result<(), HostError> { + let value = value.map_err(|e| HostError::Protocol { + message: format!("could not serialize the '{key}' extension for the worker: {e}"), + })?; + map.insert(key.to_string(), value); + Ok(()) + }; + + if let Some(request) = &extensions.request { + put("request", serde_json::to_value(request))?; + } + if let Some(agent) = &extensions.agent { + put("agent", serde_json::to_value(agent))?; + } + if let Some(http) = &extensions.http { + put("http", serde_json::to_value(sanitize_http(http)))?; + } + if let Some(security) = &extensions.security { + put("security", serde_json::to_value(security))?; + } + if let Some(delegation) = &extensions.delegation { + put("delegation", serde_json::to_value(delegation))?; + } + if let Some(mcp) = &extensions.mcp { + put("mcp", serde_json::to_value(mcp))?; + } + if let Some(completion) = &extensions.completion { + put("completion", serde_json::to_value(completion))?; + } + if let Some(provenance) = &extensions.provenance { + put("provenance", serde_json::to_value(provenance))?; + } + if let Some(llm) = &extensions.llm { + put("llm", serde_json::to_value(llm))?; + } + if let Some(framework) = &extensions.framework { + put("framework", serde_json::to_value(framework))?; + } + if let Some(meta) = &extensions.meta { + put("meta", serde_json::to_value(meta))?; + } + if let Some(custom) = &extensions.custom { + put("custom", serde_json::to_value(custom))?; + } + // `raw_credentials` deliberately absent — see the module docs. + + Ok(Value::Object(map)) +} + +/// Rebuild an `OwnedExtensions` from the worker's response, or `None`. +/// +/// `None` means "no modified extensions" and is the documented no-change +/// signal: the worker omits the field. Omission rather than an echo is +/// deliberate — a JSON round trip allocates new `Arc`s, so an echoed immutable +/// slot is indistinguishable from a forged one under `validate_immutable`'s +/// pointer check. Omitting is the only representation that reads cleanly as +/// "nothing changed". +/// +/// `inbound` is the filtered view this plugin was sent. Its `Arc`s are reused +/// for immutable slots so the executor's pointer-identity check passes; only +/// the tiers a plugin may legitimately write are taken from the wire. +/// +/// This function does not decide whether a change is *allowed* — the executor's +/// merge does, via `validate_immutable`, the monotonic label check, and the +/// write tokens carried on `inbound`. Malformed slot JSON is an error: a plugin +/// that returned a `security` object the host cannot parse has not made "no +/// change", and silently dropping it would hide the failure. +pub fn parse_returned_extensions( + response: &Value, + inbound: &Extensions, +) -> Result, HostError> { + let Some(field) = response.get(MODIFIED_EXTENSIONS_FIELD) else { + return Ok(None); + }; + owned_from_returned_slot(field, inbound) +} + +/// Rebuild an `OwnedExtensions` from an already-extracted +/// `modified_extensions` value. +/// +/// Split out so the response-conversion path can call it with the raw slot it +/// already looked up, without re-walking the response object. +pub fn owned_from_returned_slot( + field: &Value, + inbound: &Extensions, +) -> Result, HostError> { + // Explicit null is the same statement as an omitted field. `worker.py` + // serializing `None` should not read as "clear every writable slot". + if field.is_null() { + return Ok(None); + } + + let object = field.as_object().ok_or_else(|| HostError::Protocol { + message: format!( + "the worker returned a '{MODIFIED_EXTENSIONS_FIELD}' field that is not a JSON object" + ), + })?; + + // Start from the inbound view: immutable slots keep their original `Arc` + // pointers, and the write tokens the executor issued are carried over. + let mut owned = inbound.cow_copy(); + + // Whether any slot was actually taken from the wire. Pydantic serializes + // unset Optionals as `null`, so a plugin that touched nothing still sends + // every key — that must read as "no modification" rather than a no-op merge + // the executor has to validate. + let mut applied = false; + + fn slot( + object: &serde_json::Map, + key: &str, + ) -> Result, HostError> { + match object.get(key) { + None | Some(Value::Null) => Ok(None), + Some(value) => serde_json::from_value(value.clone()) + .map(Some) + .map_err(|e| HostError::Protocol { + message: format!( + "could not deserialize the '{key}' extension returned by the worker: {e}" + ), + }), + } + } + + // The three writable-but-gated slots are honored only when the executor + // issued the matching write token on the inbound view. The token is the + // host's *only* evidence that the plugin declared the write capability — + // `WriteToken::new()` is `pub(crate)` to `cpex-core`, so this crate cannot + // mint one, and a token can never be forged out of worker JSON. An edit + // without the token is dropped and the inbound value stands. + + // Guarded — `write_headers`. + if let Some(http) = slot::(object, "http")? { + if inbound.http_write_token.is_some() { + // Strip on the way back too. A plugin cannot inject a credential + // header into the pipeline through its return value. + owned.http = Some(Guarded::new(sanitize_http(&http))); + applied = true; + } + } + + // Monotonic — `append_labels`. The executor additionally checks + // `before ⊆ after` and rejects the whole return on a removal. + if let Some(security) = slot::(object, "security")? { + if inbound.labels_write_token.is_some() { + owned.security = Some(security); + applied = true; + } + } + + // Monotonic — `append_delegation`. + if let Some(delegation) = slot::(object, "delegation")? { + if inbound.delegation_write_token.is_some() { + owned.delegation = Some(delegation); + applied = true; + } + } + + // Mutable — accepted as-is. + if let Some(custom) = slot::>(object, "custom")? { + owned.custom = Some(custom); + applied = true; + } + + // Immutable slots are ignored on purpose: `owned` already holds the + // inbound `Arc`s, so a forged edit here is dropped rather than merged. + + if !applied { + return Ok(None); + } + + Ok(Some(owned)) +} + +/// Reuse the inbound `Arc` for a slot the wire must not change. +/// +/// Kept as a named helper so the intent reads at the call site in tests. +#[cfg(test)] +fn same_arc(a: &Option>, b: &Option>) -> bool { + match (a, b) { + (Some(a), Some(b)) => std::sync::Arc::ptr_eq(a, b), + (None, None) => true, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::sync::Arc; + + use cpex_core::extensions::agent::AgentExtension; + use cpex_core::extensions::monotonic::MonotonicSet; + + use super::*; + + fn http_with_headers() -> HttpExtension { + let mut http = HttpExtension::default(); + http.set_request_header("Authorization", "Bearer secret-token"); + http.set_request_header("Cookie", "session=abc"); + http.set_request_header("X-API-Key", "key-123"); + http.set_request_header("X-Request-Id", "req-1"); + http.set_response_header("Set-Cookie", "s=1"); + http.set_response_header("Authorization", "Bearer echoed"); + http.set_response_header("Content-Type", "application/json"); + http + } + + fn security_with_labels(labels: &[&str]) -> SecurityExtension { + let set: HashSet = labels.iter().map(|s| s.to_string()).collect(); + SecurityExtension { + labels: MonotonicSet::from_set(set), + ..Default::default() + } + } + + fn populated() -> Extensions { + Extensions { + agent: Some(Arc::new(AgentExtension::default())), + http: Some(Arc::new(http_with_headers())), + security: Some(Arc::new(security_with_labels(&["PII"]))), + ..Default::default() + } + } + + // -- Outbound: U1 -- + + #[test] + fn present_slots_land_on_the_wire() { + let mut task = serde_json::json!({"task_type": "run"}); + attach_extensions(&mut task, &populated()).expect("serializing a populated view"); + + let wire = task + .get(EXTENSIONS_FIELD) + .expect("the field is attached") + .as_object() + .expect("the field is an object"); + + assert!(wire.contains_key("agent"), "agent slot present"); + assert!(wire.contains_key("http"), "http slot present"); + assert!(wire.contains_key("security"), "security slot present"); + } + + #[test] + fn sensitive_request_headers_are_stripped_but_others_survive() { + let mut task = serde_json::json!({}); + attach_extensions(&mut task, &populated()).expect("serializing"); + + let headers = task[EXTENSIONS_FIELD]["http"]["request_headers"] + .as_object() + .expect("request_headers is an object"); + + // Case-insensitively: none of the three may appear under any casing. + for name in headers.keys() { + assert!( + !is_sensitive(name), + "sensitive header '{name}' must not cross the process boundary" + ); + } + assert_eq!( + headers.get("X-Request-Id").and_then(Value::as_str), + Some("req-1"), + "a non-sensitive header is preserved verbatim" + ); + } + + #[test] + fn sensitive_response_headers_are_stripped_too() { + let mut task = serde_json::json!({}); + attach_extensions(&mut task, &populated()).expect("serializing"); + + let headers = task[EXTENSIONS_FIELD]["http"]["response_headers"] + .as_object() + .expect("response_headers is an object"); + + for name in headers.keys() { + assert!( + !is_sensitive(name), + "sensitive response header '{name}' must not cross the boundary" + ); + } + assert_eq!( + headers.get("Content-Type").and_then(Value::as_str), + Some("application/json"), + ); + } + + #[test] + fn lowercase_sensitive_headers_are_stripped() { + // HTTP header names are case-insensitive; a case-sensitive filter would + // leak the token whenever the host populated it in lower case. + let mut http = HttpExtension::default(); + http.set_request_header("authorization", "Bearer sneaky"); + http.set_request_header("x-api-key", "k"); + let extensions = Extensions { + http: Some(Arc::new(http)), + ..Default::default() + }; + + let mut task = serde_json::json!({}); + attach_extensions(&mut task, &extensions).expect("serializing"); + + let headers = task[EXTENSIONS_FIELD]["http"]["request_headers"] + .as_object() + .expect("request_headers is an object"); + assert!( + headers.is_empty(), + "lower-cased sensitive headers must be stripped too, got {headers:?}" + ); + } + + #[test] + fn a_filtered_out_slot_is_absent_from_the_wire() { + // A plugin without `read_agent` gets a view with `agent: None`. The + // wire must not resurrect it. + let extensions = Extensions { + security: Some(Arc::new(security_with_labels(&["PII"]))), + ..Default::default() + }; + + let mut task = serde_json::json!({}); + attach_extensions(&mut task, &extensions).expect("serializing"); + + let wire = task[EXTENSIONS_FIELD].as_object().expect("an object"); + assert!( + !wire.contains_key("agent"), + "a slot the capabilities excluded stays excluded" + ); + assert!(wire.contains_key("security")); + } + + #[test] + fn raw_credentials_never_appear_on_the_wire() { + use cpex_core::extensions::raw_credentials::{ + RawCredentialsExtension, RawInboundToken, TokenKind, TokenRole, + }; + + let mut inbound_tokens = HashMap::new(); + inbound_tokens.insert( + TokenRole::User, + RawInboundToken::new("super-secret", "Authorization", TokenKind::Jwt), + ); + + let extensions = Extensions { + raw_credentials: Some(Arc::new(RawCredentialsExtension { + inbound_tokens, + ..Default::default() + })), + security: Some(Arc::new(security_with_labels(&["PII"]))), + ..Default::default() + }; + + let mut task = serde_json::json!({}); + attach_extensions(&mut task, &extensions).expect("serializing"); + + let wire = task[EXTENSIONS_FIELD].as_object().expect("an object"); + assert!( + !wire.contains_key("raw_credentials"), + "the credential slot belongs to the credential DTO, not this channel" + ); + let serialized = serde_json::to_string(&task).expect("re-serializing the task"); + assert!( + !serialized.contains("super-secret"), + "no token bytes anywhere in the task JSON" + ); + } + + #[test] + fn empty_extensions_attach_no_field() { + let mut task = serde_json::json!({"task_type": "run"}); + attach_extensions(&mut task, &Extensions::default()).expect("serializing an empty view"); + assert!( + task.get(EXTENSIONS_FIELD).is_none(), + "a view with no visible slots leaves the task shape unchanged" + ); + } + + // -- Inbound: U4 -- + + #[test] + fn an_absent_field_means_no_change() { + let response = serde_json::json!({"payload": {}}); + let parsed = parse_returned_extensions(&response, &populated()).expect("parsing"); + assert!( + parsed.is_none(), + "an omitted field is the documented no-change signal" + ); + } + + #[test] + fn an_explicit_null_means_no_change() { + let response = serde_json::json!({"modified_extensions": null}); + let parsed = parse_returned_extensions(&response, &populated()).expect("parsing"); + assert!(parsed.is_none(), "null reads the same as omitted"); + } + + #[test] + fn a_non_object_field_is_a_protocol_error() { + let response = serde_json::json!({"modified_extensions": "nope"}); + let err = parse_returned_extensions(&response, &populated()); + assert!(err.is_err(), "a non-object field cannot be interpreted"); + } + + #[test] + fn immutable_slots_keep_their_inbound_arcs() { + // The executor's `validate_immutable` compares by pointer. If the + // return path allocated a fresh Arc here, every out-of-process + // extension return would be rejected as tampering. + let inbound = populated(); + let response = serde_json::json!({ + "modified_extensions": {"custom": {"k": "v"}} + }); + + let owned = parse_returned_extensions(&response, &inbound) + .expect("parsing") + .expect("a returned field yields modifications"); + + assert!( + same_arc(&inbound.agent, &owned.agent), + "the agent slot must be the very same Arc the executor issued" + ); + assert!(inbound.validate_immutable(&owned), "the merge accepts it"); + } + + #[test] + fn a_forged_immutable_slot_is_dropped_not_merged() { + let inbound = populated(); + // A plugin returning a different `agent` must not change it. + let response = serde_json::json!({ + "modified_extensions": { + "agent": {"agent_id": "forged"}, + "custom": {"ok": true} + } + }); + + let owned = parse_returned_extensions(&response, &inbound) + .expect("parsing") + .expect("modifications present"); + + assert!( + same_arc(&inbound.agent, &owned.agent), + "the forged agent edit is dropped, not merged" + ); + assert!( + inbound.validate_immutable(&owned), + "so the rest of the return still merges cleanly" + ); + assert!( + owned.custom.is_some(), + "the legitimate custom edit survives" + ); + } + + #[test] + fn a_custom_change_is_taken_as_is() { + let response = serde_json::json!({ + "modified_extensions": {"custom": {"verdict": "clean", "score": 3}} + }); + let owned = parse_returned_extensions(&response, &populated()) + .expect("parsing") + .expect("modifications present"); + + let custom = owned.custom.expect("custom present"); + assert_eq!(custom.get("verdict").and_then(Value::as_str), Some("clean")); + assert_eq!(custom.get("score").and_then(Value::as_i64), Some(3)); + } + + // The three gated slots below can only be *accepted* when the executor + // issued a write token, and `WriteToken::new()` is `pub(crate)` to + // `cpex-core` — this crate cannot mint one even in a test. That is the + // security property, so these tests assert the deny side here and the + // accept side is covered end-to-end through the real executor in + // `tests/extensions_merge_e2e.rs`, where tokens come from capabilities. + + #[test] + fn a_label_append_without_the_token_is_dropped() { + let inbound = populated(); // labels: {PII} + assert!( + inbound.labels_write_token.is_none(), + "no append_labels capability was granted" + ); + + let response = serde_json::json!({ + "modified_extensions": {"security": {"labels": ["PII", "SCANNED"]}} + }); + let parsed = parse_returned_extensions(&response, &inbound).expect("parsing"); + + assert!( + parsed.is_none(), + "an unauthorized label append yields no modification, so the \ + pipeline's labels are untouched" + ); + } + + #[test] + fn a_label_removal_without_the_token_cannot_strip_labels() { + // An out-of-process plugin must not be able to launder a + // declassification through this host by returning a shorter label set. + let inbound = populated(); // labels: {PII} + let response = serde_json::json!({ + "modified_extensions": {"security": {"labels": []}} + }); + + let parsed = parse_returned_extensions(&response, &inbound).expect("parsing"); + assert!( + parsed.is_none(), + "the removal never reaches the merge, so PII cannot be stripped" + ); + } + + #[test] + fn an_http_change_needs_the_write_token() { + // No token issued — the executor withheld `write_headers`. + let inbound = populated(); + assert!(inbound.http_write_token.is_none()); + + let response = serde_json::json!({ + "modified_extensions": {"http": {"request_headers": {"X-Added": "1"}}} + }); + let parsed = parse_returned_extensions(&response, &inbound).expect("parsing"); + + assert!( + parsed.is_none(), + "an http edit without the capability is dropped entirely" + ); + } + + #[test] + fn a_delegation_change_without_the_token_is_dropped() { + let inbound = populated(); + assert!(inbound.delegation_write_token.is_none()); + + let response = serde_json::json!({ + "modified_extensions": {"delegation": {"hops": []}} + }); + let parsed = parse_returned_extensions(&response, &inbound).expect("parsing"); + + assert!( + parsed.is_none(), + "the wire cannot add a delegation slot without the capability" + ); + } + + #[test] + fn an_unauthorized_gated_write_does_not_suppress_a_legitimate_custom_write() { + // A plugin can return both. The gated slot is dropped for want of a + // token; `custom` is mutable and must still land. + let inbound = populated(); + let response = serde_json::json!({ + "modified_extensions": { + "security": {"labels": ["PII", "SCANNED"]}, + "custom": {"verdict": "clean"} + } + }); + + let owned = parse_returned_extensions(&response, &inbound) + .expect("parsing") + .expect("the custom write is a real modification"); + + assert_eq!( + owned.custom.as_ref().expect("custom present")["verdict"], + "clean" + ); + let security = owned.security.as_ref().expect("the inbound slot stands"); + assert!( + !security.labels.contains(&"SCANNED".to_string()), + "the unauthorized label append is still dropped" + ); + } + + #[test] + fn malformed_slot_json_is_an_error_not_a_silent_drop() { + let response = serde_json::json!({ + "modified_extensions": {"security": "not-an-object"} + }); + let err = parse_returned_extensions(&response, &populated()); + assert!( + err.is_err(), + "unparseable slot JSON must surface, not read as no-change" + ); + } +} diff --git a/crates/cpex-hosts-python/src/lib.rs b/crates/cpex-hosts-python/src/lib.rs index c9cf3b92..d272a882 100644 --- a/crates/cpex-hosts-python/src/lib.rs +++ b/crates/cpex-hosts-python/src/lib.rs @@ -49,10 +49,21 @@ // in-memory token directly. Production credential types keep their serde // guard and are never serialized. See the `credentials` module for the // fail-closed rules and the residual exposure this does not close. +// +// # Extensions +// +// The capability-filtered `Extensions` the executor produced for a plugin is +// serialized onto the task, so a 3-arg `(payload, context, extensions)` hook +// sees out-of-process what it would see in-process. The plugin's returned +// extensions come back through the executor's existing copy-on-write merge, +// which enforces the mutability tiers — this host adds no tier logic. Sensitive +// headers are stripped in both directions and `raw_credentials` never rides +// this channel. See the `extensions` module. pub mod conversion; pub mod credentials; pub mod error; +pub mod extensions; pub mod factory; pub mod legacy; pub mod plugin; diff --git a/crates/cpex-hosts-python/src/plugin.rs b/crates/cpex-hosts-python/src/plugin.rs index 36b2d7a4..b535fab4 100644 --- a/crates/cpex-hosts-python/src/plugin.rs +++ b/crates/cpex-hosts-python/src/plugin.rs @@ -551,6 +551,11 @@ impl AnyHookHandler for PythonHookAdapter { ) .map_err(to_plugin_error)?; + // The capability-filtered extensions, so a 3-arg + // `(payload, context, extensions)` hook sees out-of-process what it + // would see in-process. Attaches nothing when no slot is visible. + crate::extensions::attach_extensions(&mut task, extensions).map_err(to_plugin_error)?; + let worker = self.plugin.worker().await?; let response = worker.send_task(task).await.map_err(to_plugin_error)?; @@ -558,8 +563,12 @@ impl AnyHookHandler for PythonHookAdapter { // before the result is returned, so a plugin's context writes survive. apply_context_deltas(&response, ctx); - let fields = - conversion::response_to_result(self.hook_name, response).map_err(to_plugin_error)?; + // `extensions` is passed back in so the returned-extensions path can + // reuse the inbound `Arc`s for immutable slots and read the write + // tokens the executor issued. The executor's copy-on-write merge then + // validates the result against the mutability tiers. + let fields = conversion::response_to_result(self.hook_name, response, extensions) + .map_err(to_plugin_error)?; Ok(Box::new(fields)) } diff --git a/crates/cpex-hosts-python/src/testing.rs b/crates/cpex-hosts-python/src/testing.rs index cc4731b3..43ec4d9c 100644 --- a/crates/cpex-hosts-python/src/testing.rs +++ b/crates/cpex-hosts-python/src/testing.rs @@ -140,6 +140,21 @@ pub fn worker_consumes_credentials(source: &Path) -> bool { .unwrap_or(false) } +/// Whether a framework checkout's worker delivers the `extensions` field. +/// +/// The extensions channel needs a `worker.py` that reads the task's +/// `extensions` field, reconstructs a Python `Extensions`, and passes it as +/// `extensions=` to `execute_plugin`. A worker predating that change calls +/// `execute_plugin` without the argument, so every hook sees `extensions=None` +/// and an extensions test would fail for the wrong reason. Gate on the field +/// name and the keyword argument together — the name alone appears in +/// unrelated comments. +pub fn worker_delivers_extensions(source: &Path) -> bool { + std::fs::read_to_string(source.join("cpex/framework/isolated/worker.py")) + .map(|s| s.contains("EXTENSIONS_FIELD") && s.contains("extensions=")) + .unwrap_or(false) +} + /// Lay out a plugin package under `/plugins/`. /// /// Writes `__init__.py`, the plugin module, and a `requirements.txt` pointing diff --git a/crates/cpex-hosts-python/tests/extensions_merge_e2e.rs b/crates/cpex-hosts-python/tests/extensions_merge_e2e.rs new file mode 100644 index 00000000..15bbb10f --- /dev/null +++ b/crates/cpex-hosts-python/tests/extensions_merge_e2e.rs @@ -0,0 +1,718 @@ +// Location: ./crates/cpex-hosts-python/tests/extensions_merge_e2e.rs +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 +// Authors: Ted Habeck +// +// End-to-end: extensions delivered to, and returned from, an out-of-process +// Python plugin — with the mutability tiers enforced by the real executor. +// +// # Why these go through `PluginManager` rather than the adapter alone +// +// The unit tests in `src/extensions.rs` cover the wire shape and every *deny* +// outcome. They cannot cover the accept outcomes, and that is by design: +// `http`, `security`, and `delegation` writes are gated by a `WriteToken`, and +// `WriteToken::new()` is `pub(crate)` to `cpex-core`. This crate cannot mint +// one even in a test — which is exactly the property that makes the gate +// trustworthy. +// +// The only legitimate source of a token is the executor, which mints one per +// plugin from that plugin's declared capabilities. So the accept side has to be +// driven through a real pipeline: register the Python plugin with a capability +// set, invoke the hook, and assert on the merged `Extensions` that come back in +// `PipelineResult`. That also means these tests exercise the thing the plan +// actually promises — the executor's copy-on-write merge validating the return — +// instead of a host-local imitation of it. +// +// # Two layers, two skip conditions +// +// The inbound-shape test needs nothing but this crate: it drives the adapter's +// task-building step and inspects the JSON. The round-trip tests need a real +// worker, so they need python3, a `cpex` Python checkout, and a `worker.py` that +// actually consumes the `extensions` field. The Python side lives on a different +// branch (see `testing::python_source`), so those skip with a precise reason +// until it lands rather than failing for the wrong one. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use cpex_core::extensions::{Extensions, HttpExtension, SecurityExtension}; +use cpex_core::plugin::PluginConfig; +use cpex_hosts_python::extensions::{EXTENSIONS_FIELD, MODIFIED_EXTENSIONS_FIELD}; +use cpex_hosts_python::factory::KIND; +use cpex_hosts_python::testing::{ + prebuild_venv, python_source, scaffold_plugin, skip_without_python3, + worker_delivers_extensions, TempDir, +}; + +/// Fully-qualified class name of the fixture plugin. +const FIXTURE_CLASS: &str = "ext_pkg.extensions_plugin.ExtensionsAwarePlugin"; + +/// A label the host puts on the inbound extensions. Distinctive so the fixture +/// can assert it saw the real thing rather than a default-constructed object. +const INBOUND_LABEL: &str = "E2E-INBOUND-LABEL-7b21"; + +/// The label the fixture appends, to exercise the monotonic tier. +const APPENDED_LABEL: &str = "E2E-APPENDED-LABEL-c93f"; + +/// A 3-arg plugin: it reads the inbound extensions and appends a label. +/// +/// The existing isolated fixtures are all 2-arg, so none of them can observe +/// this channel at all. The 3-arg `(payload, context, extensions)` signature is +/// the form `_accepts_extensions` detects in `cpex/framework/base.py`. +/// +/// It records what it saw to a file rather than returning it, so the inbound +/// assertions do not depend on the return path also working. +const FIXTURE_PLUGIN: &str = r#" +import os + +from cpex.framework.base import Plugin +from cpex.framework.models import PluginResult + +INBOUND_LABEL = "E2E-INBOUND-LABEL-7b21" +APPENDED_LABEL = "E2E-APPENDED-LABEL-c93f" + + +class ExtensionsAwarePlugin(Plugin): + """Reports the extensions it received, then appends a security label.""" + + def __init__(self, config): + super().__init__(config) + + async def tool_pre_invoke(self, payload, context, extensions): + # A 2-arg plugin would never get here; the framework only forwards + # extensions to hooks whose signature accepts them. + if extensions is None: + verdict = "NO_EXTENSIONS" + else: + labels = set() + if extensions.security is not None: + labels = set(extensions.security.labels or []) + saw_label = INBOUND_LABEL in labels + # Sensitive headers must never arrive over this channel. + headers = {} + if extensions.http is not None: + headers = dict(extensions.http.request_headers or {}) + lowered = {k.lower() for k in headers} + leaked = lowered & {"authorization", "cookie", "x-api-key"} + verdict = "GOT_LABEL" if saw_label else "NO_LABEL" + if leaked: + verdict += f";LEAKED={sorted(leaked)}" + if "x-request-id" in lowered: + verdict += ";GOT_SAFE_HEADER" + + marker = os.path.join(os.getcwd(), "extensions_verdict.txt") + with open(marker, "w") as f: + f.write(verdict) + + # Append a label. `Extensions` is frozen, so modification means a new + # instance via model_copy — the framework's documented pattern. + modified = None + if extensions is not None and extensions.security is not None: + new_labels = list(extensions.security.labels or []) + [APPENDED_LABEL] + new_security = extensions.security.model_copy(update={"labels": new_labels}) + modified = extensions.model_copy(update={"security": new_security}) + + return PluginResult(continue_processing=True, modified_extensions=modified) +"#; + +/// Skip guard for the worker-backed tests. +fn require_worker(test_name: &str) -> Option { + if skip_without_python3(test_name) { + return None; + } + let source = match python_source() { + Ok(source) => source, + Err(reason) => { + println!("SKIP {test_name}: {reason}"); + return None; + }, + }; + if !worker_delivers_extensions(&source) { + println!( + "SKIP {test_name}: the cpex source at {} has a worker.py that does not deliver the \ + `{EXTENSIONS_FIELD}` field to execute_plugin — the inbound channel has no consumer \ + there yet", + source.display() + ); + return None; + } + Some(source) +} + +/// Scaffold the fixture and pre-build its venv. +fn setup(dir: &TempDir, source: &Path, test_name: &str) -> Option { + let plugin_dir = scaffold_plugin(dir, source, "ext_pkg", "extensions_plugin", FIXTURE_PLUGIN); + match prebuild_venv(&plugin_dir, source, FIXTURE_CLASS, "ext_pkg") { + Ok(()) => Some(plugin_dir), + Err(reason) => { + println!("SKIP {test_name}: could not prepare the plugin venv — {reason}"); + None + }, + } +} + +/// Inbound extensions: a label to read, a safe header, and three that must be +/// stripped before they reach another process. +fn inbound_extensions() -> Extensions { + let mut security = SecurityExtension::default(); + security.add_label(INBOUND_LABEL); + + let mut http = HttpExtension::default(); + http.set_request_header("Authorization", "Bearer must-not-travel"); + http.set_request_header("Cookie", "session=must-not-travel"); + http.set_request_header("X-API-Key", "must-not-travel"); + http.set_request_header("X-Request-Id", "req-e2e-1"); + + Extensions { + security: Some(Arc::new(security)), + http: Some(Arc::new(http)), + ..Default::default() + } +} + +/// The legacy `tool_pre_invoke` hook, as a type the manager can dispatch. +/// +/// `invoke_named` is generic over `HookTypeDef` to recover the payload type for +/// the typed handler path. The Python host registers type-erased handlers, so +/// only the payload type matters here — it must match what the adapter +/// serializes, which is this crate's `legacy::ToolPreInvokePayload`. +struct ToolPreInvoke; + +impl cpex_core::hooks::trait_def::HookTypeDef for ToolPreInvoke { + type Payload = cpex_hosts_python::legacy::ToolPreInvokePayload; + type Result = + cpex_core::hooks::trait_def::PluginResult; + const NAME: &'static str = "tool_pre_invoke"; +} + +fn plugin_config(capabilities: &[&str]) -> PluginConfig { + PluginConfig { + name: "extensions-aware".into(), + kind: KIND.into(), + hooks: vec!["tool_pre_invoke".into()], + capabilities: capabilities.iter().map(|c| (*c).to_string()).collect(), + version: Some("1.0.0".into()), + config: Some(serde_json::json!({ + "class_name": FIXTURE_CLASS, + "requirements_file": "requirements.txt", + "timeout_secs": 300, + })), + ..Default::default() + } +} + +// ===================================================================== +// Inbound wire shape — no worker required +// ===================================================================== + +#[test] +fn the_inbound_task_carries_visible_slots_without_sensitive_headers() { + // Drives the host's task-building step directly. This is the contract + // `worker.py` will read, so it is worth pinning independently of whether a + // worker exists yet to consume it. + let mut task = serde_json::json!({"task_type": "load_and_run_hook"}); + cpex_hosts_python::extensions::attach_extensions(&mut task, &inbound_extensions()) + .expect("the filtered view serializes"); + + let wire = task + .get(EXTENSIONS_FIELD) + .expect("the extensions field is attached") + .as_object() + .expect("it is an object"); + + let labels = wire["security"]["labels"] + .as_array() + .expect("labels is an array"); + assert!( + labels.iter().any(|l| l.as_str() == Some(INBOUND_LABEL)), + "the inbound label must reach the worker: {labels:?}" + ); + + let headers = wire["http"]["request_headers"] + .as_object() + .expect("request_headers is an object"); + for name in headers.keys() { + let lowered = name.to_ascii_lowercase(); + assert!( + !matches!(lowered.as_str(), "authorization" | "cookie" | "x-api-key"), + "sensitive header '{name}' must not cross the process boundary" + ); + } + assert_eq!( + headers.get("X-Request-Id").and_then(|v| v.as_str()), + Some("req-e2e-1"), + "a non-sensitive header still travels" + ); + + let serialized = serde_json::to_string(&task).expect("the task serializes"); + assert!( + !serialized.contains("must-not-travel"), + "no sensitive header value may appear anywhere in the task JSON" + ); +} + +#[test] +fn a_plugin_without_read_labels_gets_no_security_slot() { + // The host serializes the *filtered* view, so capability filtering the + // executor already did is what determines slot visibility. Simulate the + // filtered result: no security slot in, none out. + let filtered = Extensions { + http: Some(Arc::new(HttpExtension::default())), + ..Default::default() + }; + + let mut task = serde_json::json!({}); + cpex_hosts_python::extensions::attach_extensions(&mut task, &filtered) + .expect("serializing a filtered view"); + + let wire = task[EXTENSIONS_FIELD].as_object().expect("an object"); + assert!( + !wire.contains_key("security"), + "a slot the plugin's capabilities excluded must stay excluded" + ); +} + +// ===================================================================== +// Return path — tier enforcement through the real executor +// ===================================================================== + +#[tokio::test(flavor = "multi_thread")] +async fn an_appended_label_survives_the_merge_when_the_capability_is_declared() { + // The monotonic accept case, end to end. `append_labels` makes the executor + // mint a labels write token, the host honors the returned `security`, and + // the executor's `before ⊆ after` check accepts the addition. + let test = "an_appended_label_survives_the_merge_when_the_capability_is_declared"; + let Some(source) = require_worker(test) else { + return; + }; + let dir = TempDir::new(); + let Some(plugin_dir) = setup(&dir, &source, test) else { + return; + }; + + let (verdict, merged) = run_pipeline( + &dir, + &plugin_dir, + &["read_labels", "append_labels", "read_headers"], + ) + .await; + + assert!( + verdict.starts_with("GOT_LABEL"), + "the 3-arg hook must receive the inbound extensions; got: {verdict}" + ); + assert!( + !verdict.contains("LEAKED"), + "no sensitive header may reach the plugin: {verdict}" + ); + + let security = merged + .security + .as_ref() + .expect("security survives the merge"); + assert!( + security.has_label(APPENDED_LABEL), + "the plugin's additive label must survive the monotonic merge" + ); + assert!( + security.has_label(INBOUND_LABEL), + "the original label must still be there" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_appended_label_is_dropped_without_the_capability() { + // Same fixture, same return value — but no `append_labels`, so the executor + // mints no token and the host drops the write. The pipeline's labels are + // unchanged. + let test = "an_appended_label_is_dropped_without_the_capability"; + let Some(source) = require_worker(test) else { + return; + }; + let dir = TempDir::new(); + let Some(plugin_dir) = setup(&dir, &source, test) else { + return; + }; + + let (verdict, merged) = run_pipeline(&dir, &plugin_dir, &["read_labels", "read_headers"]).await; + + assert!( + verdict.starts_with("GOT_LABEL"), + "reading is still allowed by read_labels; got: {verdict}" + ); + + let security = merged.security.as_ref().expect("security slot present"); + assert!( + !security.has_label(APPENDED_LABEL), + "a label append without `append_labels` must not land" + ); + assert!( + security.has_label(INBOUND_LABEL), + "and the inbound label is untouched" + ); +} + +/// Register the Python plugin on `tool_pre_invoke`, run one pipeline pass, and +/// return the fixture's recorded verdict plus the merged extensions. +/// +/// Going through `PluginManager` is the point: it is what filters extensions per +/// capability, mints the write tokens, and runs the copy-on-write merge that +/// validates the plugin's return. +async fn run_pipeline( + dir: &TempDir, + plugin_dir: &Path, + capabilities: &[&str], +) -> (String, Extensions) { + use cpex_core::config::CpexConfig; + use cpex_core::manager::{ManagerConfig, PluginManager}; + + let mut config = plugin_config(capabilities); + // Point the plugin at the scaffolded temp dir and run the worker there, so + // a test run never touches the repository's real plugins directory. + config.config = Some({ + let mut base = config + .config + .take() + .unwrap_or_else(|| serde_json::json!({})); + base["plugin_dirs"] = serde_json::json!([plugin_dir.display().to_string()]); + base["worker_cwd"] = serde_json::json!(dir.path().display().to_string()); + base + }); + + let manager = PluginManager::new(ManagerConfig::default()); + manager.register_factory(KIND, Box::new(cpex_hosts_python::IsolatedVenvFactory)); + manager + .load_config(CpexConfig { + plugins: vec![config], + ..Default::default() + }) + .expect("the config loads"); + manager + .initialize() + .await + .expect("the venv is reused and the worker starts"); + + let inbound = inbound_extensions(); + let (result, _bg) = manager + .invoke_named::( + "tool_pre_invoke", + cpex_hosts_python::legacy::ToolPreInvokePayload::default(), + inbound.clone(), + None, + ) + .await; + + manager.shutdown().await; + + let verdict = std::fs::read_to_string(dir.path().join("extensions_verdict.txt")) + .unwrap_or_else(|e| panic!("the plugin recorded no verdict: {e}")); + + // `modified_extensions` is `None` when the pipeline merged nothing — which + // is itself a meaningful outcome here (a dropped write). Fall back to the + // inbound view so callers can assert "unchanged" without special-casing. + let merged = result.modified_extensions.unwrap_or(inbound); + (verdict, merged) +} + +// ===================================================================== +// Tier enforcement through the real executor, without a Python worker +// ===================================================================== +// +// The worker-backed tests above skip until the Python branch lands. These cover +// the same merge with a Rust handler standing in for the worker: it calls the +// *production* `owned_from_returned_slot` on a canned response, so the code +// under test is identical — only the subprocess is replaced. That lets the +// accept-side tier outcomes be verified now, with write tokens minted by the +// executor from real declared capabilities. + +mod fake_worker { + use std::sync::Arc; + + use async_trait::async_trait; + use cpex_core::context::PluginContext; + use cpex_core::error::PluginError; + use cpex_core::executor::ErasedResultFields; + use cpex_core::extensions::Extensions; + use cpex_core::factory::{PluginFactory, PluginInstance}; + use cpex_core::hooks::PluginPayload; + use cpex_core::plugin::{Plugin, PluginConfig}; + use cpex_core::registry::AnyHookHandler; + use serde_json::Value; + + /// Kind name for the stand-in factory. + pub const KIND: &str = "fake-worker"; + + /// A plugin whose handler returns a fixed worker-shaped response. + pub struct FakeWorkerPlugin { + cfg: PluginConfig, + } + + #[async_trait] + impl Plugin for FakeWorkerPlugin { + fn config(&self) -> &PluginConfig { + &self.cfg + } + } + + /// Handler that feeds a canned response through the production parser. + pub struct FakeWorkerHandler { + /// The `modified_extensions` JSON a worker would have returned. + pub response: Value, + } + + #[async_trait] + impl AnyHookHandler for FakeWorkerHandler { + async fn invoke( + &self, + _payload: &dyn PluginPayload, + extensions: &Extensions, + _ctx: &mut PluginContext, + ) -> Result, Box> { + // The real thing: same function the Python adapter calls, same + // inbound view the executor filtered and tokenized. + let modified_extensions = + cpex_hosts_python::extensions::owned_from_returned_slot(&self.response, extensions) + .expect("the canned response parses"); + + Ok(Box::new(ErasedResultFields { + continue_processing: true, + modified_payload: None, + modified_extensions, + violation: None, + })) + } + + fn hook_type_name(&self) -> &'static str { + "tool_pre_invoke" + } + } + + /// Factory reading the canned response out of the plugin's own config. + pub struct FakeWorkerFactory; + + impl PluginFactory for FakeWorkerFactory { + fn create(&self, config: &PluginConfig) -> Result> { + let response = config + .config + .as_ref() + .and_then(|c| c.get("response")) + .cloned() + .unwrap_or(Value::Null); + + Ok(PluginInstance { + plugin: Arc::new(FakeWorkerPlugin { + cfg: config.clone(), + }), + handlers: vec![( + "tool_pre_invoke", + Arc::new(FakeWorkerHandler { response }) as Arc, + )], + }) + } + } +} + +/// Run one pipeline pass against the stand-in worker. +/// +/// `response` is the `modified_extensions` value a real worker would return. +/// Returns the post-merge extensions the executor accepted. +async fn merge_through_executor( + capabilities: &[&str], + response: serde_json::Value, +) -> Option { + use cpex_core::config::CpexConfig; + use cpex_core::manager::{ManagerConfig, PluginManager}; + + let config = PluginConfig { + name: "fake-worker".into(), + kind: fake_worker::KIND.into(), + hooks: vec!["tool_pre_invoke".into()], + capabilities: capabilities.iter().map(|c| (*c).to_string()).collect(), + version: Some("1.0.0".into()), + config: Some(serde_json::json!({"response": response})), + ..Default::default() + }; + + let manager = PluginManager::new(ManagerConfig::default()); + manager.register_factory(fake_worker::KIND, Box::new(fake_worker::FakeWorkerFactory)); + manager + .load_config(CpexConfig { + plugins: vec![config], + ..Default::default() + }) + .expect("the config loads"); + manager.initialize().await.expect("the plugin initializes"); + + let (result, _bg) = manager + .invoke_named::( + "tool_pre_invoke", + cpex_hosts_python::legacy::ToolPreInvokePayload::default(), + inbound_extensions(), + None, + ) + .await; + + manager.shutdown().await; + result.modified_extensions +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_label_append_is_accepted_with_append_labels() { + // The monotonic accept path, through the executor's real merge. This is the + // case the unit tests structurally cannot reach: the token comes from the + // declared capability. + let merged = merge_through_executor( + &["read_labels", "append_labels"], + serde_json::json!({"security": {"labels": [INBOUND_LABEL, APPENDED_LABEL]}}), + ) + .await + .expect("the append is a real modification"); + + let security = merged.security.as_ref().expect("security survives"); + assert!( + security.has_label(APPENDED_LABEL), + "an additive label change must be accepted by the monotonic tier" + ); + assert!( + security.has_label(INBOUND_LABEL), + "the pre-existing label must remain" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_label_removal_is_rejected_by_the_monotonic_tier() { + // `before ⊆ after` fails, so the executor rejects the whole return and the + // pipeline keeps the original labels. + let merged = merge_through_executor( + &["read_labels", "append_labels"], + serde_json::json!({"security": {"labels": []}}), + ) + .await; + + if let Some(merged) = merged { + let security = merged.security.as_ref().expect("security present"); + assert!( + security.has_label(INBOUND_LABEL), + "a label removal must not strip the pipeline's labels" + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_custom_change_is_accepted_as_mutable() { + let merged = merge_through_executor( + &["read_labels"], + serde_json::json!({"custom": {"verdict": "clean"}}), + ) + .await + .expect("a custom write is a modification"); + + let custom = merged.custom.as_ref().expect("custom survives the merge"); + assert_eq!( + custom.get("verdict").and_then(|v| v.as_str()), + Some("clean"), + "the mutable tier is accepted as-is" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_forged_immutable_slot_does_not_poison_the_merge() { + // A plugin returning an `agent` it was never given must not cause the + // executor to reject the whole return — the host drops the forged slot + // before the merge sees it, so the legitimate `custom` write still lands. + let merged = merge_through_executor( + &["read_labels"], + serde_json::json!({ + "agent": {"agent_id": "forged"}, + "custom": {"verdict": "clean"} + }), + ) + .await + .expect("the custom write still counts as a modification"); + + assert_eq!( + merged + .custom + .as_ref() + .expect("custom survives") + .get("verdict") + .and_then(|v| v.as_str()), + Some("clean"), + "the immutable-tier violation must not take the valid write down with it" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_http_write_is_accepted_only_with_write_headers() { + // Guarded tier, both sides. With the capability the header lands; without + // it, the write is dropped. + let with_cap = merge_through_executor( + &["read_headers", "write_headers"], + serde_json::json!({"http": {"request_headers": {"X-Added": "1"}}}), + ) + .await + .expect("an authorized header write is a modification"); + + assert_eq!( + with_cap + .http + .as_ref() + .expect("http survives") + .get_request_header("X-Added"), + Some("1"), + "write_headers authorizes the guarded write" + ); + + let without_cap = merge_through_executor( + &["read_headers"], + serde_json::json!({"http": {"request_headers": {"X-Added": "1"}}}), + ) + .await; + + let leaked = without_cap + .as_ref() + .and_then(|m| m.http.as_ref()) + .and_then(|h| h.get_request_header("X-Added")); + assert!( + leaked.is_none(), + "without write_headers the guarded write must not land" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_returned_sensitive_header_never_reenters_the_pipeline() { + // Symmetry with the outbound strip: a plugin must not be able to inject a + // credential header back into the pipeline through its return value. + let merged = merge_through_executor( + &["read_headers", "write_headers"], + serde_json::json!({"http": {"request_headers": { + "Authorization": "Bearer injected", + "X-Fine": "yes" + }}}), + ) + .await + .expect("the header write is a modification"); + + let http = merged.http.as_ref().expect("http survives"); + assert!( + http.get_request_header("Authorization").is_none(), + "an injected credential header must be stripped on the way back" + ); + assert_eq!( + http.get_request_header("X-Fine"), + Some("yes"), + "the benign header still lands" + ); +} + +/// The return field name is a contract with the Python `PluginResult` model. +/// +/// `worker.py` serializes a `PluginResult`, whose `modified_extensions` field +/// already exists (`cpex/framework/models.py`) and is the same one the +/// in-process manager accumulates. Pinning it here means a rename on either side +/// fails a test instead of silently dropping every plugin's extension writes. +#[test] +fn the_return_field_matches_the_python_plugin_result_model() { + assert_eq!(MODIFIED_EXTENSIONS_FIELD, "modified_extensions"); + assert_eq!(EXTENSIONS_FIELD, "extensions"); +} From ca5bf7fffb8415c56a8313424cd0bda2d210f3d0 Mon Sep 17 00:00:00 2001 From: habeck Date: Fri, 31 Jul 2026 16:56:02 -0400 Subject: [PATCH 3/5] chore: ignore plugins folder Signed-off-by: habeck --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 88099657..bd4bbf83 100644 --- a/.gitignore +++ b/.gitignore @@ -277,3 +277,4 @@ tmp/ plugin-catalog # Hugo build output docs/public/ +plugins/ \ No newline at end of file From fd19bc20b7a6a3fca8c2f234844c92524bb88591 Mon Sep 17 00:00:00 2001 From: habeck Date: Fri, 31 Jul 2026 16:56:25 -0400 Subject: [PATCH 4/5] chore: test updates Signed-off-by: habeck --- crates/cpex-hosts-python/tests/config_e2e.rs | 171 ++++++++++++++++++- 1 file changed, 169 insertions(+), 2 deletions(-) diff --git a/crates/cpex-hosts-python/tests/config_e2e.rs b/crates/cpex-hosts-python/tests/config_e2e.rs index eaeb0b7c..615ded1e 100644 --- a/crates/cpex-hosts-python/tests/config_e2e.rs +++ b/crates/cpex-hosts-python/tests/config_e2e.rs @@ -18,6 +18,12 @@ // catches a config-shape or dispatch-wiring break that a direct adapter call // would not see. // +// `tool_pre_invoke` also carries populated `Extensions`, which makes this the +// only test that drives the extensions channel against an installer-written +// config — one that declares `capabilities: []`. `extensions_merge_e2e.rs` owns +// the observation and merge behaviour with a 3-arg fixture and explicit +// capability sets; what is unique here is the real config's no-capability path. +// // # Why it is `#[ignore]` // // It depends on state this repo does not create: an installed plugin with a @@ -27,7 +33,9 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; -use cpex_core::extensions::Extensions; +use cpex_core::extensions::{ + AgentExtension, Extensions, HttpExtension, RequestExtension, SecurityExtension, +}; use cpex_core::hooks::types::hook_names; use cpex_core::manager::PluginManager; use cpex_hosts_python::factory::{IsolatedVenvFactory, KIND}; @@ -49,6 +57,15 @@ const IDENTITY_RAW_TOKEN_MARKER: &str = "identity_resolve_raw_token"; /// non-empty secret value. See [`IDENTITY_RAW_TOKEN_MARKER`]. const TOKEN_DELEGATE_BEARER_MARKER: &str = "token_delegate_bearer_token"; +/// A security label on the inbound extensions. Distinctive so an assertion on it +/// cannot pass against a default-constructed `Extensions`. +const INBOUND_LABEL: &str = "CONFIG-E2E-LABEL-4f18"; + +/// A header value that must never cross the process boundary. Asserted by +/// absence from the serialized task, so it has to be unique enough that a match +/// anywhere in the JSON is conclusive. +const SENSITIVE_HEADER_VALUE: &str = "Bearer config-e2e-must-not-travel"; + /// The repository root — `plugins/config.yaml` and the installed plugin's /// `plugin_dirs` are both relative to it. fn repo_root() -> PathBuf { @@ -146,6 +163,57 @@ fn assert_allowed(result: &cpex_core::executor::PipelineResult, hook_name: &str) ); } +/// A populated `Extensions` spanning the three filter behaviours a plugin with +/// `capabilities: []` can exhibit, so one fixture exercises all of them: +/// +/// * `request` / `custom` — unrestricted immutable slots. `filter_extensions` +/// copies these through regardless of capabilities, so they are the slots that +/// actually reach a plugin declaring none. +/// * `agent` / `http` — capability-gated slots. `cpex-test-plugin` declares +/// `capabilities: []` in `plugins/config.yaml`, so the executor strips both +/// before the host ever serializes. Including them is the point: it proves the +/// gate holds on the real installer-written config rather than on a config a +/// test constructed to have no gated data in the first place. +/// * `security` — gated per sub-field. The slot survives, `labels` does not. +/// +/// The `Authorization` header is a second, independent guard: `attach_extensions` +/// strips sensitive headers on the way out, so even a config that *did* declare +/// `read_headers` must not put this value on the wire. +fn inbound_extensions() -> Extensions { + let mut security = SecurityExtension::default(); + security.add_label(INBOUND_LABEL); + security.classification = Some("internal".to_string()); + security.auth_method = Some("jwt".to_string()); + + let mut http = HttpExtension::default(); + http.set_request_header("Authorization", SENSITIVE_HEADER_VALUE); + http.set_request_header("X-Request-Id", "req-config-e2e-1"); + http.method = Some("POST".to_string()); + http.path = Some("/rpc/tools/invoke".to_string()); + + Extensions { + request: Some(Arc::new(RequestExtension { + environment: Some("test".to_string()), + request_id: Some("req-config-e2e-1".to_string()), + trace_id: Some("trace-config-e2e-1".to_string()), + ..Default::default() + })), + agent: Some(Arc::new(AgentExtension { + session_id: Some("session-config-e2e".to_string()), + agent_id: Some("agent-config-e2e".to_string()), + turn: Some(3), + ..Default::default() + })), + security: Some(Arc::new(security)), + http: Some(Arc::new(http)), + custom: Some(Arc::new(HashMap::from([( + "config_e2e".to_string(), + serde_json::json!({"source": "config_e2e.rs"}), + )]))), + ..Default::default() + } +} + // --------------------------------------------------------------------------- // Setup // @@ -224,11 +292,19 @@ async fn config_yaml_dispatches_tool_pre_invoke_to_the_installed_plugin() { headers: None, }; + // Populated extensions rather than `Extensions::default()`. The default is + // the one input that cannot fail: with every slot `None`, + // `attach_extensions` writes no field at all, so the whole inbound channel + // is untested and the assertions below hold vacuously. Passing real slots + // through the installer-written config is what makes this test cover the + // channel — see `inbound_extensions` for what each slot is there to prove. + let inbound = inbound_extensions(); + let (result, _background) = manager .invoke_by_name( hook_names::TOOL_PRE_INVOKE, Box::new(payload), - Extensions::default(), + inbound.clone(), None, ) .await; @@ -240,7 +316,98 @@ async fn config_yaml_dispatches_tool_pre_invoke_to_the_installed_plugin() { // The marker proves the plugin's hook body ran. Without it, a response // shaped like an allow is indistinguishable from the manager's // no-plugins-registered short circuit. + // + // Combined with the populated extensions above, it is also the assertion + // that carries the most weight here: the worker received a task with an + // `extensions` field, deserialized it, and still ran the hook. A slot shape + // the worker's Pydantic models reject surfaces as a validation error in + // `errors` and no marker — which is precisely the break a `default()` input + // could never detect. assert_marker_written(&marker, hook_names::TOOL_PRE_INVOKE); + + // `modified_extensions` on an allowed pipeline is the *final* extensions + // view, not a "something changed" signal — `PipelineResult::allowed_with` + // always populates it with whatever the pipeline ended up holding. So the + // meaningful assertion is that the view came back unchanged. + // + // Unchanged is the correct outcome for two independent reasons, and this is + // the only place the real installer-written config pins either: the plugin + // declares `capabilities: []`, so the executor strips every gated slot before + // the host serializes; and its `tool_pre_invoke(payload, context)` is 2-arg, + // so the framework forwards no extensions to the hook body at all. A plugin + // that receives nothing has nothing to write back. + // + // Note this is the *pipeline's* view, not the filtered one the plugin saw — + // the executor filters per plugin and merges writes back into the unfiltered + // original, so a stripped slot reappearing here is expected and correct. + let returned = result + .modified_extensions + .as_ref() + .expect("an allowed pipeline always carries its final extensions view"); + assert!( + returned.agent.is_some() && returned.request.is_some() && returned.http.is_some(), + "the pipeline's own view keeps the slots it started with; per-plugin \ + filtering must not mutate it" + ); + assert!( + returned + .security + .as_ref() + .is_some_and(|s| s.has_label(INBOUND_LABEL)), + "a plugin that never received `security.labels` cannot have dropped the \ + pipeline's label" + ); + assert_eq!( + returned.custom.as_deref(), + inbound.custom.as_deref(), + "no plugin wrote `custom`, so it must survive the pipeline byte-for-byte" + ); + + // What the plugin *would* have seen, computed with the same production + // filter the executor ran. Asserting on this rather than on the worker's + // observations is deliberate: the installed plugin's hooks are 2-arg, so it + // has no way to report what arrived, and `extensions_merge_e2e.rs` owns the + // 3-arg observation path with a purpose-built fixture. This keeps the + // capability contract pinned against the real installer-written config. + let filtered = cpex_core::extensions::filter_extensions(&inbound, &Default::default()); + assert!( + filtered.agent.is_none(), + "`agent` is capability-gated and this config declares none, so it must \ + not reach the plugin" + ); + assert!( + filtered.http.is_none(), + "`http` is capability-gated and this config declares none, so it must \ + not reach the plugin" + ); + assert!( + !filtered + .security + .as_ref() + .expect("the security slot survives; its gated sub-fields do not") + .has_label(INBOUND_LABEL), + "`security.labels` is gated under `read_labels`, which this config does \ + not declare" + ); + assert!( + filtered.request.is_some(), + "`request` is unrestricted — stripping it would silently starve every \ + plugin of request context" + ); + + // The outbound sensitive-header strip, on the path a real gateway takes. + // This holds for a second, independent reason beyond the capability filter + // above: `attach_extensions` scrubs these regardless of what the plugin + // declares, so the assertion stays meaningful if this config ever gains + // `read_headers`. + let mut task = serde_json::json!({"task_type": "load_and_run_hook"}); + cpex_hosts_python::extensions::attach_extensions(&mut task, &inbound) + .expect("the inbound view serializes"); + let wire = serde_json::to_string(&task).expect("the task serializes"); + assert!( + !wire.contains(SENSITIVE_HEADER_VALUE), + "no credential header value may appear anywhere in the task JSON: {wire}" + ); } /// The two credential-adjacent hooks dispatch through the same config → factory From 6a347f1d3d40b042a24cf379679e21822e7578e0 Mon Sep 17 00:00:00 2001 From: habeck Date: Fri, 31 Jul 2026 20:53:03 -0400 Subject: [PATCH 5/5] chore: update .gitignore Signed-off-by: habeck --- .gitignore | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bd4bbf83..29f63454 100644 --- a/.gitignore +++ b/.gitignore @@ -274,7 +274,9 @@ db_path/ tmp/ .continue -plugin-catalog # Hugo build output docs/public/ -plugins/ \ No newline at end of file +# python cpex install target(s) +plugins/ +data/ +plugin-catalog \ No newline at end of file