From 03e4cdbfa8e4158d653a2831e177a48a43fe2654 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 7 Aug 2026 17:37:02 -0400 Subject: [PATCH 01/11] feat(k8s): add namespace-per-workspace support (RFC 0011 Phase 3) Implement three workspace namespace modes for the Kubernetes compute driver: shared (default, preserves current single-namespace behavior), managed (auto-creates/deletes namespaces per workspace), and operator (pre-provisioned namespaces with dynamic discovery via label selector or drop-in allowlist file). Key changes: - WorkspaceMode enum and namespace resolution in driver config - Managed namespace lifecycle with ServiceAccount and OpenShift SCC annotation propagation - Cluster-wide sandbox CR watchers for managed/operator modes - NamespaceValidator (Exact/Prefix/Allowlist) for SA token auth - Workspace-aware credential secret storage - Helm ClusterRole for multi-namespace RBAC - Gateway config, architecture, and reference docs Signed-off-by: Derek Carr --- architecture/compute-runtimes.md | 63 ++ crates/openshell-core/src/driver_utils.rs | 3 + .../src/lib.rs | 74 ++- crates/openshell-driver-kubernetes/README.md | 14 +- .../openshell-driver-kubernetes/src/config.rs | 539 ++++++++++++++++++ .../openshell-driver-kubernetes/src/driver.rs | 532 ++++++++++++++--- crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../openshell-driver-kubernetes/src/main.rs | 26 +- crates/openshell-server/src/auth/k8s_sa.rs | 256 ++++++--- crates/openshell-server/src/lib.rs | 26 +- deploy/helm/openshell/README.md | 3 + .../helm/openshell/templates/clusterrole.yaml | 52 ++ .../openshell/templates/gateway-config.yaml | 10 + deploy/helm/openshell/templates/role.yaml | 3 + .../helm/openshell/templates/rolebinding.yaml | 3 + deploy/helm/openshell/values.yaml | 14 + docs/reference/gateway-config.mdx | 15 + 17 files changed, 1466 insertions(+), 172 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 646b6320bd..f22da681a1 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -286,5 +286,68 @@ Standalone local deployments start the gateway with a selected runtime such as Docker, Podman, or VM. The CLI can register multiple gateways and switch between them without changing the sandbox architecture. +## Workspace Namespace Modes (Kubernetes) + +The Kubernetes driver maps workspaces to namespaces through the `workspace_mode` +configuration field (`WorkspaceMode` in `crates/openshell-driver-kubernetes/src/config.rs`). +The mode controls namespace resolution, resource naming, sandbox CR watching, SA +token authentication, and RBAC requirements. + +| Mode | Namespace resolution | Resource name | Namespace lifecycle | +|---|---|---|---| +| **Shared** (default) | Single static namespace from config | `{workspace}--{name}` | None | +| **Managed** | `openshell-{gateway_id}-{workspace}` | bare sandbox name | Driver creates and deletes | +| **Operator** | Workspace name maps 1:1 to a pre-provisioned namespace | bare sandbox name | External (platform team) | + +**Shared** renders all sandboxes into one configured namespace. Resource names +embed the workspace prefix for collision avoidance. No namespace lifecycle +management. RBAC uses a namespace-scoped Role. + +**Managed** auto-creates a K8s namespace per workspace on first sandbox create. +Each new namespace receives a ServiceAccount and copies OpenShift SCC UID-range +and supplemental-group annotations from the gateway namespace when present. The +driver deletes the namespace when the last sandbox in it is removed +(`delete_namespace_if_empty`). Requires a non-empty `gateway_id` (validated as a +DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character +limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace +`create`/`delete` and ServiceAccount `create`/`get` permissions. + +**Operator** uses pre-provisioned namespaces discovered through two optional +sources: a K8s label selector (`operator_namespace_label`) and a drop-in +allowlist file (`operator_namespace_file`). At least one must be configured. +The `OperatorNamespaceAllowlist` (`Arc>>`) is populated +at runtime by background watchers and read by the namespace resolver. Sandbox +creation fails closed if the workspace is not in the current allowlist. Platform +teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as +managed mode but without namespace `create`/`delete` or ServiceAccount +permissions. + +### Watching and Querying + +Managed and operator modes set `is_multi_namespace() == true`, which switches +sandbox CR watchers from namespace-scoped `Api::namespaced` to cluster-wide +`Api::all_with`. In managed mode the driver scopes cluster-wide queries with a +`LABEL_GATEWAY_ID` label selector to support multiple gateways on the same +cluster. K8s Events are not watched in cluster-wide mode — the cluster-wide +watcher emits only sandbox CR changes, not platform events. + +### SA Token Authentication + +The gateway's `K8sServiceAccountAuthenticator` adapts its `NamespaceValidator` +per mode (`crates/openshell-server/src/auth/k8s_sa.rs`): + +- **Shared:** `Exact` — accepts only the single configured namespace. +- **Managed:** `Prefix` — accepts any namespace starting with `openshell-{gateway_id}-`. +- **Operator:** `Allowlist` — accepts namespaces present in the dynamic + `BTreeSet` populated by the label/file watchers. Starts empty (fail-closed) + until the first watcher update. + +### Credential Driver Integration + +The Kubernetes Secrets credential driver (`openshell-driver-kubernetes-secrets`) +stores secrets in workspace-specific namespaces when `workspace_mode` is managed +or operator. In shared mode, all secrets render into the single configured +namespace. + When runtime infrastructure changes, validate the relevant sandbox e2e path and update the matching driver README if a maintainer-facing constraint changes. diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 9bcca9f11d..6cc547b263 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -35,6 +35,9 @@ pub const LABEL_SANDBOX_NAMESPACE: &str = "openshell.ai/sandbox-namespace"; /// Container/pod label carrying the sandbox workspace. pub const LABEL_SANDBOX_WORKSPACE: &str = "openshell.ai/sandbox-workspace"; +/// Label carrying the gateway identity on managed namespaces. +pub const LABEL_GATEWAY_ID: &str = "openshell.ai/gateway-id"; + /// Label selector that matches all OpenShell-managed resources which carry a /// sandbox ID label. Used by list and watch operations to exclude foreign /// resources from the same namespace. diff --git a/crates/openshell-driver-kubernetes-secrets/src/lib.rs b/crates/openshell-driver-kubernetes-secrets/src/lib.rs index 65c655be16..1c59401cb1 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/lib.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/lib.rs @@ -48,10 +48,33 @@ impl CredentialDriverService { } } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +enum WorkspaceMode { + #[default] + Shared, + Managed, + Operator, +} + #[derive(Debug, Clone, PartialEq, Eq)] struct KubernetesSecretsDriverSettings { namespace: String, allow_reference_namespace: bool, + workspace_mode: WorkspaceMode, + gateway_id: String, +} + +impl KubernetesSecretsDriverSettings { + fn target_namespace(&self, workspace: &str) -> String { + match self.workspace_mode { + WorkspaceMode::Shared => self.namespace.clone(), + WorkspaceMode::Managed => { + format!("openshell-{}-{}", self.gateway_id, workspace) + } + WorkspaceMode::Operator => workspace.to_string(), + } + } } #[derive(Debug, Clone, Default, serde::Deserialize)] @@ -59,6 +82,8 @@ struct KubernetesSecretsDriverSettings { struct KubernetesSecretsDriverConfig { namespace: Option, allow_reference_namespace: bool, + workspace_mode: WorkspaceMode, + gateway_id: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -134,7 +159,10 @@ impl KubernetesSecretsCredentialDriver { credential_key: &str, ) -> Result { let reference = Self::parse_handle(handle, credential_key)?; - if reference.namespace != self.settings.namespace + // In managed/operator modes secrets live in workspace-specific + // namespaces so cross-namespace handles are expected. + if self.settings.workspace_mode == WorkspaceMode::Shared + && reference.namespace != self.settings.namespace && !self.settings.allow_reference_namespace { return Err(Status::permission_denied(format!( @@ -175,7 +203,7 @@ impl KubernetesSecretsCredentialDriver { reference } else { KubernetesSecretReference { - namespace: self.settings.namespace.clone(), + namespace: self.settings.target_namespace(&request.workspace), secret_name: managed_secret_name( &request.workspace, &request.provider_id, @@ -508,6 +536,8 @@ impl KubernetesSecretsDriverSettings { Ok(Self { namespace, allow_reference_namespace: config.allow_reference_namespace, + workspace_mode: config.workspace_mode, + gateway_id: config.gateway_id.unwrap_or_default(), }) } } @@ -842,6 +872,8 @@ mod tests { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -865,6 +897,8 @@ mod tests { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), allow_reference_namespace: true, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -1065,4 +1099,40 @@ mod tests { assert_eq!(err.code(), Code::FailedPrecondition); assert!(err.message().contains("is not managed by OpenShell")); } + + #[test] + fn target_namespace_shared_returns_static_namespace() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), + }; + assert_eq!(settings.target_namespace("team-a"), "openshell"); + assert_eq!(settings.target_namespace("team-b"), "openshell"); + } + + #[test] + fn target_namespace_managed_computes_from_workspace() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Managed, + gateway_id: "gw1".to_string(), + }; + assert_eq!(settings.target_namespace("team-a"), "openshell-gw1-team-a"); + assert_eq!(settings.target_namespace("team-b"), "openshell-gw1-team-b"); + } + + #[test] + fn target_namespace_operator_uses_workspace_name() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Operator, + gateway_id: String::new(), + }; + assert_eq!(settings.target_namespace("team-a"), "team-a"); + assert_eq!(settings.target_namespace("prod-ns"), "prod-ns"); + } } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 1356e2d932..c985e0b776 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -3,8 +3,18 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox -custom resources in the configured namespace. It runs in-process with the -gateway server. +custom resources. It runs in-process with the gateway server and supports three +workspace namespace modes via `workspace_mode`: + +- **Shared** (default): All sandboxes render into a single static namespace. + Resource names use `{workspace}--{name}` for collision avoidance. +- **Managed**: The driver auto-creates/deletes a K8s namespace per workspace + (`openshell-{gateway_id}-{workspace_name}`), creates a ServiceAccount in each, + and copies OpenShift SCC annotations from the gateway namespace when present. +- **Operator**: Workspace names map 1:1 to pre-provisioned namespaces discovered + via label selector (`operator_namespace_label`) and/or drop-in allowlist file + (`operator_namespace_file`). Sandbox creation fails closed if the workspace + namespace is not in the current allowlist. ## Runtime Model diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 5311f56436..21d3aea851 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -3,8 +3,13 @@ use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::BTreeSet; use std::path::Path; use std::str::FromStr; +use std::sync::{Arc, RwLock}; + +/// Default gateway identity used in managed-mode namespace naming. +pub const DEFAULT_GATEWAY_ID: &str = "openshell"; /// Default Kubernetes namespace for sandbox resources. pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; @@ -88,6 +93,48 @@ impl FromStr for SupervisorTopology { } } +/// How workspaces map to Kubernetes namespaces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WorkspaceMode { + /// All sandboxes render into a single statically-configured namespace. + /// Resource names use `{workspace}--{name}` for collision avoidance. + #[default] + Shared, + /// The driver creates and deletes K8s namespaces on demand using the + /// convention `openshell-{gateway_id}-{workspace_name}`. + Managed, + /// Sandboxes render into pre-existing K8s namespaces. The driver has no + /// namespace create/delete permissions. Platform teams manage namespaces + /// via their existing tooling. + Operator, +} + +impl std::fmt::Display for WorkspaceMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Shared => f.write_str("shared"), + Self::Managed => f.write_str("managed"), + Self::Operator => f.write_str("operator"), + } + } +} + +impl FromStr for WorkspaceMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "shared" => Ok(Self::Shared), + "managed" => Ok(Self::Managed), + "operator" => Ok(Self::Operator), + other => Err(format!( + "unknown workspace mode '{other}'; expected 'shared', 'managed', or 'operator'" + )), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSidecarConfig { @@ -232,7 +279,24 @@ where #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { + /// How workspaces map to Kubernetes namespaces. `"shared"` (default) + /// renders all sandboxes into `namespace`; `"managed"` creates per-workspace + /// namespaces on demand; `"operator"` uses pre-provisioned namespaces. + pub workspace_mode: WorkspaceMode, + /// Stable gateway identity used in managed-mode namespace naming + /// (`openshell-{gateway_id}-{workspace}`). Propagated from + /// `gateway_jwt.gateway_id`. + pub gateway_id: String, pub namespace: String, + /// K8s label selector for operator-mode namespace discovery (e.g., + /// `"openshell.ai/workspace=true"`). The driver watches namespaces matching + /// this label and builds the allowlist dynamically. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_namespace_label: Option, + /// Path to a drop-in JSON file mapping workspace names to namespace names. + /// Hot-reloaded on change. Delivered via `ConfigMap` volume mount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_namespace_file: Option, /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by /// the gateway's `TokenReview` bootstrap authenticator. pub service_account_name: String, @@ -332,7 +396,11 @@ pub const ANNOTATION_SCC_SUPPLEMENTAL_GROUPS: &str = "openshift.io/sa.scc.supple impl Default for KubernetesComputeConfig { fn default() -> Self { Self { + workspace_mode: WorkspaceMode::default(), + gateway_id: DEFAULT_GATEWAY_ID.to_string(), namespace: DEFAULT_K8S_NAMESPACE.to_string(), + operator_namespace_label: None, + operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), // Default empty so the gateway omits `imagePullPolicy` from pod @@ -473,6 +541,214 @@ impl KubernetesComputeConfig { } Ok(()) } + + /// Resolve the K8s namespace for a workspace. + /// + /// - **Shared:** returns the static `namespace` config field. + /// - **Managed:** computes `openshell-{gateway_id}-{workspace_name}`. + /// - **Operator:** looks up `workspace` in the dynamic allowlist. Fails + /// closed if the workspace is not found. + pub fn namespace_for_workspace( + &self, + workspace: &str, + operator_allowlist: Option<&OperatorNamespaceAllowlist>, + ) -> Result { + match self.workspace_mode { + WorkspaceMode::Shared => Ok(self.namespace.clone()), + WorkspaceMode::Managed => Ok(managed_namespace(&self.gateway_id, workspace)), + WorkspaceMode::Operator => { + let allowlist = + operator_allowlist.ok_or("operator mode requires a namespace allowlist")?; + let namespaces = allowlist.read(); + if namespaces.contains(workspace) { + Ok(workspace.to_string()) + } else { + Err(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + )) + } + } + } + } + + /// Whether the driver operates across multiple namespaces. + #[must_use] + pub fn is_multi_namespace(&self) -> bool { + !matches!(self.workspace_mode, WorkspaceMode::Shared) + } + + /// Compute the K8s resource name for a sandbox. + /// + /// - **Shared:** `{workspace}--{name}` (namespace doesn't provide isolation). + /// - **Managed/Operator:** bare sandbox name (namespace provides isolation). + #[must_use] + pub fn kube_resource_name(&self, workspace: &str, name: &str) -> String { + match self.workspace_mode { + WorkspaceMode::Shared => format!("{workspace}--{name}"), + WorkspaceMode::Managed | WorkspaceMode::Operator => name.to_string(), + } + } + + /// Validate workspace-mode-specific configuration at startup. + pub fn validate_workspace_mode(&self) -> Result<(), String> { + match self.workspace_mode { + WorkspaceMode::Shared => Ok(()), + WorkspaceMode::Managed => { + if self.gateway_id.is_empty() { + return Err("managed workspace mode requires a non-empty gateway_id".into()); + } + if !is_dns_1123_label(&self.gateway_id) { + return Err(format!( + "gateway_id '{}' is not a valid DNS-1123 label", + self.gateway_id + )); + } + // Workspace names can be up to 19 chars (MAX_ROUTABLE_NAME_LEN + // in the server crate). The managed namespace prefix + + // workspace must fit within 63 chars. + let prefix = managed_namespace_prefix(&self.gateway_id); + if prefix.len() + 19 > 63 { + return Err(format!( + "gateway_id '{}' is too long for managed mode; \ + the namespace prefix '{}' ({} chars) plus the \ + maximum workspace name (19 chars) exceeds the \ + 63-char K8s namespace limit", + self.gateway_id, + prefix, + prefix.len() + )); + } + Ok(()) + } + WorkspaceMode::Operator => { + if self.operator_namespace_label.is_none() && self.operator_namespace_file.is_none() + { + return Err("operator workspace mode requires at least one of \ + operator_namespace_label or operator_namespace_file" + .into()); + } + if let Some(ref label) = self.operator_namespace_label + && label.is_empty() + { + return Err("operator_namespace_label must not be empty when set".into()); + } + if let Some(ref file) = self.operator_namespace_file + && file.is_empty() + { + return Err("operator_namespace_file must not be empty when set".into()); + } + Ok(()) + } + } + } +} + +/// Compute the managed-mode namespace name for a workspace. +#[must_use] +pub fn managed_namespace(gateway_id: &str, workspace: &str) -> String { + format!("openshell-{gateway_id}-{workspace}") +} + +/// The managed-mode namespace prefix used for SA token validation. +#[must_use] +pub fn managed_namespace_prefix(gateway_id: &str) -> String { + format!("openshell-{gateway_id}-") +} + +/// Check whether a string is a valid DNS-1123 label (lowercase alphanumeric +/// and hyphens, 1-63 chars, must start and end with alphanumeric). +#[must_use] +pub fn is_dns_1123_label(s: &str) -> bool { + let len = s.len(); + if len == 0 || len > 63 { + return false; + } + let bytes = s.as_bytes(); + if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() { + return false; + } + if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() { + return false; + } + bytes + .iter() + .all(|&b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Validate that a workspace name produces a valid K8s namespace name in +/// managed mode (combined length <= 63, DNS-1123 compliant). +pub fn validate_managed_namespace_name(gateway_id: &str, workspace: &str) -> Result<(), String> { + let ns = managed_namespace(gateway_id, workspace); + if !is_dns_1123_label(&ns) { + return Err(format!( + "managed namespace '{ns}' (from workspace '{workspace}') is not a valid DNS-1123 label" + )); + } + Ok(()) +} + +/// Thread-safe dynamic allowlist of valid operator-mode namespaces. +/// +/// Backed by an `Arc>>` that is updated by background +/// tasks (label selector watcher, drop-in file watcher) and read by the SA +/// authenticator and namespace resolver. +#[derive(Debug, Clone)] +pub struct OperatorNamespaceAllowlist { + inner: Arc>>, +} + +impl OperatorNamespaceAllowlist { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(BTreeSet::new())), + } + } + + #[must_use] + pub fn from_set(set: BTreeSet) -> Self { + Self { + inner: Arc::new(RwLock::new(set)), + } + } + + /// Replace the entire allowlist (used by background watchers on refresh). + pub fn replace(&self, new_set: BTreeSet) { + let mut guard = self.inner.write().expect("allowlist lock poisoned"); + *guard = new_set; + } + + /// Merge additional namespaces into the allowlist. + pub fn merge(&self, additional: &BTreeSet) { + let mut guard = self.inner.write().expect("allowlist lock poisoned"); + guard.extend(additional.iter().cloned()); + } + + /// Read the current allowlist snapshot. + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.inner.read().expect("allowlist lock poisoned") + } + + /// Check whether a namespace is in the allowlist. + #[must_use] + pub fn contains(&self, namespace: &str) -> bool { + self.inner + .read() + .expect("allowlist lock poisoned") + .contains(namespace) + } + + /// Return a clone of the inner `Arc` for sharing with background tasks. + #[must_use] + pub fn shared(&self) -> Arc>> { + Arc::clone(&self.inner) + } +} + +impl Default for OperatorNamespaceAllowlist { + fn default() -> Self { + Self::new() + } } fn validate_provider_spiffe_workload_api_socket_path_value( @@ -966,4 +1242,267 @@ mod tests { let uid = cfg.resolve_sandbox_uid(None); assert_eq!(cfg.resolve_sandbox_gid(uid, None), uid); } + + // -- WorkspaceMode tests -- + + #[test] + fn default_workspace_mode_is_shared() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Shared); + } + + #[test] + fn serde_override_workspace_mode_managed() { + let json = serde_json::json!({ "workspace_mode": "managed" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Managed); + } + + #[test] + fn serde_override_workspace_mode_operator() { + let json = serde_json::json!({ "workspace_mode": "operator" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Operator); + } + + #[test] + fn serde_rejects_invalid_workspace_mode() { + let json = serde_json::json!({ "workspace_mode": "invalid" }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + + #[test] + fn workspace_mode_display_roundtrips() { + for mode in [ + WorkspaceMode::Shared, + WorkspaceMode::Managed, + WorkspaceMode::Operator, + ] { + assert_eq!(mode.to_string().parse::().unwrap(), mode); + } + } + + #[test] + fn namespace_for_workspace_shared() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Shared, + namespace: "sandbox-ns".to_string(), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", None).unwrap(), + "sandbox-ns" + ); + assert_eq!( + cfg.namespace_for_workspace("team-b", None).unwrap(), + "sandbox-ns" + ); + } + + #[test] + fn namespace_for_workspace_managed() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "gw1".to_string(), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", None).unwrap(), + "openshell-gw1-team-a" + ); + } + + #[test] + fn namespace_for_workspace_operator() { + let allowlist = OperatorNamespaceAllowlist::from_set(BTreeSet::from(["prod".to_string()])); + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("prod", Some(&allowlist)) + .unwrap(), + "prod" + ); + assert!( + cfg.namespace_for_workspace("unknown", Some(&allowlist)) + .is_err() + ); + } + + #[test] + fn kube_resource_name_shared_prefixes_workspace() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.kube_resource_name("ws", "box1"), "ws--box1"); + } + + #[test] + fn kube_resource_name_managed_uses_bare_name() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + ..KubernetesComputeConfig::default() + }; + assert_eq!(cfg.kube_resource_name("ws", "box1"), "box1"); + } + + #[test] + fn kube_resource_name_operator_uses_bare_name() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("x=y".to_string()), + ..KubernetesComputeConfig::default() + }; + assert_eq!(cfg.kube_resource_name("ws", "box1"), "box1"); + } + + #[test] + fn is_multi_namespace() { + assert!(!KubernetesComputeConfig::default().is_multi_namespace()); + assert!( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + ..KubernetesComputeConfig::default() + } + .is_multi_namespace() + ); + assert!( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("x=y".to_string()), + ..KubernetesComputeConfig::default() + } + .is_multi_namespace() + ); + } + + #[test] + fn validate_workspace_mode_shared_always_ok() { + let cfg = KubernetesComputeConfig::default(); + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_managed_requires_gateway_id() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: String::new(), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_managed_rejects_invalid_gateway_id() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "INVALID".to_string(), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_managed_rejects_long_gateway_id() { + // prefix = "openshell-{id}-" = 11 + id.len() + // 11 + 34 + 19 = 64 > 63 → rejected + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "a".repeat(34), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("too long for managed mode"), "{err}"); + } + + #[test] + fn validate_workspace_mode_managed_accepts_max_gateway_id() { + // 11 + 33 + 19 = 63 → accepted + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "a".repeat(33), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_requires_discovery() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_operator_accepts_label_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_accepts_file_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_file: Some("/etc/openshell/namespaces.json".to_string()), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn dns_1123_label_validation() { + assert!(is_dns_1123_label("openshell")); + assert!(is_dns_1123_label("my-gateway-1")); + assert!(is_dns_1123_label("a")); + assert!(!is_dns_1123_label("")); + assert!(!is_dns_1123_label("UPPER")); + assert!(!is_dns_1123_label("-starts-with-dash")); + assert!(!is_dns_1123_label("ends-with-dash-")); + assert!(!is_dns_1123_label("has_underscore")); + assert!(!is_dns_1123_label(&"a".repeat(64))); + } + + #[test] + fn managed_namespace_naming() { + assert_eq!( + managed_namespace("openshell", "default"), + "openshell-openshell-default" + ); + assert_eq!(managed_namespace("gw1", "team-a"), "openshell-gw1-team-a"); + } + + #[test] + fn validate_managed_namespace_name_accepts_valid() { + validate_managed_namespace_name("gw1", "team-a").unwrap(); + } + + #[test] + fn validate_managed_namespace_name_rejects_too_long() { + let long_workspace = "a".repeat(50); + assert!(validate_managed_namespace_name("openshell", &long_workspace).is_err()); + } + + #[test] + fn operator_allowlist_operations() { + let al = OperatorNamespaceAllowlist::new(); + assert!(!al.contains("ns1")); + + al.replace(BTreeSet::from(["ns1".to_string(), "ns2".to_string()])); + assert!(al.contains("ns1")); + assert!(al.contains("ns2")); + assert!(!al.contains("ns3")); + + al.merge(&BTreeSet::from(["ns3".to_string()])); + assert!(al.contains("ns3")); + + al.replace(BTreeSet::new()); + assert!(!al.contains("ns1")); + } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 2f1ea72a32..53f8443d28 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7,11 +7,12 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, - SupervisorTopology, + SupervisorTopology, WorkspaceMode, managed_namespace, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ - Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Volume, VolumeMount, + Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, ServiceAccount, + Volume, VolumeMount, }; use kube::api::{Api, ApiResource, DeleteParams, ListParams, PostParams, Preconditions}; use kube::core::gvk::GroupVersionKind; @@ -20,8 +21,9 @@ use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, openshell_sandbox_label_selector, + LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, + LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, + openshell_sandbox_label_selector, }; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::progress::{ @@ -450,6 +452,9 @@ impl std::fmt::Debug for KubernetesComputeDriver { impl KubernetesComputeDriver { pub async fn new(config: KubernetesComputeConfig) -> Result { + config + .validate_workspace_mode() + .map_err(KubernetesDriverError::Precondition)?; config .validate_provider_spiffe_workload_api_socket_path() .map_err(KubernetesDriverError::Precondition)?; @@ -508,6 +513,169 @@ impl KubernetesComputeDriver { &self.config.ssh_socket_path } + pub fn workspace_mode(&self) -> WorkspaceMode { + self.config.workspace_mode + } + + /// Ensure the K8s namespace for a workspace exists (managed mode only). + /// + /// Idempotent: returns the namespace name whether it was just created or + /// already existed. Also creates the sandbox `ServiceAccount` in the + /// namespace. + pub async fn ensure_namespace(&self, workspace: &str) -> Result { + let ns_name = managed_namespace(&self.config.gateway_id, workspace); + let ns_api: Api = Api::all(self.client.clone()); + + let gateway_ns_api: Api = Api::all(self.client.clone()); + let gateway_ns_annotations = match tokio::time::timeout( + KUBE_API_TIMEOUT, + gateway_ns_api.get(&self.config.namespace), + ) + .await + { + Ok(Ok(ns)) => ns.metadata.annotations.unwrap_or_default(), + _ => BTreeMap::new(), + }; + + let mut labels = BTreeMap::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_GATEWAY_ID.to_string(), self.config.gateway_id.clone()); + labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); + + let mut annotations = BTreeMap::new(); + for key in [ + crate::config::ANNOTATION_SCC_UID_RANGE, + crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, + ] { + if let Some(val) = gateway_ns_annotations.get(key) { + annotations.insert(key.to_string(), val.clone()); + } + } + + let ns = Namespace { + metadata: ObjectMeta { + name: Some(ns_name.clone()), + labels: Some(labels), + annotations: if annotations.is_empty() { + None + } else { + Some(annotations) + }, + ..Default::default() + }, + ..Default::default() + }; + + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.create(&PostParams::default(), &ns)) + .await + { + Ok(Ok(_)) => { + info!(namespace = %ns_name, workspace = %workspace, "created managed namespace"); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => { + debug!(namespace = %ns_name, "managed namespace already exists"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating namespace {ns_name}" + ))); + } + } + + self.ensure_service_account(&ns_name).await?; + + Ok(ns_name) + } + + async fn ensure_service_account(&self, namespace: &str) -> Result<(), KubernetesDriverError> { + let sa_api: Api = Api::namespaced(self.client.clone(), namespace); + let sa = ServiceAccount { + metadata: ObjectMeta { + name: Some(self.config.service_account_name.clone()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + + match tokio::time::timeout(KUBE_API_TIMEOUT, sa_api.create(&PostParams::default(), &sa)) + .await + { + Ok(Ok(_)) => { + info!(namespace = %namespace, sa = %self.config.service_account_name, "created service account"); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => {} + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating service account in {namespace}" + ))); + } + } + + Ok(()) + } + + /// Delete the managed namespace if it contains no sandboxes (managed mode + /// only). Called after sandbox deletion. + pub async fn delete_namespace_if_empty( + &self, + workspace: &str, + ) -> Result<(), KubernetesDriverError> { + let ns_name = managed_namespace(&self.config.gateway_id, workspace); + + let sandbox_api_version = self + .supported_sandbox_api_version(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let agent_api = Self::agent_sandbox_api(self.client.clone(), sandbox_api_version, &ns_name); + + let lp = ListParams::default() + .labels(&openshell_sandbox_label_selector()) + .limit(1); + let list = tokio::time::timeout(KUBE_API_TIMEOUT, agent_api.api.list(&lp)) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!("timeout listing sandboxes in {ns_name}")) + })? + .map_err(KubernetesDriverError::from_kube)?; + + if !list.items.is_empty() { + debug!(namespace = %ns_name, "namespace still has sandboxes, skipping delete"); + return Ok(()); + } + + let ns_api: Api = Api::all(self.client.clone()); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + ns_api.delete(&ns_name, &DeleteParams::default()), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace = %ns_name, workspace = %workspace, "deleted empty managed namespace"); + } + Ok(Err(KubeError::Api(api))) if api.code == 404 => { + debug!(namespace = %ns_name, "managed namespace already deleted"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout deleting namespace {ns_name}" + ))); + } + } + + Ok(()) + } + fn validate_driver_config_for_sandbox( &self, sandbox: &Sandbox, @@ -522,16 +690,70 @@ impl KubernetesComputeDriver { ) } - fn agent_sandbox_api(&self, client: Client, sandbox_api_version: &str) -> AgentSandboxApi { + fn agent_sandbox_api( + client: Client, + sandbox_api_version: &str, + namespace: &str, + ) -> AgentSandboxApi { + let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + let api = Api::namespaced_with(client, namespace, &resource); + AgentSandboxApi { api, resource } + } + + fn cluster_wide_sandbox_api(client: Client, sandbox_api_version: &str) -> AgentSandboxApi { let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); let resource = ApiResource::from_gvk(&gvk); - let api = Api::namespaced_with(client, &self.config.namespace, &resource); + let api = Api::all_with(client, &resource); AgentSandboxApi { api, resource } } - async fn supported_agent_sandbox_api(&self, client: Client) -> Result { + async fn supported_agent_sandbox_api( + &self, + client: Client, + namespace: &str, + ) -> Result { let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; - Ok(self.agent_sandbox_api(client, sandbox_api_version)) + Ok(Self::agent_sandbox_api( + client, + sandbox_api_version, + namespace, + )) + } + + async fn supported_sandbox_api_for_lookup( + &self, + client: Client, + ) -> Result { + let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; + if self.config.is_multi_namespace() { + Ok(Self::cluster_wide_sandbox_api(client, sandbox_api_version)) + } else { + Ok(Self::agent_sandbox_api( + client, + sandbox_api_version, + &self.config.namespace, + )) + } + } + + fn sandbox_lookup_selector(&self, sandbox_id: &str) -> String { + let mut selector = + format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + if self.config.workspace_mode == WorkspaceMode::Managed { + use std::fmt::Write; + write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); + } + selector + } + + fn openshell_sandbox_selector(&self) -> String { + let mut selector = openshell_sandbox_label_selector(); + if self.config.workspace_mode == WorkspaceMode::Managed { + use std::fmt::Write; + write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); + } + selector } async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { @@ -548,7 +770,11 @@ impl KubernetesComputeDriver { client: Client, ) -> Result<&'static str, String> { for sandbox_api_version in SANDBOX_VERSIONS { - let agent_sandbox_api = self.agent_sandbox_api(client.clone(), sandbox_api_version); + let agent_sandbox_api = Self::agent_sandbox_api( + client.clone(), + sandbox_api_version, + &self.config.namespace, + ); match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&ListParams::default().limit(1)), @@ -586,39 +812,27 @@ impl KubernetesComputeDriver { )) } - /// Resolve sandbox UID/GID from config or `OpenShift` SCC namespace annotations. - /// - /// Returns `(uid, gid, ns_annotations_map)`: - /// - If `sandbox_uid` is set in config, returns that (with fallback GID) - /// - Otherwise fetches the target namespace and checks for - /// `openshift.io/sa.scc.uid-range` / `openshift.io/sa.scc.supplemental-groups` - /// annotations. - /// - If neither config nor `OpenShift` is found, returns `(1000, 1000, {})` as defaults. - async fn resolve_sandbox_identity(&self) -> (u32, u32, BTreeMap) { - // Explicit config takes priority — skip namespace lookup entirely. + async fn resolve_sandbox_identity_in_namespace( + &self, + namespace: &str, + ) -> (u32, u32, BTreeMap) { if self.config.sandbox_uid.is_some() { let uid = self.config.resolve_sandbox_uid(None); let gid = self.config.resolve_sandbox_gid(uid, None); return (uid, gid, BTreeMap::new()); } - // Try to read namespace annotations for OpenShift SCC. - // Namespace is namespaced so Api::all works (it's cluster-scoped but - // can list all namespaces) and we filter by name, or use Api::namespaced. let ns_api: Api = Api::all(self.client.clone()); - match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(self.config.namespace.as_str())) - .await - { + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(namespace)).await { Ok(Ok(ns)) => { let anns = ns.metadata.annotations.unwrap_or_default(); tracing::info!( - namespace = %self.config.namespace, + namespace = %namespace, uid_range = ?anns.get(crate::config::ANNOTATION_SCC_UID_RANGE), sup_groups = ?anns.get(crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS), "Resolved namespace annotations for sandbox identity" ); let uid = self.config.resolve_sandbox_uid(Some(&anns)); - // Explicit sandbox_gid config wins; SCC annotation only applies when not set. let baseline_gid = self.config.resolve_sandbox_gid(uid, None); let gid = self.config.sandbox_gid.map_or_else( || { @@ -637,7 +851,7 @@ impl KubernetesComputeDriver { } Ok(Err(e)) => { tracing::warn!( - namespace = %self.config.namespace, + namespace = %namespace, error = %e, "Failed to fetch namespace for SCC annotations, falling back to defaults" ); @@ -647,7 +861,7 @@ impl KubernetesComputeDriver { } Err(_) => { tracing::warn!( - namespace = %self.config.namespace, + namespace = %namespace, "Namespace fetch timed out, falling back to defaults" ); let uid = DEFAULT_SANDBOX_UID; @@ -672,7 +886,15 @@ impl KubernetesComputeDriver { let _ = self .validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; - validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; + match self.config.workspace_mode { + WorkspaceMode::Shared => { + validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; + } + WorkspaceMode::Managed | WorkspaceMode::Operator => { + validate_kubernetes_dns1123_label(&sandbox.name, "sandbox name") + .map_err(tonic::Status::invalid_argument)?; + } + } let gpu_requirements = sandbox .spec .as_ref() @@ -693,15 +915,14 @@ impl KubernetesComputeDriver { pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { info!( sandbox_id = %sandbox_id, - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Fetching sandbox from Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { Ok(Ok(list)) => list.items.into_iter().next().map_or_else( @@ -710,9 +931,12 @@ impl KubernetesComputeDriver { Ok(None) }, |obj| { - Ok(sandbox_from_object(&self.config.namespace, obj) - .ok() - .map(|(_, s)| s)) + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) }, ), Ok(Err(err)) => { @@ -739,18 +963,19 @@ impl KubernetesComputeDriver { pub async fn list_sandboxes(&self) -> Result, String> { info!( - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Listing sandboxes from Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; + let selector = self.openshell_sandbox_selector(); match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api .api - .list(&ListParams::default().labels(&openshell_sandbox_label_selector())), + .list(&ListParams::default().labels(&selector)), ) .await { @@ -760,7 +985,12 @@ impl KubernetesComputeDriver { .into_iter() .filter_map(|obj| { let name = obj.metadata.name.clone().unwrap_or_default(); - match sandbox_from_object(&self.config.namespace, obj) { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + match sandbox_from_object(&ns, obj) { Ok((_, s)) => Some(s), Err(err) => { warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); @@ -778,7 +1008,6 @@ impl KubernetesComputeDriver { } Ok(Err(err)) => { warn!( - namespace = %self.config.namespace, error = %err, "Failed to list sandboxes from Kubernetes" ); @@ -786,7 +1015,6 @@ impl KubernetesComputeDriver { } Err(_elapsed) => { warn!( - namespace = %self.config.namespace, timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out listing sandboxes from Kubernetes" ); @@ -813,21 +1041,32 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::InvalidArgument)?; let name = sandbox.name.as_str(); + let workspace = sandbox.workspace.as_str(); + + let target_namespace = match self.config.workspace_mode { + WorkspaceMode::Shared => self.config.namespace.clone(), + WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, + WorkspaceMode::Operator => workspace.to_string(), + }; + info!( sandbox_id = %sandbox.id, sandbox_name = %name, - namespace = %self.config.namespace, + namespace = %target_namespace, + workspace = %workspace, + workspace_mode = %self.config.workspace_mode, "Creating sandbox in Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_agent_sandbox_api(self.client.clone(), &target_namespace) .await .map_err(KubernetesDriverError::Message)?; // Resolve sandbox UID/GID from config or OpenShift SCC namespace annotations. - let (resolved_user_id, resolved_group_id, ns_annotations) = - self.resolve_sandbox_identity().await; + let (resolved_user_id, resolved_group_id, ns_annotations) = self + .resolve_sandbox_identity_in_namespace(&target_namespace) + .await; let params = SandboxPodParams { default_image: &self.config.default_image, @@ -866,11 +1105,8 @@ impl KubernetesComputeDriver { let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) .map_err(KubernetesDriverError::InvalidArgument)?; - let kube_name = kube_resource_name(&sandbox.workspace, name); + let kube_name = self.config.kube_resource_name(workspace, name); let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); - // Copy only the SCC-related annotations onto the Sandbox CR for - // traceability. Copying the full namespace annotation map exposes - // unrelated cluster metadata and can fail with oversized annotations. let mut annotations = sandbox_annotations(sandbox); for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, @@ -882,7 +1118,7 @@ impl KubernetesComputeDriver { } obj.metadata = ObjectMeta { name: Some(kube_name), - namespace: Some(self.config.namespace.clone()), + namespace: Some(target_namespace), labels: Some(sandbox_labels(sandbox)), annotations: Some(annotations), ..Default::default() @@ -930,19 +1166,18 @@ impl KubernetesComputeDriver { pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { info!( sandbox_id = %sandbox_id, - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Deleting sandbox from Kubernetes" ); - let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, preconditions) = match tokio::time::timeout( + let (kube_name, obj_namespace, workspace, preconditions) = match tokio::time::timeout( KUBE_API_TIMEOUT, - agent_sandbox_api.api.list(&lp), + lookup_api.api.list(&lp), ) .await { @@ -950,11 +1185,22 @@ impl KubernetesComputeDriver { if let Some(obj) = list.items.into_iter().next() { match obj.metadata.name { Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); let pc = Preconditions { uid: obj.metadata.uid, resource_version: obj.metadata.resource_version, }; - (name, pc) + (name, ns, ws, pc) } None => return Ok(false), } @@ -984,15 +1230,22 @@ impl KubernetesComputeDriver { } }; + let delete_api = self + .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + .await?; let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout( - KUBE_API_TIMEOUT, - agent_sandbox_api.api.delete(&kube_name, &dp), - ) - .await - { + match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { - info!(sandbox_id = %sandbox_id, "Sandbox deleted from Kubernetes"); + info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + if self.config.workspace_mode == WorkspaceMode::Managed + && let Err(e) = self.delete_namespace_if_empty(&workspace).await + { + warn!( + workspace = %workspace, + error = %e, + "Failed to clean up empty managed namespace after sandbox deletion" + ); + } Ok(true) } Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { @@ -1023,10 +1276,9 @@ impl KubernetesComputeDriver { pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { Ok(Ok(list)) => Ok(!list.items.is_empty()), @@ -1041,9 +1293,17 @@ impl KubernetesComputeDriver { // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { + if self.config.is_multi_namespace() { + self.watch_sandboxes_cluster_wide().await + } else { + self.watch_sandboxes_single_namespace().await + } + } + + async fn watch_sandboxes_single_namespace(&self) -> Result { let namespace = self.config.namespace.clone(); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.watch_client.clone()) + .supported_agent_sandbox_api(self.watch_client.clone(), &self.config.namespace) .await?; let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); let watcher_config = watcher::Config::default().labels(&openshell_sandbox_label_selector()); @@ -1151,6 +1411,85 @@ impl KubernetesComputeDriver { Ok(Box::pin(ReceiverStream::new(rx))) } + + async fn watch_sandboxes_cluster_wide(&self) -> Result { + let sandbox_api_version = self + .supported_sandbox_api_version(self.watch_client.clone()) + .await?; + let cluster_api = + Self::cluster_wide_sandbox_api(self.watch_client.clone(), sandbox_api_version); + let selector = self.openshell_sandbox_selector(); + let watcher_config = watcher::Config::default().labels(&selector); + let mut sandbox_stream = watcher::watcher(cluster_api.api, watcher_config).boxed(); + let (tx, rx) = mpsc::channel(256); + let default_namespace = self.config.namespace.clone(); + + tokio::spawn(async move { + loop { + tokio::select! { + result = sandbox_stream.try_next() => match result { + Ok(Some(Event::Applied(obj))) => { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Ok(Some(Event::Deleted(obj))) => { + if is_openshell_managed(&obj) + && let Ok(sandbox_id) = sandbox_id_from_object(&obj) + { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Ok(Some(Event::Restarted(objs))) => { + for obj in objs { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + return; + } + } + } + } + Ok(None) => { + let _ = tx.send(Err(KubernetesDriverError::Message( + "sandbox watcher stream ended unexpectedly".to_string() + ))).await; + break; + } + Err(err) => { + let _ = tx.send(Err(KubernetesDriverError::Message(err.to_string()))).await; + break; + } + }, + () = tx.closed() => break, + } + } + }); + + Ok(Box::pin(ReceiverStream::new(rx))) + } } fn should_try_next_sandbox_api_version(err: &KubeError) -> bool { @@ -1169,10 +1508,6 @@ fn validate_gpu_request( Ok(()) } -fn kube_resource_name(workspace: &str, name: &str) -> String { - format!("{workspace}--{name}") -} - const MAX_KUBE_NAME_LEN: usize = 63; fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), tonic::Status> { @@ -5849,22 +6184,6 @@ mod tests { assert!(validate_kubernetes_dns1123_label("dotted.name", "sandbox name").is_err()); } - #[test] - fn kube_resource_name_qualifies_with_workspace() { - assert_eq!(kube_resource_name("alpha", "work"), "alpha--work"); - assert_eq!( - kube_resource_name("default", "my-sandbox"), - "default--my-sandbox" - ); - } - - #[test] - fn kube_resource_name_different_workspaces_produce_different_names() { - let alpha = kube_resource_name("alpha", "work"); - let beta = kube_resource_name("beta", "work"); - assert_ne!(alpha, beta); - } - #[test] fn kube_resource_name_length_validation_accepts_short_names() { validate_kube_resource_name_length("default", "my-sandbox").unwrap(); @@ -5961,6 +6280,37 @@ mod tests { assert!(result.unwrap_err().contains("not managed by openshell")); } + #[test] + fn sandbox_from_object_uses_object_namespace_over_fallback() { + let obj = DynamicObject { + types: None, + metadata: ObjectMeta { + name: Some("work".to_string()), + namespace: Some("openshell-gw1-team-a".to_string()), + annotations: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-cross".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ])), + labels: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-cross".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + ])), + ..Default::default() + }, + data: serde_json::json!({}), + }; + + let (_, sandbox) = sandbox_from_object("openshell", obj).unwrap(); + assert_eq!(sandbox.namespace, "openshell-gw1-team-a"); + assert_eq!(sandbox.workspace, "team-a"); + } + #[test] fn sandbox_from_object_warns_on_managed_cr_missing_workspace() { let obj = DynamicObject { diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 7c56c8de5b..d18a23a618 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,9 +6,10 @@ pub mod driver; pub mod grpc; pub use config::{ - AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, + OperatorNamespaceAllowlist, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index b7d5514ac2..c5b7659406 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,9 +10,9 @@ use tracing_subscriber::EnvFilter; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, + KubernetesSidecarConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -29,9 +29,25 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, + #[arg(long, env = "OPENSHELL_WORKSPACE_MODE", default_value = "shared")] + workspace_mode: WorkspaceMode, + + #[arg( + long, + env = "OPENSHELL_GATEWAY_ID", + default_value = DEFAULT_GATEWAY_ID + )] + gateway_id: String, + #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] sandbox_namespace: String, + #[arg(long, env = "OPENSHELL_OPERATOR_NAMESPACE_LABEL")] + operator_namespace_label: Option, + + #[arg(long, env = "OPENSHELL_OPERATOR_NAMESPACE_FILE")] + operator_namespace_file: Option, + #[arg( long, env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", @@ -133,7 +149,11 @@ async fn main() -> Result<()> { .init(); let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { + workspace_mode: args.workspace_mode, + gateway_id: args.gateway_id, namespace: args.sandbox_namespace, + operator_namespace_label: args.operator_namespace_label, + operator_namespace_file: args.operator_namespace_file, service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index eed0e5f083..54f7ed9afa 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,7 +26,8 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use std::sync::Arc; +use std::collections::BTreeSet; +use std::sync::{Arc, RwLock}; use tonic::Status; use tracing::{debug, info, warn}; @@ -135,8 +136,32 @@ impl Authenticator for K8sServiceAccountAuthenticator { } } +/// Validates the namespace extracted from an SA token username against the +/// expected set for the active workspace mode. +#[derive(Debug, Clone)] +pub enum NamespaceValidator { + /// Shared mode: accept only the single configured namespace. + Exact(String), + /// Managed mode: accept any namespace with the managed prefix + /// (`openshell-{gateway_id}-`). + Prefix(String), + /// Operator mode: accept namespaces in the dynamic allowlist. + Allowlist(Arc>>), +} + +impl NamespaceValidator { + pub fn accepts(&self, namespace: &str) -> bool { + match self { + Self::Exact(expected) => namespace == expected, + Self::Prefix(prefix) => namespace.starts_with(prefix.as_str()), + Self::Allowlist(set) => set.read().is_ok_and(|s| s.contains(namespace)), + } + } +} + #[derive(Debug)] struct TokenReviewIdentity { + namespace: String, pod_name: String, pod_uid: String, } @@ -151,59 +176,53 @@ struct SandboxOwnerReference { /// Resolver backed by the apiserver's `TokenReview` API and `kube::Client` /// for the per-pod annotation lookup. pub struct LiveK8sResolver { + client: kube::Client, token_reviews_api: Api, - pods_api: Api, - sandboxes_api_v1beta1: Api, - sandboxes_api_v1alpha1: Api, expected_audience: String, - sandbox_namespace: String, + namespace_validator: NamespaceValidator, expected_service_account: String, } impl LiveK8sResolver { pub fn new( client: kube::Client, - namespace: &str, + namespace_validator: NamespaceValidator, expected_audience: String, expected_service_account: String, ) -> Self { let token_reviews_api: Api = Api::all(client.clone()); - let pods_api: Api = Api::namespaced(client.clone(), namespace); - let sandbox_gvk_v1beta1 = - GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION_V1BETA1, SANDBOX_KIND); - let sandbox_resource_v1beta1 = ApiResource::from_gvk(&sandbox_gvk_v1beta1); - let sandbox_gvk_v1alpha1 = GroupVersionKind::gvk( - SANDBOX_API_GROUP, - SANDBOX_API_VERSION_V1ALPHA1, - SANDBOX_KIND, - ); - let sandbox_resource_v1alpha1 = ApiResource::from_gvk(&sandbox_gvk_v1alpha1); - let sandboxes_api_v1beta1: Api = - Api::namespaced_with(client.clone(), namespace, &sandbox_resource_v1beta1); - let sandboxes_api_v1alpha1: Api = - Api::namespaced_with(client, namespace, &sandbox_resource_v1alpha1); Self { + client, token_reviews_api, - pods_api, - sandboxes_api_v1beta1, - sandboxes_api_v1alpha1, expected_audience, - sandbox_namespace: namespace.to_string(), + namespace_validator, expected_service_account, } } + fn pods_api(&self, namespace: &str) -> Api { + Api::namespaced(self.client.clone(), namespace) + } + + fn sandboxes_api(&self, namespace: &str, api_version: &str) -> Api { + let gvk = GroupVersionKind::gvk(SANDBOX_API_GROUP, api_version, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + Api::namespaced_with(self.client.clone(), namespace, &resource) + } + async fn get_sandbox_cr_for_owner( &self, + namespace: &str, owner: &SandboxOwnerReference, ) -> Result, KubeError> { - let apis = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { - [&self.sandboxes_api_v1alpha1, &self.sandboxes_api_v1beta1] + let versions = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { + [SANDBOX_API_VERSION_V1ALPHA1, SANDBOX_API_VERSION_V1BETA1] } else { - [&self.sandboxes_api_v1beta1, &self.sandboxes_api_v1alpha1] + [SANDBOX_API_VERSION_V1BETA1, SANDBOX_API_VERSION_V1ALPHA1] }; - for api in apis { + for version in versions { + let api = self.sandboxes_api(namespace, version); match api.get_opt(&owner.name).await { Ok(Some(sandbox_cr)) => return Ok(Some(sandbox_cr)), Ok(None) => {} @@ -242,7 +261,7 @@ impl K8sIdentityResolver for LiveK8sResolver { let Some(identity) = token_review_identity( &status, &self.expected_audience, - &self.sandbox_namespace, + &self.namespace_validator, &self.expected_service_account, )? else { @@ -252,34 +271,30 @@ impl K8sIdentityResolver for LiveK8sResolver { info!( pod_name = %identity.pod_name, pod_uid = %identity.pod_uid, + namespace = %identity.namespace, service_account = %self.expected_service_account, "validated K8s SA token via TokenReview" ); - // Look up the pod and read its sandbox-id annotation. - let pod = self - .pods_api - .get_opt(&identity.pod_name) - .await - .map_err(|e| { - warn!( - pod = %identity.pod_name, - error = %e, - "failed to fetch sandbox pod for annotation lookup" - ); - Status::internal(format!("pod GET failed: {e}")) - })?; + let pods_api = self.pods_api(&identity.namespace); + let pod = pods_api.get_opt(&identity.pod_name).await.map_err(|e| { + warn!( + pod = %identity.pod_name, + namespace = %identity.namespace, + error = %e, + "failed to fetch sandbox pod for annotation lookup" + ); + Status::internal(format!("pod GET failed: {e}")) + })?; let Some(pod) = pod else { warn!( pod = %identity.pod_name, - "sandbox pod referenced by SA token not found in this namespace" + namespace = %identity.namespace, + "sandbox pod referenced by SA token not found" ); return Err(Status::not_found("sandbox pod not found")); }; - // Defense-in-depth: confirm the pod UID matches the SA token's - // `kubernetes.io.pod.uid`. Prevents a replayed token from a - // recreated pod with the same name. let actual_uid = pod.metadata.uid.as_deref().unwrap_or_default(); if actual_uid != identity.pod_uid { warn!( @@ -294,16 +309,19 @@ impl K8sIdentityResolver for LiveK8sResolver { let sandbox_id = pod_sandbox_id(&pod)?; let owner = sandbox_owner_reference(&pod)?; - let sandbox_cr = self.get_sandbox_cr_for_owner(&owner).await.map_err(|e| { - warn!( - pod = %identity.pod_name, - sandbox_owner = %owner.name, - sandbox_owner_api_version = %owner.api_version, - error = %e, - "failed to fetch owning Sandbox CR for pod identity validation" - ); - Status::internal(format!("sandbox GET failed: {e}")) - })?; + let sandbox_cr = self + .get_sandbox_cr_for_owner(&identity.namespace, &owner) + .await + .map_err(|e| { + warn!( + pod = %identity.pod_name, + sandbox_owner = %owner.name, + sandbox_owner_api_version = %owner.api_version, + error = %e, + "failed to fetch owning Sandbox CR for pod identity validation" + ); + Status::internal(format!("sandbox GET failed: {e}")) + })?; let Some(sandbox_cr) = sandbox_cr else { warn!( pod = %identity.pod_name, @@ -327,7 +345,7 @@ impl K8sIdentityResolver for LiveK8sResolver { fn token_review_identity( status: &TokenReviewStatus, expected_audience: &str, - sandbox_namespace: &str, + namespace_validator: &NamespaceValidator, expected_service_account: &str, ) -> Result, Status> { if status.authenticated != Some(true) { @@ -356,13 +374,20 @@ fn token_review_identity( .username .as_deref() .ok_or_else(|| Status::permission_denied("TokenReview response missing username"))?; - let expected_username = - format!("system:serviceaccount:{sandbox_namespace}:{expected_service_account}"); - if username != expected_username { + + let (namespace, sa_name) = parse_sa_username(username).ok_or_else(|| { warn!( username = %username, - sandbox_namespace = %sandbox_namespace, - service_account = %expected_service_account, + "K8s TokenReview username is not a service account" + ); + Status::permission_denied("SA token username format not recognized") + })?; + + if sa_name != expected_service_account { + warn!( + username = %username, + service_account = %sa_name, + expected = %expected_service_account, "K8s TokenReview principal is not the configured sandbox service account" ); return Err(Status::permission_denied( @@ -370,9 +395,33 @@ fn token_review_identity( )); } + if !namespace_validator.accepts(&namespace) { + warn!( + username = %username, + namespace = %namespace, + "K8s TokenReview SA namespace not accepted by workspace mode validator" + ); + return Err(Status::permission_denied( + "SA token is not from an accepted sandbox namespace", + )); + } + let pod_name = user_extra_one(user, POD_NAME_EXTRA)?; let pod_uid = user_extra_one(user, POD_UID_EXTRA)?; - Ok(Some(TokenReviewIdentity { pod_name, pod_uid })) + Ok(Some(TokenReviewIdentity { + namespace, + pod_name, + pod_uid, + })) +} + +fn parse_sa_username(username: &str) -> Option<(String, String)> { + let rest = username.strip_prefix("system:serviceaccount:")?; + let (namespace, sa_name) = rest.split_once(':')?; + if namespace.is_empty() || sa_name.is_empty() { + return None; + } + Some((namespace.to_string(), sa_name.to_string())) } #[allow(clippy::result_large_err)] @@ -664,6 +713,10 @@ mod tests { cr } + fn exact_validator(ns: &str) -> NamespaceValidator { + NamespaceValidator::Exact(ns.to_string()) + } + #[test] fn token_review_identity_extracts_pod_binding() { let status = token_review_status( @@ -676,10 +729,12 @@ mod tests { ], ); - let identity = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let validator = exact_validator("openshell"); + let identity = token_review_identity(&status, "openshell-gateway", &validator, "default") .unwrap() .expect("authenticated token should resolve"); + assert_eq!(identity.namespace, "openshell"); assert_eq!(identity.pod_name, "openshell-sandbox-a"); assert_eq!(identity.pod_uid, "uid-a"); } @@ -691,9 +746,10 @@ mod tests { error: Some("invalid audience".to_string()), ..Default::default() }; + let validator = exact_validator("openshell"); assert!( - token_review_identity(&status, "openshell-gateway", "openshell", "default") + token_review_identity(&status, "openshell-gateway", &validator, "default") .unwrap() .is_none() ); @@ -710,8 +766,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("wrong audience must fail closed"); assert_eq!(err.code(), tonic::Code::Unauthenticated); } @@ -727,8 +784,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("other namespace must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -744,8 +802,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("other service account must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -758,12 +817,71 @@ mod tests { "system:serviceaccount:openshell:default", vec![], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("non pod-bound tokens must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } + #[test] + fn namespace_validator_exact_accepts_matching() { + let v = NamespaceValidator::Exact("openshell".to_string()); + assert!(v.accepts("openshell")); + assert!(!v.accepts("other")); + } + + #[test] + fn namespace_validator_prefix_accepts_managed_namespaces() { + let v = NamespaceValidator::Prefix("openshell-gw1-".to_string()); + assert!(v.accepts("openshell-gw1-workspace-a")); + assert!(v.accepts("openshell-gw1-default")); + assert!(!v.accepts("openshell-gw2-workspace-a")); + assert!(!v.accepts("other")); + } + + #[test] + fn namespace_validator_allowlist_accepts_known_namespaces() { + let set = Arc::new(RwLock::new(BTreeSet::from([ + "ns-a".to_string(), + "ns-b".to_string(), + ]))); + let v = NamespaceValidator::Allowlist(set); + assert!(v.accepts("ns-a")); + assert!(v.accepts("ns-b")); + assert!(!v.accepts("ns-c")); + } + + #[test] + fn token_review_identity_prefix_validator_accepts_managed_namespace() { + let status = token_review_status( + true, + vec!["openshell-gateway"], + "system:serviceaccount:openshell-gw1-workspace-a:default", + vec![ + (POD_NAME_EXTRA, "openshell-sandbox-a"), + (POD_UID_EXTRA, "uid-a"), + ], + ); + let validator = NamespaceValidator::Prefix("openshell-gw1-".to_string()); + + let identity = token_review_identity(&status, "openshell-gateway", &validator, "default") + .unwrap() + .expect("managed namespace token should resolve"); + assert_eq!(identity.namespace, "openshell-gw1-workspace-a"); + } + + #[test] + fn parse_sa_username_extracts_namespace_and_sa() { + let (ns, sa) = parse_sa_username("system:serviceaccount:openshell:default").unwrap(); + assert_eq!(ns, "openshell"); + assert_eq!(sa, "default"); + + assert!(parse_sa_username("system:node:nodename").is_none()); + assert!(parse_sa_username("system:serviceaccount::default").is_none()); + assert!(parse_sa_username("system:serviceaccount:ns:").is_none()); + } + #[test] fn pod_sandbox_id_requires_annotation() { assert_eq!( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 5cd06d3900..b669a457dc 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -454,13 +454,33 @@ pub(crate) async fn run_server( // namespace and service account used by the Kubernetes driver. let kubernetes_config = compute::driver_config::kubernetes_config_for_k8s_sa_bootstrap(config_file.as_ref())?; - let sandbox_namespace = kubernetes_config.namespace; - let sandbox_service_account = kubernetes_config.service_account_name; + let sandbox_namespace = kubernetes_config.namespace.clone(); + let sandbox_service_account = kubernetes_config.service_account_name.clone(); + let namespace_validator = match kubernetes_config.workspace_mode { + openshell_driver_kubernetes::WorkspaceMode::Shared => { + auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace) + } + openshell_driver_kubernetes::WorkspaceMode::Managed => { + auth::k8s_sa::NamespaceValidator::Prefix( + openshell_driver_kubernetes::managed_namespace_prefix( + &kubernetes_config.gateway_id, + ), + ) + } + openshell_driver_kubernetes::WorkspaceMode::Operator => { + // The operator allowlist is populated at runtime by the label + // watcher and file watcher. An empty initial set is fail-closed + // until the watcher populates it. + auth::k8s_sa::NamespaceValidator::Allowlist(Arc::new(std::sync::RwLock::new( + std::collections::BTreeSet::new(), + ))) + } + }; match kube::Client::try_default().await { Ok(client) => { let resolver = Arc::new(auth::k8s_sa::LiveK8sResolver::new( client, - &sandbox_namespace, + namespace_validator, "openshell-gateway".to_string(), sandbox_service_account.clone(), )); diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 7096a8ca74..19e26562ed 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -225,6 +225,9 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | +| server.drivers.kubernetes.operatorNamespaceFile | operator mode | `""` | Path to a drop-in JSON file mapping workspace names to namespace names. Hot-reloaded on change. | +| server.drivers.kubernetes.operatorNamespaceLabel | operator mode | `""` | K8s label selector for namespace discovery. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | | server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 073c8835ec..2acfaa2dad 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -25,9 +26,60 @@ rules: - list - watch # Read namespace annotations for OpenShift SCC UID/GID range resolution. + # Managed/operator modes additionally need list+watch for cluster-wide + # namespace discovery. Managed mode needs create+delete for namespace + # lifecycle. - apiGroups: - "" resources: - namespaces verbs: - get + {{- if ne $workspaceMode "shared" }} + - list + - watch + {{- end }} + {{- if eq $workspaceMode "managed" }} + - create + - delete + {{- end }} + {{- if ne $workspaceMode "shared" }} + # Cluster-wide sandbox CRD access for managed/operator workspace modes. + - apiGroups: + - agents.x-k8s.io + resources: + - sandboxes + - sandboxes/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + {{- end }} + {{- if eq $workspaceMode "managed" }} + # ServiceAccount creation in managed namespaces. + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + {{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index e22b5e7485..454affd0d3 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -127,8 +127,16 @@ data: {{- end }} [openshell.drivers.kubernetes] + workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} + operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} + {{- end }} + {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} + operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} + {{- end }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} @@ -178,6 +186,8 @@ data: [openshell.credential_drivers.kubernetes-secrets] namespace = {{ include "openshell.credentialKubernetesSecretsNamespace" . | quote }} allow_reference_namespace = {{ .Values.server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace }} + workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} {{- end }} {{- if .Values.server.credentialDrivers.vault.enabled }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 5ecc4428ad..4ccc5e3d96 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if eq $workspaceMode "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -42,3 +44,4 @@ rules: - pods verbs: - get +{{- end }} diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index e5233f753c..9bf7c73fab 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if eq $workspaceMode "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -16,3 +18,4 @@ subjects: - kind: ServiceAccount name: {{ include "openshell.serviceAccountName" . }} namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 39205df1bf..ab13c55632 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -223,6 +223,20 @@ server: # the field, "RuntimeDefault" to force the runtime default profile, or # "Localhost/profile-name" for an operator-managed localhost profile. appArmorProfile: "Unconfined" + # Kubernetes compute driver settings. + drivers: + kubernetes: + # -- How workspaces map to Kubernetes namespaces. + # "shared" (default): all sandboxes in a single namespace. + # "managed": auto-creates per-workspace namespaces. + # "operator": uses pre-provisioned namespaces. + workspaceMode: "shared" + # -- (operator mode) K8s label selector for namespace discovery. + # The driver watches namespaces matching this label. + operatorNamespaceLabel: "" + # -- (operator mode) Path to a drop-in JSON file mapping workspace + # names to namespace names. Hot-reloaded on change. + operatorNamespaceFile: "" # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. disableTls: false diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 2cd10b8a0b..b201b6f198 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -413,6 +413,14 @@ key_path = "/etc/openshell-tls/server/tls.key" client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" [openshell.drivers.kubernetes] +# Workspace isolation mode. "shared" renders all sandboxes into a single +# namespace. "managed" auto-creates a K8s namespace per workspace +# (openshell-{gateway_id}-{workspace}). "operator" maps each workspace to a +# pre-provisioned namespace discovered via label selector or drop-in file. +workspace_mode = "shared" +# Gateway identity used in managed-mode namespace naming. Defaults to the +# gateway JWT gateway_id. Must be a DNS-1123 label. +# gateway_id = "openshell" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" @@ -455,6 +463,13 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # back to 1000 on non-OpenShift clusters. # sandbox_uid = 1500 # sandbox_gid = 1500 +# Operator-mode namespace discovery. At least one must be set when +# workspace_mode = "operator". Both can be combined. +# operator_namespace_label discovers namespaces matching a K8s label selector. +# operator_namespace_label = "openshell.ai/workspace=true" +# operator_namespace_file reads allowed namespaces from a JSON/YAML file +# (hot-reloaded on change, e.g. via ConfigMap volume mount). +# operator_namespace_file = "/etc/openshell/workspace-namespaces.json" [openshell.drivers.kubernetes.sidecar] # UID used by relaxed long-running network sidecars. Strict process/binary-aware From 485c65f491999c28baca6922d941550deb12f1de Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Sat, 8 Aug 2026 13:32:15 -0400 Subject: [PATCH 02/11] test(k8s): add e2e tests for workspace namespace modes Add end-to-end tests for managed and operator workspace modes introduced in RFC 0011 Phase 3. The managed mode tests verify namespace creation with correct labels, ServiceAccount provisioning, sandbox CR placement, and namespace survival with remaining sandboxes. The operator mode tests verify rejection of unlabeled and nonexistent namespaces. The positive operator path (sandbox in labeled namespace) is known to fail due to an RBAC gap and will be addressed separately. Also fixes Helm 4 compatibility: move SPDX license headers inside conditional guards in 8 chart templates to prevent empty comment-only documents, and fix a trailing whitespace trimmer in clusterrole.yaml that concatenated the license header with apiVersion. Adds cleanup sweep in with-kube-gateway.sh to remove managed and operator namespaces before Helm uninstall, and mise tasks for running each mode independently. Signed-off-by: Derek Carr --- .../ci/values-workspace-managed.yaml | 9 + .../ci/values-workspace-operator.yaml | 10 + .../openshell/templates/cert-manager-pki.yaml | 3 +- .../helm/openshell/templates/clusterrole.yaml | 2 +- .../templates/credential-secrets-role.yaml | 3 +- .../credential-secrets-rolebinding.yaml | 3 +- .../helm/openshell/templates/deployment.yaml | 4 +- deploy/helm/openshell/templates/gateway.yaml | 3 +- .../helm/openshell/templates/grpcroute.yaml | 3 +- deploy/helm/openshell/templates/role.yaml | 5 +- .../helm/openshell/templates/rolebinding.yaml | 5 +- e2e/rust/Cargo.toml | 12 + e2e/rust/tests/workspace_namespace_managed.rs | 301 ++++++++++++++++++ .../tests/workspace_namespace_operator.rs | 263 +++++++++++++++ e2e/with-kube-gateway.sh | 16 + tasks/test.toml | 10 + 16 files changed, 633 insertions(+), 19 deletions(-) create mode 100644 deploy/helm/openshell/ci/values-workspace-managed.yaml create mode 100644 deploy/helm/openshell/ci/values-workspace-operator.yaml create mode 100644 e2e/rust/tests/workspace_namespace_managed.rs create mode 100644 e2e/rust/tests/workspace_namespace_operator.rs diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml new file mode 100644 index 0000000000..9b8911fbe7 --- /dev/null +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: deploy the gateway in managed workspace mode. +# Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. +server: + drivers: + kubernetes: + workspaceMode: "managed" diff --git a/deploy/helm/openshell/ci/values-workspace-operator.yaml b/deploy/helm/openshell/ci/values-workspace-operator.yaml new file mode 100644 index 0000000000..8d895e4e98 --- /dev/null +++ b/deploy/helm/openshell/ci/values-workspace-operator.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: deploy the gateway in operator workspace mode. +# Namespaces must be pre-provisioned and labeled before sandbox creation. +server: + drivers: + kubernetes: + workspaceMode: "operator" + operatorNamespaceLabel: "openshell.ai/e2e-operator-workspace=true" diff --git a/deploy/helm/openshell/templates/cert-manager-pki.yaml b/deploy/helm/openshell/templates/cert-manager-pki.yaml index fdd702a305..5f9e5f36f1 100644 --- a/deploy/helm/openshell/templates/cert-manager-pki.yaml +++ b/deploy/helm/openshell/templates/cert-manager-pki.yaml @@ -1,7 +1,6 @@ +{{- if .Values.certManager.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if .Values.certManager.enabled }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 2acfaa2dad..66102ab751 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/deploy/helm/openshell/templates/credential-secrets-role.yaml b/deploy/helm/openshell/templates/credential-secrets-role.yaml index 72f0528cb2..f6187c9acb 100644 --- a/deploy/helm/openshell/templates/credential-secrets-role.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-role.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml index 4274fa6e1a..3a9ee0bddc 100644 --- a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/deploy/helm/openshell/templates/deployment.yaml b/deploy/helm/openshell/templates/deployment.yaml index e937979370..f94900b136 100644 --- a/deploy/helm/openshell/templates/deployment.yaml +++ b/deploy/helm/openshell/templates/deployment.yaml @@ -1,7 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 {{- include "openshell.validateValues" . }} {{- if eq (include "openshell.workloadKind" .) "deployment" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/helm/openshell/templates/gateway.yaml b/deploy/helm/openshell/templates/gateway.yaml index f431ffbbd1..2b78595053 100644 --- a/deploy/helm/openshell/templates/gateway.yaml +++ b/deploy/helm/openshell/templates/gateway.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.grpcRoute.enabled .Values.grpcRoute.gateway.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.grpcRoute.enabled .Values.grpcRoute.gateway.create }} apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: diff --git a/deploy/helm/openshell/templates/grpcroute.yaml b/deploy/helm/openshell/templates/grpcroute.yaml index 8fde5458cd..362067fda3 100644 --- a/deploy/helm/openshell/templates/grpcroute.yaml +++ b/deploy/helm/openshell/templates/grpcroute.yaml @@ -1,7 +1,6 @@ +{{- if .Values.grpcRoute.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if .Values.grpcRoute.enabled }} apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 4ccc5e3d96..af80989072 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,8 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - {{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} {{- if eq $workspaceMode "shared" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index 9bf7c73fab..381473a58b 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,8 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - {{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} {{- if eq $workspaceMode "shared" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 3353f07af7..a65d452110 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -29,6 +29,8 @@ e2e-gpu = ["e2e"] e2e-docker-gpu = ["e2e-docker", "e2e-gpu"] e2e-kubernetes = ["e2e"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] +e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] +e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] @@ -134,6 +136,16 @@ name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" required-features = ["e2e-host-gateway"] +[[test]] +name = "workspace_namespace_managed" +path = "tests/workspace_namespace_managed.rs" +required-features = ["e2e-kubernetes-workspace-managed"] + +[[test]] +name = "workspace_namespace_operator" +path = "tests/workspace_namespace_operator.rs" +required-features = ["e2e-kubernetes-workspace-operator"] + [[test]] name = "gpu" path = "tests/gpu.rs" diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs new file mode 100644 index 0000000000..4bfc793a23 --- /dev/null +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-workspace-managed")] + +//! E2E tests for managed workspace mode. +//! +//! The gateway is deployed with `workspace_mode = "managed"`, which +//! auto-creates a K8s namespace per workspace (`openshell-{gateway_id}-{ws}`) +//! and deletes it when the last sandbox is removed. +//! +//! Namespace cleanup after sandbox deletion is best-effort and depends on +//! controller finalization timing. These tests focus on verifiable behavior: +//! namespace creation, labels, ServiceAccount provisioning, and sandbox CR +//! placement in the correct namespace. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; +use openshell_e2e::harness::output::strip_ansi; + +fn kube_context() -> String { + std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") +} + +async fn kubectl(args: &[&str]) -> (bool, String) { + let context = kube_context(); + let output = tokio::process::Command::new("kubectl") + .arg("--context") + .arg(&context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn kubectl"); + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) +} + +fn managed_namespace(workspace: &str) -> String { + format!("openshell-openshell-{workspace}") +} + +async fn run_cli(args: &[&str]) -> (bool, String) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + let output = cmd.output().await.expect("failed to spawn openshell"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), strip_ansi(&combined)) +} + +fn unique_workspace(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + % 100_000; + format!("{prefix}-{ts}") +} + +struct ManagedCleanup { + workspace: String, + sandboxes: Vec, +} + +impl Drop for ManagedCleanup { + fn drop(&mut self) { + let bin = openshell_bin(); + for sb in &self.sandboxes { + let _ = std::process::Command::new(&bin) + .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = std::process::Command::new(&bin) + .args(["workspace", "delete", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); + if !context.is_empty() { + let ns = managed_namespace(&self.workspace); + let _ = std::process::Command::new("kubectl") + .args([ + "--context", + &context, + "delete", + "namespace", + &ns, + "--ignore-not-found", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} + +#[tokio::test] +async fn managed_creates_namespace_with_labels() { + let ws = unique_workspace("mgd"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["mgd-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create a sandbox — this triggers namespace creation. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "mgd-sb", + "--", + "echo", + "managed-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("managed-ok"), + "sandbox output missing expected string: {out}" + ); + + // Verify the managed namespace was created. + let (ok, out) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should exist: {out}"); + + // Verify labels on the namespace. + let (ok, label_out) = + kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; + assert!(ok, "failed to read namespace labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/managed-by"), + "namespace missing managed-by label: {label_out}" + ); + assert!( + label_out.contains("openshell.ai/gateway-id"), + "namespace missing gateway-id label: {label_out}" + ); + + // Verify the ServiceAccount was created in the managed namespace. + let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; + assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); + + // Verify sandbox CR is in the managed namespace (not the gateway namespace). + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns, + "-o", + "name", + ]) + .await; + assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox CR name mismatch: {out}" + ); +} + +#[tokio::test] +async fn managed_namespace_survives_with_remaining_sandboxes() { + let ws = unique_workspace("mgd2"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["sb-a".into(), "sb-b".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create two sandboxes. + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws, "--name", "sb-a", "--", "echo", "a", + ]) + .await; + assert!(ok, "sandbox sb-a create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws, "--name", "sb-b", "--", "echo", "b", + ]) + .await; + assert!(ok, "sandbox sb-b create failed: {out}"); + + // Delete first sandbox — namespace should survive because sb-b still exists. + let (ok, out) = run_cli(&["sandbox", "delete", "sb-a", "--workspace", &ws]).await; + assert!(ok, "sandbox sb-a delete failed: {out}"); + + // Brief wait, then verify the namespace still exists. + tokio::time::sleep(Duration::from_secs(3)).await; + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should still exist with sb-b"); + + // Verify sb-b's CR is still in the managed namespace. + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns, + "-o", + "name", + ]) + .await; + assert!(ok, "sandbox CRs should still exist in {ns}: {out}"); + assert!( + out.contains("sb-b"), + "sb-b CR should still be present: {out}" + ); +} + +#[tokio::test] +async fn managed_isolates_workspaces_into_separate_namespaces() { + let ws_a = unique_workspace("iso-a"); + let ws_b = unique_workspace("iso-b"); + let ns_a = managed_namespace(&ws_a); + let ns_b = managed_namespace(&ws_b); + let _cleanup_a = ManagedCleanup { + workspace: ws_a.clone(), + sandboxes: vec!["sb-iso-a".into()], + }; + let _cleanup_b = ManagedCleanup { + workspace: ws_b.clone(), + sandboxes: vec!["sb-iso-b".into()], + }; + + // Create two workspaces with sandboxes. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_a]).await; + assert!(ok, "workspace A create failed: {out}"); + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_b]).await; + assert!(ok, "workspace B create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws_a, "--name", "sb-iso-a", "--", "echo", "a", + ]) + .await; + assert!(ok, "sandbox A create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", "create", "--workspace", &ws_b, "--name", "sb-iso-b", "--", "echo", "b", + ]) + .await; + assert!(ok, "sandbox B create failed: {out}"); + + // Verify each workspace has its own namespace. + assert_ne!(ns_a, ns_b, "namespaces should differ"); + + let (ok, _) = kubectl(&["get", "namespace", &ns_a]).await; + assert!(ok, "namespace {ns_a} should exist"); + let (ok, _) = kubectl(&["get", "namespace", &ns_b]).await; + assert!(ok, "namespace {ns_b} should exist"); + + // Verify sandbox CRs are in the correct namespaces (no cross-contamination). + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns_a, + "-o", + "name", + ]) + .await; + assert!(ok, "failed to list CRs in {ns_a}: {out}"); + assert!(out.contains("sb-iso-a"), "sb-iso-a should be in {ns_a}"); + assert!(!out.contains("sb-iso-b"), "sb-iso-b should NOT be in {ns_a}"); + + let (ok, out) = kubectl(&[ + "get", + "sandbox.agents.x-k8s.io", + "-n", + &ns_b, + "-o", + "name", + ]) + .await; + assert!(ok, "failed to list CRs in {ns_b}: {out}"); + assert!(out.contains("sb-iso-b"), "sb-iso-b should be in {ns_b}"); + assert!(!out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}"); +} diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs new file mode 100644 index 0000000000..172414e100 --- /dev/null +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-workspace-operator")] + +//! E2E tests for operator workspace mode. +//! +//! The gateway is deployed with `workspace_mode = "operator"` and +//! `operator_namespace_label = "openshell.ai/e2e-operator-workspace=true"`. +//! Namespaces must be pre-provisioned and labeled before sandbox creation. +//! The gateway discovers valid namespaces via the label selector. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; +use openshell_e2e::harness::output::strip_ansi; + +const OPERATOR_LABEL: &str = "openshell.ai/e2e-operator-workspace=true"; +const SA_NAME: &str = "openshell-sandbox"; + +fn kube_context() -> String { + std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") +} + +async fn kubectl(args: &[&str]) -> (bool, String) { + let context = kube_context(); + let output = tokio::process::Command::new("kubectl") + .arg("--context") + .arg(&context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn kubectl"); + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) +} + +async fn run_cli(args: &[&str]) -> (bool, String) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + let output = cmd.output().await.expect("failed to spawn openshell"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), strip_ansi(&combined)) +} + +fn unique_namespace(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + % 100_000; + format!("{prefix}-{ts}") +} + +async fn provision_operator_namespace(name: &str) { + let (ok, out) = kubectl(&["create", "namespace", name]).await; + assert!(ok, "failed to create namespace {name}: {out}"); + + let (ok, out) = kubectl(&["label", "namespace", name, OPERATOR_LABEL]).await; + assert!(ok, "failed to label namespace {name}: {out}"); + + let (ok, out) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", name]).await; + assert!(ok, "failed to create SA in {name}: {out}"); +} + +async fn delete_namespace(name: &str) { + let _ = kubectl(&[ + "delete", + "namespace", + name, + "--ignore-not-found", + "--wait=false", + ]) + .await; +} + +struct OperatorCleanup { + workspace: String, + namespace: String, + sandboxes: Vec, +} + +impl Drop for OperatorCleanup { + fn drop(&mut self) { + let bin = openshell_bin(); + for sb in &self.sandboxes { + let _ = std::process::Command::new(&bin) + .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = std::process::Command::new(&bin) + .args(["workspace", "delete", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); + if !context.is_empty() { + let _ = std::process::Command::new("kubectl") + .args([ + "--context", + &context, + "delete", + "namespace", + &self.namespace, + "--ignore-not-found", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} + +#[tokio::test] +async fn operator_sandbox_in_labeled_namespace() { + let ns = unique_namespace("op"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["op-sb".into()], + }; + + // Pre-provision the namespace with the operator label and ServiceAccount. + provision_operator_namespace(&ns).await; + + // Wait for the gateway's namespace watcher to discover it. + tokio::time::sleep(Duration::from_secs(5)).await; + + // Create a workspace matching the namespace name (operator mode: 1:1 mapping). + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create a sandbox in the workspace. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "op-sb", + "--", + "echo", + "operator-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("operator-ok"), + "sandbox output missing expected string: {out}" + ); + + // Verify the sandbox CR lives in the pre-provisioned namespace. + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; + assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); + assert!( + out.contains("op-sb"), + "sandbox CR name should be bare 'op-sb', got: {out}" + ); + + // Clean up. + let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox delete failed: {out}"); + + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; + assert!(ok, "workspace delete failed: {out}"); + + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_rejects_unlabeled_namespace() { + let ns = unique_namespace("opun"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec![], + }; + + // Create namespace WITHOUT the operator label. + let (ok, out) = kubectl(&["create", "namespace", &ns]).await; + assert!(ok, "failed to create namespace: {out}"); + + // Create the ServiceAccount (not the label — that's the point). + let (ok, _) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", &ns]).await; + assert!(ok, "failed to create SA"); + + // Create workspace. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Attempt sandbox creation — should fail because namespace is not in the allowlist. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "should-fail", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for unlabeled namespace, but succeeded: {out}" + ); + + // Clean up. + let _ = run_cli(&["workspace", "delete", &ns]).await; + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_rejects_nonexistent_namespace() { + let ns = unique_namespace("opne"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec![], + }; + + // Create workspace with no matching namespace at all. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Attempt sandbox creation — should fail. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "should-fail", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for nonexistent namespace, but succeeded: {out}" + ); + + // Clean up. + let _ = run_cli(&["workspace", "delete", &ns]).await; +} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index cde230daaf..b8a7621a12 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -270,6 +270,22 @@ cleanup() { cleanup_vault_fixture fi + # Sweep managed-mode and operator-mode workspace namespaces before + # uninstalling the Helm release (ClusterRole still needed for deletion). + if command -v kubectl >/dev/null 2>&1 && [ -n "${KUBE_CONTEXT}" ]; then + for label in "openshell.ai/managed-by=openshell" \ + "openshell.ai/e2e-operator-workspace=true"; do + ns_list="$(kctl get namespaces -l "${label}" -o name 2>/dev/null || true)" + if [ -n "${ns_list}" ]; then + echo "Cleaning up namespaces with label ${label}..." + echo "${ns_list}" | while read -r ns_ref; do + kctl delete "${ns_ref}" --wait=false --ignore-not-found \ + 2>/dev/null || true + done + fi + done + fi + if [ "${HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then if command -v helm >/dev/null 2>&1; then helmctl uninstall "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait \ diff --git a/tasks/test.toml b/tasks/test.toml index ed0d17d7af..df409c27d8 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -156,6 +156,16 @@ description = "Run Kubernetes e2e for provider credential storage backed by Kube env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:workspace-managed"] +description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +run = "e2e/rust/e2e-kubernetes.sh" + +["e2e:kubernetes:workspace-operator"] +description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh" From 628a4b7f4ff51f1fa223fca3b1b7c1b74cb522e9 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Sat, 8 Aug 2026 15:04:54 -0400 Subject: [PATCH 03/11] feat(k8s): add operator namespace label watcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spawn a background kube::runtime::watcher in the K8s driver that watches namespaces matching the configured label selector and populates the OperatorNamespaceAllowlist at runtime. The driver owns the allowlist and exposes its Arc so the server can share the same set with the SA token authenticator. create_sandbox now gates pod creation on the allowlist in operator mode — workspaces whose namespace is not yet labeled are rejected at resource render time rather than silently proceeding. Workspace lifecycle itself is unaffected; only sandbox (resource) creation is gated. Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 112 +++++++++++++++++- crates/openshell-server/src/compute/mod.rs | 17 ++- crates/openshell-server/src/lib.rs | 42 ++++--- e2e/rust/tests/workspace_namespace_managed.rs | 97 +++++++-------- 4 files changed, 200 insertions(+), 68 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 53f8443d28..543d2c884a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -6,8 +6,8 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, - SupervisorTopology, WorkspaceMode, managed_namespace, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ @@ -438,6 +438,7 @@ pub struct KubernetesComputeDriver { watch_client: Client, sandbox_api_version: Arc>, config: KubernetesComputeConfig, + operator_allowlist: Option, } impl std::fmt::Debug for KubernetesComputeDriver { @@ -485,11 +486,26 @@ impl KubernetesComputeDriver { let watch_client = Client::try_from(watch_kube_config).map_err(KubernetesDriverError::from_kube)?; + let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) { + config.operator_namespace_label.as_ref().map(|label| { + let allowlist = OperatorNamespaceAllowlist::new(); + spawn_namespace_label_watcher( + watch_client.clone(), + label.clone(), + allowlist.clone(), + ); + allowlist + }) + } else { + None + }; + Ok(Self { client, watch_client, sandbox_api_version: Arc::new(OnceCell::new()), config, + operator_allowlist, }) } @@ -501,6 +517,10 @@ impl KubernetesComputeDriver { )) } + pub fn operator_allowlist(&self) -> Option<&OperatorNamespaceAllowlist> { + self.operator_allowlist.as_ref() + } + pub fn default_image(&self) -> &str { &self.config.default_image } @@ -1046,7 +1066,16 @@ impl KubernetesComputeDriver { let target_namespace = match self.config.workspace_mode { WorkspaceMode::Shared => self.config.namespace.clone(), WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, - WorkspaceMode::Operator => workspace.to_string(), + WorkspaceMode::Operator => { + if let Some(ref allowlist) = self.operator_allowlist + && !allowlist.contains(workspace) + { + return Err(KubernetesDriverError::InvalidArgument(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + workspace.to_string() + } }; info!( @@ -3479,6 +3508,83 @@ fn condition_from_value(value: &serde_json::Value) -> Option { }) } +fn spawn_namespace_label_watcher( + client: Client, + label_selector: String, + allowlist: OperatorNamespaceAllowlist, +) { + let ns_api: Api = Api::all(client); + let watcher_config = watcher::Config::default().labels(&label_selector); + + tokio::spawn(async move { + loop { + let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + + loop { + match stream.try_next().await { + Ok(Some(Event::Applied(ns))) => { + if let Some(name) = ns.metadata.name.as_deref() { + let inner = allowlist.shared(); + let mut guard = inner.write().expect("allowlist lock poisoned"); + if guard.insert(name.to_string()) { + let count = guard.len(); + drop(guard); + info!( + namespace = name, + total = count, + "operator namespace added to allowlist" + ); + } + } + } + Ok(Some(Event::Deleted(ns))) => { + if let Some(name) = ns.metadata.name.as_deref() { + let inner = allowlist.shared(); + let mut guard = inner.write().expect("allowlist lock poisoned"); + if guard.remove(name) { + let count = guard.len(); + drop(guard); + info!( + namespace = name, + total = count, + "operator namespace removed from allowlist" + ); + } + } + } + Ok(Some(Event::Restarted(namespaces))) => { + let names: std::collections::BTreeSet = namespaces + .into_iter() + .filter_map(|ns| ns.metadata.name) + .collect(); + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist replaced from full relist" + ); + } + Ok(None) => { + warn!("operator namespace watcher stream ended unexpectedly"); + break; + } + Err(err) => { + warn!(error = %err, "operator namespace watcher stream error"); + break; + } + } + } + + tokio::time::sleep(Duration::from_secs(2)).await; + } + }); + + info!( + label_selector = %label_selector, + "operator namespace label watcher started" + ); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a1c33e49ff..36fb2aff7d 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -45,6 +45,7 @@ use openshell_core::{ObjectLabels, ObjectWorkspace}; use openshell_driver_docker::DockerComputeDriver; use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, + OperatorNamespaceAllowlist, }; use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; @@ -749,12 +750,21 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - ) -> Result { + ) -> Result< + ( + Self, + Option>>>, + ), + ComputeError, + > { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; + let operator_allowlist_arc = driver + .operator_allowlist() + .map(OperatorNamespaceAllowlist::shared); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); - Self::from_driver( + let runtime = Self::from_driver( ComputeDriverKind::Kubernetes.as_str().to_string(), driver, None, @@ -766,7 +776,8 @@ impl ComputeRuntime { tracing_log_bus, supervisor_sessions, ) - .await + .await?; + Ok((runtime, operator_allowlist_arc)) } pub(crate) async fn new_remote_driver( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index b669a457dc..a0c686ae42 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -349,7 +349,7 @@ pub(crate) async fn run_server( gateway_tls_enabled: config.tls.is_some(), endpoint_overrides: &config.compute_driver_endpoints, }; - let compute = build_compute_runtime( + let (compute, operator_allowlist) = build_compute_runtime( &config, driver_startup, store.clone(), @@ -468,12 +468,12 @@ pub(crate) async fn run_server( ) } openshell_driver_kubernetes::WorkspaceMode::Operator => { - // The operator allowlist is populated at runtime by the label - // watcher and file watcher. An empty initial set is fail-closed - // until the watcher populates it. - auth::k8s_sa::NamespaceValidator::Allowlist(Arc::new(std::sync::RwLock::new( - std::collections::BTreeSet::new(), - ))) + // Share the driver's allowlist Arc so the SA authenticator and + // the driver's namespace label watcher use the same set. + let allowlist = operator_allowlist.clone().unwrap_or_else(|| { + Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())) + }); + auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) } }; match kube::Client::try_default().await { @@ -860,6 +860,8 @@ async fn terminate_signal() { // Internal wiring helper: each argument is a distinct piece of runtime state // that must be passed through, so the count is justified. #[allow(clippy::too_many_arguments)] +type OperatorAllowlistArc = Option>>>; + async fn build_compute_runtime( config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, @@ -868,16 +870,16 @@ async fn build_compute_runtime( sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, -) -> Result { +) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { let driver = configured_compute_driver(config, driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); - let runtime = match driver { + let (runtime, operator_allowlist) = match driver { ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => { warn_if_kubernetes_sandbox_jwt_expiry_disabled(config); let k8s_config = compute::driver_config::kubernetes_config_from_context(driver_startup)?; - ComputeRuntime::new_kubernetes( + let (rt, allowlist) = ComputeRuntime::new_kubernetes( k8s_config, store, sandbox_index, @@ -886,10 +888,12 @@ async fn build_compute_runtime( supervisor_sessions.clone(), ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, allowlist) } ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => { let docker_config = compute::driver_config::docker_config_from_context(driver_startup)?; - ComputeRuntime::new_docker( + let rt = ComputeRuntime::new_docker( config.clone(), docker_config, store, @@ -899,10 +903,12 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => { let podman_config = compute::driver_config::podman_config_from_context(driver_startup)?; - ComputeRuntime::new_podman( + let rt = ComputeRuntime::new_podman( podman_config, store, sandbox_index, @@ -911,6 +917,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => { let vm_config = compute::driver_config::vm_config_from_context(driver_startup)?; @@ -918,7 +926,7 @@ async fn build_compute_runtime( .file .and_then(|file| file.openshell.gateway.otlp.as_ref()); let endpoint = compute::vm::spawn(config, &vm_config, otlp_config).await?; - ComputeRuntime::new_remote_driver( + let rt = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -927,6 +935,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -939,7 +949,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - ComputeRuntime::new_remote_driver( + let rt = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -948,10 +958,12 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } }; - runtime.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + Ok((runtime, operator_allowlist)) } #[derive(Debug, Clone)] diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 4bfc793a23..f18827478b 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -163,20 +163,9 @@ async fn managed_creates_namespace_with_labels() { assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); // Verify sandbox CR is in the managed namespace (not the gateway namespace). - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns, - "-o", - "name", - ]) - .await; + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); - assert!( - out.contains("mgd-sb"), - "sandbox CR name mismatch: {out}" - ); + assert!(out.contains("mgd-sb"), "sandbox CR name mismatch: {out}"); } #[tokio::test] @@ -193,13 +182,29 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { // Create two sandboxes. let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws, "--name", "sb-a", "--", "echo", "a", + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "sb-a", + "--", + "echo", + "a", ]) .await; assert!(ok, "sandbox sb-a create failed: {out}"); let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws, "--name", "sb-b", "--", "echo", "b", + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "sb-b", + "--", + "echo", + "b", ]) .await; assert!(ok, "sandbox sb-b create failed: {out}"); @@ -215,15 +220,7 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { assert!(ok, "managed namespace {ns} should still exist with sb-b"); // Verify sb-b's CR is still in the managed namespace. - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns, - "-o", - "name", - ]) - .await; + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CRs should still exist in {ns}: {out}"); assert!( out.contains("sb-b"), @@ -253,13 +250,29 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { assert!(ok, "workspace B create failed: {out}"); let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws_a, "--name", "sb-iso-a", "--", "echo", "a", + "sandbox", + "create", + "--workspace", + &ws_a, + "--name", + "sb-iso-a", + "--", + "echo", + "a", ]) .await; assert!(ok, "sandbox A create failed: {out}"); let (ok, out) = run_cli(&[ - "sandbox", "create", "--workspace", &ws_b, "--name", "sb-iso-b", "--", "echo", "b", + "sandbox", + "create", + "--workspace", + &ws_b, + "--name", + "sb-iso-b", + "--", + "echo", + "b", ]) .await; assert!(ok, "sandbox B create failed: {out}"); @@ -273,29 +286,19 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { assert!(ok, "namespace {ns_b} should exist"); // Verify sandbox CRs are in the correct namespaces (no cross-contamination). - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns_a, - "-o", - "name", - ]) - .await; + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_a, "-o", "name"]).await; assert!(ok, "failed to list CRs in {ns_a}: {out}"); assert!(out.contains("sb-iso-a"), "sb-iso-a should be in {ns_a}"); - assert!(!out.contains("sb-iso-b"), "sb-iso-b should NOT be in {ns_a}"); - - let (ok, out) = kubectl(&[ - "get", - "sandbox.agents.x-k8s.io", - "-n", - &ns_b, - "-o", - "name", - ]) - .await; + assert!( + !out.contains("sb-iso-b"), + "sb-iso-b should NOT be in {ns_a}" + ); + + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_b, "-o", "name"]).await; assert!(ok, "failed to list CRs in {ns_b}: {out}"); assert!(out.contains("sb-iso-b"), "sb-iso-b should be in {ns_b}"); - assert!(!out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}"); + assert!( + !out.contains("sb-iso-a"), + "sb-iso-a should NOT be in {ns_b}" + ); } From 6a067bbb29ef5fdde7d395fd248deef8b9d2ee69 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Sat, 8 Aug 2026 15:59:29 -0400 Subject: [PATCH 04/11] fix(k8s): harden operator mode and address review findings Close the fail-open gap in operator mode when only operator_namespace_file is configured: the allowlist is now created unconditionally in operator mode (fail-closed from startup). Implement the namespace file watcher using the notify crate, following the TLS hot-reload pattern (parent-directory watch, 1s debounce, ConfigMap symlink-swap safe). The file format is a JSON array of namespace name strings. Additional fixes from the 10-reviewer audit: - Change allowlist rejection from InvalidArgument to FailedPrecondition so callers know the request may succeed later once the namespace is provisioned. - NamespaceValidator::Allowlist now holds the OperatorNamespaceAllowlist newtype instead of a raw Arc>, eliminating silent denial on RwLock poison. - Verify LABEL_MANAGED_BY and LABEL_GATEWAY_ID ownership before deleting a managed namespace. - Replace fixed 5s sleep in operator e2e test with a 30s poll loop. - Add Helm validation for workspaceMode values. - Fix Helm README type column and description for operator fields. - Add insert/remove methods to OperatorNamespaceAllowlist; label watcher now uses them instead of reaching through shared(). - Reject configs with both operator_namespace_label and operator_namespace_file set. Signed-off-by: Derek Carr --- Cargo.lock | 1 + crates/openshell-driver-kubernetes/Cargo.toml | 1 + .../openshell-driver-kubernetes/src/config.rs | 28 ++- .../openshell-driver-kubernetes/src/driver.rs | 201 +++++++++++++++--- crates/openshell-server/src/auth/k8s_sa.rs | 14 +- crates/openshell-server/src/compute/mod.rs | 12 +- crates/openshell-server/src/lib.rs | 8 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/templates/_helpers.tpl | 4 + deploy/helm/openshell/values.yaml | 6 +- .../tests/workspace_namespace_operator.rs | 47 ++-- 11 files changed, 245 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acf5fff2c7..8861233b9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3785,6 +3785,7 @@ dependencies = [ "kube", "kube-runtime", "miette", + "notify", "openshell-core", "openshell-policy", "prost", diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 2c02f864ab..9be2b1c76b 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } +notify = "8" [dev-dependencies] temp-env = "0.3" diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 21d3aea851..8849dde43e 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -293,8 +293,8 @@ pub struct KubernetesComputeConfig { /// this label and builds the allowlist dynamically. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_label: Option, - /// Path to a drop-in JSON file mapping workspace names to namespace names. - /// Hot-reloaded on change. Delivered via `ConfigMap` volume mount. + /// Path to a JSON file containing an array of namespace names allowed in + /// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_file: Option, /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by @@ -623,10 +623,16 @@ impl KubernetesComputeConfig { WorkspaceMode::Operator => { if self.operator_namespace_label.is_none() && self.operator_namespace_file.is_none() { - return Err("operator workspace mode requires at least one of \ + return Err("operator workspace mode requires exactly one of \ operator_namespace_label or operator_namespace_file" .into()); } + if self.operator_namespace_label.is_some() && self.operator_namespace_file.is_some() + { + return Err("operator workspace mode requires exactly one of \ + operator_namespace_label or operator_namespace_file, not both" + .into()); + } if let Some(ref label) = self.operator_namespace_label && label.is_empty() { @@ -738,6 +744,22 @@ impl OperatorNamespaceAllowlist { .contains(namespace) } + /// Insert a namespace into the allowlist. Returns `true` if it was new. + pub fn insert(&self, name: String) -> bool { + self.inner + .write() + .expect("allowlist lock poisoned") + .insert(name) + } + + /// Remove a namespace from the allowlist. Returns `true` if it was present. + pub fn remove(&self, name: &str) -> bool { + self.inner + .write() + .expect("allowlist lock poisoned") + .remove(name) + } + /// Return a clone of the inner `Arc` for sharing with background tasks. #[must_use] pub fn shared(&self) -> Arc>> { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 543d2c884a..1f492139ec 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -41,7 +41,7 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use serde::Deserialize; use std::collections::{BTreeMap, HashSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -487,15 +487,21 @@ impl KubernetesComputeDriver { Client::try_from(watch_kube_config).map_err(KubernetesDriverError::from_kube)?; let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) { - config.operator_namespace_label.as_ref().map(|label| { - let allowlist = OperatorNamespaceAllowlist::new(); + let allowlist = OperatorNamespaceAllowlist::new(); + + if let Some(ref label) = config.operator_namespace_label { spawn_namespace_label_watcher( watch_client.clone(), label.clone(), allowlist.clone(), ); - allowlist - }) + } + + if let Some(ref path) = config.operator_namespace_file { + spawn_namespace_file_watcher(path.into(), allowlist.clone()); + } + + Some(allowlist) } else { None }; @@ -673,6 +679,36 @@ impl KubernetesComputeDriver { } let ns_api: Api = Api::all(self.client.clone()); + + let ns = match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { + Ok(Ok(ns)) => ns, + Ok(Err(KubeError::Api(api))) if api.code == 404 => { + debug!(namespace = %ns_name, "managed namespace already deleted"); + return Ok(()); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout getting namespace {ns_name}" + ))); + } + }; + + let labels = ns.metadata.labels.as_ref(); + let is_owned = labels + .and_then(|l| l.get(LABEL_MANAGED_BY)) + .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) + && labels + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == &self.config.gateway_id); + if !is_owned { + debug!( + namespace = %ns_name, + "namespace not owned by this gateway, skipping delete" + ); + return Ok(()); + } + match tokio::time::timeout( KUBE_API_TIMEOUT, ns_api.delete(&ns_name, &DeleteParams::default()), @@ -1070,7 +1106,7 @@ impl KubernetesComputeDriver { if let Some(ref allowlist) = self.operator_allowlist && !allowlist.contains(workspace) { - return Err(KubernetesDriverError::InvalidArgument(format!( + return Err(KubernetesDriverError::Precondition(format!( "workspace '{workspace}' is not in the operator namespace allowlist" ))); } @@ -3523,33 +3559,20 @@ fn spawn_namespace_label_watcher( loop { match stream.try_next().await { Ok(Some(Event::Applied(ns))) => { - if let Some(name) = ns.metadata.name.as_deref() { - let inner = allowlist.shared(); - let mut guard = inner.write().expect("allowlist lock poisoned"); - if guard.insert(name.to_string()) { - let count = guard.len(); - drop(guard); - info!( - namespace = name, - total = count, - "operator namespace added to allowlist" - ); - } + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.insert(name.to_string()) + { + info!(namespace = name, "operator namespace added to allowlist"); } } Ok(Some(Event::Deleted(ns))) => { - if let Some(name) = ns.metadata.name.as_deref() { - let inner = allowlist.shared(); - let mut guard = inner.write().expect("allowlist lock poisoned"); - if guard.remove(name) { - let count = guard.len(); - drop(guard); - info!( - namespace = name, - total = count, - "operator namespace removed from allowlist" - ); - } + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); } } Ok(Some(Event::Restarted(namespaces))) => { @@ -3581,10 +3604,126 @@ fn spawn_namespace_label_watcher( info!( label_selector = %label_selector, - "operator namespace label watcher started" + "operator namespace label watcher spawned" ); } +fn load_namespace_file(path: &Path) -> Result, String> { + let contents = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let names: Vec = serde_json::from_str(&contents) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + Ok(names.into_iter().collect()) +} + +fn spawn_namespace_file_watcher(path: PathBuf, allowlist: OperatorNamespaceAllowlist) { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + path = %path.display(), + total = count, + "operator namespace allowlist loaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to load initial operator namespace file, allowlist empty" + ); + } + } + + let watch_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let debounce = Duration::from_secs(1); + + tokio::spawn(async move { + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut watcher = + match notify::recommended_watcher(move |res: Result| { + if let Ok(event) = res + && matches!( + event.kind, + notify::EventKind::Modify(_) | notify::EventKind::Create(_) + ) + { + let _ = tx.send(()); + } + }) { + Ok(w) => w, + Err(e) => { + warn!( + error = %e, + "failed to start operator namespace file watcher, hot-reload disabled" + ); + return; + } + }; + + if let Err(e) = notify::Watcher::watch( + &mut watcher, + &watch_dir, + notify::RecursiveMode::NonRecursive, + ) { + warn!( + error = %e, + dir = %watch_dir.display(), + "failed to watch operator namespace file directory, hot-reload disabled" + ); + return; + } + + info!( + path = %path.display(), + "operator namespace file watcher started" + ); + + loop { + let got_event = rx.recv().await.is_some(); + if !got_event { + warn!("operator namespace file watcher disconnected"); + break; + } + + loop { + tokio::select! { + () = tokio::time::sleep(debounce) => { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist reloaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to reload operator namespace file, keeping existing allowlist" + ); + } + } + break; + } + r = rx.recv() => { + if r.is_some() { + continue; + } + warn!("operator namespace file watcher disconnected"); + return; + } + } + } + } + }); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 54f7ed9afa..32cb2e119c 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,8 +26,8 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use std::collections::BTreeSet; -use std::sync::{Arc, RwLock}; +use openshell_driver_kubernetes::OperatorNamespaceAllowlist; +use std::sync::Arc; use tonic::Status; use tracing::{debug, info, warn}; @@ -146,7 +146,7 @@ pub enum NamespaceValidator { /// (`openshell-{gateway_id}-`). Prefix(String), /// Operator mode: accept namespaces in the dynamic allowlist. - Allowlist(Arc>>), + Allowlist(OperatorNamespaceAllowlist), } impl NamespaceValidator { @@ -154,7 +154,7 @@ impl NamespaceValidator { match self { Self::Exact(expected) => namespace == expected, Self::Prefix(prefix) => namespace.starts_with(prefix.as_str()), - Self::Allowlist(set) => set.read().is_ok_and(|s| s.contains(namespace)), + Self::Allowlist(al) => al.contains(namespace), } } } @@ -842,11 +842,11 @@ mod tests { #[test] fn namespace_validator_allowlist_accepts_known_namespaces() { - let set = Arc::new(RwLock::new(BTreeSet::from([ + let al = OperatorNamespaceAllowlist::from_set(std::collections::BTreeSet::from([ "ns-a".to_string(), "ns-b".to_string(), - ]))); - let v = NamespaceValidator::Allowlist(set); + ])); + let v = NamespaceValidator::Allowlist(al); assert!(v.accepts("ns-a")); assert!(v.accepts("ns-b")); assert!(!v.accepts("ns-c")); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 36fb2aff7d..071309cf25 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -750,19 +750,11 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - ) -> Result< - ( - Self, - Option>>>, - ), - ComputeError, - > { + ) -> Result<(Self, Option), ComputeError> { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; - let operator_allowlist_arc = driver - .operator_allowlist() - .map(OperatorNamespaceAllowlist::shared); + let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); let runtime = Self::from_driver( ComputeDriverKind::Kubernetes.as_str().to_string(), diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a0c686ae42..96a17c7315 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -468,11 +468,7 @@ pub(crate) async fn run_server( ) } openshell_driver_kubernetes::WorkspaceMode::Operator => { - // Share the driver's allowlist Arc so the SA authenticator and - // the driver's namespace label watcher use the same set. - let allowlist = operator_allowlist.clone().unwrap_or_else(|| { - Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())) - }); + let allowlist = operator_allowlist.clone().unwrap_or_default(); auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) } }; @@ -860,7 +856,7 @@ async fn terminate_signal() { // Internal wiring helper: each argument is a distinct piece of runtime state // that must be passed through, so the count is justified. #[allow(clippy::too_many_arguments)] -type OperatorAllowlistArc = Option>>>; +type OperatorAllowlistArc = Option; async fn build_compute_runtime( config: &Config, diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 19e26562ed..a7f6dfda15 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -225,8 +225,8 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | -| server.drivers.kubernetes.operatorNamespaceFile | operator mode | `""` | Path to a drop-in JSON file mapping workspace names to namespace names. Hot-reloaded on change. | -| server.drivers.kubernetes.operatorNamespaceLabel | operator mode | `""` | K8s label selector for namespace discovery. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | +| server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | | server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3764fa6d7a..548418abc6 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -247,6 +247,10 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (eq $workloadKind "statefulset") (gt $replicaCount 1) (not (get $workload "allowMultiReplicaStatefulSet" | default false)) -}} {{- fail "replicaCount > 1 with workload.kind=statefulset requires workload.allowMultiReplicaStatefulSet=true; use workload.kind=deployment for external database-backed multi-replica gateways." -}} {{- end -}} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} +{{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} +{{- end -}} {{- $credentialDrivers := list -}} {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} {{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index ab13c55632..81ec18c036 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -231,11 +231,11 @@ server: # "managed": auto-creates per-workspace namespaces. # "operator": uses pre-provisioned namespaces. workspaceMode: "shared" - # -- (operator mode) K8s label selector for namespace discovery. + # -- K8s label selector for namespace discovery in operator mode. # The driver watches namespaces matching this label. operatorNamespaceLabel: "" - # -- (operator mode) Path to a drop-in JSON file mapping workspace - # names to namespace names. Hot-reloaded on change. + # -- Path to a JSON file containing an array of namespace names + # allowed in operator mode. Hot-reloaded on change. operatorNamespaceFile: "" # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index 172414e100..a382de446c 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -139,30 +139,39 @@ async fn operator_sandbox_in_labeled_namespace() { // Pre-provision the namespace with the operator label and ServiceAccount. provision_operator_namespace(&ns).await; - // Wait for the gateway's namespace watcher to discover it. - tokio::time::sleep(Duration::from_secs(5)).await; - // Create a workspace matching the namespace name (operator mode: 1:1 mapping). let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; assert!(ok, "workspace create failed: {out}"); - // Create a sandbox in the workspace. - let (ok, out) = run_cli(&[ - "sandbox", - "create", - "--workspace", - &ns, - "--name", - "op-sb", - "--", - "echo", - "operator-ok", - ]) - .await; - assert!(ok, "sandbox create failed: {out}"); + // Poll until the gateway's namespace watcher discovers the labeled namespace + // and sandbox creation succeeds (up to 30s). + let mut sandbox_out = String::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "op-sb", + "--", + "echo", + "operator-ok", + ]) + .await; + if ok { + sandbox_out = out; + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } assert!( - out.contains("operator-ok"), - "sandbox output missing expected string: {out}" + sandbox_out.contains("operator-ok"), + "sandbox output missing expected string: {sandbox_out}" ); // Verify the sandbox CR lives in the pre-provisioned namespace. From b9ab04ad4dc7809889392cd380bdc26bef8b02ed Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Mon, 10 Aug 2026 19:22:02 -0400 Subject: [PATCH 05/11] feat(k8s): add workspace-level compute driver RPCs and harden RBAC Decouple namespace lifecycle from sandbox lifecycle by adding EnsureWorkspace/DeleteWorkspace RPCs to the ComputeDriver service. Namespace creation now happens before credential storage and namespace deletion happens on workspace delete, fixing credential storage in managed workspace mode. - Add EnsureWorkspace and DeleteWorkspace proto RPCs with implementations across all compute drivers (K8s managed delegates to ensure_namespace/delete_namespace_if_empty; others no-op) - Wire ensure_workspace into provider create/update/refresh paths so the namespace exists before the credential driver writes secrets - Wire delete_workspace into workspace deletion for cleanup - Remove delete_namespace_if_empty from sandbox deletion path - Scope ClusterRole secrets access to non-shared workspace modes - Add TODO for TLS cert hot-reload in sandbox gRPC client - Harden e2e tests with control-plane sandbox resolution assertions - Fix docker image save --platform flag for OCI index manifests Signed-off-by: Derek Carr --- crates/openshell-core/src/grpc_client.rs | 4 + crates/openshell-driver-docker/src/lib.rs | 19 ++- .../openshell-driver-kubernetes/src/driver.rs | 152 ++++++++++++++++-- .../openshell-driver-kubernetes/src/grpc.rs | 47 +++++- crates/openshell-driver-podman/src/grpc.rs | 26 ++- crates/openshell-driver-vm/src/driver.rs | 33 ++-- crates/openshell-server/src/compute/mod.rs | 103 +++++++++++- crates/openshell-server/src/grpc/provider.rs | 7 + crates/openshell-server/src/grpc/workspace.rs | 4 + .../openshell-server/src/provider_refresh.rs | 57 +++++-- crates/openshell-server/src/test_support.rs | 28 +++- .../helm/openshell/templates/clusterrole.yaml | 22 +++ e2e/rust/tests/workspace_namespace_managed.rs | 62 +++++++ .../tests/workspace_namespace_operator.rs | 25 ++- proto/compute_driver.proto | 21 +++ tasks/scripts/helm-k3s-local.sh | 6 +- 16 files changed, 558 insertions(+), 58 deletions(-) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 579ee4a5b3..432493267d 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -146,6 +146,10 @@ async fn build_plain_channel(endpoint: &str) -> Result { let tls_enabled = endpoint.starts_with("https://"); + // TODO: TLS certs are loaded once here and never re-read. The gateway + // server side supports hot-reload (ArcSwap + notify in tls.rs). The + // supervisor should do the same so that cert-manager rotations take + // effect without restarting the sandbox. if tls_enabled { let ca_path = std::env::var(sandbox_env::TLS_CA) .into_diagnostic() diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index dd4d9ef0f0..27ce29061c 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -38,8 +38,9 @@ use openshell_core::progress::{ }; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, + DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, @@ -1550,6 +1551,20 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } impl DockerProvisioningFailure { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 1f492139ec..5d40afcb0c 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -11,8 +11,8 @@ use crate::config::{ }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ - Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, ServiceAccount, - Volume, VolumeMount, + Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Secret, + ServiceAccount, Volume, VolumeMount, }; use kube::api::{Api, ApiResource, DeleteParams, ListParams, PostParams, Preconditions}; use kube::core::gvk::GroupVersionKind; @@ -548,6 +548,15 @@ impl KubernetesComputeDriver { /// Idempotent: returns the namespace name whether it was just created or /// already existed. Also creates the sandbox `ServiceAccount` in the /// namespace. + /// + /// TODO: no `NetworkPolicy` is created here. The Helm-managed namespace gets + /// an SSH-isolation policy (port 2222 restricted to the gateway pod), but + /// managed-mode namespaces do not. Current risk is low: only sandbox pods + /// from the same workspace run in this namespace, so there is no lateral + /// movement target. A same-cluster `namespaceSelector` policy would also + /// break cross-cluster topologies where the gateway is external. Add a + /// configurable `NetworkPolicy` when mixed-workload or cross-cluster managed + /// namespaces are supported. pub async fn ensure_namespace(&self, workspace: &str) -> Result { let ns_name = managed_namespace(&self.config.gateway_id, workspace); let ns_api: Api = Api::all(self.client.clone()); @@ -649,8 +658,108 @@ impl KubernetesComputeDriver { Ok(()) } + /// Ensure the client TLS Secret exists in `namespace` by copying it from + /// the gateway's Helm release namespace. Idempotent: creates the Secret on + /// first call, updates it on subsequent calls to pick up cert rotations. + /// No-op when `client_tls_secret_name` is empty (TLS disabled). + async fn ensure_tls_secret(&self, namespace: &str) -> Result<(), KubernetesDriverError> { + if self.config.client_tls_secret_name.is_empty() { + return Ok(()); + } + + let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let source = match tokio::time::timeout( + KUBE_API_TIMEOUT, + source_api.get(&self.config.client_tls_secret_name), + ) + .await + { + Ok(Ok(s)) => s, + Ok(Err(e)) => { + warn!( + secret = %self.config.client_tls_secret_name, + source_namespace = %self.config.namespace, + error = %e, + "failed to read source TLS secret" + ); + return Err(KubernetesDriverError::from_kube(e)); + } + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading TLS secret {} from {}", + self.config.client_tls_secret_name, self.config.namespace + ))); + } + }; + + let target_api: Api = Api::namespaced(self.client.clone(), namespace); + let copy = Secret { + metadata: ObjectMeta { + name: Some(self.config.client_tls_secret_name.clone()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + }; + + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.create(&PostParams::default(), ©), + ) + .await + { + Ok(Ok(_)) => { + info!( + namespace = %namespace, + secret = %self.config.client_tls_secret_name, + "created TLS secret copy" + ); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => { + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.replace( + &self.config.client_tls_secret_name, + &PostParams::default(), + ©, + ), + ) + .await + { + Ok(Ok(_)) => { + debug!( + namespace = %namespace, + secret = %self.config.client_tls_secret_name, + "updated TLS secret copy" + ); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout updating TLS secret in {namespace}" + ))); + } + } + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating TLS secret in {namespace}" + ))); + } + } + + Ok(()) + } + /// Delete the managed namespace if it contains no sandboxes (managed mode - /// only). Called after sandbox deletion. + /// only). Called via the `DeleteWorkspace` RPC after workspace deletion. pub async fn delete_namespace_if_empty( &self, workspace: &str, @@ -1114,6 +1223,10 @@ impl KubernetesComputeDriver { } }; + if self.config.is_multi_namespace() { + self.ensure_tls_secret(&target_namespace).await?; + } + info!( sandbox_id = %sandbox.id, sandbox_name = %name, @@ -1184,7 +1297,7 @@ impl KubernetesComputeDriver { obj.metadata = ObjectMeta { name: Some(kube_name), namespace: Some(target_namespace), - labels: Some(sandbox_labels(sandbox)), + labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() }; @@ -1240,7 +1353,7 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, workspace, preconditions) = match tokio::time::timeout( + let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( KUBE_API_TIMEOUT, lookup_api.api.list(&lp), ) @@ -1302,15 +1415,6 @@ impl KubernetesComputeDriver { match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); - if self.config.workspace_mode == WorkspaceMode::Managed - && let Err(e) = self.delete_namespace_if_empty(&workspace).await - { - warn!( - workspace = %workspace, - error = %e, - "Failed to clean up empty managed namespace after sandbox deletion" - ); - } Ok(true) } Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { @@ -1586,7 +1690,7 @@ fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), Ok(()) } -fn sandbox_labels(sandbox: &Sandbox) -> BTreeMap { +fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { let mut labels = BTreeMap::new(); labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); @@ -1598,6 +1702,9 @@ fn sandbox_labels(sandbox: &Sandbox) -> BTreeMap { LABEL_MANAGED_BY.to_string(), LABEL_MANAGED_BY_VALUE.to_string(), ); + if let Some(gw_id) = gateway_id { + labels.insert(LABEL_GATEWAY_ID.to_string(), gw_id.to_string()); + } labels } @@ -6589,7 +6696,7 @@ mod tests { workspace: "alpha".to_string(), ..Default::default() }; - let labels = sandbox_labels(&sandbox); + let labels = sandbox_labels(&sandbox, None); assert_eq!(labels.get(LABEL_SANDBOX_ID).unwrap(), "uuid-1"); assert_eq!(labels.get(LABEL_SANDBOX_NAME).unwrap(), "work"); assert_eq!(labels.get(LABEL_SANDBOX_WORKSPACE).unwrap(), "alpha"); @@ -6597,6 +6704,19 @@ mod tests { labels.get(LABEL_MANAGED_BY).unwrap(), LABEL_MANAGED_BY_VALUE ); + assert!(!labels.contains_key(LABEL_GATEWAY_ID)); + } + + #[test] + fn sandbox_labels_includes_gateway_id_when_provided() { + let sandbox = Sandbox { + id: "uuid-1".to_string(), + name: "work".to_string(), + workspace: "alpha".to_string(), + ..Default::default() + }; + let labels = sandbox_labels(&sandbox, Some("gw-42")); + assert_eq!(labels.get(LABEL_GATEWAY_ID).unwrap(), "gw-42"); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 6eeb51cd73..b748361a16 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -6,16 +6,19 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_server::ComputeDriver, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; use crate::KubernetesComputeDriver; +use crate::WorkspaceMode; #[derive(Debug, Clone)] pub struct ComputeDriverService { @@ -150,6 +153,40 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok(Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + let workspace = request.into_inner().workspace; + if workspace.is_empty() { + return Err(Status::invalid_argument("workspace is required")); + } + if self.driver.workspace_mode() == WorkspaceMode::Managed { + self.driver + .ensure_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + let workspace = request.into_inner().workspace; + if workspace.is_empty() { + return Err(Status::invalid_argument("workspace is required")); + } + if self.driver.workspace_mode() == WorkspaceMode::Managed { + self.driver + .delete_namespace_if_empty(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 2d0792d447..16ee7a129a 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -6,11 +6,13 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_server::ComputeDriver, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; @@ -154,6 +156,20 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok(Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 9b6c0dd6ce..49fbd255bd 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -38,15 +38,16 @@ use openshell_core::progress::{ }; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, - DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, - DriverSandboxTemplate as SandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, watch_sandboxes_event, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition as SandboxCondition, + DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, + DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, + EnsureWorkspaceRequest, EnsureWorkspaceResponse, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, + WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, + WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -3253,6 +3254,20 @@ impl ComputeDriver for VmDriver { let stream: Self::WatchSandboxesStream = Box::pin(ReceiverStream::new(out_rx)); Ok(Response::new(stream)) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(target_os = "linux")] diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 071309cf25..632f4c2996 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -26,9 +26,10 @@ use futures::{Stream, StreamExt}; use hyper_util::rt::TokioIo; use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ - CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, - DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, - DriverSandboxTemplate, GatewayListenerRequirement as ProtoGatewayListenerRequirement, + CreateSandboxRequest, DeleteSandboxRequest, DeleteWorkspaceRequest, DeleteWorkspaceResponse, + DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, + DriverSandboxSpec, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement as ProtoGatewayListenerRequirement, GetCapabilitiesRequest, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, @@ -543,6 +544,22 @@ impl ComputeDriver for RemoteComputeDriver { let stream = response.into_inner(); Ok(tonic::Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.ensure_workspace(request).await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.delete_workspace(request).await + } } #[derive(Clone)] @@ -843,6 +860,30 @@ impl ComputeRuntime { &self.gateway_listener_requirements } + pub(crate) async fn ensure_workspace(&self, workspace: &str) -> Result<(), Status> { + let workspace = workspace.to_string(); + self.driver + .call("driver.ensure_workspace", None, |driver| async move { + driver + .ensure_workspace(Request::new(EnsureWorkspaceRequest { workspace })) + .await + }) + .await + .map(|_| ()) + } + + pub(crate) async fn delete_workspace(&self, workspace: &str) -> Result<(), Status> { + let workspace = workspace.to_string(); + self.driver + .call("driver.delete_workspace", None, |driver| async move { + driver + .delete_workspace(Request::new(DeleteWorkspaceRequest { workspace })) + .await + }) + .await + .map(|_| ()) + } + pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { let driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; @@ -3194,6 +3235,20 @@ impl ComputeDriver for NoopTestDriver { ) -> Result, Status> { Ok(tonic::Response::new(Box::pin(futures::stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] @@ -3464,6 +3519,20 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(Box::pin(stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } #[derive(Clone)] @@ -3679,6 +3748,20 @@ mod tests { UnboundedReceiverStream::new(receiver), ))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } async fn test_runtime(driver: SharedComputeDriver) -> ComputeRuntime { @@ -4438,6 +4521,20 @@ mod tests { ) -> Result, Status> { self.0.watch_sandboxes(request).await } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.0.ensure_workspace(request).await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.0.delete_workspace(request).await + } } let runtime = test_runtime(Arc::new(FailingDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index d0a201b2f9..59950f9d78 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -1806,6 +1806,9 @@ pub(super) async fn handle_create_provider( metadata.workspace.clone_from(&workspace); } let provider_type = provider.r#type.clone(); + if state.credentials.stores_provider_credentials() && !provider.credentials.is_empty() { + state.compute.ensure_workspace(&workspace).await?; + } let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -2918,6 +2921,9 @@ pub(super) async fn handle_update_provider( provider .credential_expires_at_ms .extend(req.credential_expires_at_ms); + if state.credentials.stores_provider_credentials() && !provider.credentials.is_empty() { + state.compute.ensure_workspace(&workspace).await?; + } let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3333,6 +3339,7 @@ pub(super) async fn handle_rotate_provider_credential( state.store.as_ref(), &workspace, Some(&state.credentials), + Some(&state.compute), provider_name, credential_key, ) diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index a22a195226..f2863511f1 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -435,6 +435,10 @@ pub(super) async fn handle_delete_workspace( } })?; + if deleted && let Err(e) = state.compute.delete_workspace(&name).await { + tracing::warn!(workspace = %name, error = %e, "failed to delete workspace platform resources"); + } + Ok(Response::new(DeleteWorkspaceResponse { deleted })) } diff --git a/crates/openshell-server/src/provider_refresh.rs b/crates/openshell-server/src/provider_refresh.rs index 9a655babb4..dc039265f6 100644 --- a/crates/openshell-server/src/provider_refresh.rs +++ b/crates/openshell-server/src/provider_refresh.rs @@ -343,6 +343,7 @@ pub async fn refresh_provider_credential( store: &Store, workspace: &str, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, provider_name: &str, credential_key: &str, ) -> Result { @@ -448,6 +449,7 @@ pub async fn refresh_provider_credential( store, workspace, credentials, + compute, &provider, credential_key, &minted, @@ -511,6 +513,7 @@ async fn apply_minted_credential( store: &Store, workspace: &str, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, provider: &Provider, credential_key: &str, minted: &MintedCredential, @@ -520,6 +523,9 @@ async fn apply_minted_credential( let staged_handles = if let Some(credentials) = credentials && credentials.stores_provider_credentials() { + if let Some(compute) = compute { + compute.ensure_workspace(workspace).await?; + } let mut creds_to_store = HashMap::from([(credential_key.to_string(), minted.access_token.clone())]); for (key, value) in &minted.additional_credentials { @@ -1115,8 +1121,12 @@ pub fn spawn_refresh_worker(state: std::sync::Arc, interval: ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { ticker.tick().await; - if let Err(err) = - run_refresh_worker_tick(state.store.as_ref(), Some(&state.credentials)).await + if let Err(err) = run_refresh_worker_tick( + state.store.as_ref(), + Some(&state.credentials), + Some(&state.compute), + ) + .await { warn!(error = %err, "provider credential refresh worker tick failed"); } @@ -1136,6 +1146,7 @@ pub fn spawn_refresh_worker(state: std::sync::Arc, interval: async fn run_refresh_worker_tick( store: &Store, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, ) -> Result<(), Status> { let now_ms = current_time_ms(); let states = list_all_refresh_states(store).await.inspect_err(|_| { @@ -1200,6 +1211,7 @@ async fn run_refresh_worker_tick( store, state.object_workspace(), credentials, + compute, &state.provider_name, &state.credential_key, ) @@ -1328,6 +1340,7 @@ mod tests { &store, "default", None, + None, "my-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1398,6 +1411,7 @@ mod tests { &store, "default", Some(&credentials), + None, "my-stored-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1498,6 +1512,7 @@ mod tests { &store, "default", None, + None, "refreshing-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1578,6 +1593,7 @@ mod tests { &store, "default", None, + None, "my-delegated-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1672,6 +1688,7 @@ mod tests { &store, "default", None, + None, "my-drive", "GOOGLE_DRIVE_ACCESS_TOKEN", ) @@ -1715,7 +1732,7 @@ mod tests { .unwrap(); put_refresh_state(&store, &state).await.unwrap(); - run_refresh_worker_tick(&store, None).await.unwrap(); + run_refresh_worker_tick(&store, None, None).await.unwrap(); let stored_state = get_refresh_state( &store, @@ -1751,7 +1768,7 @@ mod tests { let store = test_store().await; let traced = test_exporter::install_traced(); - run_refresh_worker_tick(&store, None).await.unwrap(); + run_refresh_worker_tick(&store, None, None).await.unwrap(); let spans = traced.finished_spans(); let root = spans @@ -1873,6 +1890,7 @@ mod tests { &store, "default", None, + None, "aws-sts-test", "AWS_ACCESS_KEY_ID", ) @@ -1970,6 +1988,7 @@ mod tests { &store, "default", None, + None, "aws-sts-custom", "AWS_ACCESS_KEY_ID", ) @@ -2046,6 +2065,7 @@ mod tests { &store, "default", None, + None, "aws-sts-partial", "AWS_ACCESS_KEY_ID", ) @@ -2088,9 +2108,17 @@ mod tests { ]), }; - apply_minted_credential(&store, "default", None, &prov, "AWS_ACCESS_KEY_ID", &minted) - .await - .unwrap(); + apply_minted_credential( + &store, + "default", + None, + None, + &prov, + "AWS_ACCESS_KEY_ID", + &minted, + ) + .await + .unwrap(); let stored = store .get_message_by_name::("default", "aws-test") @@ -2163,6 +2191,7 @@ mod tests { &store, "default", Some(&credentials), + None, &prov, "AWS_ACCESS_KEY_ID", &minted, @@ -2266,6 +2295,7 @@ mod tests { &store, "default", Some(&credentials), + None, &refreshing_provider, "AWS_ACCESS_KEY_ID", &minted, @@ -2376,6 +2406,7 @@ mod tests { &store, "default", None, + None, "aws-sts-session", "AWS_ACCESS_KEY_ID", ) @@ -2444,6 +2475,7 @@ mod tests { &store, "default", None, + None, "aws-sts-lonesession", "AWS_ACCESS_KEY_ID", ) @@ -2526,8 +2558,14 @@ mod tests { .unwrap(); put_refresh_state(&store, &state).await.unwrap(); - let rotate = - refresh_provider_credential(&store, "default", None, "aws-race", "AWS_ACCESS_KEY_ID"); + let rotate = refresh_provider_credential( + &store, + "default", + None, + None, + "aws-race", + "AWS_ACCESS_KEY_ID", + ); let interfere = async { // Wait until the rotation is inside the STS call (its state read has // already happened), then delete the refresh and release STS. @@ -2645,6 +2683,7 @@ mod tests { &store, "default", None, + None, "aws-superseded", "AWS_ACCESS_KEY_ID", ); diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index d1aa10da11..ffd5ec116e 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -8,12 +8,14 @@ use futures::{Stream, stream}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverSandbox, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverSandbox, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_server::ComputeDriver, + gateway_listener_requirement::Selector, }; use std::collections::HashMap; #[cfg(unix)] @@ -380,4 +382,18 @@ impl ComputeDriver for FakeComputeDriver { self.with_state(|state| state.calls.push(FakeComputeDriverCall::WatchSandboxes)); Ok(Response::new(Box::pin(stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 66102ab751..c05667e29f 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -73,6 +73,28 @@ rules: verbs: - get {{- end }} + {{- if ne $workspaceMode "shared" }} + # TLS secret sync: read the source Secret in the release namespace and + # create/update copies in workspace namespaces so sandbox pods can mount + # client TLS material for mTLS to the gateway. + {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # Managed mode with kubernetes-secrets credential driver: credentials are + # stored as Secrets in workspace namespaces, requiring patch+delete in + # addition to the TLS sync verbs. + {{- end }} + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + - patch + - delete + {{- end }} + {{- end }} {{- if eq $workspaceMode "managed" }} # ServiceAccount creation in managed namespaces. - apiGroups: diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index f18827478b..1fa9bd1744 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -166,6 +166,33 @@ async fn managed_creates_namespace_with_labels() { let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); assert!(out.contains("mgd-sb"), "sandbox CR name mismatch: {out}"); + + // Verify the sandbox is resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox list should find mgd-sb via control plane: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "get", "mgd-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox get failed: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox get should resolve mgd-sb via control plane: {out}" + ); + + // Verify sandbox delete works through the control plane (uses sandbox_lookup_selector). + let (ok, out) = run_cli(&["sandbox", "delete", "mgd-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox delete failed: {out}"); + + // Verify sandbox is gone from the control plane after deletion. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list after delete failed: {out}"); + assert!( + !out.contains("mgd-sb"), + "sandbox list should NOT find mgd-sb after deletion: {out}" + ); } #[tokio::test] @@ -226,6 +253,18 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { out.contains("sb-b"), "sb-b CR should still be present: {out}" ); + + // Verify sb-b is still resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("sb-b"), + "sandbox list should find sb-b via control plane: {out}" + ); + assert!( + !out.contains("sb-a"), + "sandbox list should NOT find deleted sb-a: {out}" + ); } #[tokio::test] @@ -301,4 +340,27 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { !out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}" ); + + // Verify workspace isolation through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_a]).await; + assert!(ok, "sandbox list ws_a failed: {out}"); + assert!( + out.contains("sb-iso-a"), + "sandbox list ws_a should find sb-iso-a: {out}" + ); + assert!( + !out.contains("sb-iso-b"), + "sandbox list ws_a should NOT find sb-iso-b: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_b]).await; + assert!(ok, "sandbox list ws_b failed: {out}"); + assert!( + out.contains("sb-iso-b"), + "sandbox list ws_b should find sb-iso-b: {out}" + ); + assert!( + !out.contains("sb-iso-a"), + "sandbox list ws_b should NOT find sb-iso-a: {out}" + ); } diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index a382de446c..ddd94f8a1d 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -182,10 +182,33 @@ async fn operator_sandbox_in_labeled_namespace() { "sandbox CR name should be bare 'op-sb', got: {out}" ); - // Clean up. + // Verify sandbox is resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("op-sb"), + "sandbox list should find op-sb via control plane: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "get", "op-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox get failed: {out}"); + assert!( + out.contains("op-sb"), + "sandbox get should resolve op-sb via control plane: {out}" + ); + + // Verify sandbox delete works through the control plane. let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; assert!(ok, "sandbox delete failed: {out}"); + // Verify sandbox is gone after deletion. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; + assert!(ok, "sandbox list after delete failed: {out}"); + assert!( + !out.contains("op-sb"), + "sandbox list should NOT find op-sb after deletion: {out}" + ); + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; assert!(ok, "workspace delete failed: {out}"); diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index e3f18af19f..2a0a420c70 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -48,6 +48,13 @@ service ComputeDriver { // Stream sandbox observations from the platform. rpc WatchSandboxes(WatchSandboxesRequest) returns (stream WatchSandboxesEvent); + + // Ensure platform resources for a workspace exist (e.g. namespace). + // Idempotent: succeeds if resources already exist. + rpc EnsureWorkspace(EnsureWorkspaceRequest) returns (EnsureWorkspaceResponse); + + // Tear down platform resources for a workspace. + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); } message GetCapabilitiesRequest {} @@ -327,3 +334,17 @@ message WatchSandboxesEvent { WatchSandboxesPlatformEvent platform_event = 3; } } + +message EnsureWorkspaceRequest { + // Workspace identifier used by the gateway. + string workspace = 1; +} + +message EnsureWorkspaceResponse {} + +message DeleteWorkspaceRequest { + // Workspace identifier used by the gateway. + string workspace = 1; +} + +message DeleteWorkspaceResponse {} diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index f9ac186f52..82b8d5cfc8 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -230,11 +230,13 @@ preload_sandbox_image() { docker pull --platform "${platform}" "${PRELOAD_SANDBOX_IMAGE}" fi + # Save without --platform: the platform-specific pull already constrained the + # local image, and --platform fails on OCI index (multi-arch) manifests. tmp="$(mktemp "${TMPDIR:-/tmp}/openshell-sandbox-image.XXXXXX")" - if ! docker image save --platform "${platform}" -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}"; then + if ! docker image save -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}"; then echo "Pulling sandbox image for ${platform}..." docker pull --platform "${platform}" "${PRELOAD_SANDBOX_IMAGE}" - docker image save --platform "${platform}" -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}" + docker image save -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}" fi if ! k3d image import "${tmp}" --cluster "${CLUSTER_NAME}"; then From c655db19ad46e1dd75f75d51c4643072f681c1e6 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Mon, 10 Aug 2026 21:49:33 -0400 Subject: [PATCH 06/11] fix(k8s): address re-review findings and add test coverage - Use server-side apply for TLS secret sync (fixes second sandbox creation failure when TLS is enabled) - Scope gateway-ID label selector unconditionally across all workspace modes (fixes operator reads/watches/deletes seeing foreign sandboxes) - Validate operator allowlist in EnsureWorkspace and DeleteWorkspace RPCs (prevents credential writes to namespaces outside the allowlist) - Extend ClusterRole secrets patch+delete to all non-shared modes with credential driver enabled (fixes operator credential storage RBAC) - Validate namespace ownership on 409 conflict in ensure_namespace (prevents adopting unowned namespaces in managed mode) - Replace delete_namespace_if_empty with unconditional delete_namespace letting Kubernetes cascade cleanup (fixes stuck terminating CRs) - Strengthen NetworkPolicy TODO to cover both managed and operator modes - Extract selector and ownership logic into testable free functions - Add unit tests for gateway-ID selectors and namespace ownership - Add Helm ClusterRole RBAC tests for operator credential driver Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 222 +++++++++++------- .../openshell-driver-kubernetes/src/grpc.rs | 44 +++- .../helm/openshell/templates/clusterrole.yaml | 9 +- .../openshell/tests/clusterrole_test.yaml | 44 ++++ 4 files changed, 217 insertions(+), 102 deletions(-) create mode 100644 deploy/helm/openshell/tests/clusterrole_test.yaml diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 5d40afcb0c..3c8dfd53f8 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -14,7 +14,9 @@ use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Secret, ServiceAccount, Volume, VolumeMount, }; -use kube::api::{Api, ApiResource, DeleteParams, ListParams, PostParams, Preconditions}; +use kube::api::{ + Api, ApiResource, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions, +}; use kube::core::gvk::GroupVersionKind; use kube::core::{DynamicObject, ObjectMeta}; use kube::runtime::watcher::{self, Event}; @@ -549,13 +551,15 @@ impl KubernetesComputeDriver { /// already existed. Also creates the sandbox `ServiceAccount` in the /// namespace. /// - /// TODO: no `NetworkPolicy` is created here. The Helm-managed namespace gets - /// an SSH-isolation policy (port 2222 restricted to the gateway pod), but - /// managed-mode namespaces do not. Current risk is low: only sandbox pods - /// from the same workspace run in this namespace, so there is no lateral - /// movement target. A same-cluster `namespaceSelector` policy would also - /// break cross-cluster topologies where the gateway is external. Add a - /// configurable `NetworkPolicy` when mixed-workload or cross-cluster managed + /// TODO: no `NetworkPolicy` is created in dynamic namespaces. The + /// Helm-managed static namespace gets an SSH-isolation policy (port 2222 + /// restricted to the gateway pod), but managed and operator namespaces do + /// not. In managed mode, risk is low: only sandbox pods from the same + /// workspace run in the namespace, so there is no lateral movement target. + /// In operator mode, the admin owns the namespace and is responsible for + /// applying appropriate policies. A same-cluster `namespaceSelector` policy + /// would also break cross-cluster topologies where the gateway is external. + /// Add a configurable `NetworkPolicy` when mixed-workload or cross-cluster /// namespaces are supported. pub async fn ensure_namespace(&self, workspace: &str) -> Result { let ns_name = managed_namespace(&self.config.gateway_id, workspace); @@ -611,6 +615,24 @@ impl KubernetesComputeDriver { info!(namespace = %ns_name, workspace = %workspace, "created managed namespace"); } Ok(Err(KubeError::Api(api))) if api.code == 409 => { + let existing = + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { + Ok(Ok(ns)) => ns, + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading namespace {ns_name}" + ))); + } + }; + if !is_namespace_owned_by_gateway( + existing.metadata.labels.as_ref(), + &self.config.gateway_id, + ) { + return Err(KubernetesDriverError::Precondition(format!( + "namespace {ns_name} exists but is not owned by this gateway" + ))); + } debug!(namespace = %ns_name, "managed namespace already exists"); } Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), @@ -710,7 +732,11 @@ impl KubernetesComputeDriver { match tokio::time::timeout( KUBE_API_TIMEOUT, - target_api.create(&PostParams::default(), ©), + target_api.patch( + &self.config.client_tls_secret_name, + &PatchParams::apply("openshell"), + &Patch::Apply(©), + ), ) .await { @@ -718,39 +744,13 @@ impl KubernetesComputeDriver { info!( namespace = %namespace, secret = %self.config.client_tls_secret_name, - "created TLS secret copy" + "applied TLS secret copy" ); } - Ok(Err(KubeError::Api(api))) if api.code == 409 => { - match tokio::time::timeout( - KUBE_API_TIMEOUT, - target_api.replace( - &self.config.client_tls_secret_name, - &PostParams::default(), - ©, - ), - ) - .await - { - Ok(Ok(_)) => { - debug!( - namespace = %namespace, - secret = %self.config.client_tls_secret_name, - "updated TLS secret copy" - ); - } - Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), - Err(_) => { - return Err(KubernetesDriverError::Message(format!( - "timeout updating TLS secret in {namespace}" - ))); - } - } - } Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), Err(_) => { return Err(KubernetesDriverError::Message(format!( - "timeout creating TLS secret in {namespace}" + "timeout applying TLS secret in {namespace}" ))); } } @@ -758,35 +758,11 @@ impl KubernetesComputeDriver { Ok(()) } - /// Delete the managed namespace if it contains no sandboxes (managed mode - /// only). Called via the `DeleteWorkspace` RPC after workspace deletion. - pub async fn delete_namespace_if_empty( - &self, - workspace: &str, - ) -> Result<(), KubernetesDriverError> { + /// Delete the managed namespace and all its contents (managed mode only). + /// Called via the `DeleteWorkspace` RPC after workspace deletion. + /// Kubernetes cascades namespace deletion to all resources within it. + pub async fn delete_namespace(&self, workspace: &str) -> Result<(), KubernetesDriverError> { let ns_name = managed_namespace(&self.config.gateway_id, workspace); - - let sandbox_api_version = self - .supported_sandbox_api_version(self.client.clone()) - .await - .map_err(KubernetesDriverError::Message)?; - let agent_api = Self::agent_sandbox_api(self.client.clone(), sandbox_api_version, &ns_name); - - let lp = ListParams::default() - .labels(&openshell_sandbox_label_selector()) - .limit(1); - let list = tokio::time::timeout(KUBE_API_TIMEOUT, agent_api.api.list(&lp)) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!("timeout listing sandboxes in {ns_name}")) - })? - .map_err(KubernetesDriverError::from_kube)?; - - if !list.items.is_empty() { - debug!(namespace = %ns_name, "namespace still has sandboxes, skipping delete"); - return Ok(()); - } - let ns_api: Api = Api::all(self.client.clone()); let ns = match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { @@ -803,14 +779,7 @@ impl KubernetesComputeDriver { } }; - let labels = ns.metadata.labels.as_ref(); - let is_owned = labels - .and_then(|l| l.get(LABEL_MANAGED_BY)) - .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) - && labels - .and_then(|l| l.get(LABEL_GATEWAY_ID)) - .is_some_and(|v| v == &self.config.gateway_id); - if !is_owned { + if !is_namespace_owned_by_gateway(ns.metadata.labels.as_ref(), &self.config.gateway_id) { debug!( namespace = %ns_name, "namespace not owned by this gateway, skipping delete" @@ -825,7 +794,7 @@ impl KubernetesComputeDriver { .await { Ok(Ok(_)) => { - info!(namespace = %ns_name, workspace = %workspace, "deleted empty managed namespace"); + info!(namespace = %ns_name, workspace = %workspace, "deleted managed namespace"); } Ok(Err(KubeError::Api(api))) if api.code == 404 => { debug!(namespace = %ns_name, "managed namespace already deleted"); @@ -903,22 +872,11 @@ impl KubernetesComputeDriver { } fn sandbox_lookup_selector(&self, sandbox_id: &str) -> String { - let mut selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); - if self.config.workspace_mode == WorkspaceMode::Managed { - use std::fmt::Write; - write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); - } - selector + sandbox_lookup_selector_for(sandbox_id, &self.config.gateway_id) } fn openshell_sandbox_selector(&self) -> String { - let mut selector = openshell_sandbox_label_selector(); - if self.config.workspace_mode == WorkspaceMode::Managed { - use std::fmt::Write; - write!(selector, ",{LABEL_GATEWAY_ID}={}", self.config.gateway_id).unwrap(); - } - selector + openshell_sandbox_selector_for(&self.config.gateway_id) } async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { @@ -1690,6 +1648,31 @@ fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), Ok(()) } +fn is_namespace_owned_by_gateway( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|l| l.get(LABEL_MANAGED_BY)) + .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) + && labels + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == gateway_id) +} + +fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { + format!( + "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" + ) +} + +fn openshell_sandbox_selector_for(gateway_id: &str) -> String { + use std::fmt::Write; + let mut selector = openshell_sandbox_label_selector(); + write!(selector, ",{LABEL_GATEWAY_ID}={gateway_id}").unwrap(); + selector +} + fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { let mut labels = BTreeMap::new(); labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -6783,4 +6766,69 @@ mod tests { .is_none() ); } + + #[test] + fn sandbox_lookup_selector_always_includes_gateway_id() { + let sel = sandbox_lookup_selector_for("sb-123", "gw-42"); + assert!( + sel.contains(&format!("{LABEL_GATEWAY_ID}=gw-42")), + "selector must include gateway ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_SANDBOX_ID}=sb-123")), + "selector must include sandbox ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}")), + "selector must include managed-by: {sel}" + ); + } + + #[test] + fn openshell_sandbox_selector_always_includes_gateway_id() { + let sel = openshell_sandbox_selector_for("gw-99"); + assert!( + sel.contains(&format!("{LABEL_GATEWAY_ID}=gw-99")), + "selector must include gateway ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}")), + "selector must include managed-by: {sel}" + ); + } + + #[test] + fn namespace_owned_with_correct_labels() { + let labels = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gw-1".to_string()), + ]); + assert!(is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_missing_managed_by() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-1".to_string())]); + assert!(!is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_wrong_gateway_id() { + let labels = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gw-other".to_string()), + ]); + assert!(!is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_no_labels() { + assert!(!is_namespace_owned_by_gateway(None, "gw-1")); + } } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index b748361a16..d3f0f51b2d 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -162,11 +162,23 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } - if self.driver.workspace_mode() == WorkspaceMode::Managed { - self.driver - .ensure_namespace(&workspace) - .await - .map_err(|e| Status::internal(e.to_string()))?; + match self.driver.workspace_mode() { + WorkspaceMode::Managed => { + self.driver + .ensure_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + WorkspaceMode::Operator => { + if let Some(allowlist) = self.driver.operator_allowlist() + && !allowlist.contains(&workspace) + { + return Err(Status::permission_denied(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + } + WorkspaceMode::Shared => {} } Ok(Response::new(EnsureWorkspaceResponse {})) } @@ -179,11 +191,23 @@ impl ComputeDriver for ComputeDriverService { if workspace.is_empty() { return Err(Status::invalid_argument("workspace is required")); } - if self.driver.workspace_mode() == WorkspaceMode::Managed { - self.driver - .delete_namespace_if_empty(&workspace) - .await - .map_err(|e| Status::internal(e.to_string()))?; + match self.driver.workspace_mode() { + WorkspaceMode::Managed => { + self.driver + .delete_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + WorkspaceMode::Operator => { + if let Some(allowlist) = self.driver.operator_allowlist() + && !allowlist.contains(&workspace) + { + return Err(Status::permission_denied(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + } + WorkspaceMode::Shared => {} } Ok(Response::new(DeleteWorkspaceResponse {})) } diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index c05667e29f..299ee9ca3d 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -77,10 +77,9 @@ rules: # TLS secret sync: read the source Secret in the release namespace and # create/update copies in workspace namespaces so sandbox pods can mount # client TLS material for mTLS to the gateway. - {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - # Managed mode with kubernetes-secrets credential driver: credentials are - # stored as Secrets in workspace namespaces, requiring patch+delete in - # addition to the TLS sync verbs. + {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # kubernetes-secrets credential driver: credentials are stored as Secrets + # in workspace namespaces, requiring patch+delete in addition to TLS sync. {{- end }} - apiGroups: - "" @@ -90,7 +89,7 @@ rules: - get - create - update - {{- if and (eq $workspaceMode "managed") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - patch - delete {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml new file mode 100644 index 0000000000..c2363b08eb --- /dev/null +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: ClusterRole RBAC +templates: + - templates/clusterrole.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: grants secrets patch and delete when credential driver is enabled (operator) + set: + server.drivers.kubernetes.workspaceMode: operator + server.credentialDrivers.kubernetesSecrets.enabled: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch", "delete"] + + - it: omits secrets patch and delete when credential driver is disabled (operator) + set: + server.drivers.kubernetes.workspaceMode: operator + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + + - it: omits secrets rule entirely in shared mode + set: + server.drivers.kubernetes.workspaceMode: shared + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + any: true From 11d9c216b09778116b1edae44ebceea63bd70929 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Tue, 11 Aug 2026 09:54:17 -0400 Subject: [PATCH 07/11] ci(k8s): add workspace managed and operator mode e2e to CI Wire the existing e2e:kubernetes:workspace-managed and e2e:kubernetes:workspace-operator mise tasks into the branch-e2e workflow so they run alongside the other core Kubernetes e2e suites. Both are gated by run_core_e2e and included in the Core E2E result gate. Signed-off-by: Derek Carr --- .github/workflows/branch-e2e.yml | 36 ++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 3d68746b82..37154c75df 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -182,6 +182,34 @@ jobs: extra-helm-values: ${{ matrix.extra_helm_values }} cli-artifact-prefix: rust-binary-cli + kubernetes-workspace-managed-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (workspace managed mode) + e2e-task: e2e:kubernetes:workspace-managed + cli-artifact-prefix: rust-binary-cli + + kubernetes-workspace-operator-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (workspace operator mode) + e2e-task: e2e:kubernetes:workspace-operator + cli-artifact-prefix: rust-binary-cli + kubernetes-ha-e2e: needs: [pr_metadata, build-gateway, build-supervisor, build-cli] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' @@ -212,7 +240,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -224,6 +252,8 @@ jobs: BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} E2E_RESULT: ${{ needs.e2e.result }} KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} + KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }} + KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }} run: | set -euo pipefail failed=0 @@ -233,7 +263,9 @@ jobs: "build-cli:$BUILD_CLI_RESULT" \ "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ "e2e:$E2E_RESULT" \ - "kubernetes-e2e:$KUBERNETES_E2E_RESULT"; do + "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ + "kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \ + "kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do name="${item%%:*}" result="${item#*:}" if [ "$result" != "success" ]; then From dfd33068830456ddca41334ace628c6d6ca8bc4c Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Tue, 11 Aug 2026 12:56:54 -0400 Subject: [PATCH 08/11] test(k8s): add e2e tests for workspace namespace modes Add 7 new e2e tests covering workspace namespace lifecycle, TLS secret copying, ownership conflict detection, DNS-1123 validation, operator namespace preservation, and dynamic label watcher behavior. Fix async sandbox deletion race condition in existing tests by polling sandbox list instead of asserting immediately after delete. Signed-off-by: Derek Carr --- e2e/rust/tests/workspace_namespace_managed.rs | 355 +++++++++++++++++- .../tests/workspace_namespace_operator.rs | 169 ++++++++- 2 files changed, 504 insertions(+), 20 deletions(-) diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 1fa9bd1744..6bc0564ccf 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -70,6 +70,22 @@ fn unique_workspace(prefix: &str) -> String { format!("{prefix}-{ts}") } +async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; + if ok && !out.contains(sandbox) { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + struct ManagedCleanup { workspace: String, sandboxes: Vec, @@ -186,13 +202,8 @@ async fn managed_creates_namespace_with_labels() { let (ok, out) = run_cli(&["sandbox", "delete", "mgd-sb", "--workspace", &ws]).await; assert!(ok, "sandbox delete failed: {out}"); - // Verify sandbox is gone from the control plane after deletion. - let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; - assert!(ok, "sandbox list after delete failed: {out}"); - assert!( - !out.contains("mgd-sb"), - "sandbox list should NOT find mgd-sb after deletion: {out}" - ); + // Wait for the sandbox CR to be fully removed (deletion is asynchronous). + wait_sandbox_gone(&ws, "mgd-sb").await; } #[tokio::test] @@ -240,8 +251,8 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { let (ok, out) = run_cli(&["sandbox", "delete", "sb-a", "--workspace", &ws]).await; assert!(ok, "sandbox sb-a delete failed: {out}"); - // Brief wait, then verify the namespace still exists. - tokio::time::sleep(Duration::from_secs(3)).await; + // Wait for sb-a to be fully removed before checking namespace state. + wait_sandbox_gone(&ws, "sb-a").await; let (ok, _) = kubectl(&["get", "namespace", &ns]).await; assert!(ok, "managed namespace {ns} should still exist with sb-b"); @@ -261,10 +272,6 @@ async fn managed_namespace_survives_with_remaining_sandboxes() { out.contains("sb-b"), "sandbox list should find sb-b via control plane: {out}" ); - assert!( - !out.contains("sb-a"), - "sandbox list should NOT find deleted sb-a: {out}" - ); } #[tokio::test] @@ -364,3 +371,325 @@ async fn managed_isolates_workspaces_into_separate_namespaces() { "sandbox list ws_b should NOT find sb-iso-a: {out}" ); } + +#[tokio::test] +async fn managed_workspace_delete_removes_namespace() { + let ws = unique_workspace("mgddel"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["del-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "del-sb", + "--", + "echo", + "del-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("del-ok"), + "sandbox output missing expected string: {out}" + ); + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "managed namespace {ns} should exist after sandbox create" + ); + + let (ok, out) = run_cli(&["sandbox", "delete", "del-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox delete failed: {out}"); + + wait_sandbox_gone(&ws, "del-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; + assert!(ok, "workspace delete failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (exists, _) = kubectl(&["get", "namespace", &ns]).await; + if !exists { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("managed namespace {ns} still exists 30s after workspace delete"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn managed_tls_secret_copied_to_namespace() { + let (ok, config_out) = kubectl(&[ + "get", + "configmap", + "openshell-config", + "-n", + "openshell", + "-o", + "jsonpath={.data.gateway\\.toml}", + ]) + .await; + if !ok || !config_out.contains("client_tls_secret_name") { + eprintln!("SKIP: client_tls_secret_name not configured; TLS secret copying disabled"); + return; + } + + let ws = unique_workspace("mgdtls"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["tls-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "tls-sb", + "--", + "echo", + "tls-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("tls-ok"), + "sandbox output missing expected string: {out}" + ); + + let (ok, out) = kubectl(&["get", "secret", "openshell-client-tls", "-n", &ns]).await; + assert!( + ok, + "TLS secret openshell-client-tls should be copied to managed namespace {ns}: {out}" + ); + + let (ok, label_out) = kubectl(&[ + "get", + "secret", + "openshell-client-tls", + "-n", + &ns, + "-o", + "jsonpath={.metadata.labels}", + ]) + .await; + assert!(ok, "failed to read TLS secret labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/managed-by"), + "copied TLS secret missing managed-by label: {label_out}" + ); +} + +#[tokio::test] +async fn managed_rejects_namespace_owned_by_different_gateway() { + let ws = unique_workspace("mgdown"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![], + }; + + let (ok, out) = kubectl(&["create", "namespace", &ns]).await; + assert!(ok, "failed to pre-create namespace {ns}: {out}"); + + let (ok, out) = kubectl(&[ + "label", + "namespace", + &ns, + "openshell.ai/managed-by=openshell", + "openshell.ai/gateway-id=wrong-gateway", + ]) + .await; + assert!(ok, "failed to label namespace: {out}"); + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "conflict-sb", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for namespace owned by different gateway, but succeeded: {out}" + ); +} + +#[tokio::test] +async fn managed_full_lifecycle_with_multiple_sandboxes() { + let ws = unique_workspace("mgdlc"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["lc-a".into(), "lc-b".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "lc-a", + "--", + "echo", + "a", + ]) + .await; + assert!(ok, "sandbox lc-a create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "lc-b", + "--", + "echo", + "b", + ]) + .await; + assert!(ok, "sandbox lc-b create failed: {out}"); + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should exist"); + + let (ok, out) = run_cli(&["sandbox", "delete", "lc-a", "--workspace", &ws]).await; + assert!(ok, "sandbox lc-a delete failed: {out}"); + + wait_sandbox_gone(&ws, "lc-a").await; + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "managed namespace {ns} should still exist with lc-b remaining" + ); + + let (ok, out) = run_cli(&["sandbox", "delete", "lc-b", "--workspace", &ws]).await; + assert!(ok, "sandbox lc-b delete failed: {out}"); + + wait_sandbox_gone(&ws, "lc-b").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; + assert!(ok, "workspace delete failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (exists, _) = kubectl(&["get", "namespace", &ns]).await; + if !exists { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("managed namespace {ns} still exists 30s after full lifecycle cleanup"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn managed_rejects_invalid_dns1123_sandbox_name() { + let ws = unique_workspace("mgddns"); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "my_bad_name", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox with underscore name should be rejected: {out}" + ); + assert!( + out.contains("lowercase alphanumeric"), + "error should mention character constraint: {out}" + ); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "MyBadName", + "--", + "echo", + "nope", + ]) + .await; + assert!(!ok, "sandbox with uppercase name should be rejected: {out}"); + assert!( + out.contains("lowercase alphanumeric"), + "error should mention character constraint: {out}" + ); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "trailing-", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox with trailing hyphen should be rejected: {out}" + ); + let normalized: String = out + .chars() + .filter(|c| *c != '│') + .collect::() + .split_whitespace() + .collect::>() + .join(" "); + assert!( + normalized.contains("must not start or end with a hyphen"), + "error should mention hyphen constraint: {out}" + ); +} diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs index ddd94f8a1d..f421aa6ada 100644 --- a/e2e/rust/tests/workspace_namespace_operator.rs +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -65,6 +65,22 @@ fn unique_namespace(prefix: &str) -> String { format!("{prefix}-{ts}") } +async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; + if ok && !out.contains(sandbox) { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + async fn provision_operator_namespace(name: &str) { let (ok, out) = kubectl(&["create", "namespace", name]).await; assert!(ok, "failed to create namespace {name}: {out}"); @@ -201,13 +217,8 @@ async fn operator_sandbox_in_labeled_namespace() { let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; assert!(ok, "sandbox delete failed: {out}"); - // Verify sandbox is gone after deletion. - let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; - assert!(ok, "sandbox list after delete failed: {out}"); - assert!( - !out.contains("op-sb"), - "sandbox list should NOT find op-sb after deletion: {out}" - ); + // Wait for the sandbox CR to be fully removed (deletion is asynchronous). + wait_sandbox_gone(&ns, "op-sb").await; let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; assert!(ok, "workspace delete failed: {out}"); @@ -293,3 +304,147 @@ async fn operator_rejects_nonexistent_namespace() { // Clean up. let _ = run_cli(&["workspace", "delete", &ns]).await; } + +#[tokio::test] +async fn operator_workspace_delete_preserves_namespace() { + let ns = unique_namespace("opdel"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["opdel-sb".into()], + }; + + provision_operator_namespace(&ns).await; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "opdel-sb", + "--", + "echo", + "opdel-ok", + ]) + .await; + if ok { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let (ok, out) = run_cli(&["sandbox", "delete", "opdel-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox delete failed: {out}"); + + wait_sandbox_gone(&ns, "opdel-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; + assert!(ok, "workspace delete failed: {out}"); + + let (ok, out) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "operator namespace {ns} should still exist after workspace delete: {out}" + ); + + let (ok, label_out) = + kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; + assert!(ok, "failed to read namespace labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/e2e-operator-workspace"), + "operator label should be intact after workspace delete: {label_out}" + ); + + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_label_removal_blocks_sandbox_creation() { + let ns = unique_namespace("oplbl"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["lbl-sb1".into()], + }; + + provision_operator_namespace(&ns).await; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "lbl-sb1", + "--", + "echo", + "lbl-ok", + ]) + .await; + if ok { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let (ok, out) = run_cli(&["sandbox", "delete", "lbl-sb1", "--workspace", &ns]).await; + assert!(ok, "sandbox lbl-sb1 delete failed: {out}"); + + wait_sandbox_gone(&ns, "lbl-sb1").await; + + let (ok, out) = kubectl(&[ + "label", + "namespace", + &ns, + "openshell.ai/e2e-operator-workspace-", + ]) + .await; + assert!(ok, "failed to remove operator label: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, _out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "lbl-sb2", + "--", + "echo", + "should-fail", + ]) + .await; + if !ok { + break; + } + // Sandbox was created despite label removal — clean it up and retry. + let _ = run_cli(&["sandbox", "delete", "lbl-sb2", "--workspace", &ns]).await; + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox creation still succeeds 30s after operator label removal; \ + watcher did not remove namespace from allowlist" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + delete_namespace(&ns).await; +} From 2a581bf14c09d7a12374ee839e67becc46fb0ef5 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Wed, 12 Aug 2026 11:33:38 -0400 Subject: [PATCH 09/11] fix(k8s): grant secrets/patch unconditionally and backfill gateway-id labels Address two review findings: 1. RBAC: server-side apply (PATCH) is used for TLS secret sync in multi-namespace modes, but the ClusterRole only granted patch when the kubernetes-secrets credential driver was enabled. Grant patch unconditionally for non-shared modes since TLS sync always needs it; keep delete gated on the credential driver. 2. Upgrade safety: the new gateway-id label selector would orphan legacy Sandbox CRs that predate its introduction. Add a startup backfill in shared mode that patches any managed Sandbox CR missing the gateway-id label before the driver begins serving requests. Signed-off-by: Derek Carr --- .../openshell-driver-kubernetes/src/driver.rs | 72 +++++++++++++++++++ crates/openshell-server/src/compute/mod.rs | 5 +- .../helm/openshell/templates/clusterrole.yaml | 14 ++-- .../openshell/tests/clusterrole_test.yaml | 4 +- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 3c8dfd53f8..f8069cfac6 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -545,6 +545,78 @@ impl KubernetesComputeDriver { self.config.workspace_mode } + /// Backfill the `openshell.ai/gateway-id` label on Sandbox CRs that + /// predate its introduction. Runs once at startup in shared mode so that + /// label-selector based lookups continue to find legacy resources. + pub async fn backfill_gateway_id_labels(&self) { + let sandbox_api = match self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + { + Ok(api) => api, + Err(e) => { + warn!(error = %e, "skipping gateway-id label backfill: cannot resolve Sandbox API"); + return; + } + }; + + let selector = openshell_sandbox_label_selector(); + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + sandbox_api + .api + .list(&ListParams::default().labels(&selector)), + ) + .await + { + Ok(Ok(list)) => list, + Ok(Err(e)) => { + warn!(error = %e, "skipping gateway-id label backfill: list failed"); + return; + } + Err(_) => { + warn!("skipping gateway-id label backfill: list timed out"); + return; + } + }; + + let gateway_id = &self.config.gateway_id; + for obj in &list { + let has_label = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == gateway_id); + if has_label { + continue; + } + let name = match obj.metadata.name.as_deref() { + Some(n) => n, + None => continue, + }; + let patch = serde_json::json!({ + "metadata": { + "labels": { + LABEL_GATEWAY_ID: gateway_id + } + } + }); + match sandbox_api + .api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + { + Ok(_) => { + info!(sandbox = %name, gateway_id, "backfilled gateway-id label"); + } + Err(e) => { + warn!(sandbox = %name, error = %e, "failed to backfill gateway-id label"); + } + } + } + } + /// Ensure the K8s namespace for a workspace exists (managed mode only). /// /// Idempotent: returns the namespace name whether it was just created or diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 632f4c2996..07fb822a05 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -46,7 +46,7 @@ use openshell_core::{ObjectLabels, ObjectWorkspace}; use openshell_driver_docker::DockerComputeDriver; use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, - OperatorNamespaceAllowlist, + OperatorNamespaceAllowlist, WorkspaceMode, }; use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; @@ -771,6 +771,9 @@ impl ComputeRuntime { let driver = KubernetesComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; + if driver.workspace_mode() == WorkspaceMode::Shared { + driver.backfill_gateway_id_labels().await; + } let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); let runtime = Self::from_driver( diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 299ee9ca3d..0f62d8e915 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -74,13 +74,11 @@ rules: - get {{- end }} {{- if ne $workspaceMode "shared" }} - # TLS secret sync: read the source Secret in the release namespace and - # create/update copies in workspace namespaces so sandbox pods can mount - # client TLS material for mTLS to the gateway. - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - # kubernetes-secrets credential driver: credentials are stored as Secrets - # in workspace namespaces, requiring patch+delete in addition to TLS sync. - {{- end }} + # Secrets access for multi-namespace modes: + # - TLS sync uses server-side apply (patch) to copy the client TLS Secret + # into workspace namespaces so sandbox pods can mount mTLS material. + # - The kubernetes-secrets credential driver stores credentials as Secrets + # in workspace namespaces, additionally requiring delete for cleanup. - apiGroups: - "" resources: @@ -89,8 +87,8 @@ rules: - get - create - update - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - patch + {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - delete {{- end }} {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index c2363b08eb..efaa117f44 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -21,7 +21,7 @@ tests: resources: ["secrets"] verbs: ["get", "create", "update", "patch", "delete"] - - it: omits secrets patch and delete when credential driver is disabled (operator) + - it: grants secrets patch for TLS sync even when credential driver is disabled (operator) set: server.drivers.kubernetes.workspaceMode: operator asserts: @@ -30,7 +30,7 @@ tests: content: apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "update"] + verbs: ["get", "create", "update", "patch"] - it: omits secrets rule entirely in shared mode set: From ddb67aeb94cb6d50a96b115ae7f7d6433430f81b Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Wed, 12 Aug 2026 12:41:34 -0400 Subject: [PATCH 10/11] fix(k8s): address workspace namespace review findings Signed-off-by: Derek Carr --- architecture/compute-runtimes.md | 18 +- .../openshell-driver-kubernetes/src/config.rs | 63 ++- .../openshell-driver-kubernetes/src/driver.rs | 390 +++++++++++++++--- crates/openshell-driver-kubernetes/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 35 +- crates/openshell-server/src/compute/mod.rs | 40 +- crates/openshell-server/src/grpc/mod.rs | 31 +- crates/openshell-server/src/grpc/workspace.rs | 73 +++- deploy/helm/openshell/README.md | 2 +- .../ci/values-workspace-managed.yaml | 2 + .../helm/openshell/templates/clusterrole.yaml | 12 + .../openshell/templates/gateway-config.yaml | 5 + .../openshell/tests/clusterrole_test.yaml | 23 ++ .../openshell/tests/gateway_config_test.yaml | 7 + deploy/helm/openshell/values.yaml | 3 +- docs/reference/gateway-config.mdx | 11 + docs/reference/sandbox-compute-drivers.mdx | 3 +- e2e/rust/tests/workspace_namespace_managed.rs | 56 ++- e2e/with-kube-gateway.sh | 8 + tasks/test.toml | 2 +- 20 files changed, 715 insertions(+), 73 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f22da681a1..9cf6b20a67 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -304,14 +304,24 @@ embed the workspace prefix for collision avoidance. No namespace lifecycle management. RBAC uses a namespace-scoped Role. **Managed** auto-creates a K8s namespace per workspace on first sandbox create. -Each new namespace receives a ServiceAccount and copies OpenShift SCC UID-range -and supplemental-group annotations from the gateway namespace when present. The -driver deletes the namespace when the last sandbox in it is removed -(`delete_namespace_if_empty`). Requires a non-empty `gateway_id` (validated as a +Each new namespace receives a ServiceAccount and the configured gateway-only +SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the +driver's source namespace on every sandbox create so registry credential +rotations propagate. The namespace also copies OpenShift SCC UID-range and +supplemental-group annotations from the gateway namespace when present. The +driver deletes the namespace during workspace deletion. The workspace remains +durably `Terminating` until the Kubernetes API accepts namespace cleanup, so a +transient failure can be retried. Namespace deletion uses the fetched UID as a +precondition to avoid deleting a replacement namespace. Requires a non-empty +`gateway_id` (validated as a DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace `create`/`delete` and ServiceAccount `create`/`get` permissions. +Operator mode does not create NetworkPolicies or copy image-pull Secrets. +Platform teams must apply the gateway ingress boundary and provision configured +image-pull Secrets in every operator-managed namespace. + **Operator** uses pre-provisioned namespaces discovered through two optional sources: a K8s label selector (`operator_namespace_label`) and a drop-in allowlist file (`operator_namespace_file`). At least one must be configured. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 8849dde43e..8064079e98 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -3,7 +3,7 @@ use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, RwLock}; @@ -304,6 +304,10 @@ pub struct KubernetesComputeConfig { pub image_pull_policy: String, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, + /// Managed-mode SSH ingress isolation. When enabled, the driver creates a + /// `NetworkPolicy` in each managed workspace namespace that permits TCP 2222 + /// only from gateway pods matching this peer. + pub managed_ssh_ingress: ManagedSshIngressConfig, /// Image that provides the `openshell-sandbox` supervisor binary. /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. @@ -373,6 +377,14 @@ pub struct KubernetesComputeConfig { pub sandbox_gid: Option, } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ManagedSshIngressConfig { + pub enabled: bool, + pub gateway_namespace: String, + pub gateway_pod_selector: BTreeMap, +} + /// Lower bound enforced by kubelet for projected SA tokens. pub const MIN_SA_TOKEN_TTL_SECS: i64 = 600; @@ -409,6 +421,7 @@ impl Default for KubernetesComputeConfig { // is Podman vocabulary and is not a valid Kubernetes value. image_pull_policy: String::new(), image_pull_secrets: Vec::new(), + managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), @@ -473,7 +486,7 @@ impl KubernetesComputeConfig { /// 3. Fallback defaults: UID=`1000`, GID=UID pub fn resolve_sandbox_uid( &self, - namespace_annotations: Option<&std::collections::BTreeMap>, + namespace_annotations: Option<&BTreeMap>, ) -> u32 { if let Some(uid) = self.sandbox_uid { return uid; @@ -491,7 +504,7 @@ impl KubernetesComputeConfig { pub fn resolve_sandbox_gid( &self, resolved_uid: u32, - _namespace_annotations: Option<&std::collections::BTreeMap>, + _namespace_annotations: Option<&BTreeMap>, ) -> u32 { self.sandbox_gid .or(self.sandbox_uid) @@ -618,6 +631,18 @@ impl KubernetesComputeConfig { prefix.len() )); } + if self.managed_ssh_ingress.enabled { + if self.managed_ssh_ingress.gateway_namespace.is_empty() { + return Err( + "managed SSH ingress isolation requires gateway_namespace".into() + ); + } + if self.managed_ssh_ingress.gateway_pod_selector.is_empty() { + return Err( + "managed SSH ingress isolation requires gateway_pod_selector".into(), + ); + } + } Ok(()) } WorkspaceMode::Operator => { @@ -1449,6 +1474,38 @@ mod tests { cfg.validate_workspace_mode().unwrap(); } + #[test] + fn validate_workspace_mode_managed_requires_complete_ssh_ingress_peer() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway".to_string(), + gateway_pod_selector: BTreeMap::new(), + }, + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("gateway_pod_selector"), "{err}"); + } + + #[test] + fn validate_workspace_mode_managed_accepts_complete_ssh_ingress_peer() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway".to_string(), + gateway_pod_selector: BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )]), + }, + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + #[test] fn validate_workspace_mode_operator_requires_discovery() { let cfg = KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f8069cfac6..0787cba3ea 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -14,6 +14,12 @@ use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Secret, ServiceAccount, Volume, VolumeMount, }; +use k8s_openapi::api::networking::v1::{ + NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, + NetworkPolicySpec, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use kube::api::{ Api, ApiResource, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions, }; @@ -54,6 +60,8 @@ use tracing::{debug, info, warn}; pub type WatchStream = Pin> + Send>>; +const MANAGED_SSH_NETWORK_POLICY_NAME: &str = "openshell-sandbox-ssh"; + #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { #[error("sandbox already exists")] @@ -548,17 +556,11 @@ impl KubernetesComputeDriver { /// Backfill the `openshell.ai/gateway-id` label on Sandbox CRs that /// predate its introduction. Runs once at startup in shared mode so that /// label-selector based lookups continue to find legacy resources. - pub async fn backfill_gateway_id_labels(&self) { - let sandbox_api = match self + pub async fn backfill_gateway_id_labels(&self) -> Result<(), KubernetesDriverError> { + let sandbox_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await - { - Ok(api) => api, - Err(e) => { - warn!(error = %e, "skipping gateway-id label backfill: cannot resolve Sandbox API"); - return; - } - }; + .map_err(KubernetesDriverError::Message)?; let selector = openshell_sandbox_label_selector(); let list = match tokio::time::timeout( @@ -570,30 +572,21 @@ impl KubernetesComputeDriver { .await { Ok(Ok(list)) => list, - Ok(Err(e)) => { - warn!(error = %e, "skipping gateway-id label backfill: list failed"); - return; - } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), Err(_) => { - warn!("skipping gateway-id label backfill: list timed out"); - return; + return Err(KubernetesDriverError::Message( + "timeout listing Sandbox resources for gateway-id label backfill".to_string(), + )); } }; let gateway_id = &self.config.gateway_id; for obj in &list { - let has_label = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_GATEWAY_ID)) - .is_some_and(|v| v == gateway_id); - if has_label { + if !gateway_id_label_needs_backfill(obj.metadata.labels.as_ref(), gateway_id) { continue; } - let name = match obj.metadata.name.as_deref() { - Some(n) => n, - None => continue, + let Some(name) = obj.metadata.name.as_deref() else { + continue; }; let patch = serde_json::json!({ "metadata": { @@ -602,19 +595,27 @@ impl KubernetesComputeDriver { } } }); - match sandbox_api - .api - .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) - .await + match tokio::time::timeout( + KUBE_API_TIMEOUT, + sandbox_api + .api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await { - Ok(_) => { + Ok(Ok(_)) => { info!(sandbox = %name, gateway_id, "backfilled gateway-id label"); } - Err(e) => { - warn!(sandbox = %name, error = %e, "failed to backfill gateway-id label"); + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout backfilling gateway-id label on Sandbox {name}" + ))); } } } + + Ok(()) } /// Ensure the K8s namespace for a workspace exists (managed mode only). @@ -623,16 +624,6 @@ impl KubernetesComputeDriver { /// already existed. Also creates the sandbox `ServiceAccount` in the /// namespace. /// - /// TODO: no `NetworkPolicy` is created in dynamic namespaces. The - /// Helm-managed static namespace gets an SSH-isolation policy (port 2222 - /// restricted to the gateway pod), but managed and operator namespaces do - /// not. In managed mode, risk is low: only sandbox pods from the same - /// workspace run in the namespace, so there is no lateral movement target. - /// In operator mode, the admin owns the namespace and is responsible for - /// applying appropriate policies. A same-cluster `namespaceSelector` policy - /// would also break cross-cluster topologies where the gateway is external. - /// Add a configurable `NetworkPolicy` when mixed-workload or cross-cluster - /// namespaces are supported. pub async fn ensure_namespace(&self, workspace: &str) -> Result { let ns_name = managed_namespace(&self.config.gateway_id, workspace); let ns_api: Api = Api::all(self.client.clone()); @@ -716,10 +707,42 @@ impl KubernetesComputeDriver { } self.ensure_service_account(&ns_name).await?; + self.ensure_managed_ssh_network_policy(&ns_name).await?; Ok(ns_name) } + async fn ensure_managed_ssh_network_policy( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + if !self.config.managed_ssh_ingress.enabled { + return Ok(()); + } + + let policy = managed_ssh_network_policy(namespace, &self.config); + let policy_api: Api = Api::namespaced(self.client.clone(), namespace); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policy_api.patch( + MANAGED_SSH_NETWORK_POLICY_NAME, + &PatchParams::apply("openshell"), + &Patch::Apply(&policy), + ), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace, "applied managed sandbox SSH NetworkPolicy"); + Ok(()) + } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message(format!( + "timeout applying SSH NetworkPolicy in {namespace}" + ))), + } + } + async fn ensure_service_account(&self, namespace: &str) -> Result<(), KubernetesDriverError> { let sa_api: Api = Api::namespaced(self.client.clone(), namespace); let sa = ServiceAccount { @@ -830,6 +853,62 @@ impl KubernetesComputeDriver { Ok(()) } + /// Copy the explicitly configured image-pull Secrets into a managed + /// workspace namespace. Server-side apply refreshes rotated credentials + /// without forcibly taking fields owned by another manager. + async fn ensure_image_pull_secrets( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let target_api: Api = Api::namespaced(self.client.clone(), namespace); + + for secret_name in &self.config.image_pull_secrets { + let source = match tokio::time::timeout(KUBE_API_TIMEOUT, source_api.get(secret_name)) + .await + { + Ok(Ok(secret)) => secret, + Ok(Err(KubeError::Api(error))) if error.code == 404 => { + return Err(KubernetesDriverError::Precondition(format!( + "configured image-pull Secret {secret_name} does not exist in source namespace {}", + self.config.namespace + ))); + } + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading image-pull Secret {secret_name} from {}", + self.config.namespace + ))); + } + }; + + let copy = image_pull_secret_copy(secret_name, namespace, source); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.patch( + secret_name, + &PatchParams::apply("openshell"), + &Patch::Apply(©), + ), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace, secret = %secret_name, "applied image-pull Secret copy"); + } + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout applying image-pull Secret {secret_name} in {namespace}" + ))); + } + } + } + + Ok(()) + } + /// Delete the managed namespace and all its contents (managed mode only). /// Called via the `DeleteWorkspace` RPC after workspace deletion. /// Kubernetes cascades namespace deletion to all resources within it. @@ -859,11 +938,14 @@ impl KubernetesComputeDriver { return Ok(()); } - match tokio::time::timeout( - KUBE_API_TIMEOUT, - ns_api.delete(&ns_name, &DeleteParams::default()), - ) - .await + let namespace_uid = ns.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message(format!( + "namespace {ns_name} has no UID; refusing an unguarded delete" + )) + })?; + let delete_params = namespace_delete_params(namespace_uid); + + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.delete(&ns_name, &delete_params)).await { Ok(Ok(_)) => { info!(namespace = %ns_name, workspace = %workspace, "deleted managed namespace"); @@ -1240,7 +1322,11 @@ impl KubernetesComputeDriver { let target_namespace = match self.config.workspace_mode { WorkspaceMode::Shared => self.config.namespace.clone(), - WorkspaceMode::Managed => self.ensure_namespace(workspace).await?, + WorkspaceMode::Managed => { + let namespace = self.ensure_namespace(workspace).await?; + self.ensure_image_pull_secrets(&namespace).await?; + namespace + } WorkspaceMode::Operator => { if let Some(ref allowlist) = self.operator_allowlist && !allowlist.contains(workspace) @@ -1732,6 +1818,22 @@ fn is_namespace_owned_by_gateway( .is_some_and(|v| v == gateway_id) } +fn gateway_id_label_needs_backfill( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|labels| labels.get(LABEL_GATEWAY_ID)) + .is_none_or(|value| value != gateway_id) +} + +fn namespace_delete_params(uid: String) -> DeleteParams { + DeleteParams::default().preconditions(Preconditions { + uid: Some(uid), + resource_version: None, + }) +} + fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { format!( "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" @@ -1763,6 +1865,70 @@ fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(MANAGED_SSH_NETWORK_POLICY_NAME.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string()]), + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + config.managed_ssh_ingress.gateway_namespace.clone(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(config.managed_ssh_ingress.gateway_pod_selector.clone()), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(2222)), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + ..Default::default() + }), + status: None, + } +} + +fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(secret_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + } +} + fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { let mut annotations = BTreeMap::new(); annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -6869,6 +7035,123 @@ mod tests { ); } + #[test] + fn gateway_id_backfill_adopts_unlabelled_sandbox() { + let labels = BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )]); + assert!(gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn gateway_id_backfill_adopts_sandbox_from_previous_gateway() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-old".to_string())]); + assert!(gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn gateway_id_backfill_skips_sandbox_already_owned_by_gateway() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-1".to_string())]); + assert!(!gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn managed_ssh_policy_allows_only_gateway_peer_on_port_2222() { + let config = KubernetesComputeConfig { + managed_ssh_ingress: crate::config::ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway-ns".to_string(), + gateway_pod_selector: BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )]), + }, + ..KubernetesComputeConfig::default() + }; + let policy = managed_ssh_network_policy("workspace-ns", &config); + let spec = policy.spec.unwrap(); + assert_eq!( + spec.policy_types.as_deref(), + Some(["Ingress".to_string()].as_slice()) + ); + let ingress = &spec.ingress.unwrap()[0]; + assert_eq!( + ingress.ports.as_ref().unwrap()[0].port, + Some(IntOrString::Int(2222)) + ); + let peer = &ingress.from.as_ref().unwrap()[0]; + assert_eq!( + peer.namespace_selector + .as_ref() + .unwrap() + .match_labels + .as_ref() + .unwrap() + .get("kubernetes.io/metadata.name") + .map(String::as_str), + Some("gateway-ns") + ); + assert_eq!( + peer.pod_selector + .as_ref() + .unwrap() + .match_labels + .as_ref() + .unwrap() + .get("app.kubernetes.io/name") + .map(String::as_str), + Some("openshell") + ); + } + + #[test] + fn image_pull_secret_copy_keeps_only_portable_secret_fields() { + let source: Secret = serde_json::from_value(serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "gateway", + "uid": "source-uid", + "resourceVersion": "42", + "labels": { "source-only": "true" }, + "annotations": { "source-only": "true" }, + "finalizers": ["example.test/finalizer"] + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + })) + .unwrap(); + + let copy = image_pull_secret_copy("regcred", "workspace", source); + assert_eq!(copy.metadata.name.as_deref(), Some("regcred")); + assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); + assert_eq!( + copy.type_.as_deref(), + Some("kubernetes.io/dockerconfigjson") + ); + assert!( + copy.data + .as_ref() + .unwrap() + .contains_key(".dockerconfigjson") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_MANAGED_BY) + .map(String::as_str), + Some(LABEL_MANAGED_BY_VALUE) + ); + assert!(copy.metadata.uid.is_none()); + assert!(copy.metadata.resource_version.is_none()); + assert!(copy.metadata.annotations.is_none()); + assert!(copy.metadata.finalizers.is_none()); + } + #[test] fn namespace_owned_with_correct_labels() { let labels = BTreeMap::from([ @@ -6903,4 +7186,15 @@ mod tests { fn namespace_not_owned_no_labels() { assert!(!is_namespace_owned_by_gateway(None, "gw-1")); } + + #[test] + fn namespace_delete_is_guarded_by_fetched_uid() { + let params = namespace_delete_params("namespace-uid".to_string()); + assert_eq!( + params + .preconditions + .and_then(|preconditions| preconditions.uid), + Some("namespace-uid".to_string()) + ); + } } diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index d18a23a618..1a234385c6 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,8 +8,8 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - OperatorNamespaceAllowlist, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + ManagedSshIngressConfig, OperatorNamespaceAllowlist, SupervisorSideloadMethod, + SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index c5b7659406..7cd883e529 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -3,6 +3,7 @@ use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; +use std::collections::BTreeMap; use std::net::SocketAddr; use tracing::info; use tracing_subscriber::EnvFilter; @@ -12,7 +13,8 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesSidecarConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, + WorkspaceMode, }; #[derive(Parser, Debug)] @@ -68,6 +70,19 @@ struct Args { )] sandbox_image_pull_secrets: Vec, + #[arg(long, env = "OPENSHELL_MANAGED_SSH_INGRESS_ENABLED")] + managed_ssh_ingress_enabled: bool, + + #[arg(long, env = "OPENSHELL_MANAGED_SSH_GATEWAY_NAMESPACE")] + managed_ssh_gateway_namespace: Option, + + #[arg( + long, + env = "OPENSHELL_MANAGED_SSH_GATEWAY_POD_SELECTOR", + value_delimiter = ',' + )] + managed_ssh_gateway_pod_selector: Vec, + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] grpc_endpoint: Option, @@ -148,6 +163,19 @@ async fn main() -> Result<()> { ) .init(); + let managed_ssh_gateway_pod_selector = args + .managed_ssh_gateway_pod_selector + .iter() + .map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.to_string(), value.to_string())) + .ok_or_else(|| { + miette::miette!("managed SSH gateway pod selector must use key=value: {entry}") + }) + }) + .collect::>>()?; + let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { workspace_mode: args.workspace_mode, gateway_id: args.gateway_id, @@ -158,6 +186,11 @@ async fn main() -> Result<()> { default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), image_pull_secrets: args.sandbox_image_pull_secrets, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: args.managed_ssh_ingress_enabled, + gateway_namespace: args.managed_ssh_gateway_namespace.unwrap_or_default(), + gateway_pod_selector: managed_ssh_gateway_pod_selector, + }, supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 07fb822a05..48fb4e3713 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -772,7 +772,10 @@ impl ComputeRuntime { .await .map_err(|err| ComputeError::Message(err.to_string()))?; if driver.workspace_mode() == WorkspaceMode::Shared { - driver.backfill_gateway_id_labels().await; + driver + .backfill_gateway_id_labels() + .await + .map_err(|err| ComputeError::Message(err.to_string()))?; } let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); @@ -3140,7 +3143,18 @@ fn is_terminal_failure_reason(reason: &str) -> bool { #[cfg(test)] #[derive(Debug, Default)] -pub struct NoopTestDriver; +pub struct NoopTestDriver { + workspace_delete_failures: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +impl NoopTestDriver { + pub fn failing_workspace_deletes(count: usize) -> Self { + Self { + workspace_delete_failures: std::sync::atomic::AtomicUsize::new(count), + } + } +} #[cfg(test)] #[tonic::async_trait] @@ -3250,6 +3264,17 @@ impl ComputeDriver for NoopTestDriver { &self, _request: Request, ) -> Result, Status> { + if self + .workspace_delete_failures + .fetch_update( + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + |remaining| remaining.checked_sub(1), + ) + .is_ok() + { + return Err(Status::unavailable("injected workspace cleanup failure")); + } Ok(tonic::Response::new(DeleteWorkspaceResponse {})) } } @@ -3261,8 +3286,17 @@ pub async fn new_test_runtime(store: Arc) -> ComputeRuntime { #[cfg(test)] pub async fn new_test_runtime_for_driver(store: Arc, driver_name: &str) -> ComputeRuntime { + new_test_runtime_with_driver(store, driver_name, Arc::new(NoopTestDriver::default())).await +} + +#[cfg(test)] +pub async fn new_test_runtime_with_driver( + store: Arc, + driver_name: &str, + driver: Arc, +) -> ComputeRuntime { ComputeRuntime { - driver: TracedDriver::new(Arc::new(NoopTestDriver), "test".to_string()), + driver: TracedDriver::new(driver, "test".to_string()), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index d84ca41557..9e9873df7c 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -738,7 +738,9 @@ pub mod test_support { use crate::ServerState; use crate::auth::identity::{Identity, IdentityProvider}; use crate::auth::principal::{Principal, UserPrincipal}; - use crate::compute::{new_test_runtime, new_test_runtime_for_driver}; + use crate::compute::{ + NoopTestDriver, new_test_runtime, new_test_runtime_for_driver, new_test_runtime_with_driver, + }; use crate::persistence::Store; use crate::sandbox_index::SandboxIndex; use crate::sandbox_watch::SandboxWatchBus; @@ -798,6 +800,33 @@ pub mod test_support { None, )) } + + /// Build a test state whose compute driver fails the requested number of + /// workspace cleanup calls before succeeding. + pub async fn test_server_state_with_workspace_cleanup_failures( + failures: usize, + ) -> Arc { + let store = Arc::new( + Store::connect("sqlite::memory:?cache=shared") + .await + .unwrap(), + ); + crate::ensure_default_workspace(&store).await.unwrap(); + let driver = Arc::new(NoopTestDriver::failing_workspace_deletes(failures)); + let compute = new_test_runtime_with_driver(store.clone(), "test", driver).await; + Arc::new(ServerState::new( + Config::new(None) + .with_database_url("sqlite::memory:?cache=shared") + .with_credential_drivers(["test-static"]), + store, + compute, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + None, + )) + } } // --------------------------------------------------------------------------- diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index f2863511f1..d83ffab0e8 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -423,6 +423,15 @@ pub(super) async fn handle_delete_workspace( .await .map_err(|e| Status::internal(format!("delete workspace members failed: {e}")))?; + // Keep the terminating workspace durable until platform cleanup has been + // accepted. A failed cleanup can then be retried through this same path. + state.compute.delete_workspace(&name).await.map_err(|e| { + Status::new( + e.code(), + format!("delete workspace platform resources failed: {e}"), + ) + })?; + let deleted = state .store .delete_if(Workspace::object_type(), &ws_id, delete_version) @@ -435,10 +444,6 @@ pub(super) async fn handle_delete_workspace( } })?; - if deleted && let Err(e) = state.compute.delete_workspace(&name).await { - tracing::warn!(workspace = %name, error = %e, "failed to delete workspace platform resources"); - } - Ok(Response::new(DeleteWorkspaceResponse { deleted })) } @@ -620,7 +625,9 @@ mod tests { use openshell_core::proto::datamodel::v1::ObjectMeta; use tonic::{Code, Request}; - use crate::grpc::test_support::{authed_request, test_server_state}; + use crate::grpc::test_support::{ + authed_request, test_server_state, test_server_state_with_workspace_cleanup_failures, + }; #[tokio::test] async fn create_workspace_returns_metadata() { @@ -1326,6 +1333,62 @@ mod tests { assert!(resp.deleted); } + #[tokio::test] + async fn delete_workspace_retains_terminating_record_when_platform_cleanup_fails() { + let state = test_server_state_with_workspace_cleanup_failures(1).await; + + handle_create_workspace( + &state, + Request::new(CreateWorkspaceRequest { + name: "cleanup-retry".to_string(), + labels: HashMap::new(), + }), + ) + .await + .unwrap(); + + let err = handle_delete_workspace( + &state, + Request::new(DeleteWorkspaceRequest { + name: "cleanup-retry".to_string(), + }), + ) + .await + .unwrap_err(); + assert_eq!(err.code(), Code::Unavailable); + + let retained: Workspace = state + .store + .get_message_by_name("", "cleanup-retry") + .await + .unwrap() + .expect("workspace must remain durable after cleanup failure"); + assert_ne!( + retained.metadata.unwrap().deletion_timestamp_ms, + 0, + "retained workspace must remain terminating" + ); + + let retry = handle_delete_workspace( + &state, + Request::new(DeleteWorkspaceRequest { + name: "cleanup-retry".to_string(), + }), + ) + .await + .unwrap() + .into_inner(); + assert!(retry.deleted); + assert!( + state + .store + .get_message_by_name::("", "cleanup-retry") + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn create_workspace_persists_labels_for_selector() { let state = test_server_state().await; diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index a7f6dfda15..71f6d96112 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -176,7 +176,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | image.tag | string | `""` | Gateway image tag. Defaults to the chart appVersion when empty. | | imagePullSecrets | list | `[]` | Image pull secrets attached to gateway and helper pods. | | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | -| networkPolicy.enabled | bool | `true` | Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. | +| networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | | pkiInitJob.enabled | bool | `true` | Run a pre-install/pre-upgrade Job that creates gateway and client mTLS Secrets. When certManager.enabled=true, cert-manager owns TLS and this same hook runs in JWT-only mode even if pkiInitJob.enabled remains true. | | pkiInitJob.serverDnsNames | list | `[]` | Extra DNS SANs to append to the server certificate. | diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml index 9b8911fbe7..e9f88846c4 100644 --- a/deploy/helm/openshell/ci/values-workspace-managed.yaml +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -4,6 +4,8 @@ # E2E overlay: deploy the gateway in managed workspace mode. # Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. server: + sandboxImagePullSecrets: + - name: e2e-regcred drivers: kubernetes: workspaceMode: "managed" diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 0f62d8e915..a3b486c03c 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -101,4 +101,16 @@ rules: verbs: - create - get + {{- if .Values.networkPolicy.enabled }} + # Apply gateway-only SSH ingress isolation in managed namespaces. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - create + - patch + - update + {{- end }} {{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 454affd0d3..57518e3fda 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -171,6 +171,11 @@ data: supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + [openshell.drivers.kubernetes.managed_ssh_ingress] + enabled = {{ .Values.networkPolicy.enabled }} + gateway_namespace = {{ .Release.Namespace | quote }} + gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } + [openshell.drivers.kubernetes.sidecar] proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index efaa117f44..e0bb908182 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -9,6 +9,29 @@ release: namespace: my-namespace tests: + - it: grants managed namespace NetworkPolicy apply permissions + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create", "patch", "update"] + + - it: omits managed NetworkPolicy permissions when isolation is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + networkPolicy.enabled: false + asserts: + - notContains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + any: true + - it: grants secrets patch and delete when credential driver is enabled (operator) set: server.drivers.kubernetes.workspaceMode: operator diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index f98c321fee..d77b408efe 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -159,6 +159,13 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' + - it: configures managed SSH isolation with the gateway peer + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.managed_ssh_ingress\].*?enabled\s*=\s*true.*?gateway_namespace\s*=\s*"my-namespace".*?gateway_pod_selector\s*=.*?app\.kubernetes\.io/name.*?openshell' + - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 81ec18c036..0eebee810c 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -383,7 +383,8 @@ server: # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway only. networkPolicy: - # -- Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. + # -- Restrict SSH ingress on sandbox pods to the gateway. In managed mode, + # the driver applies the equivalent policy to each workspace namespace. enabled: true # Built-in TLS PKI bootstrap via a pre-install/pre-upgrade hook Job. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index b201b6f198..e8f39f1a6f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -429,6 +429,7 @@ image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "IfNotPresent" + # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. supervisor_sideload_method = "image-volume" @@ -471,6 +472,11 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # (hot-reloaded on change, e.g. via ConfigMap volume mount). # operator_namespace_file = "/etc/openshell/workspace-namespaces.json" +[openshell.drivers.kubernetes.managed_ssh_ingress] +enabled = true +gateway_namespace = "openshell" +gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } + [openshell.drivers.kubernetes.sidecar] # UID used by relaxed long-running network sidecars. Strict process/binary-aware # sidecars run as UID 0 so Kubernetes grants the required /proc inspection @@ -484,6 +490,11 @@ proxy_uid = 1337 process_binary_aware_network_policy = true ``` +In managed workspace mode, the Kubernetes driver copies each explicitly named +`image_pull_secrets` Secret from `namespace` into the managed workspace +namespace on sandbox creation. Shared and operator modes require the Secret to +already exist in the sandbox namespace. + ### Docker Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index ea4c1a37b0..832f41fd59 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -317,7 +317,8 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the gateway TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | -| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image pull secrets to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | +| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | +| `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs index 6bc0564ccf..a92562a6c9 100644 --- a/e2e/rust/tests/workspace_namespace_managed.rs +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -11,8 +11,8 @@ //! //! Namespace cleanup after sandbox deletion is best-effort and depends on //! controller finalization timing. These tests focus on verifiable behavior: -//! namespace creation, labels, ServiceAccount provisioning, and sandbox CR -//! placement in the correct namespace. +//! namespace creation, labels, ServiceAccount and SSH NetworkPolicy +//! provisioning, and sandbox CR placement in the correct namespace. use std::process::Stdio; use std::time::Duration; @@ -178,6 +178,58 @@ async fn managed_creates_namespace_with_labels() { let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); + // The managed driver copies only explicitly configured image-pull Secrets + // from the gateway namespace into the workspace namespace. + let (ok, copied_secret) = kubectl(&[ + "get", + "secret", + "e2e-regcred", + "-n", + &ns, + "-o", + "jsonpath={.type}", + ]) + .await; + assert!( + ok && copied_secret.contains("kubernetes.io/dockerconfigjson"), + "configured image-pull Secret should be copied into {ns}: {copied_secret}" + ); + + // Verify SSH ingress is restricted to the gateway peer. Because Kubernetes + // NetworkPolicies are allowlists, the absence of a sandbox peer here + // denies sandbox-to-sandbox TCP 2222 traffic. + let (ok, policy) = kubectl(&[ + "get", + "networkpolicy", + "openshell-sandbox-ssh", + "-n", + &ns, + "-o", + "json", + ]) + .await; + assert!( + ok, + "managed SSH NetworkPolicy should exist in {ns}: {policy}" + ); + let policy: serde_json::Value = + serde_json::from_str(&policy).expect("managed SSH NetworkPolicy should be valid JSON"); + assert_eq!( + policy["spec"]["podSelector"]["matchLabels"]["openshell.ai/managed-by"], + "openshell" + ); + assert_eq!(policy["spec"]["ingress"][0]["ports"][0]["port"], 2222); + assert_eq!( + policy["spec"]["ingress"][0]["from"][0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "openshell" + ); + assert!( + policy["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + ["app.kubernetes.io/name"] + .is_string(), + "SSH ingress peer must select gateway pods: {policy}" + ); + // Verify sandbox CR is in the managed namespace (not the gateway namespace). let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index b8a7621a12..14eb42bc1b 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -762,6 +762,14 @@ else --wait --timeout 5m HELM_INSTALLED=1 + if [ -n "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET:-}" ]; then + kctl -n "${NAMESPACE}" create secret docker-registry \ + "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET}" \ + --docker-server=registry.example.test \ + --docker-username=e2e-user \ + --docker-password=e2e-password + fi + LOCAL_PORT="$(e2e_pick_port)" echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ diff --git a/tasks/test.toml b/tasks/test.toml index df409c27d8..406e1eadc6 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -158,7 +158,7 @@ run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-managed"] description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-operator"] From e0df6ce0251f7775250221d70ae4fca22d2eff3d Mon Sep 17 00:00:00 2001 From: Dhiraj Bokde Date: Sun, 9 Aug 2026 22:27:51 -0700 Subject: [PATCH 11/11] feat(k8s): map workspaces to operator namespaces Signed-off-by: Dhiraj Bokde --- architecture/compute-runtimes.md | 16 ++-- .../src/lib.rs | 66 +++++++++++++--- crates/openshell-driver-kubernetes/README.md | 9 ++- .../openshell-driver-kubernetes/src/config.rs | 79 ++++++++++++++++--- .../openshell-driver-kubernetes/src/driver.rs | 24 +++--- .../openshell-driver-kubernetes/src/main.rs | 1 + deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 6 ++ .../openshell/tests/gateway_config_test.yaml | 10 +++ deploy/helm/openshell/values.yaml | 2 + docs/reference/gateway-config.mdx | 6 +- 11 files changed, 177 insertions(+), 43 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 9cf6b20a67..97b5554a20 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -297,7 +297,7 @@ token authentication, and RBAC requirements. |---|---|---|---| | **Shared** (default) | Single static namespace from config | `{workspace}--{name}` | None | | **Managed** | `openshell-{gateway_id}-{workspace}` | bare sandbox name | Driver creates and deletes | -| **Operator** | Workspace name maps 1:1 to a pre-provisioned namespace | bare sandbox name | External (platform team) | +| **Operator** | Workspace resolves to a pre-provisioned namespace | bare sandbox name | External (platform team) | **Shared** renders all sandboxes into one configured namespace. Resource names embed the workspace prefix for collision avoidance. No namespace lifecycle @@ -322,12 +322,16 @@ Operator mode does not create NetworkPolicies or copy image-pull Secrets. Platform teams must apply the gateway ingress boundary and provision configured image-pull Secrets in every operator-managed namespace. -**Operator** uses pre-provisioned namespaces discovered through two optional -sources: a K8s label selector (`operator_namespace_label`) and a drop-in -allowlist file (`operator_namespace_file`). At least one must be configured. +**Operator** uses pre-provisioned namespaces selected through a K8s label +selector (`operator_namespace_label`), a drop-in allowlist file +(`operator_namespace_file`), or an explicit workspace-to-namespace map +(`operator_workspace_namespaces`). Exactly one must be configured. Label and +file sources preserve the 1:1 workspace/namespace convention; the map allows +platform operators to keep application-facing workspace names independent of +cluster namespace naming conventions. The `OperatorNamespaceAllowlist` (`Arc>>`) is populated -at runtime by background watchers and read by the namespace resolver. Sandbox -creation fails closed if the workspace is not in the current allowlist. Platform +from the configured source and read by the namespace resolver. Sandbox creation +fails closed if the workspace does not resolve to an allowed namespace. Platform teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as managed mode but without namespace `create`/`delete` or ServiceAccount permissions. diff --git a/crates/openshell-driver-kubernetes-secrets/src/lib.rs b/crates/openshell-driver-kubernetes-secrets/src/lib.rs index 1c59401cb1..7b53db84a4 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/lib.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/lib.rs @@ -63,16 +63,26 @@ struct KubernetesSecretsDriverSettings { allow_reference_namespace: bool, workspace_mode: WorkspaceMode, gateway_id: String, + operator_workspace_namespaces: BTreeMap, } impl KubernetesSecretsDriverSettings { - fn target_namespace(&self, workspace: &str) -> String { + fn target_namespace(&self, workspace: &str) -> Result { match self.workspace_mode { - WorkspaceMode::Shared => self.namespace.clone(), - WorkspaceMode::Managed => { - format!("openshell-{}-{}", self.gateway_id, workspace) + WorkspaceMode::Shared => Ok(self.namespace.clone()), + WorkspaceMode::Managed => Ok(format!("openshell-{}-{}", self.gateway_id, workspace)), + WorkspaceMode::Operator if self.operator_workspace_namespaces.is_empty() => { + Ok(workspace.to_string()) } - WorkspaceMode::Operator => workspace.to_string(), + WorkspaceMode::Operator => self + .operator_workspace_namespaces + .get(workspace) + .cloned() + .ok_or_else(|| { + Status::failed_precondition(format!( + "workspace '{workspace}' has no configured operator namespace" + )) + }), } } } @@ -84,6 +94,7 @@ struct KubernetesSecretsDriverConfig { allow_reference_namespace: bool, workspace_mode: WorkspaceMode, gateway_id: Option, + operator_workspace_namespaces: BTreeMap, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -203,7 +214,7 @@ impl KubernetesSecretsCredentialDriver { reference } else { KubernetesSecretReference { - namespace: self.settings.target_namespace(&request.workspace), + namespace: self.settings.target_namespace(&request.workspace)?, secret_name: managed_secret_name( &request.workspace, &request.provider_id, @@ -538,6 +549,7 @@ impl KubernetesSecretsDriverSettings { allow_reference_namespace: config.allow_reference_namespace, workspace_mode: config.workspace_mode, gateway_id: config.gateway_id.unwrap_or_default(), + operator_workspace_namespaces: config.operator_workspace_namespaces, }) } } @@ -874,6 +886,7 @@ mod tests { allow_reference_namespace: false, workspace_mode: WorkspaceMode::Shared, gateway_id: String::new(), + operator_workspace_namespaces: BTreeMap::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -899,6 +912,7 @@ mod tests { allow_reference_namespace: true, workspace_mode: WorkspaceMode::Shared, gateway_id: String::new(), + operator_workspace_namespaces: BTreeMap::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -1107,9 +1121,10 @@ mod tests { allow_reference_namespace: false, workspace_mode: WorkspaceMode::Shared, gateway_id: String::new(), + operator_workspace_namespaces: BTreeMap::new(), }; - assert_eq!(settings.target_namespace("team-a"), "openshell"); - assert_eq!(settings.target_namespace("team-b"), "openshell"); + assert_eq!(settings.target_namespace("team-a").unwrap(), "openshell"); + assert_eq!(settings.target_namespace("team-b").unwrap(), "openshell"); } #[test] @@ -1119,9 +1134,16 @@ mod tests { allow_reference_namespace: false, workspace_mode: WorkspaceMode::Managed, gateway_id: "gw1".to_string(), + operator_workspace_namespaces: BTreeMap::new(), }; - assert_eq!(settings.target_namespace("team-a"), "openshell-gw1-team-a"); - assert_eq!(settings.target_namespace("team-b"), "openshell-gw1-team-b"); + assert_eq!( + settings.target_namespace("team-a").unwrap(), + "openshell-gw1-team-a" + ); + assert_eq!( + settings.target_namespace("team-b").unwrap(), + "openshell-gw1-team-b" + ); } #[test] @@ -1131,8 +1153,28 @@ mod tests { allow_reference_namespace: false, workspace_mode: WorkspaceMode::Operator, gateway_id: String::new(), + operator_workspace_namespaces: BTreeMap::new(), }; - assert_eq!(settings.target_namespace("team-a"), "team-a"); - assert_eq!(settings.target_namespace("prod-ns"), "prod-ns"); + assert_eq!(settings.target_namespace("team-a").unwrap(), "team-a"); + assert_eq!(settings.target_namespace("prod-ns").unwrap(), "prod-ns"); + } + + #[test] + fn target_namespace_operator_uses_explicit_mapping() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Operator, + gateway_id: String::new(), + operator_workspace_namespaces: BTreeMap::from([( + "team-a".to_string(), + "platform-team-a".to_string(), + )]), + }; + assert_eq!( + settings.target_namespace("team-a").unwrap(), + "platform-team-a" + ); + assert!(settings.target_namespace("unknown").is_err()); } } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index c985e0b776..05de561a8f 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -11,10 +11,11 @@ workspace namespace modes via `workspace_mode`: - **Managed**: The driver auto-creates/deletes a K8s namespace per workspace (`openshell-{gateway_id}-{workspace_name}`), creates a ServiceAccount in each, and copies OpenShift SCC annotations from the gateway namespace when present. -- **Operator**: Workspace names map 1:1 to pre-provisioned namespaces discovered - via label selector (`operator_namespace_label`) and/or drop-in allowlist file - (`operator_namespace_file`). Sandbox creation fails closed if the workspace - namespace is not in the current allowlist. +- **Operator**: Workspaces resolve to pre-provisioned namespaces through a label + selector (`operator_namespace_label`), drop-in allowlist file + (`operator_namespace_file`), or explicit map + (`operator_workspace_namespaces`). Sandbox creation fails closed when no + allowed namespace resolves for the workspace. ## Runtime Model diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 8064079e98..aec01a8462 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -297,6 +297,10 @@ pub struct KubernetesComputeConfig { /// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_file: Option, + /// Optional explicit workspace-to-namespace mapping for operator mode. + /// When set, workspace names do not need to match namespace names. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub operator_workspace_namespaces: BTreeMap, /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by /// the gateway's `TokenReview` bootstrap authenticator. pub service_account_name: String, @@ -413,6 +417,7 @@ impl Default for KubernetesComputeConfig { namespace: DEFAULT_K8S_NAMESPACE.to_string(), operator_namespace_label: None, operator_namespace_file: None, + operator_workspace_namespaces: BTreeMap::new(), service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), // Default empty so the gateway omits `imagePullPolicy` from pod @@ -572,12 +577,22 @@ impl KubernetesComputeConfig { WorkspaceMode::Operator => { let allowlist = operator_allowlist.ok_or("operator mode requires a namespace allowlist")?; + let namespace = if self.operator_workspace_namespaces.is_empty() { + workspace + } else { + self.operator_workspace_namespaces + .get(workspace) + .map(String::as_str) + .ok_or_else(|| { + format!("workspace '{workspace}' has no configured operator namespace") + })? + }; let namespaces = allowlist.read(); - if namespaces.contains(workspace) { - Ok(workspace.to_string()) + if namespaces.contains(namespace) { + Ok(namespace.to_string()) } else { Err(format!( - "workspace '{workspace}' is not in the operator namespace allowlist" + "workspace '{workspace}' does not resolve to an allowed operator namespace" )) } } @@ -646,16 +661,19 @@ impl KubernetesComputeConfig { Ok(()) } WorkspaceMode::Operator => { - if self.operator_namespace_label.is_none() && self.operator_namespace_file.is_none() - { + let source_count = usize::from(self.operator_namespace_label.is_some()) + + usize::from(self.operator_namespace_file.is_some()) + + usize::from(!self.operator_workspace_namespaces.is_empty()); + if source_count == 0 { return Err("operator workspace mode requires exactly one of \ - operator_namespace_label or operator_namespace_file" + operator_namespace_label, operator_namespace_file, or \ + operator_workspace_namespaces" .into()); } - if self.operator_namespace_label.is_some() && self.operator_namespace_file.is_some() - { + if source_count > 1 { return Err("operator workspace mode requires exactly one of \ - operator_namespace_label or operator_namespace_file, not both" + operator_namespace_label, operator_namespace_file, or \ + operator_workspace_namespaces" .into()); } if let Some(ref label) = self.operator_namespace_label @@ -668,6 +686,13 @@ impl KubernetesComputeConfig { { return Err("operator_namespace_file must not be empty when set".into()); } + for namespace in self.operator_workspace_namespaces.values() { + if !is_dns_1123_label(namespace) { + return Err(format!( + "operator namespace '{namespace}' is not a valid DNS-1123 label" + )); + } + } Ok(()) } } @@ -1379,6 +1404,29 @@ mod tests { ); } + #[test] + fn namespace_for_workspace_operator_uses_explicit_mapping() { + let allowlist = + OperatorNamespaceAllowlist::from_set(BTreeSet::from(["platform-team-a".to_string()])); + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_workspace_namespaces: BTreeMap::from([( + "team-a".to_string(), + "platform-team-a".to_string(), + )]), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", Some(&allowlist)) + .unwrap(), + "platform-team-a" + ); + assert!( + cfg.namespace_for_workspace("unknown", Some(&allowlist)) + .is_err() + ); + } + #[test] fn kube_resource_name_shared_prefixes_workspace() { let cfg = KubernetesComputeConfig::default(); @@ -1535,6 +1583,19 @@ mod tests { cfg.validate_workspace_mode().unwrap(); } + #[test] + fn validate_workspace_mode_operator_accepts_mapping_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_workspace_namespaces: BTreeMap::from([( + "team-a".to_string(), + "platform-team-a".to_string(), + )]), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + #[test] fn dns_1123_label_validation() { assert!(is_dns_1123_label("openshell")); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0787cba3ea..0b67671612 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -499,6 +499,16 @@ impl KubernetesComputeDriver { let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) { let allowlist = OperatorNamespaceAllowlist::new(); + if !config.operator_workspace_namespaces.is_empty() { + allowlist.replace( + config + .operator_workspace_namespaces + .values() + .cloned() + .collect(), + ); + } + if let Some(ref label) = config.operator_namespace_label { spawn_namespace_label_watcher( watch_client.clone(), @@ -1327,16 +1337,10 @@ impl KubernetesComputeDriver { self.ensure_image_pull_secrets(&namespace).await?; namespace } - WorkspaceMode::Operator => { - if let Some(ref allowlist) = self.operator_allowlist - && !allowlist.contains(workspace) - { - return Err(KubernetesDriverError::Precondition(format!( - "workspace '{workspace}' is not in the operator namespace allowlist" - ))); - } - workspace.to_string() - } + WorkspaceMode::Operator => self + .config + .namespace_for_workspace(workspace, self.operator_allowlist.as_ref()) + .map_err(KubernetesDriverError::Precondition)?, }; if self.config.is_multi_namespace() { diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 7cd883e529..6c5153dfea 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -182,6 +182,7 @@ async fn main() -> Result<()> { namespace: args.sandbox_namespace, operator_namespace_label: args.operator_namespace_label, operator_namespace_file: args.operator_namespace_file, + operator_workspace_namespaces: BTreeMap::new(), service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 71f6d96112..0c9ee58d60 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -227,6 +227,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | | server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | | server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.operatorWorkspaceNamespaces | object | `{}` | Explicit workspace-to-namespace mappings for operator mode. | | server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 57518e3fda..e1373866a8 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -137,6 +137,9 @@ data: {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} {{- end }} + {{- with .Values.server.drivers.kubernetes.operatorWorkspaceNamespaces }} + operator_workspace_namespaces = { {{- range $index, $workspace := keys . | sortAlpha }}{{ if $index }}, {{ end }}{{ $workspace | quote }} = {{ index $.Values.server.drivers.kubernetes.operatorWorkspaceNamespaces $workspace | quote }}{{- end }} } + {{- end }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} @@ -193,6 +196,9 @@ data: allow_reference_namespace = {{ .Values.server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} + {{- with .Values.server.drivers.kubernetes.operatorWorkspaceNamespaces }} + operator_workspace_namespaces = { {{- range $index, $workspace := keys . | sortAlpha }}{{ if $index }}, {{ end }}{{ $workspace | quote }} = {{ index $.Values.server.drivers.kubernetes.operatorWorkspaceNamespaces $workspace | quote }}{{- end }} } + {{- end }} {{- end }} {{- if .Values.server.credentialDrivers.vault.enabled }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index d77b408efe..a542003783 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -508,3 +508,13 @@ tests: - matchRegex: path: spec.template.spec.volumes[1].name pattern: '^sandbox-jwt$' + + - it: renders operator workspace namespace mappings + template: templates/gateway-config.yaml + set: + server.drivers.kubernetes.workspaceMode: operator + server.drivers.kubernetes.operatorWorkspaceNamespaces.team-a: platform-team-a + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'operator_workspace_namespaces\s*=\s*\{"team-a"\s*=\s*"platform-team-a"\s*\}' diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 0eebee810c..17cfc8817c 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -237,6 +237,8 @@ server: # -- Path to a JSON file containing an array of namespace names # allowed in operator mode. Hot-reloaded on change. operatorNamespaceFile: "" + # -- Explicit workspace-to-namespace mappings for operator mode. + operatorWorkspaceNamespaces: {} # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. disableTls: false diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e8f39f1a6f..ce626a8b26 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -464,13 +464,15 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # back to 1000 on non-OpenShift clusters. # sandbox_uid = 1500 # sandbox_gid = 1500 -# Operator-mode namespace discovery. At least one must be set when -# workspace_mode = "operator". Both can be combined. +# Operator-mode namespace selection. Exactly one source must be set when +# workspace_mode = "operator". # operator_namespace_label discovers namespaces matching a K8s label selector. # operator_namespace_label = "openshell.ai/workspace=true" # operator_namespace_file reads allowed namespaces from a JSON/YAML file # (hot-reloaded on change, e.g. via ConfigMap volume mount). # operator_namespace_file = "/etc/openshell/workspace-namespaces.json" +# Explicit mapping for platforms where workspace and namespace names differ. +# operator_workspace_namespaces = { "team-a" = "platform-team-a" } [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = true