diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 3d68746b82..37154c75df 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -182,6 +182,34 @@ jobs: extra-helm-values: ${{ matrix.extra_helm_values }} cli-artifact-prefix: rust-binary-cli + kubernetes-workspace-managed-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (workspace managed mode) + e2e-task: e2e:kubernetes:workspace-managed + cli-artifact-prefix: rust-binary-cli + + kubernetes-workspace-operator-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (workspace operator mode) + e2e-task: e2e:kubernetes:workspace-operator + cli-artifact-prefix: rust-binary-cli + kubernetes-ha-e2e: needs: [pr_metadata, build-gateway, build-supervisor, build-cli] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' @@ -212,7 +240,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -224,6 +252,8 @@ jobs: BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} E2E_RESULT: ${{ needs.e2e.result }} KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} + KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }} + KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }} run: | set -euo pipefail failed=0 @@ -233,7 +263,9 @@ jobs: "build-cli:$BUILD_CLI_RESULT" \ "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ "e2e:$E2E_RESULT" \ - "kubernetes-e2e:$KUBERNETES_E2E_RESULT"; do + "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ + "kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \ + "kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do name="${item%%:*}" result="${item#*:}" if [ "$result" != "success" ]; then diff --git a/Cargo.lock b/Cargo.lock index 37b9140600..6b1430934c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3777,6 +3777,7 @@ dependencies = [ "kube", "kube-runtime", "miette", + "notify", "openshell-core", "openshell-policy", "prost", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e4224232dd..42c53abc6a 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -309,5 +309,99 @@ Standalone local deployments start the gateway with a selected runtime such as Docker, Podman, or VM. The CLI can register multiple gateways and switch between them without changing the sandbox architecture. +## Workspace Namespace Modes (Kubernetes) + +The Kubernetes driver maps workspaces to namespaces through the `workspace_mode` +configuration field (`WorkspaceMode` in `crates/openshell-driver-kubernetes/src/config.rs`). +The mode controls namespace resolution, resource naming, sandbox CR watching, SA +token authentication, and RBAC requirements. + +| Mode | Namespace resolution | Resource name | Namespace lifecycle | +|---|---|---|---| +| **Shared** (default) | Single static namespace from config | `{workspace}--{name}` | None | +| **Managed** | `openshell-{gateway_id}-{workspace}` | bare sandbox name | Driver creates and deletes | +| **Operator** | Workspace name maps 1:1 to a pre-provisioned namespace | bare sandbox name | External (platform team) | + +**Shared** renders all sandboxes into one configured namespace. Resource names +embed the workspace prefix for collision avoidance. No namespace lifecycle +management. RBAC uses a namespace-scoped Role. + +**Managed** auto-creates a K8s namespace per workspace on first sandbox create. +Each new namespace receives a ServiceAccount and the configured gateway-only +SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the +driver's source namespace on every sandbox create so registry credential +rotations propagate. The namespace also copies OpenShift SCC UID-range and +supplemental-group annotations from the gateway namespace when present. The +driver deletes the namespace during workspace deletion. The workspace remains +durably `Terminating` until the Kubernetes API accepts namespace cleanup, so a +transient failure can be retried. Namespace deletion uses the fetched UID as a +precondition to avoid deleting a replacement namespace. Requires a non-empty +`gateway_id` (validated as a +DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character +limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace +`create`/`delete` and ServiceAccount `create`/`get` permissions. + +Secret copies use server-side apply. Kubernetes authorizes an apply to an +existing Secret as `patch`, but also requires `create` authorization when the +target does not exist. RBAC cannot constrain `create` by `resourceNames`, so +managed mode grants cluster-wide Secret `create` while keeping source reads and +subsequent patches restricted to the explicitly configured TLS and image-pull +Secret names. The driver exercises `create` only in gateway-owned managed +namespaces. This depends on the managed-mode ownership invariant described +below; the gateway ServiceAccount must not be shared with unrelated workloads. + +Operator mode does not create NetworkPolicies or copy image-pull Secrets. +Platform teams must apply the gateway ingress boundary and provision configured +image-pull Secrets in every operator-managed namespace. + +**Operator** uses pre-provisioned namespaces discovered through two optional +sources: a K8s label selector (`operator_namespace_label`) and a drop-in +allowlist file (`operator_namespace_file`). At least one must be configured. +The `OperatorNamespaceAllowlist` (`Arc>>`) is populated +at runtime by background watchers and read by the namespace resolver. Sandbox +creation fails closed if the workspace is not in the current allowlist. Platform +teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as +managed mode but without namespace `create`/`delete` or ServiceAccount +permissions. + +### Watching and Querying + +Managed and operator modes set `is_multi_namespace() == true`, which switches +sandbox CR watchers from namespace-scoped `Api::namespaced` to cluster-wide +`Api::all_with`. In managed mode the driver scopes cluster-wide queries with a +`LABEL_GATEWAY_ID` label selector to support multiple gateways on the same +cluster. K8s Events are not watched in cluster-wide mode — the cluster-wide +watcher emits only sandbox CR changes, not platform events. + +### SA Token Authentication + +The gateway's `K8sServiceAccountAuthenticator` adapts its `NamespaceValidator` +per mode (`crates/openshell-server/src/auth/k8s_sa.rs`): + +- **Shared:** `Exact` — accepts only the single configured namespace. +- **Managed:** `Prefix` — accepts any namespace starting with `openshell-{gateway_id}-`. +- **Operator:** `Allowlist` — accepts namespaces present in the dynamic + `BTreeSet` populated by the label/file watchers. Starts empty (fail-closed) + until the first watcher update. + +These checks rely on an ownership invariant. In shared and managed modes, the +gateway and its trusted Agent Sandbox controller exclusively administer the +sandbox namespace, Sandbox CRs, sandbox pods, and configured sandbox +ServiceAccount. Other principals must not create or mutate those resources or +use that ServiceAccount. In operator mode, the platform operator retains +namespace lifecycle ownership, but must preserve the same exclusive control of +Sandbox CRs and the pods and ServiceAccount used for sandbox token bootstrap. +An allowlisted namespace is therefore a trust grant, not a tenant isolation +boundary. Kubernetes owner references alone do not prove which controller +created a pod, so admitting principals that can fabricate that resource chain +would allow them to claim an existing sandbox identity. + +### Credential Driver Integration + +The Kubernetes Secrets credential driver (`openshell-driver-kubernetes-secrets`) +stores secrets in workspace-specific namespaces when `workspace_mode` is managed +or operator. In shared mode, all secrets render into the single configured +namespace. + When runtime infrastructure changes, validate the relevant sandbox e2e path and update the matching driver README if a maintainer-facing constraint changes. diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 79f2eed3aa..be0d8beb36 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -35,6 +35,9 @@ pub const LABEL_SANDBOX_NAMESPACE: &str = "openshell.ai/sandbox-namespace"; /// Container/pod label carrying the sandbox workspace. pub const LABEL_SANDBOX_WORKSPACE: &str = "openshell.ai/sandbox-workspace"; +/// Label carrying the gateway identity on managed namespaces. +pub const LABEL_GATEWAY_ID: &str = "openshell.ai/gateway-id"; + /// Label selector that matches all OpenShell-managed resources which carry a /// sandbox ID label. Used by list and watch operations to exclude foreign /// resources from the same namespace. diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 1640fd6cf7..80ab0435e1 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -146,6 +146,10 @@ async fn build_plain_channel(endpoint: &str) -> Result { let tls_enabled = endpoint.starts_with("https://"); + // TODO: TLS certs are loaded once here and never re-read. The gateway + // server side supports hot-reload (ArcSwap + notify in tls.rs). The + // supervisor should do the same so that cert-manager rotations take + // effect without restarting the sandbox. if tls_enabled { let ca_path = std::env::var(sandbox_env::TLS_CA) .into_diagnostic() diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 4621852fc9..9ac14c1871 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -38,8 +38,9 @@ use openshell_core::progress::{ }; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, + DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, @@ -1761,6 +1762,20 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } impl DockerProvisioningFailure { diff --git a/crates/openshell-driver-kubernetes-secrets/src/lib.rs b/crates/openshell-driver-kubernetes-secrets/src/lib.rs index 65c655be16..91224212bd 100644 --- a/crates/openshell-driver-kubernetes-secrets/src/lib.rs +++ b/crates/openshell-driver-kubernetes-secrets/src/lib.rs @@ -48,10 +48,33 @@ impl CredentialDriverService { } } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +enum WorkspaceMode { + #[default] + Shared, + Managed, + Operator, +} + #[derive(Debug, Clone, PartialEq, Eq)] struct KubernetesSecretsDriverSettings { namespace: String, allow_reference_namespace: bool, + workspace_mode: WorkspaceMode, + gateway_id: String, +} + +impl KubernetesSecretsDriverSettings { + fn target_namespace(&self, workspace: &str) -> String { + match self.workspace_mode { + WorkspaceMode::Shared => self.namespace.clone(), + WorkspaceMode::Managed => { + format!("openshell-{}-{}", self.gateway_id, workspace) + } + WorkspaceMode::Operator => workspace.to_string(), + } + } } #[derive(Debug, Clone, Default, serde::Deserialize)] @@ -59,6 +82,8 @@ struct KubernetesSecretsDriverSettings { struct KubernetesSecretsDriverConfig { namespace: Option, allow_reference_namespace: bool, + workspace_mode: WorkspaceMode, + gateway_id: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -134,7 +159,10 @@ impl KubernetesSecretsCredentialDriver { credential_key: &str, ) -> Result { let reference = Self::parse_handle(handle, credential_key)?; - if reference.namespace != self.settings.namespace + // In managed/operator modes secrets live in workspace-specific + // namespaces so cross-namespace handles are expected. + if self.settings.workspace_mode == WorkspaceMode::Shared + && reference.namespace != self.settings.namespace && !self.settings.allow_reference_namespace { return Err(Status::permission_denied(format!( @@ -175,7 +203,7 @@ impl KubernetesSecretsCredentialDriver { reference } else { KubernetesSecretReference { - namespace: self.settings.namespace.clone(), + namespace: self.settings.target_namespace(&request.workspace), secret_name: managed_secret_name( &request.workspace, &request.provider_id, @@ -504,10 +532,29 @@ impl KubernetesSecretsDriverSettings { } None => default_namespace(), }; + let gateway_id = config.gateway_id.unwrap_or_default(); + if config.workspace_mode == WorkspaceMode::Managed { + let gateway_id = trimmed_config_string("gateway_id", &gateway_id)?; + if !is_dns_label(gateway_id) { + return Err(Error::config( + "[openshell.credential_drivers.kubernetes-secrets] gateway_id must be a DNS-1123 label in managed workspace mode", + )); + } + // Workspace names are limited to 19 characters by the gateway. + // Keep the longest generated namespace within Kubernetes' 63-char + // DNS label limit, matching the Kubernetes compute driver. + if "openshell-".len() + gateway_id.len() + 1 + 19 > 63 { + return Err(Error::config( + "[openshell.credential_drivers.kubernetes-secrets] gateway_id is too long for managed workspace mode", + )); + } + } Ok(Self { namespace, allow_reference_namespace: config.allow_reference_namespace, + workspace_mode: config.workspace_mode, + gateway_id, }) } } @@ -778,6 +825,50 @@ mod tests { assert!(err.to_string().contains("namespace")); } + #[test] + fn settings_managed_mode_requires_gateway_id() { + let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + }) + .unwrap_err(); + + assert!(err.to_string().contains("gateway_id")); + } + + #[test] + fn settings_managed_mode_rejects_invalid_gateway_id() { + let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + gateway_id = "Invalid_Gateway" + }) + .unwrap_err(); + + assert!(err.to_string().contains("DNS-1123")); + } + + #[test] + fn settings_managed_mode_rejects_gateway_id_that_is_too_long() { + let gateway_id = "a".repeat(35); + let err = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + gateway_id = gateway_id + }) + .unwrap_err(); + + assert!(err.to_string().contains("too long")); + } + + #[test] + fn settings_managed_mode_accepts_valid_gateway_id() { + let settings = KubernetesSecretsDriverSettings::from_table(&toml::toml! { + workspace_mode = "managed" + gateway_id = "gateway-1" + }) + .unwrap(); + + assert_eq!(settings.gateway_id, "gateway-1"); + } + #[test] fn handle_resolves_secret_reference() { let reference = KubernetesSecretsCredentialDriver::parse_handle( @@ -842,6 +933,8 @@ mod tests { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -865,6 +958,8 @@ mod tests { let settings = KubernetesSecretsDriverSettings { namespace: "openshell".to_string(), allow_reference_namespace: true, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), }; let reference = KubernetesSecretsCredentialDriver::parse_handle( &handle("v1:other-namespace:provider-secret"), @@ -1065,4 +1160,40 @@ mod tests { assert_eq!(err.code(), Code::FailedPrecondition); assert!(err.message().contains("is not managed by OpenShell")); } + + #[test] + fn target_namespace_shared_returns_static_namespace() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Shared, + gateway_id: String::new(), + }; + assert_eq!(settings.target_namespace("team-a"), "openshell"); + assert_eq!(settings.target_namespace("team-b"), "openshell"); + } + + #[test] + fn target_namespace_managed_computes_from_workspace() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Managed, + gateway_id: "gw1".to_string(), + }; + assert_eq!(settings.target_namespace("team-a"), "openshell-gw1-team-a"); + assert_eq!(settings.target_namespace("team-b"), "openshell-gw1-team-b"); + } + + #[test] + fn target_namespace_operator_uses_workspace_name() { + let settings = KubernetesSecretsDriverSettings { + namespace: "openshell".to_string(), + allow_reference_namespace: false, + workspace_mode: WorkspaceMode::Operator, + gateway_id: String::new(), + }; + assert_eq!(settings.target_namespace("team-a"), "team-a"); + assert_eq!(settings.target_namespace("prod-ns"), "prod-ns"); + } } diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index f9f5bba398..714b7d05c9 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } +notify = "8" [dev-dependencies] temp-env = "0.3" diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index d9c0e17fd8..ec37c4e67a 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -3,8 +3,31 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox -custom resources in the configured namespace. It runs in-process with the -gateway server. +custom resources. It runs in-process with the gateway server and supports three +workspace namespace modes via `workspace_mode`: + +- **Shared** (default): All sandboxes render into a single static namespace. + Resource names use `{workspace}--{name}` for collision avoidance. +- **Managed**: The driver auto-creates/deletes a K8s namespace per workspace + (`openshell-{gateway_id}-{workspace_name}`), creates a ServiceAccount in each, + and copies OpenShift SCC annotations from the gateway namespace when present. +- **Operator**: Workspace names map 1:1 to pre-provisioned namespaces discovered + through exactly one source: either a label selector + (`operator_namespace_label`) or a drop-in allowlist file + (`operator_namespace_file`). Sandbox creation fails closed if the workspace + namespace is not in the current allowlist. Workspace deletion only removes + gateway state; it never deletes or otherwise accesses the operator-managed + Kubernetes namespace. + +Workspace namespace modes assume exclusive control of the sandbox identity +resource chain. In shared and managed modes, only the gateway and its trusted +Agent Sandbox controller may administer the sandbox namespace, Sandbox CRs, +sandbox pods, or configured sandbox ServiceAccount. In operator mode, the +platform operator owns namespace lifecycle but must prevent other principals +from creating or mutating Sandbox CRs, creating sandbox pods with fabricated +owner references, or using the configured sandbox ServiceAccount. Treat adding +a namespace to the operator allowlist as granting this trust; the allowlist is +not a tenant isolation boundary. ## Runtime Model diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 4ca02bd71c..bfa08a6642 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -3,8 +3,13 @@ use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::str::FromStr; +use std::sync::{Arc, RwLock}; + +/// Default gateway identity used in managed-mode namespace naming. +pub const DEFAULT_GATEWAY_ID: &str = "openshell"; /// Default Kubernetes namespace for sandbox resources. pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; @@ -88,6 +93,48 @@ impl FromStr for SupervisorTopology { } } +/// How workspaces map to Kubernetes namespaces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WorkspaceMode { + /// All sandboxes render into a single statically-configured namespace. + /// Resource names use `{workspace}--{name}` for collision avoidance. + #[default] + Shared, + /// The driver creates and deletes K8s namespaces on demand using the + /// convention `openshell-{gateway_id}-{workspace_name}`. + Managed, + /// Sandboxes render into pre-existing K8s namespaces. The driver has no + /// namespace create/delete permissions. Platform teams manage namespaces + /// via their existing tooling. + Operator, +} + +impl std::fmt::Display for WorkspaceMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Shared => f.write_str("shared"), + Self::Managed => f.write_str("managed"), + Self::Operator => f.write_str("operator"), + } + } +} + +impl FromStr for WorkspaceMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "shared" => Ok(Self::Shared), + "managed" => Ok(Self::Managed), + "operator" => Ok(Self::Operator), + other => Err(format!( + "unknown workspace mode '{other}'; expected 'shared', 'managed', or 'operator'" + )), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSidecarConfig { @@ -232,7 +279,24 @@ where #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { + /// How workspaces map to Kubernetes namespaces. `"shared"` (default) + /// renders all sandboxes into `namespace`; `"managed"` creates per-workspace + /// namespaces on demand; `"operator"` uses pre-provisioned namespaces. + pub workspace_mode: WorkspaceMode, + /// Stable gateway identity used in managed-mode namespace naming + /// (`openshell-{gateway_id}-{workspace}`). Propagated from + /// `gateway_jwt.gateway_id`. + pub gateway_id: String, pub namespace: String, + /// K8s label selector for operator-mode namespace discovery (e.g., + /// `"openshell.ai/workspace=true"`). The driver watches namespaces matching + /// this label and builds the allowlist dynamically. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_namespace_label: Option, + /// Path to a JSON file containing an array of namespace names allowed in + /// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_namespace_file: Option, /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by /// the gateway's `TokenReview` bootstrap authenticator. pub service_account_name: String, @@ -240,6 +304,10 @@ pub struct KubernetesComputeConfig { pub image_pull_policy: String, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, + /// Managed-mode SSH ingress isolation. When enabled, the driver creates a + /// `NetworkPolicy` in each managed workspace namespace that permits TCP 2222 + /// only from gateway pods matching this peer. + pub managed_ssh_ingress: ManagedSshIngressConfig, /// Image that provides the `openshell-sandbox` supervisor binary. /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. @@ -328,6 +396,14 @@ pub struct KubernetesComputeConfig { pub sandbox_gid: Option, } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ManagedSshIngressConfig { + pub enabled: bool, + pub gateway_namespace: String, + pub gateway_pod_selector: BTreeMap, +} + /// Lower bound enforced by kubelet for projected SA tokens. pub const MIN_SA_TOKEN_TTL_SECS: i64 = 600; @@ -351,7 +427,11 @@ pub const ANNOTATION_SCC_SUPPLEMENTAL_GROUPS: &str = "openshift.io/sa.scc.supple impl Default for KubernetesComputeConfig { fn default() -> Self { Self { + workspace_mode: WorkspaceMode::default(), + gateway_id: DEFAULT_GATEWAY_ID.to_string(), namespace: DEFAULT_K8S_NAMESPACE.to_string(), + operator_namespace_label: None, + operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), // Default empty so the gateway omits `imagePullPolicy` from pod @@ -360,6 +440,7 @@ impl Default for KubernetesComputeConfig { // is Podman vocabulary and is not a valid Kubernetes value. image_pull_policy: String::new(), image_pull_secrets: Vec::new(), + managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), @@ -525,7 +606,7 @@ impl KubernetesComputeConfig { /// 3. Fallback defaults: UID=`1000`, GID=UID pub fn resolve_sandbox_uid( &self, - namespace_annotations: Option<&std::collections::BTreeMap>, + namespace_annotations: Option<&BTreeMap>, ) -> u32 { if let Some(uid) = self.sandbox_uid { return uid; @@ -543,7 +624,7 @@ impl KubernetesComputeConfig { pub fn resolve_sandbox_gid( &self, resolved_uid: u32, - _namespace_annotations: Option<&std::collections::BTreeMap>, + _namespace_annotations: Option<&BTreeMap>, ) -> u32 { self.sandbox_gid .or(self.sandbox_uid) @@ -593,6 +674,251 @@ impl KubernetesComputeConfig { } Ok(()) } + + /// Resolve the K8s namespace for a workspace. + /// + /// - **Shared:** returns the static `namespace` config field. + /// - **Managed:** computes `openshell-{gateway_id}-{workspace_name}`. + /// - **Operator:** looks up `workspace` in the dynamic allowlist. Fails + /// closed if the workspace is not found. + pub fn namespace_for_workspace( + &self, + workspace: &str, + operator_allowlist: Option<&OperatorNamespaceAllowlist>, + ) -> Result { + match self.workspace_mode { + WorkspaceMode::Shared => Ok(self.namespace.clone()), + WorkspaceMode::Managed => Ok(managed_namespace(&self.gateway_id, workspace)), + WorkspaceMode::Operator => { + let allowlist = + operator_allowlist.ok_or("operator mode requires a namespace allowlist")?; + let namespaces = allowlist.read(); + if namespaces.contains(workspace) { + Ok(workspace.to_string()) + } else { + Err(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + )) + } + } + } + } + + /// Whether the driver operates across multiple namespaces. + #[must_use] + pub fn is_multi_namespace(&self) -> bool { + !matches!(self.workspace_mode, WorkspaceMode::Shared) + } + + /// Compute the K8s resource name for a sandbox. + /// + /// - **Shared:** `{workspace}--{name}` (namespace doesn't provide isolation). + /// - **Managed/Operator:** bare sandbox name (namespace provides isolation). + #[must_use] + pub fn kube_resource_name(&self, workspace: &str, name: &str) -> String { + match self.workspace_mode { + WorkspaceMode::Shared => format!("{workspace}--{name}"), + WorkspaceMode::Managed | WorkspaceMode::Operator => name.to_string(), + } + } + + /// Validate workspace-mode-specific configuration at startup. + pub fn validate_workspace_mode(&self) -> Result<(), String> { + match self.workspace_mode { + WorkspaceMode::Shared => Ok(()), + WorkspaceMode::Managed => { + if self.gateway_id.is_empty() { + return Err("managed workspace mode requires a non-empty gateway_id".into()); + } + if !is_dns_1123_label(&self.gateway_id) { + return Err(format!( + "gateway_id '{}' is not a valid DNS-1123 label", + self.gateway_id + )); + } + // Workspace names can be up to 19 chars (MAX_ROUTABLE_NAME_LEN + // in the server crate). The managed namespace prefix + + // workspace must fit within 63 chars. + let prefix = managed_namespace_prefix(&self.gateway_id); + if prefix.len() + 19 > 63 { + return Err(format!( + "gateway_id '{}' is too long for managed mode; \ + the namespace prefix '{}' ({} chars) plus the \ + maximum workspace name (19 chars) exceeds the \ + 63-char K8s namespace limit", + self.gateway_id, + prefix, + prefix.len() + )); + } + if self.managed_ssh_ingress.enabled { + if self.managed_ssh_ingress.gateway_namespace.is_empty() { + return Err( + "managed SSH ingress isolation requires gateway_namespace".into() + ); + } + if self.managed_ssh_ingress.gateway_pod_selector.is_empty() { + return Err( + "managed SSH ingress isolation requires gateway_pod_selector".into(), + ); + } + } + Ok(()) + } + WorkspaceMode::Operator => { + if self.operator_namespace_label.is_none() && self.operator_namespace_file.is_none() + { + return Err("operator workspace mode requires exactly one of \ + operator_namespace_label or operator_namespace_file" + .into()); + } + if self.operator_namespace_label.is_some() && self.operator_namespace_file.is_some() + { + return Err("operator workspace mode requires exactly one of \ + operator_namespace_label or operator_namespace_file, not both" + .into()); + } + if let Some(ref label) = self.operator_namespace_label + && label.is_empty() + { + return Err("operator_namespace_label must not be empty when set".into()); + } + if let Some(ref file) = self.operator_namespace_file + && file.is_empty() + { + return Err("operator_namespace_file must not be empty when set".into()); + } + Ok(()) + } + } + } +} + +/// Compute the managed-mode namespace name for a workspace. +#[must_use] +pub fn managed_namespace(gateway_id: &str, workspace: &str) -> String { + format!("openshell-{gateway_id}-{workspace}") +} + +/// The managed-mode namespace prefix used for SA token validation. +#[must_use] +pub fn managed_namespace_prefix(gateway_id: &str) -> String { + format!("openshell-{gateway_id}-") +} + +/// Check whether a string is a valid DNS-1123 label (lowercase alphanumeric +/// and hyphens, 1-63 chars, must start and end with alphanumeric). +#[must_use] +pub fn is_dns_1123_label(s: &str) -> bool { + let len = s.len(); + if len == 0 || len > 63 { + return false; + } + let bytes = s.as_bytes(); + if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() { + return false; + } + if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() { + return false; + } + bytes + .iter() + .all(|&b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Validate that a workspace name produces a valid K8s namespace name in +/// managed mode (combined length <= 63, DNS-1123 compliant). +pub fn validate_managed_namespace_name(gateway_id: &str, workspace: &str) -> Result<(), String> { + let ns = managed_namespace(gateway_id, workspace); + if !is_dns_1123_label(&ns) { + return Err(format!( + "managed namespace '{ns}' (from workspace '{workspace}') is not a valid DNS-1123 label" + )); + } + Ok(()) +} + +/// Thread-safe dynamic allowlist of valid operator-mode namespaces. +/// +/// Backed by an `Arc>>` that is updated by background +/// tasks (label selector watcher, drop-in file watcher) and read by the SA +/// authenticator and namespace resolver. +#[derive(Debug, Clone)] +pub struct OperatorNamespaceAllowlist { + inner: Arc>>, +} + +impl OperatorNamespaceAllowlist { + fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet> { + self.inner + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(BTreeSet::new())), + } + } + + #[must_use] + pub fn from_set(set: BTreeSet) -> Self { + Self { + inner: Arc::new(RwLock::new(set)), + } + } + + /// Replace the entire allowlist (used by background watchers on refresh). + pub fn replace(&self, new_set: BTreeSet) { + let mut guard = self.write_guard(); + *guard = new_set; + } + + /// Merge additional namespaces into the allowlist. + pub fn merge(&self, additional: &BTreeSet) { + let mut guard = self.write_guard(); + guard.extend(additional.iter().cloned()); + } + + /// Read the current allowlist snapshot. + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.read_guard() + } + + /// Check whether a namespace is in the allowlist. + #[must_use] + pub fn contains(&self, namespace: &str) -> bool { + self.read_guard().contains(namespace) + } + + /// Insert a namespace into the allowlist. Returns `true` if it was new. + pub fn insert(&self, name: String) -> bool { + self.write_guard().insert(name) + } + + /// Remove a namespace from the allowlist. Returns `true` if it was present. + pub fn remove(&self, name: &str) -> bool { + self.write_guard().remove(name) + } + + /// Return a clone of the inner `Arc` for sharing with background tasks. + #[must_use] + pub fn shared(&self) -> Arc>> { + Arc::clone(&self.inner) + } +} + +impl Default for OperatorNamespaceAllowlist { + fn default() -> Self { + Self::new() + } } fn is_dns1123_subdomain(value: &str) -> bool { @@ -1176,6 +1502,46 @@ mod tests { } } + // -- WorkspaceMode tests -- + + #[test] + fn default_workspace_mode_is_shared() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Shared); + } + + #[test] + fn serde_override_workspace_mode_managed() { + let json = serde_json::json!({ "workspace_mode": "managed" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Managed); + } + + #[test] + fn serde_override_workspace_mode_operator() { + let json = serde_json::json!({ "workspace_mode": "operator" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_mode, WorkspaceMode::Operator); + } + + #[test] + fn serde_rejects_invalid_workspace_mode() { + let json = serde_json::json!({ "workspace_mode": "invalid" }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + + #[test] + fn workspace_mode_display_roundtrips() { + for mode in [ + WorkspaceMode::Shared, + WorkspaceMode::Managed, + WorkspaceMode::Operator, + ] { + assert_eq!(mode.to_string().parse::().unwrap(), mode); + } + } + #[test] fn upstream_proxy_config_rejects_unsupported_proxy_scheme() { let cfg = KubernetesComputeConfig { @@ -1261,4 +1627,306 @@ mod tests { }; assert!(cfg.validate_upstream_proxy_config().is_ok()); } + + #[test] + fn namespace_for_workspace_shared() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Shared, + namespace: "sandbox-ns".to_string(), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", None).unwrap(), + "sandbox-ns" + ); + assert_eq!( + cfg.namespace_for_workspace("team-b", None).unwrap(), + "sandbox-ns" + ); + } + + #[test] + fn namespace_for_workspace_managed() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "gw1".to_string(), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("team-a", None).unwrap(), + "openshell-gw1-team-a" + ); + } + + #[test] + fn namespace_for_workspace_operator() { + let allowlist = OperatorNamespaceAllowlist::from_set(BTreeSet::from(["prod".to_string()])); + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + assert_eq!( + cfg.namespace_for_workspace("prod", Some(&allowlist)) + .unwrap(), + "prod" + ); + assert!( + cfg.namespace_for_workspace("unknown", Some(&allowlist)) + .is_err() + ); + } + + #[test] + fn namespace_for_workspace_operator_requires_allowlist() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + + let err = cfg.namespace_for_workspace("prod", None).unwrap_err(); + assert_eq!(err, "operator mode requires a namespace allowlist"); + } + + #[test] + fn kube_resource_name_shared_prefixes_workspace() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.kube_resource_name("ws", "box1"), "ws--box1"); + } + + #[test] + fn kube_resource_name_managed_uses_bare_name() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + ..KubernetesComputeConfig::default() + }; + assert_eq!(cfg.kube_resource_name("ws", "box1"), "box1"); + } + + #[test] + fn kube_resource_name_operator_uses_bare_name() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("x=y".to_string()), + ..KubernetesComputeConfig::default() + }; + assert_eq!(cfg.kube_resource_name("ws", "box1"), "box1"); + } + + #[test] + fn is_multi_namespace() { + assert!(!KubernetesComputeConfig::default().is_multi_namespace()); + assert!( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + ..KubernetesComputeConfig::default() + } + .is_multi_namespace() + ); + assert!( + KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("x=y".to_string()), + ..KubernetesComputeConfig::default() + } + .is_multi_namespace() + ); + } + + #[test] + fn validate_workspace_mode_shared_always_ok() { + let cfg = KubernetesComputeConfig::default(); + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_managed_requires_gateway_id() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: String::new(), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_managed_rejects_invalid_gateway_id() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "INVALID".to_string(), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_managed_rejects_long_gateway_id() { + // prefix = "openshell-{id}-" = 11 + id.len() + // 11 + 34 + 19 = 64 > 63 → rejected + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "a".repeat(34), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("too long for managed mode"), "{err}"); + } + + #[test] + fn validate_workspace_mode_managed_accepts_max_gateway_id() { + // 11 + 33 + 19 = 63 → accepted + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + gateway_id: "a".repeat(33), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_managed_requires_complete_ssh_ingress_peer() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway".to_string(), + gateway_pod_selector: BTreeMap::new(), + }, + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("gateway_pod_selector"), "{err}"); + } + + #[test] + fn validate_workspace_mode_managed_accepts_complete_ssh_ingress_peer() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Managed, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway".to_string(), + gateway_pod_selector: BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )]), + }, + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_requires_discovery() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_workspace_mode().is_err()); + } + + #[test] + fn validate_workspace_mode_operator_accepts_label_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_accepts_file_only() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_file: Some("/etc/openshell/namespaces.json".to_string()), + ..KubernetesComputeConfig::default() + }; + cfg.validate_workspace_mode().unwrap(); + } + + #[test] + fn validate_workspace_mode_operator_rejects_label_and_file() { + let cfg = KubernetesComputeConfig { + workspace_mode: WorkspaceMode::Operator, + operator_namespace_label: Some("openshell.ai/workspace=true".to_string()), + operator_namespace_file: Some("/etc/openshell/namespaces.json".to_string()), + ..KubernetesComputeConfig::default() + }; + + let err = cfg.validate_workspace_mode().unwrap_err(); + assert!(err.contains("not both"), "{err}"); + } + + #[test] + fn dns_1123_label_validation() { + assert!(is_dns_1123_label("openshell")); + assert!(is_dns_1123_label("my-gateway-1")); + assert!(is_dns_1123_label("a")); + assert!(!is_dns_1123_label("")); + assert!(!is_dns_1123_label("UPPER")); + assert!(!is_dns_1123_label("-starts-with-dash")); + assert!(!is_dns_1123_label("ends-with-dash-")); + assert!(!is_dns_1123_label("has_underscore")); + assert!(!is_dns_1123_label(&"a".repeat(64))); + } + + #[test] + fn managed_namespace_naming() { + assert_eq!( + managed_namespace("openshell", "default"), + "openshell-openshell-default" + ); + assert_eq!(managed_namespace("gw1", "team-a"), "openshell-gw1-team-a"); + } + + #[test] + fn validate_managed_namespace_name_accepts_valid() { + validate_managed_namespace_name("gw1", "team-a").unwrap(); + } + + #[test] + fn validate_managed_namespace_name_rejects_invalid_workspace_characters() { + let err = validate_managed_namespace_name("gw1", "INVALID").unwrap_err(); + assert!(err.contains("not a valid DNS-1123 label")); + } + + #[test] + fn validate_managed_namespace_name_rejects_too_long() { + let long_workspace = "a".repeat(50); + assert!(validate_managed_namespace_name("openshell", &long_workspace).is_err()); + } + + #[test] + fn operator_allowlist_operations() { + let al = OperatorNamespaceAllowlist::new(); + assert!(!al.contains("ns1")); + + al.replace(BTreeSet::from(["ns1".to_string(), "ns2".to_string()])); + assert!(al.contains("ns1")); + assert!(al.contains("ns2")); + assert!(!al.contains("ns3")); + + al.merge(&BTreeSet::from(["ns3".to_string()])); + assert!(al.contains("ns3")); + + al.replace(BTreeSet::new()); + assert!(!al.contains("ns1")); + } + + #[test] + fn operator_allowlist_recovers_from_poisoned_lock() { + let al = OperatorNamespaceAllowlist::from_set(BTreeSet::from(["ns1".to_string()])); + let shared = al.shared(); + let _ = std::thread::spawn(move || { + let _guard = shared.write().unwrap(); + panic!("poison allowlist lock"); + }) + .join(); + + assert!(al.contains("ns1")); + assert!(al.insert("ns2".to_string())); + assert!(al.read().contains("ns2")); + assert!(al.remove("ns1")); + } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0965c3676e..ddc7fe2a4f 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -6,14 +6,21 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, - SupervisorTopology, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, + managed_namespace, validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{ - Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Volume, - VolumeMount, + Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, + ServiceAccount, Volume, VolumeMount, }; +use k8s_openapi::api::networking::v1::{ + NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, + NetworkPolicySpec, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use kube::api::{ Api, ApiResource, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions, }; @@ -23,8 +30,9 @@ use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, openshell_sandbox_label_selector, + LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, + LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, + openshell_sandbox_label_selector, }; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::progress::{ @@ -42,10 +50,10 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use serde::Deserialize; use std::collections::{BTreeMap, HashSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use tokio::sync::{OnceCell, mpsc}; use tokio_stream::wrappers::ReceiverStream; use tracing::{debug, info, warn}; @@ -53,10 +61,14 @@ use tracing::{debug, info, warn}; pub type WatchStream = Pin> + Send>>; +const MANAGED_SSH_NETWORK_POLICY_NAME: &str = "openshell-sandbox-ssh"; + #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { #[error("sandbox already exists")] AlreadyExists, + #[error("sandbox not found")] + NotFound, #[error("{0}")] InvalidArgument(String), #[error("{0}")] @@ -78,6 +90,7 @@ impl From for openshell_core::ComputeDriverError { fn from(err: KubernetesDriverError) -> Self { match err { KubernetesDriverError::AlreadyExists => Self::AlreadyExists, + KubernetesDriverError::NotFound => Self::NotFound, KubernetesDriverError::InvalidArgument(m) => Self::InvalidArgument(m), KubernetesDriverError::Precondition(m) => Self::Precondition(m), KubernetesDriverError::Message(m) => Self::Message(m), @@ -344,23 +357,13 @@ fn validate_kubernetes_driver_volume_mounts( Ok(()) } -// TODO: replace with an openshell_core Kubernetes-name helper once available. -fn is_dns_label(label: &str) -> bool { - if label.is_empty() || label.len() > 63 || label.starts_with('-') || label.ends_with('-') { - return false; - } - label - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') -} - // TODO: replace with an openshell_core Kubernetes-name helper once available. fn is_dns_subdomain(value: &str) -> bool { - value.len() <= 253 && value.split('.').all(is_dns_label) + value.len() <= 253 && value.split('.').all(is_dns_1123_label) } fn validate_kubernetes_dns1123_label(value: &str, field: &str) -> Result<(), String> { - if !is_dns_label(value) { + if !is_dns_1123_label(value) { return Err(format!( "{field} must be a DNS-1123 label: use lowercase alphanumeric characters or '-', start and end with an alphanumeric character, and use at most 63 characters" )); @@ -450,6 +453,7 @@ pub struct KubernetesComputeDriver { watch_client: Client, sandbox_api_version: Arc>, config: KubernetesComputeConfig, + operator_allowlist: Option, } impl std::fmt::Debug for KubernetesComputeDriver { @@ -463,7 +467,13 @@ impl std::fmt::Debug for KubernetesComputeDriver { } impl KubernetesComputeDriver { - pub async fn new(config: KubernetesComputeConfig) -> Result { + pub async fn new( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + ) -> Result { + config + .validate_workspace_mode() + .map_err(KubernetesDriverError::Precondition)?; config .validate_provider_spiffe_workload_api_socket_path() .map_err(KubernetesDriverError::Precondition)?; @@ -497,12 +507,40 @@ impl KubernetesComputeDriver { let watch_client = Client::try_from(watch_kube_config).map_err(KubernetesDriverError::from_kube)?; - Ok(Self { + let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) { + let allowlist = OperatorNamespaceAllowlist::new(); + + if let Some(ref label) = config.operator_namespace_label { + spawn_namespace_label_watcher( + watch_client.clone(), + label.clone(), + allowlist.clone(), + shutdown_rx.clone(), + ); + } + + if let Some(ref path) = config.operator_namespace_file { + spawn_namespace_file_watcher(path.into(), allowlist.clone(), shutdown_rx.clone()); + } + + Some(allowlist) + } else { + None + }; + + let driver = Self { client, watch_client, sandbox_api_version: Arc::new(OnceCell::new()), config, - }) + operator_allowlist, + }; + + if driver.workspace_mode() == WorkspaceMode::Shared { + driver.backfill_gateway_id_labels().await?; + } + + Ok(driver) } pub fn capabilities(&self) -> Result { @@ -513,6 +551,10 @@ impl KubernetesComputeDriver { )) } + pub fn operator_allowlist(&self) -> Option<&OperatorNamespaceAllowlist> { + self.operator_allowlist.as_ref() + } + pub fn default_image(&self) -> &str { &self.config.default_image } @@ -525,6 +567,437 @@ impl KubernetesComputeDriver { &self.config.ssh_socket_path } + pub fn workspace_mode(&self) -> WorkspaceMode { + self.config.workspace_mode + } + + pub(crate) fn validate_workspace_namespace( + &self, + workspace: &str, + ) -> Result<(), KubernetesDriverError> { + if self.config.workspace_mode == WorkspaceMode::Managed { + validate_managed_namespace_name(&self.config.gateway_id, workspace) + .map_err(KubernetesDriverError::InvalidArgument)?; + } + Ok(()) + } + + /// Backfill the `openshell.ai/gateway-id` label on Sandbox CRs that + /// predate its introduction. Runs once at startup in shared mode so that + /// label-selector based lookups continue to find legacy resources. + async fn backfill_gateway_id_labels(&self) -> Result<(), KubernetesDriverError> { + let sandbox_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + + let selector = openshell_sandbox_label_selector(); + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + sandbox_api + .api + .list(&ListParams::default().labels(&selector)), + ) + .await + { + Ok(Ok(list)) => list, + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message( + "timeout listing Sandbox resources for gateway-id label backfill".to_string(), + )); + } + }; + + let gateway_id = &self.config.gateway_id; + for obj in &list { + if !gateway_id_label_needs_backfill(obj.metadata.labels.as_ref(), gateway_id) { + continue; + } + let Some(name) = obj.metadata.name.as_deref() else { + continue; + }; + let patch = serde_json::json!({ + "metadata": { + "labels": { + LABEL_GATEWAY_ID: gateway_id + } + } + }); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + sandbox_api + .api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => { + info!(sandbox = %name, gateway_id, "backfilled gateway-id label"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout backfilling gateway-id label on Sandbox {name}" + ))); + } + } + } + + Ok(()) + } + + /// Ensure the K8s namespace for a workspace exists (managed mode only). + /// + /// Idempotent: returns the namespace name whether it was just created or + /// already existed. Also creates the sandbox `ServiceAccount` in the + /// namespace. + /// + pub async fn ensure_namespace(&self, workspace: &str) -> Result { + let ns_name = managed_namespace(&self.config.gateway_id, workspace); + let ns_api: Api = Api::all(self.client.clone()); + + let gateway_ns_annotations = match tokio::time::timeout( + KUBE_API_TIMEOUT, + ns_api.get(&self.config.namespace), + ) + .await + { + Ok(Ok(ns)) => ns.metadata.annotations.unwrap_or_default(), + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout getting gateway namespace {} for SCC annotations", + self.config.namespace + ))); + } + }; + + let mut labels = BTreeMap::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_GATEWAY_ID.to_string(), self.config.gateway_id.clone()); + labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); + + let mut annotations = BTreeMap::new(); + for key in [ + crate::config::ANNOTATION_SCC_UID_RANGE, + crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, + ] { + if let Some(val) = gateway_ns_annotations.get(key) { + annotations.insert(key.to_string(), val.clone()); + } + } + + let ns = Namespace { + metadata: ObjectMeta { + name: Some(ns_name.clone()), + labels: Some(labels), + annotations: if annotations.is_empty() { + None + } else { + Some(annotations) + }, + ..Default::default() + }, + ..Default::default() + }; + + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.create(&PostParams::default(), &ns)) + .await + { + Ok(Ok(_)) => { + info!(namespace = %ns_name, workspace = %workspace, "created managed namespace"); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => { + let existing = + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { + Ok(Ok(ns)) => ns, + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading namespace {ns_name}" + ))); + } + }; + if !is_namespace_owned_by_gateway( + existing.metadata.labels.as_ref(), + &self.config.gateway_id, + ) { + return Err(KubernetesDriverError::Precondition(format!( + "namespace {ns_name} exists but is not owned by this gateway" + ))); + } + debug!(namespace = %ns_name, "managed namespace already exists"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating namespace {ns_name}" + ))); + } + } + + self.ensure_service_account(&ns_name).await?; + self.ensure_managed_ssh_network_policy(&ns_name).await?; + + Ok(ns_name) + } + + async fn ensure_managed_ssh_network_policy( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + if !self.config.managed_ssh_ingress.enabled { + return Ok(()); + } + + let policy = managed_ssh_network_policy(namespace, &self.config); + let policy_api: Api = Api::namespaced(self.client.clone(), namespace); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policy_api.patch( + MANAGED_SSH_NETWORK_POLICY_NAME, + &PatchParams::apply("openshell"), + &Patch::Apply(&policy), + ), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace, "applied managed sandbox SSH NetworkPolicy"); + Ok(()) + } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message(format!( + "timeout applying SSH NetworkPolicy in {namespace}" + ))), + } + } + + async fn ensure_service_account(&self, namespace: &str) -> Result<(), KubernetesDriverError> { + let sa_api: Api = Api::namespaced(self.client.clone(), namespace); + let sa = ServiceAccount { + metadata: ObjectMeta { + name: Some(self.config.service_account_name.clone()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + + match tokio::time::timeout(KUBE_API_TIMEOUT, sa_api.create(&PostParams::default(), &sa)) + .await + { + Ok(Ok(_)) => { + info!(namespace = %namespace, sa = %self.config.service_account_name, "created service account"); + } + Ok(Err(KubeError::Api(api))) if api.code == 409 => {} + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout creating service account in {namespace}" + ))); + } + } + + Ok(()) + } + + /// Ensure the client TLS Secret exists in `namespace` by copying it from + /// the gateway's Helm release namespace. Idempotent: creates the Secret on + /// first call, updates it on subsequent calls to pick up cert rotations. + /// No-op when `client_tls_secret_name` is empty (TLS disabled). + async fn ensure_tls_secret(&self, namespace: &str) -> Result<(), KubernetesDriverError> { + if self.config.client_tls_secret_name.is_empty() { + return Ok(()); + } + + let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let source = match tokio::time::timeout( + KUBE_API_TIMEOUT, + source_api.get(&self.config.client_tls_secret_name), + ) + .await + { + Ok(Ok(s)) => s, + Ok(Err(e)) => { + warn!( + secret = %self.config.client_tls_secret_name, + source_namespace = %self.config.namespace, + error = %e, + "failed to read source TLS secret" + ); + return Err(KubernetesDriverError::from_kube(e)); + } + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading TLS secret {} from {}", + self.config.client_tls_secret_name, self.config.namespace + ))); + } + }; + + let target_api: Api = Api::namespaced(self.client.clone(), namespace); + let copy = Secret { + metadata: ObjectMeta { + name: Some(self.config.client_tls_secret_name.clone()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + }; + + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.patch( + &self.config.client_tls_secret_name, + &PatchParams::apply("openshell"), + &Patch::Apply(©), + ), + ) + .await + { + Ok(Ok(_)) => { + info!( + namespace = %namespace, + secret = %self.config.client_tls_secret_name, + "applied TLS secret copy" + ); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout applying TLS secret in {namespace}" + ))); + } + } + + Ok(()) + } + + /// Copy the explicitly configured image-pull Secrets into a managed + /// workspace namespace. Server-side apply refreshes rotated credentials + /// without forcibly taking fields owned by another manager. + async fn ensure_image_pull_secrets( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let target_api: Api = Api::namespaced(self.client.clone(), namespace); + + for secret_name in &self.config.image_pull_secrets { + let source = match tokio::time::timeout(KUBE_API_TIMEOUT, source_api.get(secret_name)) + .await + { + Ok(Ok(secret)) => secret, + Ok(Err(KubeError::Api(error))) if error.code == 404 => { + return Err(KubernetesDriverError::Precondition(format!( + "configured image-pull Secret {secret_name} does not exist in source namespace {}", + self.config.namespace + ))); + } + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout reading image-pull Secret {secret_name} from {}", + self.config.namespace + ))); + } + }; + + let copy = image_pull_secret_copy(secret_name, namespace, source); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + target_api.patch( + secret_name, + &PatchParams::apply("openshell"), + &Patch::Apply(©), + ), + ) + .await + { + Ok(Ok(_)) => { + info!(namespace, secret = %secret_name, "applied image-pull Secret copy"); + } + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout applying image-pull Secret {secret_name} in {namespace}" + ))); + } + } + } + + Ok(()) + } + + /// Delete the managed namespace and all its contents (managed mode only). + /// Called via the `DeleteWorkspace` RPC after workspace deletion. + /// Kubernetes cascades namespace deletion to all resources within it. + pub async fn delete_namespace(&self, workspace: &str) -> Result<(), KubernetesDriverError> { + let ns_name = managed_namespace(&self.config.gateway_id, workspace); + let ns_api: Api = Api::all(self.client.clone()); + + let ns = match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(&ns_name)).await { + Ok(Ok(ns)) => ns, + Ok(Err(KubeError::Api(api))) if api.code == 404 => { + debug!(namespace = %ns_name, "managed namespace already deleted"); + return Ok(()); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout getting namespace {ns_name}" + ))); + } + }; + + if !is_namespace_owned_by_gateway(ns.metadata.labels.as_ref(), &self.config.gateway_id) { + debug!( + namespace = %ns_name, + "namespace not owned by this gateway, skipping delete" + ); + return Ok(()); + } + + let namespace_uid = ns.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message(format!( + "namespace {ns_name} has no UID; refusing an unguarded delete" + )) + })?; + let delete_params = namespace_delete_params(namespace_uid); + + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.delete(&ns_name, &delete_params)).await + { + Ok(Ok(_)) => { + info!(namespace = %ns_name, workspace = %workspace, "deleted managed namespace"); + } + Ok(Err(KubeError::Api(api))) if api.code == 404 => { + debug!(namespace = %ns_name, "managed namespace already deleted"); + } + Ok(Err(e)) => return Err(KubernetesDriverError::from_kube(e)), + Err(_) => { + return Err(KubernetesDriverError::Message(format!( + "timeout deleting namespace {ns_name}" + ))); + } + } + + Ok(()) + } + fn validate_driver_config_for_sandbox( &self, sandbox: &Sandbox, @@ -539,16 +1012,59 @@ impl KubernetesComputeDriver { ) } - fn agent_sandbox_api(&self, client: Client, sandbox_api_version: &str) -> AgentSandboxApi { + fn agent_sandbox_api( + client: Client, + sandbox_api_version: &str, + namespace: &str, + ) -> AgentSandboxApi { let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); let resource = ApiResource::from_gvk(&gvk); - let api = Api::namespaced_with(client, &self.config.namespace, &resource); + let api = Api::namespaced_with(client, namespace, &resource); AgentSandboxApi { api, resource } } - async fn supported_agent_sandbox_api(&self, client: Client) -> Result { + fn cluster_wide_sandbox_api(client: Client, sandbox_api_version: &str) -> AgentSandboxApi { + let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + let api = Api::all_with(client, &resource); + AgentSandboxApi { api, resource } + } + + async fn supported_agent_sandbox_api( + &self, + client: Client, + namespace: &str, + ) -> Result { + let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; + Ok(Self::agent_sandbox_api( + client, + sandbox_api_version, + namespace, + )) + } + + async fn supported_sandbox_api_for_lookup( + &self, + client: Client, + ) -> Result { let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; - Ok(self.agent_sandbox_api(client, sandbox_api_version)) + if self.config.is_multi_namespace() { + Ok(Self::cluster_wide_sandbox_api(client, sandbox_api_version)) + } else { + Ok(Self::agent_sandbox_api( + client, + sandbox_api_version, + &self.config.namespace, + )) + } + } + + fn sandbox_lookup_selector(&self, sandbox_id: &str) -> String { + sandbox_lookup_selector_for(sandbox_id, &self.config.gateway_id) + } + + fn openshell_sandbox_selector(&self) -> String { + openshell_sandbox_selector_for(&self.config.gateway_id) } async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { @@ -565,7 +1081,11 @@ impl KubernetesComputeDriver { client: Client, ) -> Result<&'static str, String> { for sandbox_api_version in SANDBOX_VERSIONS { - let agent_sandbox_api = self.agent_sandbox_api(client.clone(), sandbox_api_version); + let agent_sandbox_api = Self::agent_sandbox_api( + client.clone(), + sandbox_api_version, + &self.config.namespace, + ); match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&ListParams::default().limit(1)), @@ -603,39 +1123,27 @@ impl KubernetesComputeDriver { )) } - /// Resolve sandbox UID/GID from config or `OpenShift` SCC namespace annotations. - /// - /// Returns `(uid, gid, ns_annotations_map)`: - /// - If `sandbox_uid` is set in config, returns that (with fallback GID) - /// - Otherwise fetches the target namespace and checks for - /// `openshift.io/sa.scc.uid-range` / `openshift.io/sa.scc.supplemental-groups` - /// annotations. - /// - If neither config nor `OpenShift` is found, returns `(1000, 1000, {})` as defaults. - async fn resolve_sandbox_identity(&self) -> (u32, u32, BTreeMap) { - // Explicit config takes priority — skip namespace lookup entirely. + async fn resolve_sandbox_identity_in_namespace( + &self, + namespace: &str, + ) -> (u32, u32, BTreeMap) { if self.config.sandbox_uid.is_some() { let uid = self.config.resolve_sandbox_uid(None); let gid = self.config.resolve_sandbox_gid(uid, None); return (uid, gid, BTreeMap::new()); } - // Try to read namespace annotations for OpenShift SCC. - // Namespace is namespaced so Api::all works (it's cluster-scoped but - // can list all namespaces) and we filter by name, or use Api::namespaced. let ns_api: Api = Api::all(self.client.clone()); - match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(self.config.namespace.as_str())) - .await - { + match tokio::time::timeout(KUBE_API_TIMEOUT, ns_api.get(namespace)).await { Ok(Ok(ns)) => { let anns = ns.metadata.annotations.unwrap_or_default(); tracing::info!( - namespace = %self.config.namespace, + namespace = %namespace, uid_range = ?anns.get(crate::config::ANNOTATION_SCC_UID_RANGE), sup_groups = ?anns.get(crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS), "Resolved namespace annotations for sandbox identity" ); let uid = self.config.resolve_sandbox_uid(Some(&anns)); - // Explicit sandbox_gid config wins; SCC annotation only applies when not set. let baseline_gid = self.config.resolve_sandbox_gid(uid, None); let gid = self.config.sandbox_gid.map_or_else( || { @@ -654,7 +1162,7 @@ impl KubernetesComputeDriver { } Ok(Err(e)) => { tracing::warn!( - namespace = %self.config.namespace, + namespace = %namespace, error = %e, "Failed to fetch namespace for SCC annotations, falling back to defaults" ); @@ -664,7 +1172,7 @@ impl KubernetesComputeDriver { } Err(_) => { tracing::warn!( - namespace = %self.config.namespace, + namespace = %namespace, "Namespace fetch timed out, falling back to defaults" ); let uid = DEFAULT_SANDBOX_UID; @@ -689,7 +1197,15 @@ impl KubernetesComputeDriver { let _ = self .validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; - validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; + match self.config.workspace_mode { + WorkspaceMode::Shared => { + validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; + } + WorkspaceMode::Managed | WorkspaceMode::Operator => { + validate_kubernetes_dns1123_label(&sandbox.name, "sandbox name") + .map_err(tonic::Status::invalid_argument)?; + } + } let gpu_requirements = sandbox .spec .as_ref() @@ -710,15 +1226,14 @@ impl KubernetesComputeDriver { pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { info!( sandbox_id = %sandbox_id, - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Fetching sandbox from Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { Ok(Ok(list)) => list.items.into_iter().next().map_or_else( @@ -727,9 +1242,12 @@ impl KubernetesComputeDriver { Ok(None) }, |obj| { - Ok(sandbox_from_object(&self.config.namespace, obj) - .ok() - .map(|(_, s)| s)) + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) }, ), Ok(Err(err)) => { @@ -756,18 +1274,19 @@ impl KubernetesComputeDriver { pub async fn list_sandboxes(&self) -> Result, String> { info!( - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Listing sandboxes from Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; + let selector = self.openshell_sandbox_selector(); match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api .api - .list(&ListParams::default().labels(&openshell_sandbox_label_selector())), + .list(&ListParams::default().labels(&selector)), ) .await { @@ -777,7 +1296,12 @@ impl KubernetesComputeDriver { .into_iter() .filter_map(|obj| { let name = obj.metadata.name.clone().unwrap_or_default(); - match sandbox_from_object(&self.config.namespace, obj) { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + match sandbox_from_object(&ns, obj) { Ok((_, s)) => Some(s), Err(err) => { warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); @@ -795,7 +1319,6 @@ impl KubernetesComputeDriver { } Ok(Err(err)) => { warn!( - namespace = %self.config.namespace, error = %err, "Failed to list sandboxes from Kubernetes" ); @@ -803,7 +1326,6 @@ impl KubernetesComputeDriver { } Err(_elapsed) => { warn!( - namespace = %self.config.namespace, timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out listing sandboxes from Kubernetes" ); @@ -830,21 +1352,50 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::InvalidArgument)?; let name = sandbox.name.as_str(); + let workspace = sandbox.workspace.as_str(); + self.validate_workspace_namespace(workspace)?; + + let target_namespace = match self.config.workspace_mode { + WorkspaceMode::Shared => self.config.namespace.clone(), + WorkspaceMode::Managed => { + let namespace = self.ensure_namespace(workspace).await?; + self.ensure_image_pull_secrets(&namespace).await?; + namespace + } + WorkspaceMode::Operator => { + if let Some(ref allowlist) = self.operator_allowlist + && !allowlist.contains(workspace) + { + return Err(KubernetesDriverError::Precondition(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + workspace.to_string() + } + }; + + if self.config.is_multi_namespace() { + self.ensure_tls_secret(&target_namespace).await?; + } + info!( sandbox_id = %sandbox.id, sandbox_name = %name, - namespace = %self.config.namespace, + namespace = %target_namespace, + workspace = %workspace, + workspace_mode = %self.config.workspace_mode, "Creating sandbox in Kubernetes" ); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_agent_sandbox_api(self.client.clone(), &target_namespace) .await .map_err(KubernetesDriverError::Message)?; // Resolve sandbox UID/GID from config or OpenShift SCC namespace annotations. - let (resolved_user_id, resolved_group_id, ns_annotations) = - self.resolve_sandbox_identity().await; + let (resolved_user_id, resolved_group_id, ns_annotations) = self + .resolve_sandbox_identity_in_namespace(&target_namespace) + .await; let params = SandboxPodParams { default_image: &self.config.default_image, @@ -889,11 +1440,8 @@ impl KubernetesComputeDriver { let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) .map_err(KubernetesDriverError::InvalidArgument)?; - let kube_name = kube_resource_name(&sandbox.workspace, name); + let kube_name = self.config.kube_resource_name(workspace, name); let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); - // Copy only the SCC-related annotations onto the Sandbox CR for - // traceability. Copying the full namespace annotation map exposes - // unrelated cluster metadata and can fail with oversized annotations. let mut annotations = sandbox_annotations(sandbox); for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, @@ -905,8 +1453,8 @@ impl KubernetesComputeDriver { } obj.metadata = ObjectMeta { name: Some(kube_name), - namespace: Some(self.config.namespace.clone()), - labels: Some(sandbox_labels(sandbox)), + namespace: Some(target_namespace), + labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() }; @@ -950,22 +1498,22 @@ impl KubernetesComputeDriver { } } - pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), String> { - let (agent_sandbox_api, kube_name, pod_name, stop_timeout) = self + pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self .patch_sandbox_operating_state(sandbox_id, false) .await?; let legacy_pod_api = (agent_sandbox_api.resource.version == SANDBOX_VERSION_V1ALPHA1) - .then(|| Api::::namespaced(self.client.clone(), &self.config.namespace)); + .then(|| Api::::namespaced(self.client.clone(), &namespace)); let deadline = tokio::time::Instant::now() + stop_timeout; let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; loop { let now = tokio::time::Instant::now(); if now >= deadline { - return Err(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes sandbox to stop", stop_timeout.as_secs() - )); + ))); } let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); let object = tokio::time::timeout( @@ -974,36 +1522,38 @@ impl KubernetesComputeDriver { ) .await .map_err(|_| { - format!( + KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API while checking sandbox stop", request_timeout.as_secs() - ) + )) })? - .map_err(|err| err.to_string())?; + .map_err(KubernetesDriverError::from_kube)?; if kubernetes_sandbox_has_stopped_condition(&object) { return Ok(()); } if let Some(error) = kubernetes_sandbox_stop_failure(&object) { - return Err(error); + return Err(KubernetesDriverError::Message(error)); } if let Some(pod_api) = legacy_pod_api.as_ref() - && kubernetes_sandbox_pod_is_gone(pod_api, &pod_name, deadline).await? + && kubernetes_sandbox_pod_is_gone(pod_api, &pod_name, deadline) + .await + .map_err(KubernetesDriverError::Message)? { return Ok(()); } let now = tokio::time::Instant::now(); if now >= deadline { - return Err(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes sandbox to stop", stop_timeout.as_secs() - )); + ))); } tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; poll_interval = next_stop_poll_interval(poll_interval); } } - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), String> { + pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { self.patch_sandbox_operating_state(sandbox_id, true) .await .map(|_| ()) @@ -1013,36 +1563,45 @@ impl KubernetesComputeDriver { &self, sandbox_id: &str, running: bool, - ) -> Result<(AgentSandboxApi, String, String, Duration), String> { - let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) - .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); let list = tokio::time::timeout( KUBE_API_TIMEOUT, - agent_sandbox_api + lookup_api .api .list(&ListParams::default().labels(&selector)), ) .await .map_err(|_| { - format!( + KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ) + )) })? - .map_err(|err| err.to_string())?; + .map_err(KubernetesDriverError::from_kube)?; let object = list .items .into_iter() .next() - .ok_or_else(|| "sandbox not found".to_string())?; - let stop_timeout = kubernetes_sandbox_stop_timeout(&object); - let kube_name = object + .ok_or(KubernetesDriverError::NotFound)?; + let namespace = object .metadata - .name - .ok_or_else(|| "sandbox resource has no name".to_string())?; + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let agent_sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); + let stop_timeout = kubernetes_sandbox_stop_timeout(&object); + let kube_name = object.metadata.name.ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; let pod_name = object .metadata .annotations @@ -1066,12 +1625,12 @@ impl KubernetesComputeDriver { ) .await .map_err(|_| { - format!( + KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ) + )) })? - .map_err(|err| err.to_string())?; + .map_err(KubernetesDriverError::from_kube)?; info!( sandbox_id, @@ -1079,25 +1638,30 @@ impl KubernetesComputeDriver { running, "Updated Kubernetes sandbox operating state" ); - Ok((agent_sandbox_api, kube_name, pod_name, stop_timeout)) + Ok(( + agent_sandbox_api, + kube_name, + pod_name, + namespace, + stop_timeout, + )) } pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { info!( sandbox_id = %sandbox_id, - namespace = %self.config.namespace, + workspace_mode = %self.config.workspace_mode, "Deleting sandbox from Kubernetes" ); - let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, preconditions) = match tokio::time::timeout( + let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( KUBE_API_TIMEOUT, - agent_sandbox_api.api.list(&lp), + lookup_api.api.list(&lp), ) .await { @@ -1105,11 +1669,22 @@ impl KubernetesComputeDriver { if let Some(obj) = list.items.into_iter().next() { match obj.metadata.name { Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); let pc = Preconditions { uid: obj.metadata.uid, resource_version: obj.metadata.resource_version, }; - (name, pc) + (name, ns, ws, pc) } None => return Ok(false), } @@ -1139,15 +1714,13 @@ impl KubernetesComputeDriver { } }; + let delete_api = self + .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + .await?; let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout( - KUBE_API_TIMEOUT, - agent_sandbox_api.api.delete(&kube_name, &dp), - ) - .await - { + match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { - info!(sandbox_id = %sandbox_id, "Sandbox deleted from Kubernetes"); + info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); Ok(true) } Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { @@ -1178,10 +1751,9 @@ impl KubernetesComputeDriver { pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { let agent_sandbox_api = self - .supported_agent_sandbox_api(self.client.clone()) + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let selector = - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id}"); + let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { Ok(Ok(list)) => Ok(!list.items.is_empty()), @@ -1196,9 +1768,17 @@ impl KubernetesComputeDriver { // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { + if self.config.is_multi_namespace() { + self.watch_sandboxes_cluster_wide().await + } else { + self.watch_sandboxes_single_namespace().await + } + } + + async fn watch_sandboxes_single_namespace(&self) -> Result { let namespace = self.config.namespace.clone(); let agent_sandbox_api = self - .supported_agent_sandbox_api(self.watch_client.clone()) + .supported_agent_sandbox_api(self.watch_client.clone(), &self.config.namespace) .await?; let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); let watcher_config = watcher::Config::default().labels(&openshell_sandbox_label_selector()); @@ -1284,13 +1864,92 @@ impl KubernetesComputeDriver { } } } - Ok(Some(Event::Deleted(_))) => {} - Ok(Some(Event::Restarted(_))) => { - debug!(namespace = %namespace, "Kubernetes event watcher restarted"); - } + Ok(Some(Event::Deleted(_))) => {} + Ok(Some(Event::Restarted(_))) => { + debug!(namespace = %namespace, "Kubernetes event watcher restarted"); + } + Ok(None) => { + let _ = tx.send(Err(KubernetesDriverError::Message( + "kubernetes event watcher stream ended".to_string() + ))).await; + break; + } + Err(err) => { + let _ = tx.send(Err(KubernetesDriverError::Message(err.to_string()))).await; + break; + } + }, + () = tx.closed() => break, + } + } + }); + + Ok(Box::pin(ReceiverStream::new(rx))) + } + + async fn watch_sandboxes_cluster_wide(&self) -> Result { + let sandbox_api_version = self + .supported_sandbox_api_version(self.watch_client.clone()) + .await?; + let cluster_api = + Self::cluster_wide_sandbox_api(self.watch_client.clone(), sandbox_api_version); + let selector = self.openshell_sandbox_selector(); + let watcher_config = watcher::Config::default().labels(&selector); + let mut sandbox_stream = watcher::watcher(cluster_api.api, watcher_config).boxed(); + let (tx, rx) = mpsc::channel(256); + let default_namespace = self.config.namespace.clone(); + + tokio::spawn(async move { + loop { + tokio::select! { + result = sandbox_stream.try_next() => match result { + Ok(Some(Event::Applied(obj))) => { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Ok(Some(Event::Deleted(obj))) => { + if is_openshell_managed(&obj) + && let Ok(sandbox_id) = sandbox_id_from_object(&obj) + { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Ok(Some(Event::Restarted(objs))) => { + for obj in objs { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + return; + } + } + } + } Ok(None) => { let _ = tx.send(Err(KubernetesDriverError::Message( - "kubernetes event watcher stream ended".to_string() + "sandbox watcher stream ended unexpectedly".to_string() ))).await; break; } @@ -1324,10 +1983,6 @@ fn validate_gpu_request( Ok(()) } -fn kube_resource_name(workspace: &str, name: &str) -> String { - format!("{workspace}--{name}") -} - const MAX_KUBE_NAME_LEN: usize = 63; fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), tonic::Status> { @@ -1341,7 +1996,48 @@ fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), Ok(()) } -fn sandbox_labels(sandbox: &Sandbox) -> BTreeMap { +fn is_namespace_owned_by_gateway( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|l| l.get(LABEL_MANAGED_BY)) + .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) + && labels + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == gateway_id) +} + +fn gateway_id_label_needs_backfill( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|labels| labels.get(LABEL_GATEWAY_ID)) + .is_none_or(|value| value != gateway_id) +} + +fn namespace_delete_params(uid: String) -> DeleteParams { + DeleteParams::default().preconditions(Preconditions { + uid: Some(uid), + resource_version: None, + }) +} + +fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { + format!( + "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" + ) +} + +fn openshell_sandbox_selector_for(gateway_id: &str) -> String { + use std::fmt::Write; + let mut selector = openshell_sandbox_label_selector(); + write!(selector, ",{LABEL_GATEWAY_ID}={gateway_id}").unwrap(); + selector +} + +fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { let mut labels = BTreeMap::new(); labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); @@ -1353,9 +2049,76 @@ fn sandbox_labels(sandbox: &Sandbox) -> BTreeMap { LABEL_MANAGED_BY.to_string(), LABEL_MANAGED_BY_VALUE.to_string(), ); + if let Some(gw_id) = gateway_id { + labels.insert(LABEL_GATEWAY_ID.to_string(), gw_id.to_string()); + } labels } +fn managed_ssh_network_policy(namespace: &str, config: &KubernetesComputeConfig) -> NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(MANAGED_SSH_NETWORK_POLICY_NAME.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string()]), + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + config.managed_ssh_ingress.gateway_namespace.clone(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(config.managed_ssh_ingress.gateway_pod_selector.clone()), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(2222)), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + ..Default::default() + }), + status: None, + } +} + +fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(secret_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + } +} + fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { let mut annotations = BTreeMap::new(); annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -3529,6 +4292,241 @@ fn condition_from_value(value: &serde_json::Value) -> Option { }) } +fn spawn_namespace_label_watcher( + client: Client, + label_selector: String, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + let ns_api: Api = Api::all(client); + let watcher_config = watcher::Config::default().labels(&label_selector); + let jitter_seed = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |duration| { + duration.as_secs() ^ u64::from(duration.subsec_nanos()) + }); + + tokio::spawn(async move { + let mut retry_attempt = 0; + loop { + let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + + loop { + let event = tokio::select! { + result = stream.try_next() => result, + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + match event { + Ok(Some(Event::Applied(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.insert(name.to_string()) + { + info!(namespace = name, "operator namespace added to allowlist"); + } + } + Ok(Some(Event::Deleted(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); + } + } + Ok(Some(Event::Restarted(namespaces))) => { + retry_attempt = 0; + let names: std::collections::BTreeSet = namespaces + .into_iter() + .filter_map(|ns| ns.metadata.name) + .collect(); + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist replaced from full relist" + ); + } + Ok(None) => { + warn!("operator namespace watcher stream ended unexpectedly"); + break; + } + Err(err) => { + warn!(error = %err, "operator namespace watcher stream error"); + break; + } + } + } + + let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); + warn!(?retry_delay, "operator namespace watcher reconnecting"); + tokio::select! { + () = tokio::time::sleep(retry_delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + retry_attempt = retry_attempt.saturating_add(1); + } + }); + + info!( + label_selector = %label_selector, + "operator namespace label watcher spawned" + ); +} + +fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { + let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); + let max_jitter_secs = base_secs / 4; + let mixed_seed = + jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); + let jitter_secs = mixed_seed % (max_jitter_secs + 1); + Duration::from_secs(base_secs + jitter_secs) +} + +fn load_namespace_file(path: &Path) -> Result, String> { + let contents = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let names: Vec = serde_json::from_str(&contents) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + Ok(names.into_iter().collect()) +} + +fn spawn_namespace_file_watcher( + path: PathBuf, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + path = %path.display(), + total = count, + "operator namespace allowlist loaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to load initial operator namespace file, allowlist empty" + ); + } + } + + let watch_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let debounce = Duration::from_secs(1); + + tokio::spawn(async move { + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut watcher = + match notify::recommended_watcher(move |res: Result| { + if let Ok(event) = res + && matches!( + event.kind, + notify::EventKind::Modify(_) | notify::EventKind::Create(_) + ) + { + let _ = tx.send(()); + } + }) { + Ok(w) => w, + Err(e) => { + warn!( + error = %e, + "failed to start operator namespace file watcher, hot-reload disabled" + ); + return; + } + }; + + if let Err(e) = notify::Watcher::watch( + &mut watcher, + &watch_dir, + notify::RecursiveMode::NonRecursive, + ) { + warn!( + error = %e, + dir = %watch_dir.display(), + "failed to watch operator namespace file directory, hot-reload disabled" + ); + return; + } + + info!( + path = %path.display(), + "operator namespace file watcher started" + ); + + loop { + let got_event = tokio::select! { + event = rx.recv() => event.is_some(), + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + if !got_event { + warn!("operator namespace file watcher disconnected"); + break; + } + + loop { + tokio::select! { + () = tokio::time::sleep(debounce) => { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist reloaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to reload operator namespace file, keeping existing allowlist" + ); + } + } + break; + } + r = rx.recv() => { + if r.is_some() { + continue; + } + warn!("operator namespace file watcher disconnected"); + return; + } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + } + } + }); +} + #[cfg(test)] mod tests { use super::*; @@ -6348,22 +7346,6 @@ mod tests { assert!(validate_kubernetes_dns1123_label("dotted.name", "sandbox name").is_err()); } - #[test] - fn kube_resource_name_qualifies_with_workspace() { - assert_eq!(kube_resource_name("alpha", "work"), "alpha--work"); - assert_eq!( - kube_resource_name("default", "my-sandbox"), - "default--my-sandbox" - ); - } - - #[test] - fn kube_resource_name_different_workspaces_produce_different_names() { - let alpha = kube_resource_name("alpha", "work"); - let beta = kube_resource_name("beta", "work"); - assert_ne!(alpha, beta); - } - #[test] fn kube_resource_name_length_validation_accepts_short_names() { validate_kube_resource_name_length("default", "my-sandbox").unwrap(); @@ -6460,6 +7442,37 @@ mod tests { assert!(result.unwrap_err().contains("not managed by openshell")); } + #[test] + fn sandbox_from_object_uses_object_namespace_over_fallback() { + let obj = DynamicObject { + types: None, + metadata: ObjectMeta { + name: Some("work".to_string()), + namespace: Some("openshell-gw1-team-a".to_string()), + annotations: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-cross".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ])), + labels: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-cross".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + ])), + ..Default::default() + }, + data: serde_json::json!({}), + }; + + let (_, sandbox) = sandbox_from_object("openshell", obj).unwrap(); + assert_eq!(sandbox.namespace, "openshell-gw1-team-a"); + assert_eq!(sandbox.workspace, "team-a"); + } + #[test] fn sandbox_from_object_warns_on_managed_cr_missing_workspace() { let obj = DynamicObject { @@ -6493,7 +7506,7 @@ mod tests { workspace: "alpha".to_string(), ..Default::default() }; - let labels = sandbox_labels(&sandbox); + let labels = sandbox_labels(&sandbox, None); assert_eq!(labels.get(LABEL_SANDBOX_ID).unwrap(), "uuid-1"); assert_eq!(labels.get(LABEL_SANDBOX_NAME).unwrap(), "work"); assert_eq!(labels.get(LABEL_SANDBOX_WORKSPACE).unwrap(), "alpha"); @@ -6501,6 +7514,19 @@ mod tests { labels.get(LABEL_MANAGED_BY).unwrap(), LABEL_MANAGED_BY_VALUE ); + assert!(!labels.contains_key(LABEL_GATEWAY_ID)); + } + + #[test] + fn sandbox_labels_includes_gateway_id_when_provided() { + let sandbox = Sandbox { + id: "uuid-1".to_string(), + name: "work".to_string(), + workspace: "alpha".to_string(), + ..Default::default() + }; + let labels = sandbox_labels(&sandbox, Some("gw-42")); + assert_eq!(labels.get(LABEL_GATEWAY_ID).unwrap(), "gw-42"); } #[test] @@ -6664,4 +7690,220 @@ mod tests { ); assert_eq!(volume["secret"]["defaultMode"], 0o440); } + + #[test] + fn sandbox_lookup_selector_always_includes_gateway_id() { + let sel = sandbox_lookup_selector_for("sb-123", "gw-42"); + assert!( + sel.contains(&format!("{LABEL_GATEWAY_ID}=gw-42")), + "selector must include gateway ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_SANDBOX_ID}=sb-123")), + "selector must include sandbox ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}")), + "selector must include managed-by: {sel}" + ); + } + + #[test] + fn openshell_sandbox_selector_always_includes_gateway_id() { + let sel = openshell_sandbox_selector_for("gw-99"); + assert!( + sel.contains(&format!("{LABEL_GATEWAY_ID}=gw-99")), + "selector must include gateway ID: {sel}" + ); + assert!( + sel.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}")), + "selector must include managed-by: {sel}" + ); + } + + #[test] + fn gateway_id_backfill_adopts_unlabelled_sandbox() { + let labels = BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )]); + assert!(gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn gateway_id_backfill_adopts_sandbox_from_previous_gateway() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-old".to_string())]); + assert!(gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn gateway_id_backfill_skips_sandbox_already_owned_by_gateway() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-1".to_string())]); + assert!(!gateway_id_label_needs_backfill(Some(&labels), "gw-1")); + } + + #[test] + fn managed_ssh_policy_allows_only_gateway_peer_on_port_2222() { + let config = KubernetesComputeConfig { + managed_ssh_ingress: crate::config::ManagedSshIngressConfig { + enabled: true, + gateway_namespace: "gateway-ns".to_string(), + gateway_pod_selector: BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )]), + }, + ..KubernetesComputeConfig::default() + }; + let policy = managed_ssh_network_policy("workspace-ns", &config); + let spec = policy.spec.unwrap(); + assert_eq!( + spec.policy_types.as_deref(), + Some(["Ingress".to_string()].as_slice()) + ); + let ingress = &spec.ingress.unwrap()[0]; + assert_eq!( + ingress.ports.as_ref().unwrap()[0].port, + Some(IntOrString::Int(2222)) + ); + let peer = &ingress.from.as_ref().unwrap()[0]; + assert_eq!( + peer.namespace_selector + .as_ref() + .unwrap() + .match_labels + .as_ref() + .unwrap() + .get("kubernetes.io/metadata.name") + .map(String::as_str), + Some("gateway-ns") + ); + assert_eq!( + peer.pod_selector + .as_ref() + .unwrap() + .match_labels + .as_ref() + .unwrap() + .get("app.kubernetes.io/name") + .map(String::as_str), + Some("openshell") + ); + } + + #[test] + fn image_pull_secret_copy_keeps_only_portable_secret_fields() { + let source: Secret = serde_json::from_value(serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "gateway", + "uid": "source-uid", + "resourceVersion": "42", + "labels": { "source-only": "true" }, + "annotations": { "source-only": "true" }, + "finalizers": ["example.test/finalizer"] + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + })) + .unwrap(); + + let copy = image_pull_secret_copy("regcred", "workspace", source); + assert_eq!(copy.metadata.name.as_deref(), Some("regcred")); + assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); + assert_eq!( + copy.type_.as_deref(), + Some("kubernetes.io/dockerconfigjson") + ); + assert!( + copy.data + .as_ref() + .unwrap() + .contains_key(".dockerconfigjson") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_MANAGED_BY) + .map(String::as_str), + Some(LABEL_MANAGED_BY_VALUE) + ); + assert!(copy.metadata.uid.is_none()); + assert!(copy.metadata.resource_version.is_none()); + assert!(copy.metadata.annotations.is_none()); + assert!(copy.metadata.finalizers.is_none()); + } + + #[test] + fn namespace_owned_with_correct_labels() { + let labels = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gw-1".to_string()), + ]); + assert!(is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_missing_managed_by() { + let labels = BTreeMap::from([(LABEL_GATEWAY_ID.to_string(), "gw-1".to_string())]); + assert!(!is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_wrong_gateway_id() { + let labels = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gw-other".to_string()), + ]); + assert!(!is_namespace_owned_by_gateway(Some(&labels), "gw-1")); + } + + #[test] + fn namespace_not_owned_no_labels() { + assert!(!is_namespace_owned_by_gateway(None, "gw-1")); + } + + #[test] + fn namespace_delete_is_guarded_by_fetched_uid() { + let params = namespace_delete_params("namespace-uid".to_string()); + assert_eq!( + params + .preconditions + .and_then(|preconditions| preconditions.uid), + Some("namespace-uid".to_string()) + ); + } + + #[test] + fn namespace_watcher_retry_delay_is_bounded_exponential_with_jitter() { + let seed = 42; + let expected_ranges = [(2, 2), (4, 5), (8, 10), (16, 20), (24, 30), (24, 30)]; + + for (attempt, (minimum, maximum)) in expected_ranges.into_iter().enumerate() { + let attempt = u32::try_from(attempt).unwrap(); + let delay = namespace_watcher_retry_delay(attempt, seed).as_secs(); + assert!( + (minimum..=maximum).contains(&delay), + "attempt {attempt} produced {delay}s" + ); + } + } + + #[test] + fn namespace_watcher_retry_delay_uses_seeded_jitter() { + assert_ne!( + namespace_watcher_retry_delay(3, 1), + namespace_watcher_retry_delay(3, 2) + ); + } } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index ef2e2686e4..ce23cf0189 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -6,17 +6,19 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_server::ComputeDriver, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; use crate::KubernetesComputeDriver; +use crate::WorkspaceMode; #[derive(Debug, Clone)] pub struct ComputeDriverService { @@ -122,7 +124,7 @@ impl ComputeDriver for ComputeDriverService { self.driver .stop_sandbox(&request.sandbox_id) .await - .map_err(kubernetes_lifecycle_status)?; + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; Ok(Response::new(StopSandboxResponse {})) } @@ -137,7 +139,7 @@ impl ComputeDriver for ComputeDriverService { self.driver .start_sandbox(&request.sandbox_id) .await - .map_err(kubernetes_lifecycle_status)?; + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; Ok(Response::new(StartSandboxResponse {})) } @@ -172,18 +174,67 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok(Response::new(Box::pin(stream))) } -} -fn kubernetes_lifecycle_status(message: String) -> Status { - if message == "sandbox not found" { - Status::not_found(message) - } else { - Status::internal(message) + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + let workspace = request.into_inner().workspace; + if workspace.is_empty() { + return Err(Status::invalid_argument("workspace is required")); + } + self.driver + .validate_workspace_namespace(&workspace) + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; + match self.driver.workspace_mode() { + WorkspaceMode::Managed => { + self.driver + .ensure_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + WorkspaceMode::Operator => { + if let Some(allowlist) = self.driver.operator_allowlist() + && !allowlist.contains(&workspace) + { + return Err(Status::permission_denied(format!( + "workspace '{workspace}' is not in the operator namespace allowlist" + ))); + } + } + WorkspaceMode::Shared => {} + } + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + let workspace = request.into_inner().workspace; + if workspace.is_empty() { + return Err(Status::invalid_argument("workspace is required")); + } + if workspace_delete_requires_namespace_access(self.driver.workspace_mode()) { + self.driver + .validate_workspace_namespace(&workspace) + .map_err(|error| Status::from(openshell_core::ComputeDriverError::from(error)))?; + self.driver + .delete_namespace(&workspace) + .await + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(Response::new(DeleteWorkspaceResponse {})) } } +fn workspace_delete_requires_namespace_access(mode: WorkspaceMode) -> bool { + matches!(mode, WorkspaceMode::Managed) +} + #[cfg(test)] mod tests { + use super::{WorkspaceMode, workspace_delete_requires_namespace_access}; use crate::KubernetesDriverError; use openshell_core::ComputeDriverError; use tonic::Status; @@ -199,6 +250,17 @@ mod tests { assert_eq!(status.message(), "sandbox agent pod IP is not available"); } + #[test] + fn invalid_workspace_driver_errors_map_to_invalid_argument_status() { + let status: Status = ComputeDriverError::from(KubernetesDriverError::InvalidArgument( + "managed namespace is invalid".to_string(), + )) + .into(); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert_eq!(status.message(), "managed namespace is invalid"); + } + #[test] fn already_exists_driver_errors_map_to_already_exists_status() { let status: Status = ComputeDriverError::from(KubernetesDriverError::AlreadyExists).into(); @@ -206,4 +268,25 @@ mod tests { assert_eq!(status.code(), tonic::Code::AlreadyExists); assert_eq!(status.message(), "sandbox already exists"); } + + #[test] + fn not_found_driver_errors_map_to_not_found_status() { + let status: Status = ComputeDriverError::from(KubernetesDriverError::NotFound).into(); + + assert_eq!(status.code(), tonic::Code::NotFound); + assert_eq!(status.message(), "sandbox not found"); + } + + #[test] + fn only_managed_workspace_delete_accesses_the_namespace() { + assert!(workspace_delete_requires_namespace_access( + WorkspaceMode::Managed + )); + assert!(!workspace_delete_requires_namespace_access( + WorkspaceMode::Operator + )); + assert!(!workspace_delete_requires_namespace_access( + WorkspaceMode::Shared + )); + } } diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 7c56c8de5b..1a234385c6 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,9 +6,10 @@ pub mod driver; pub mod grpc; pub use config::{ - AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, + ManagedSshIngressConfig, OperatorNamespaceAllowlist, SupervisorSideloadMethod, + SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 99df4ea165..30b4fcada9 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -3,6 +3,7 @@ use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; +use std::collections::BTreeMap; use std::net::SocketAddr; use tracing::info; use tracing_subscriber::EnvFilter; @@ -10,9 +11,10 @@ use tracing_subscriber::EnvFilter; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, + KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, + WorkspaceMode, }; #[derive(Parser, Debug)] @@ -30,9 +32,25 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, + #[arg(long, env = "OPENSHELL_WORKSPACE_MODE", default_value = "shared")] + workspace_mode: WorkspaceMode, + + #[arg( + long, + env = "OPENSHELL_GATEWAY_ID", + default_value = DEFAULT_GATEWAY_ID + )] + gateway_id: String, + #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] sandbox_namespace: String, + #[arg(long, env = "OPENSHELL_OPERATOR_NAMESPACE_LABEL")] + operator_namespace_label: Option, + + #[arg(long, env = "OPENSHELL_OPERATOR_NAMESPACE_FILE")] + operator_namespace_file: Option, + #[arg( long, env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", @@ -53,6 +71,19 @@ struct Args { )] sandbox_image_pull_secrets: Vec, + #[arg(long, env = "OPENSHELL_MANAGED_SSH_INGRESS_ENABLED")] + managed_ssh_ingress_enabled: bool, + + #[arg(long, env = "OPENSHELL_MANAGED_SSH_GATEWAY_NAMESPACE")] + managed_ssh_gateway_namespace: Option, + + #[arg( + long, + env = "OPENSHELL_MANAGED_SSH_GATEWAY_POD_SELECTOR", + value_delimiter = ',' + )] + managed_ssh_gateway_pod_selector: Vec, + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] grpc_endpoint: Option, @@ -148,6 +179,29 @@ struct Args { sandbox_gid: Option, } +async fn shutdown_signal() { + #[cfg(unix)] + { + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = terminate => {} + } + } + + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} + #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); @@ -157,58 +211,88 @@ async fn main() -> Result<()> { ) .init(); - let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { - namespace: args.sandbox_namespace, - service_account_name: args.sandbox_service_account, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), - image_pull_secrets: args.sandbox_image_pull_secrets, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, + let managed_ssh_gateway_pod_selector = args + .managed_ssh_gateway_pod_selector + .iter() + .map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.to_string(), value.to_string())) + .ok_or_else(|| { + miette::miette!("managed SSH gateway pod selector must use key=value: {entry}") + }) + }) + .collect::>>()?; + + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let driver = KubernetesComputeDriver::new( + KubernetesComputeConfig { + workspace_mode: args.workspace_mode, + gateway_id: args.gateway_id, + namespace: args.sandbox_namespace, + operator_namespace_label: args.operator_namespace_label, + operator_namespace_file: args.operator_namespace_file, + service_account_name: args.sandbox_service_account, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_secrets: args.sandbox_image_pull_secrets, + managed_ssh_ingress: ManagedSshIngressConfig { + enabled: args.managed_ssh_ingress_enabled, + gateway_namespace: args.managed_ssh_gateway_namespace.unwrap_or_default(), + gateway_pod_selector: managed_ssh_gateway_pod_selector, + }, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_sideload_method: args.supervisor_sideload_method, + topology: args.topology, + sidecar: KubernetesSidecarConfig { + proxy_uid: args.sidecar_proxy_uid, + process_binary_aware_network_policy: args + .sidecar_process_binary_aware_network_policy, + }, + https_proxy: args.https_proxy, + no_proxy: args.no_proxy, + proxy_auth_secret_name: args.proxy_auth_secret_name, + proxy_auth_secret_key: args.proxy_auth_secret_key, + proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + ssh_socket_path: args.sandbox_ssh_socket_path, + client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), + host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), + enable_user_namespaces: args.enable_user_namespaces, + app_armor_profile: args.app_armor_profile, + workspace_default_storage_size: std::env::var( + "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", + ) + .unwrap_or_else(|_| { + openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() + }), + workspace_storage_class: std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") + .unwrap_or_default(), + default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") + .unwrap_or_default(), + sa_token_ttl_secs: args.sa_token_ttl_secs, + provider_spiffe_workload_api_socket_path: args + .provider_spiffe_workload_api_socket_path + .unwrap_or_default(), + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, }, - https_proxy: args.https_proxy, - no_proxy: args.no_proxy, - proxy_auth_secret_name: args.proxy_auth_secret_name, - proxy_auth_secret_key: args.proxy_auth_secret_key, - proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), - proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - ssh_socket_path: args.sandbox_ssh_socket_path, - client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), - host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), - enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, - workspace_default_storage_size: std::env::var( - "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", - ) - .unwrap_or_else(|_| { - openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() - }), - workspace_storage_class: std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") - .unwrap_or_default(), - default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") - .unwrap_or_default(), - sa_token_ttl_secs: args.sa_token_ttl_secs, - provider_spiffe_workload_api_socket_path: args - .provider_spiffe_workload_api_socket_path - .unwrap_or_default(), - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) + shutdown_rx, + ) .await .into_diagnostic()?; info!(address = %args.bind_address, "Starting Kubernetes compute driver"); tonic::transport::Server::builder() .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve(args.bind_address) + .serve_with_shutdown(args.bind_address, async move { + shutdown_signal().await; + let _ = shutdown_tx.send(true); + }) .await .into_diagnostic() } diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 8d34660514..19d0b55254 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -6,12 +6,13 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_server::ComputeDriver, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; @@ -170,6 +171,20 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok(Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index af914ec467..4dd2ea059b 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -38,15 +38,17 @@ use openshell_core::progress::{ }; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, - DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, - DriverSandboxTemplate as SandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, - WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition as SandboxCondition, + DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, + DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, + EnsureWorkspaceRequest, EnsureWorkspaceResponse, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, + compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -3432,6 +3434,20 @@ impl ComputeDriver for VmDriver { let stream: Self::WatchSandboxesStream = Box::pin(ReceiverStream::new(out_rx)); Ok(Response::new(stream)) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(target_os = "linux")] diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index eed0e5f083..32cb2e119c 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,6 +26,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; +use openshell_driver_kubernetes::OperatorNamespaceAllowlist; use std::sync::Arc; use tonic::Status; use tracing::{debug, info, warn}; @@ -135,8 +136,32 @@ impl Authenticator for K8sServiceAccountAuthenticator { } } +/// Validates the namespace extracted from an SA token username against the +/// expected set for the active workspace mode. +#[derive(Debug, Clone)] +pub enum NamespaceValidator { + /// Shared mode: accept only the single configured namespace. + Exact(String), + /// Managed mode: accept any namespace with the managed prefix + /// (`openshell-{gateway_id}-`). + Prefix(String), + /// Operator mode: accept namespaces in the dynamic allowlist. + Allowlist(OperatorNamespaceAllowlist), +} + +impl NamespaceValidator { + pub fn accepts(&self, namespace: &str) -> bool { + match self { + Self::Exact(expected) => namespace == expected, + Self::Prefix(prefix) => namespace.starts_with(prefix.as_str()), + Self::Allowlist(al) => al.contains(namespace), + } + } +} + #[derive(Debug)] struct TokenReviewIdentity { + namespace: String, pod_name: String, pod_uid: String, } @@ -151,59 +176,53 @@ struct SandboxOwnerReference { /// Resolver backed by the apiserver's `TokenReview` API and `kube::Client` /// for the per-pod annotation lookup. pub struct LiveK8sResolver { + client: kube::Client, token_reviews_api: Api, - pods_api: Api, - sandboxes_api_v1beta1: Api, - sandboxes_api_v1alpha1: Api, expected_audience: String, - sandbox_namespace: String, + namespace_validator: NamespaceValidator, expected_service_account: String, } impl LiveK8sResolver { pub fn new( client: kube::Client, - namespace: &str, + namespace_validator: NamespaceValidator, expected_audience: String, expected_service_account: String, ) -> Self { let token_reviews_api: Api = Api::all(client.clone()); - let pods_api: Api = Api::namespaced(client.clone(), namespace); - let sandbox_gvk_v1beta1 = - GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION_V1BETA1, SANDBOX_KIND); - let sandbox_resource_v1beta1 = ApiResource::from_gvk(&sandbox_gvk_v1beta1); - let sandbox_gvk_v1alpha1 = GroupVersionKind::gvk( - SANDBOX_API_GROUP, - SANDBOX_API_VERSION_V1ALPHA1, - SANDBOX_KIND, - ); - let sandbox_resource_v1alpha1 = ApiResource::from_gvk(&sandbox_gvk_v1alpha1); - let sandboxes_api_v1beta1: Api = - Api::namespaced_with(client.clone(), namespace, &sandbox_resource_v1beta1); - let sandboxes_api_v1alpha1: Api = - Api::namespaced_with(client, namespace, &sandbox_resource_v1alpha1); Self { + client, token_reviews_api, - pods_api, - sandboxes_api_v1beta1, - sandboxes_api_v1alpha1, expected_audience, - sandbox_namespace: namespace.to_string(), + namespace_validator, expected_service_account, } } + fn pods_api(&self, namespace: &str) -> Api { + Api::namespaced(self.client.clone(), namespace) + } + + fn sandboxes_api(&self, namespace: &str, api_version: &str) -> Api { + let gvk = GroupVersionKind::gvk(SANDBOX_API_GROUP, api_version, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + Api::namespaced_with(self.client.clone(), namespace, &resource) + } + async fn get_sandbox_cr_for_owner( &self, + namespace: &str, owner: &SandboxOwnerReference, ) -> Result, KubeError> { - let apis = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { - [&self.sandboxes_api_v1alpha1, &self.sandboxes_api_v1beta1] + let versions = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { + [SANDBOX_API_VERSION_V1ALPHA1, SANDBOX_API_VERSION_V1BETA1] } else { - [&self.sandboxes_api_v1beta1, &self.sandboxes_api_v1alpha1] + [SANDBOX_API_VERSION_V1BETA1, SANDBOX_API_VERSION_V1ALPHA1] }; - for api in apis { + for version in versions { + let api = self.sandboxes_api(namespace, version); match api.get_opt(&owner.name).await { Ok(Some(sandbox_cr)) => return Ok(Some(sandbox_cr)), Ok(None) => {} @@ -242,7 +261,7 @@ impl K8sIdentityResolver for LiveK8sResolver { let Some(identity) = token_review_identity( &status, &self.expected_audience, - &self.sandbox_namespace, + &self.namespace_validator, &self.expected_service_account, )? else { @@ -252,34 +271,30 @@ impl K8sIdentityResolver for LiveK8sResolver { info!( pod_name = %identity.pod_name, pod_uid = %identity.pod_uid, + namespace = %identity.namespace, service_account = %self.expected_service_account, "validated K8s SA token via TokenReview" ); - // Look up the pod and read its sandbox-id annotation. - let pod = self - .pods_api - .get_opt(&identity.pod_name) - .await - .map_err(|e| { - warn!( - pod = %identity.pod_name, - error = %e, - "failed to fetch sandbox pod for annotation lookup" - ); - Status::internal(format!("pod GET failed: {e}")) - })?; + let pods_api = self.pods_api(&identity.namespace); + let pod = pods_api.get_opt(&identity.pod_name).await.map_err(|e| { + warn!( + pod = %identity.pod_name, + namespace = %identity.namespace, + error = %e, + "failed to fetch sandbox pod for annotation lookup" + ); + Status::internal(format!("pod GET failed: {e}")) + })?; let Some(pod) = pod else { warn!( pod = %identity.pod_name, - "sandbox pod referenced by SA token not found in this namespace" + namespace = %identity.namespace, + "sandbox pod referenced by SA token not found" ); return Err(Status::not_found("sandbox pod not found")); }; - // Defense-in-depth: confirm the pod UID matches the SA token's - // `kubernetes.io.pod.uid`. Prevents a replayed token from a - // recreated pod with the same name. let actual_uid = pod.metadata.uid.as_deref().unwrap_or_default(); if actual_uid != identity.pod_uid { warn!( @@ -294,16 +309,19 @@ impl K8sIdentityResolver for LiveK8sResolver { let sandbox_id = pod_sandbox_id(&pod)?; let owner = sandbox_owner_reference(&pod)?; - let sandbox_cr = self.get_sandbox_cr_for_owner(&owner).await.map_err(|e| { - warn!( - pod = %identity.pod_name, - sandbox_owner = %owner.name, - sandbox_owner_api_version = %owner.api_version, - error = %e, - "failed to fetch owning Sandbox CR for pod identity validation" - ); - Status::internal(format!("sandbox GET failed: {e}")) - })?; + let sandbox_cr = self + .get_sandbox_cr_for_owner(&identity.namespace, &owner) + .await + .map_err(|e| { + warn!( + pod = %identity.pod_name, + sandbox_owner = %owner.name, + sandbox_owner_api_version = %owner.api_version, + error = %e, + "failed to fetch owning Sandbox CR for pod identity validation" + ); + Status::internal(format!("sandbox GET failed: {e}")) + })?; let Some(sandbox_cr) = sandbox_cr else { warn!( pod = %identity.pod_name, @@ -327,7 +345,7 @@ impl K8sIdentityResolver for LiveK8sResolver { fn token_review_identity( status: &TokenReviewStatus, expected_audience: &str, - sandbox_namespace: &str, + namespace_validator: &NamespaceValidator, expected_service_account: &str, ) -> Result, Status> { if status.authenticated != Some(true) { @@ -356,13 +374,20 @@ fn token_review_identity( .username .as_deref() .ok_or_else(|| Status::permission_denied("TokenReview response missing username"))?; - let expected_username = - format!("system:serviceaccount:{sandbox_namespace}:{expected_service_account}"); - if username != expected_username { + + let (namespace, sa_name) = parse_sa_username(username).ok_or_else(|| { warn!( username = %username, - sandbox_namespace = %sandbox_namespace, - service_account = %expected_service_account, + "K8s TokenReview username is not a service account" + ); + Status::permission_denied("SA token username format not recognized") + })?; + + if sa_name != expected_service_account { + warn!( + username = %username, + service_account = %sa_name, + expected = %expected_service_account, "K8s TokenReview principal is not the configured sandbox service account" ); return Err(Status::permission_denied( @@ -370,9 +395,33 @@ fn token_review_identity( )); } + if !namespace_validator.accepts(&namespace) { + warn!( + username = %username, + namespace = %namespace, + "K8s TokenReview SA namespace not accepted by workspace mode validator" + ); + return Err(Status::permission_denied( + "SA token is not from an accepted sandbox namespace", + )); + } + let pod_name = user_extra_one(user, POD_NAME_EXTRA)?; let pod_uid = user_extra_one(user, POD_UID_EXTRA)?; - Ok(Some(TokenReviewIdentity { pod_name, pod_uid })) + Ok(Some(TokenReviewIdentity { + namespace, + pod_name, + pod_uid, + })) +} + +fn parse_sa_username(username: &str) -> Option<(String, String)> { + let rest = username.strip_prefix("system:serviceaccount:")?; + let (namespace, sa_name) = rest.split_once(':')?; + if namespace.is_empty() || sa_name.is_empty() { + return None; + } + Some((namespace.to_string(), sa_name.to_string())) } #[allow(clippy::result_large_err)] @@ -664,6 +713,10 @@ mod tests { cr } + fn exact_validator(ns: &str) -> NamespaceValidator { + NamespaceValidator::Exact(ns.to_string()) + } + #[test] fn token_review_identity_extracts_pod_binding() { let status = token_review_status( @@ -676,10 +729,12 @@ mod tests { ], ); - let identity = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let validator = exact_validator("openshell"); + let identity = token_review_identity(&status, "openshell-gateway", &validator, "default") .unwrap() .expect("authenticated token should resolve"); + assert_eq!(identity.namespace, "openshell"); assert_eq!(identity.pod_name, "openshell-sandbox-a"); assert_eq!(identity.pod_uid, "uid-a"); } @@ -691,9 +746,10 @@ mod tests { error: Some("invalid audience".to_string()), ..Default::default() }; + let validator = exact_validator("openshell"); assert!( - token_review_identity(&status, "openshell-gateway", "openshell", "default") + token_review_identity(&status, "openshell-gateway", &validator, "default") .unwrap() .is_none() ); @@ -710,8 +766,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("wrong audience must fail closed"); assert_eq!(err.code(), tonic::Code::Unauthenticated); } @@ -727,8 +784,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("other namespace must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -744,8 +802,9 @@ mod tests { (POD_UID_EXTRA, "uid-a"), ], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("other service account must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -758,12 +817,71 @@ mod tests { "system:serviceaccount:openshell:default", vec![], ); + let validator = exact_validator("openshell"); - let err = token_review_identity(&status, "openshell-gateway", "openshell", "default") + let err = token_review_identity(&status, "openshell-gateway", &validator, "default") .expect_err("non pod-bound tokens must be rejected"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } + #[test] + fn namespace_validator_exact_accepts_matching() { + let v = NamespaceValidator::Exact("openshell".to_string()); + assert!(v.accepts("openshell")); + assert!(!v.accepts("other")); + } + + #[test] + fn namespace_validator_prefix_accepts_managed_namespaces() { + let v = NamespaceValidator::Prefix("openshell-gw1-".to_string()); + assert!(v.accepts("openshell-gw1-workspace-a")); + assert!(v.accepts("openshell-gw1-default")); + assert!(!v.accepts("openshell-gw2-workspace-a")); + assert!(!v.accepts("other")); + } + + #[test] + fn namespace_validator_allowlist_accepts_known_namespaces() { + let al = OperatorNamespaceAllowlist::from_set(std::collections::BTreeSet::from([ + "ns-a".to_string(), + "ns-b".to_string(), + ])); + let v = NamespaceValidator::Allowlist(al); + assert!(v.accepts("ns-a")); + assert!(v.accepts("ns-b")); + assert!(!v.accepts("ns-c")); + } + + #[test] + fn token_review_identity_prefix_validator_accepts_managed_namespace() { + let status = token_review_status( + true, + vec!["openshell-gateway"], + "system:serviceaccount:openshell-gw1-workspace-a:default", + vec![ + (POD_NAME_EXTRA, "openshell-sandbox-a"), + (POD_UID_EXTRA, "uid-a"), + ], + ); + let validator = NamespaceValidator::Prefix("openshell-gw1-".to_string()); + + let identity = token_review_identity(&status, "openshell-gateway", &validator, "default") + .unwrap() + .expect("managed namespace token should resolve"); + assert_eq!(identity.namespace, "openshell-gw1-workspace-a"); + } + + #[test] + fn parse_sa_username_extracts_namespace_and_sa() { + let (ns, sa) = parse_sa_username("system:serviceaccount:openshell:default").unwrap(); + assert_eq!(ns, "openshell"); + assert_eq!(sa, "default"); + + assert!(parse_sa_username("system:node:nodename").is_none()); + assert!(parse_sa_username("system:serviceaccount::default").is_none()); + assert!(parse_sa_username("system:serviceaccount:ns:").is_none()); + } + #[test] fn pod_sandbox_id_requires_annotation() { assert_eq!( diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index bc98670b89..e09b63de09 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -32,9 +32,10 @@ use futures::{Stream, StreamExt}; use hyper_util::rt::TokioIo; use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ - CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, - DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, - DriverSandboxTemplate, GatewayListenerRequirement as ProtoGatewayListenerRequirement, + CreateSandboxRequest, DeleteSandboxRequest, DeleteWorkspaceRequest, DeleteWorkspaceResponse, + DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, + DriverSandboxSpec, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement as ProtoGatewayListenerRequirement, GetCapabilitiesRequest, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, @@ -53,6 +54,7 @@ use openshell_driver_docker::DockerComputeDriver; #[cfg(not(target_os = "windows"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, + OperatorNamespaceAllowlist, }; #[cfg(not(target_os = "windows"))] use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; @@ -566,6 +568,22 @@ impl ComputeDriver for RemoteComputeDriver { let stream = response.into_inner(); Ok(tonic::Response::new(Box::pin(stream))) } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.ensure_workspace(request).await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.delete_workspace(request).await + } } #[derive(Clone)] @@ -775,12 +793,14 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - ) -> Result { - let driver = KubernetesComputeDriver::new(config) + shutdown_rx: watch::Receiver, + ) -> Result<(Self, Option), ComputeError> { + let driver = KubernetesComputeDriver::new(config, shutdown_rx) .await .map_err(|err| ComputeError::Message(err.to_string()))?; + let operator_allowlist_arc = driver.operator_allowlist().cloned(); let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); - Self::from_driver( + let runtime = Self::from_driver( ComputeDriverKind::Kubernetes.as_str().to_string(), driver, None, @@ -792,7 +812,8 @@ impl ComputeRuntime { tracing_log_bus, supervisor_sessions, ) - .await + .await?; + Ok((runtime, operator_allowlist_arc)) } pub(crate) async fn new_remote_driver( @@ -867,6 +888,40 @@ impl ComputeRuntime { &self.gateway_listener_requirements } + pub(crate) async fn ensure_workspace(&self, workspace: &str) -> Result<(), Status> { + let workspace = workspace.to_string(); + match self + .driver + .call("driver.ensure_workspace", None, |driver| async move { + driver + .ensure_workspace(Request::new(EnsureWorkspaceRequest { workspace })) + .await + }) + .await + { + Ok(_) => Ok(()), + Err(status) if status.code() == Code::Unimplemented => Ok(()), + Err(status) => Err(status), + } + } + + pub(crate) async fn delete_workspace(&self, workspace: &str) -> Result<(), Status> { + let workspace = workspace.to_string(); + match self + .driver + .call("driver.delete_workspace", None, |driver| async move { + driver + .delete_workspace(Request::new(DeleteWorkspaceRequest { workspace })) + .await + }) + .await + { + Ok(_) => Ok(()), + Err(status) if status.code() == Code::Unimplemented => Ok(()), + Err(status) => Err(status), + } + } + pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { let driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; @@ -3770,7 +3825,18 @@ fn is_terminal_failure_reason(reason: &str) -> bool { #[cfg(test)] #[derive(Debug, Default)] -pub struct NoopTestDriver; +pub struct NoopTestDriver { + workspace_delete_failures: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +impl NoopTestDriver { + pub fn failing_workspace_deletes(count: usize) -> Self { + Self { + workspace_delete_failures: std::sync::atomic::AtomicUsize::new(count), + } + } +} #[cfg(test)] #[tonic::async_trait] @@ -3878,6 +3944,31 @@ impl ComputeDriver for NoopTestDriver { ) -> Result, Status> { Ok(tonic::Response::new(Box::pin(futures::stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + if self + .workspace_delete_failures + .fetch_update( + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + |remaining| remaining.checked_sub(1), + ) + .is_ok() + { + return Err(Status::unavailable("injected workspace cleanup failure")); + } + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } #[cfg(test)] @@ -3887,8 +3978,17 @@ pub async fn new_test_runtime(store: Arc) -> ComputeRuntime { #[cfg(test)] pub async fn new_test_runtime_for_driver(store: Arc, driver_name: &str) -> ComputeRuntime { + new_test_runtime_with_driver(store, driver_name, Arc::new(NoopTestDriver::default())).await +} + +#[cfg(test)] +pub async fn new_test_runtime_with_driver( + store: Arc, + driver_name: &str, + driver: Arc, +) -> ComputeRuntime { ComputeRuntime { - driver: TracedDriver::new(Arc::new(NoopTestDriver), "test".to_string()), + driver: TracedDriver::new(driver, "test".to_string()), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), @@ -4042,6 +4142,7 @@ mod tests { struct TestDriver { listed_sandboxes: Vec, current_sandboxes: Vec, + workspace_rpcs_unimplemented: bool, } #[tonic::async_trait] @@ -4155,6 +4256,38 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(Box::pin(stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + if self.workspace_rpcs_unimplemented { + return Err(Status::unimplemented("workspace lifecycle is unsupported")); + } + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + if self.workspace_rpcs_unimplemented { + return Err(Status::unimplemented("workspace lifecycle is unsupported")); + } + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } + } + + #[tokio::test] + async fn workspace_lifecycle_allows_legacy_driver_without_workspace_rpcs() { + let runtime = test_runtime(Arc::new(TestDriver { + workspace_rpcs_unimplemented: true, + ..Default::default() + })) + .await; + + runtime.ensure_workspace("legacy").await.unwrap(); + runtime.delete_workspace("legacy").await.unwrap(); } #[derive(Clone)] @@ -4481,6 +4614,20 @@ mod tests { UnboundedReceiverStream::new(receiver), ))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new(DeleteWorkspaceResponse {})) + } } async fn test_runtime(driver: SharedComputeDriver) -> ComputeRuntime { @@ -5247,6 +5394,20 @@ mod tests { ) -> Result, Status> { self.0.watch_sandboxes(request).await } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.0.ensure_workspace(request).await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.0.delete_workspace(request).await + } } let runtime = test_runtime(Arc::new(FailingDriver::default())).await; @@ -7487,6 +7648,7 @@ mod tests { #[tokio::test] async fn reconcile_store_with_backend_applies_driver_snapshot() { let runtime = test_runtime(Arc::new(TestDriver { + workspace_rpcs_unimplemented: false, listed_sandboxes: vec![DriverSandbox { id: "sb-1".to_string(), name: "sandbox-a".to_string(), @@ -7674,6 +7836,7 @@ mod tests { #[tokio::test] async fn reconcile_store_with_backend_does_not_recreate_missing_record_from_snapshot() { let runtime = test_runtime(Arc::new(TestDriver { + workspace_rpcs_unimplemented: false, listed_sandboxes: vec![DriverSandbox { id: "sb-1".to_string(), name: "sandbox-a".to_string(), diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index a0ea195442..2c52acbe12 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -753,7 +753,9 @@ pub mod test_support { use crate::ServerState; use crate::auth::identity::{Identity, IdentityProvider}; use crate::auth::principal::{Principal, UserPrincipal}; - use crate::compute::{new_test_runtime, new_test_runtime_for_driver}; + use crate::compute::{ + NoopTestDriver, new_test_runtime, new_test_runtime_for_driver, new_test_runtime_with_driver, + }; use crate::persistence::Store; use crate::sandbox_index::SandboxIndex; use crate::sandbox_watch::SandboxWatchBus; @@ -813,6 +815,33 @@ pub mod test_support { None, )) } + + /// Build a test state whose compute driver fails the requested number of + /// workspace cleanup calls before succeeding. + pub async fn test_server_state_with_workspace_cleanup_failures( + failures: usize, + ) -> Arc { + let store = Arc::new( + Store::connect("sqlite::memory:?cache=shared") + .await + .unwrap(), + ); + crate::ensure_default_workspace(&store).await.unwrap(); + let driver = Arc::new(NoopTestDriver::failing_workspace_deletes(failures)); + let compute = new_test_runtime_with_driver(store.clone(), "test", driver).await; + Arc::new(ServerState::new( + Config::new(None) + .with_database_url("sqlite::memory:?cache=shared") + .with_credential_drivers(["test-static"]), + store, + compute, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + None, + )) + } } // --------------------------------------------------------------------------- diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 6dc4318a8a..4d67eee018 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -2078,6 +2078,9 @@ pub(super) async fn handle_create_provider( metadata.workspace.clone_from(&workspace); } let provider_type = provider.r#type.clone(); + if state.credentials.stores_provider_credentials() && !provider.credentials.is_empty() { + state.compute.ensure_workspace(&workspace).await?; + } let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3233,6 +3236,9 @@ pub(super) async fn handle_update_provider( provider .credential_expires_at_ms .extend(req.credential_expires_at_ms); + if state.credentials.stores_provider_credentials() && !provider.credentials.is_empty() { + state.compute.ensure_workspace(&workspace).await?; + } let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3648,6 +3654,7 @@ pub(super) async fn handle_rotate_provider_credential( state.store.as_ref(), &workspace, Some(&state.credentials), + Some(&state.compute), provider_name, credential_key, ) diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index a22a195226..d83ffab0e8 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -423,6 +423,15 @@ pub(super) async fn handle_delete_workspace( .await .map_err(|e| Status::internal(format!("delete workspace members failed: {e}")))?; + // Keep the terminating workspace durable until platform cleanup has been + // accepted. A failed cleanup can then be retried through this same path. + state.compute.delete_workspace(&name).await.map_err(|e| { + Status::new( + e.code(), + format!("delete workspace platform resources failed: {e}"), + ) + })?; + let deleted = state .store .delete_if(Workspace::object_type(), &ws_id, delete_version) @@ -616,7 +625,9 @@ mod tests { use openshell_core::proto::datamodel::v1::ObjectMeta; use tonic::{Code, Request}; - use crate::grpc::test_support::{authed_request, test_server_state}; + use crate::grpc::test_support::{ + authed_request, test_server_state, test_server_state_with_workspace_cleanup_failures, + }; #[tokio::test] async fn create_workspace_returns_metadata() { @@ -1322,6 +1333,62 @@ mod tests { assert!(resp.deleted); } + #[tokio::test] + async fn delete_workspace_retains_terminating_record_when_platform_cleanup_fails() { + let state = test_server_state_with_workspace_cleanup_failures(1).await; + + handle_create_workspace( + &state, + Request::new(CreateWorkspaceRequest { + name: "cleanup-retry".to_string(), + labels: HashMap::new(), + }), + ) + .await + .unwrap(); + + let err = handle_delete_workspace( + &state, + Request::new(DeleteWorkspaceRequest { + name: "cleanup-retry".to_string(), + }), + ) + .await + .unwrap_err(); + assert_eq!(err.code(), Code::Unavailable); + + let retained: Workspace = state + .store + .get_message_by_name("", "cleanup-retry") + .await + .unwrap() + .expect("workspace must remain durable after cleanup failure"); + assert_ne!( + retained.metadata.unwrap().deletion_timestamp_ms, + 0, + "retained workspace must remain terminating" + ); + + let retry = handle_delete_workspace( + &state, + Request::new(DeleteWorkspaceRequest { + name: "cleanup-retry".to_string(), + }), + ) + .await + .unwrap() + .into_inner(); + assert!(retry.deleted); + assert!( + state + .store + .get_message_by_name::("", "cleanup-retry") + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn create_workspace_persists_labels_for_selector() { let state = test_server_state().await; diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 8ecf69a7a1..dbdfb1ac43 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -284,6 +284,7 @@ pub(crate) async fn run_server( config_file, guest_tls, } = startup; + let (shutdown_tx, shutdown_rx) = watch::channel(false); auth::descriptor_authz::init() .map_err(|error| Error::config(format!("invalid gRPC authorization metadata: {error}")))?; @@ -349,7 +350,7 @@ pub(crate) async fn run_server( gateway_tls_enabled: config.tls.is_some(), endpoint_overrides: &config.compute_driver_endpoints, }; - let compute = build_compute_runtime( + let (compute, operator_allowlist) = build_compute_runtime( &config, driver_startup, store.clone(), @@ -357,6 +358,7 @@ pub(crate) async fn run_server( sandbox_watch_bus.clone(), tracing_log_bus.clone(), supervisor_sessions.clone(), + shutdown_rx.clone(), ) .await?; let gateway_interceptors = @@ -457,13 +459,29 @@ pub(crate) async fn run_server( compute::driver_config::builtin::kubernetes_config_for_k8s_sa_bootstrap( config_file.as_ref(), )?; - let sandbox_namespace = kubernetes_config.namespace; - let sandbox_service_account = kubernetes_config.service_account_name; + let sandbox_namespace = kubernetes_config.namespace.clone(); + let sandbox_service_account = kubernetes_config.service_account_name.clone(); + let namespace_validator = match kubernetes_config.workspace_mode { + openshell_driver_kubernetes::WorkspaceMode::Shared => { + auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace) + } + openshell_driver_kubernetes::WorkspaceMode::Managed => { + auth::k8s_sa::NamespaceValidator::Prefix( + openshell_driver_kubernetes::managed_namespace_prefix( + &kubernetes_config.gateway_id, + ), + ) + } + openshell_driver_kubernetes::WorkspaceMode::Operator => { + let allowlist = operator_allowlist.clone().unwrap_or_default(); + auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) + } + }; match kube::Client::try_default().await { Ok(client) => { let resolver = Arc::new(auth::k8s_sa::LiveK8sResolver::new( client, - &sandbox_namespace, + namespace_validator, "openshell-gateway".to_string(), sandbox_service_account.clone(), )); @@ -485,8 +503,6 @@ pub(crate) async fn run_server( let state = Arc::new(state); - let (shutdown_tx, shutdown_rx) = watch::channel(false); - // Start sandboxes that were stopped during the previous gateway // shutdown so the running compute state matches the persisted store. // Runs before watchers spawn so the watch loop sees the post-start @@ -853,6 +869,9 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com // Internal wiring helper: each argument is a distinct piece of runtime state // that must be passed through, so the count is justified. +#[allow(clippy::too_many_arguments)] +type OperatorAllowlistArc = Option; + #[allow(clippy::too_many_arguments)] async fn build_compute_runtime( config: &Config, @@ -862,33 +881,41 @@ async fn build_compute_runtime( sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, -) -> Result { + shutdown_rx: watch::Receiver, +) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { let driver = configured_compute_driver(config, driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); - let runtime = match driver { + let (runtime, operator_allowlist) = match driver { #[cfg(target_os = "windows")] - ConfiguredComputeDriver::Builtin(driver) => Err(unsupported_builtin_compute_driver(driver)), + ConfiguredComputeDriver::Builtin(driver) => { + return Err(Error::execution( + unsupported_builtin_compute_driver(driver).to_string(), + )); + } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => { warn_if_kubernetes_sandbox_jwt_expiry_disabled(config); let k8s_config = compute::driver_config::builtin::kubernetes_config_from_context(driver_startup)?; - ComputeRuntime::new_kubernetes( + let (rt, allowlist) = ComputeRuntime::new_kubernetes( k8s_config, store, sandbox_index, sandbox_watch_bus, tracing_log_bus, supervisor_sessions.clone(), + shutdown_rx, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, allowlist) } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => { let docker_config = compute::driver_config::builtin::docker_config_from_context(driver_startup)?; - ComputeRuntime::new_docker( + let rt = ComputeRuntime::new_docker( config.clone(), docker_config, store, @@ -898,12 +925,14 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => { let podman_config = compute::driver_config::builtin::podman_config_from_context(driver_startup)?; - ComputeRuntime::new_podman( + let rt = ComputeRuntime::new_podman( podman_config, store, sandbox_index, @@ -912,6 +941,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } #[cfg(not(target_os = "windows"))] ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => { @@ -921,7 +952,7 @@ async fn build_compute_runtime( .file .and_then(|file| file.openshell.gateway.otlp.as_ref()); let endpoint = compute::vm::spawn(config, &vm_config, otlp_config).await?; - ComputeRuntime::new_remote_driver( + let rt = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -930,6 +961,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -942,7 +975,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - ComputeRuntime::new_remote_driver( + let rt = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -951,10 +984,12 @@ async fn build_compute_runtime( supervisor_sessions, ) .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + (rt, None) } }; - runtime.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + Ok((runtime, operator_allowlist)) } #[derive(Debug, Clone)] diff --git a/crates/openshell-server/src/provider_refresh.rs b/crates/openshell-server/src/provider_refresh.rs index 9a655babb4..dc039265f6 100644 --- a/crates/openshell-server/src/provider_refresh.rs +++ b/crates/openshell-server/src/provider_refresh.rs @@ -343,6 +343,7 @@ pub async fn refresh_provider_credential( store: &Store, workspace: &str, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, provider_name: &str, credential_key: &str, ) -> Result { @@ -448,6 +449,7 @@ pub async fn refresh_provider_credential( store, workspace, credentials, + compute, &provider, credential_key, &minted, @@ -511,6 +513,7 @@ async fn apply_minted_credential( store: &Store, workspace: &str, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, provider: &Provider, credential_key: &str, minted: &MintedCredential, @@ -520,6 +523,9 @@ async fn apply_minted_credential( let staged_handles = if let Some(credentials) = credentials && credentials.stores_provider_credentials() { + if let Some(compute) = compute { + compute.ensure_workspace(workspace).await?; + } let mut creds_to_store = HashMap::from([(credential_key.to_string(), minted.access_token.clone())]); for (key, value) in &minted.additional_credentials { @@ -1115,8 +1121,12 @@ pub fn spawn_refresh_worker(state: std::sync::Arc, interval: ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { ticker.tick().await; - if let Err(err) = - run_refresh_worker_tick(state.store.as_ref(), Some(&state.credentials)).await + if let Err(err) = run_refresh_worker_tick( + state.store.as_ref(), + Some(&state.credentials), + Some(&state.compute), + ) + .await { warn!(error = %err, "provider credential refresh worker tick failed"); } @@ -1136,6 +1146,7 @@ pub fn spawn_refresh_worker(state: std::sync::Arc, interval: async fn run_refresh_worker_tick( store: &Store, credentials: Option<&crate::credentials::CredentialRuntime>, + compute: Option<&crate::compute::ComputeRuntime>, ) -> Result<(), Status> { let now_ms = current_time_ms(); let states = list_all_refresh_states(store).await.inspect_err(|_| { @@ -1200,6 +1211,7 @@ async fn run_refresh_worker_tick( store, state.object_workspace(), credentials, + compute, &state.provider_name, &state.credential_key, ) @@ -1328,6 +1340,7 @@ mod tests { &store, "default", None, + None, "my-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1398,6 +1411,7 @@ mod tests { &store, "default", Some(&credentials), + None, "my-stored-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1498,6 +1512,7 @@ mod tests { &store, "default", None, + None, "refreshing-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1578,6 +1593,7 @@ mod tests { &store, "default", None, + None, "my-delegated-graph", "MS_GRAPH_ACCESS_TOKEN", ) @@ -1672,6 +1688,7 @@ mod tests { &store, "default", None, + None, "my-drive", "GOOGLE_DRIVE_ACCESS_TOKEN", ) @@ -1715,7 +1732,7 @@ mod tests { .unwrap(); put_refresh_state(&store, &state).await.unwrap(); - run_refresh_worker_tick(&store, None).await.unwrap(); + run_refresh_worker_tick(&store, None, None).await.unwrap(); let stored_state = get_refresh_state( &store, @@ -1751,7 +1768,7 @@ mod tests { let store = test_store().await; let traced = test_exporter::install_traced(); - run_refresh_worker_tick(&store, None).await.unwrap(); + run_refresh_worker_tick(&store, None, None).await.unwrap(); let spans = traced.finished_spans(); let root = spans @@ -1873,6 +1890,7 @@ mod tests { &store, "default", None, + None, "aws-sts-test", "AWS_ACCESS_KEY_ID", ) @@ -1970,6 +1988,7 @@ mod tests { &store, "default", None, + None, "aws-sts-custom", "AWS_ACCESS_KEY_ID", ) @@ -2046,6 +2065,7 @@ mod tests { &store, "default", None, + None, "aws-sts-partial", "AWS_ACCESS_KEY_ID", ) @@ -2088,9 +2108,17 @@ mod tests { ]), }; - apply_minted_credential(&store, "default", None, &prov, "AWS_ACCESS_KEY_ID", &minted) - .await - .unwrap(); + apply_minted_credential( + &store, + "default", + None, + None, + &prov, + "AWS_ACCESS_KEY_ID", + &minted, + ) + .await + .unwrap(); let stored = store .get_message_by_name::("default", "aws-test") @@ -2163,6 +2191,7 @@ mod tests { &store, "default", Some(&credentials), + None, &prov, "AWS_ACCESS_KEY_ID", &minted, @@ -2266,6 +2295,7 @@ mod tests { &store, "default", Some(&credentials), + None, &refreshing_provider, "AWS_ACCESS_KEY_ID", &minted, @@ -2376,6 +2406,7 @@ mod tests { &store, "default", None, + None, "aws-sts-session", "AWS_ACCESS_KEY_ID", ) @@ -2444,6 +2475,7 @@ mod tests { &store, "default", None, + None, "aws-sts-lonesession", "AWS_ACCESS_KEY_ID", ) @@ -2526,8 +2558,14 @@ mod tests { .unwrap(); put_refresh_state(&store, &state).await.unwrap(); - let rotate = - refresh_provider_credential(&store, "default", None, "aws-race", "AWS_ACCESS_KEY_ID"); + let rotate = refresh_provider_credential( + &store, + "default", + None, + None, + "aws-race", + "AWS_ACCESS_KEY_ID", + ); let interfere = async { // Wait until the rotation is inside the STS call (its state read has // already happened), then delete the refresh and release STS. @@ -2645,6 +2683,7 @@ mod tests { &store, "default", None, + None, "aws-superseded", "AWS_ACCESS_KEY_ID", ); diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 016e8d56af..f8124ded6c 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -8,13 +8,14 @@ use futures::{Stream, stream}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverSandbox, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_server::ComputeDriver, - gateway_listener_requirement::Selector, + DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverSandbox, EnsureWorkspaceRequest, + EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, }; use std::collections::HashMap; #[cfg(unix)] @@ -400,4 +401,18 @@ impl ComputeDriver for FakeComputeDriver { self.with_state(|state| state.calls.push(FakeComputeDriverCall::WatchSandboxes)); Ok(Response::new(Box::pin(stream::empty()))) } + + async fn ensure_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(EnsureWorkspaceResponse {})) + } + + async fn delete_workspace( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DeleteWorkspaceResponse {})) + } } diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 8752b863d9..93dab354b6 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -177,7 +177,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | image.tag | string | `""` | Gateway image tag. Defaults to the chart appVersion when empty. | | imagePullSecrets | list | `[]` | Image pull secrets attached to gateway and helper pods. | | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | -| networkPolicy.enabled | bool | `true` | Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. | +| networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | | openshiftRoute.annotations | object | `{}` | Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). | | openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | @@ -229,6 +229,9 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | +| server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | +| server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | | server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml new file mode 100644 index 0000000000..e9f88846c4 --- /dev/null +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: deploy the gateway in managed workspace mode. +# Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. +server: + sandboxImagePullSecrets: + - name: e2e-regcred + drivers: + kubernetes: + workspaceMode: "managed" diff --git a/deploy/helm/openshell/ci/values-workspace-operator.yaml b/deploy/helm/openshell/ci/values-workspace-operator.yaml new file mode 100644 index 0000000000..8d895e4e98 --- /dev/null +++ b/deploy/helm/openshell/ci/values-workspace-operator.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: deploy the gateway in operator workspace mode. +# Namespaces must be pre-provisioned and labeled before sandbox creation. +server: + drivers: + kubernetes: + workspaceMode: "operator" + operatorNamespaceLabel: "openshell.ai/e2e-operator-workspace=true" diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3764fa6d7a..548418abc6 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -247,6 +247,10 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (eq $workloadKind "statefulset") (gt $replicaCount 1) (not (get $workload "allowMultiReplicaStatefulSet" | default false)) -}} {{- fail "replicaCount > 1 with workload.kind=statefulset requires workload.allowMultiReplicaStatefulSet=true; use workload.kind=deployment for external database-backed multi-replica gateways." -}} {{- end -}} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} +{{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} +{{- end -}} {{- $credentialDrivers := list -}} {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} {{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} diff --git a/deploy/helm/openshell/templates/cert-manager-pki.yaml b/deploy/helm/openshell/templates/cert-manager-pki.yaml index 2cd8aeabbf..838534e62e 100644 --- a/deploy/helm/openshell/templates/cert-manager-pki.yaml +++ b/deploy/helm/openshell/templates/cert-manager-pki.yaml @@ -1,7 +1,6 @@ +{{- if .Values.certManager.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if .Values.certManager.enabled }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 073c8835ec..eb1ed8e1d0 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -25,9 +26,124 @@ rules: - list - watch # Read namespace annotations for OpenShift SCC UID/GID range resolution. + # Managed/operator modes additionally need list+watch for cluster-wide + # namespace discovery. Managed mode needs create+delete for namespace + # lifecycle. - apiGroups: - "" resources: - namespaces verbs: - get + {{- if ne $workspaceMode "shared" }} + - list + - watch + {{- end }} + {{- if eq $workspaceMode "managed" }} + - create + - delete + {{- end }} + {{- if ne $workspaceMode "shared" }} + # Cluster-wide sandbox CRD access for managed/operator workspace modes. + - apiGroups: + - agents.x-k8s.io + resources: + - sandboxes + - sandboxes/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + {{- end }} + {{- $copiedSecretNames := list }} + {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} + {{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }} + {{- end }} + {{- if eq $workspaceMode "managed" }} + {{- range .Values.server.sandboxImagePullSecrets }} + {{- if .name }} + {{- $copiedSecretNames = append $copiedSecretNames .name }} + {{- end }} + {{- end }} + {{- end }} + {{- $copiedSecretNames = uniq $copiedSecretNames }} + {{- if $copiedSecretNames }} + # Copy only explicitly configured TLS and image-pull Secrets into workspace + # namespaces. Server-side apply authorizes these requests as patch operations. + - apiGroups: + - "" + resources: + - secrets + resourceNames: + {{- range $copiedSecretNames }} + - {{ . | quote }} + {{- end }} + verbs: + - get + - patch + # Server-side apply uses PATCH for an existing Secret, but the API server + # additionally authorizes CREATE when the named Secret does not exist yet. + # Kubernetes RBAC cannot restrict CREATE by resourceNames because create + # authorization happens before the object name is available to RBAC. Keep + # reads and mutations name-restricted above; this broader grant is required + # only to create the initial copy in a gateway-owned managed namespace. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + {{- end }} + {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + # The kubernetes-secrets credential driver uses dynamic hashed names in + # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - patch + - delete + {{- end }} + {{- if eq $workspaceMode "managed" }} + # ServiceAccount creation in managed namespaces. + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + {{- if .Values.networkPolicy.enabled }} + # Apply gateway-only SSH ingress isolation in managed namespaces. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - create + - patch + - update + {{- end }} + {{- end }} diff --git a/deploy/helm/openshell/templates/credential-secrets-role.yaml b/deploy/helm/openshell/templates/credential-secrets-role.yaml index 72f0528cb2..f6187c9acb 100644 --- a/deploy/helm/openshell/templates/credential-secrets-role.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-role.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml index 4274fa6e1a..3a9ee0bddc 100644 --- a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/deploy/helm/openshell/templates/deployment.yaml b/deploy/helm/openshell/templates/deployment.yaml index e937979370..f94900b136 100644 --- a/deploy/helm/openshell/templates/deployment.yaml +++ b/deploy/helm/openshell/templates/deployment.yaml @@ -1,7 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 {{- include "openshell.validateValues" . }} {{- if eq (include "openshell.workloadKind" .) "deployment" }} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 40f5748df4..9d24dbd917 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -135,8 +135,16 @@ data: {{- end }} [openshell.drivers.kubernetes] + workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} + operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} + {{- end }} + {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} + operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} + {{- end }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} @@ -189,6 +197,11 @@ data: supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + [openshell.drivers.kubernetes.managed_ssh_ingress] + enabled = {{ .Values.networkPolicy.enabled }} + gateway_namespace = {{ .Release.Namespace | quote }} + gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } + [openshell.drivers.kubernetes.sidecar] proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} @@ -204,6 +217,8 @@ data: [openshell.credential_drivers.kubernetes-secrets] namespace = {{ include "openshell.credentialKubernetesSecretsNamespace" . | quote }} allow_reference_namespace = {{ .Values.server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace }} + workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} {{- end }} {{- if .Values.server.credentialDrivers.vault.enabled }} diff --git a/deploy/helm/openshell/templates/gateway.yaml b/deploy/helm/openshell/templates/gateway.yaml index f431ffbbd1..2b78595053 100644 --- a/deploy/helm/openshell/templates/gateway.yaml +++ b/deploy/helm/openshell/templates/gateway.yaml @@ -1,7 +1,6 @@ +{{- if and .Values.grpcRoute.enabled .Values.grpcRoute.gateway.create }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if and .Values.grpcRoute.enabled .Values.grpcRoute.gateway.create }} apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: diff --git a/deploy/helm/openshell/templates/grpcroute.yaml b/deploy/helm/openshell/templates/grpcroute.yaml index 8fde5458cd..362067fda3 100644 --- a/deploy/helm/openshell/templates/grpcroute.yaml +++ b/deploy/helm/openshell/templates/grpcroute.yaml @@ -1,7 +1,6 @@ +{{- if .Values.grpcRoute.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -{{- if .Values.grpcRoute.enabled }} apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 5ecc4428ad..af80989072 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,6 +1,7 @@ +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if eq $workspaceMode "shared" }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -42,3 +43,4 @@ rules: - pods verbs: - get +{{- end }} diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index e5233f753c..381473a58b 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,6 +1,7 @@ +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- if eq $workspaceMode "shared" }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -16,3 +17,4 @@ subjects: - kind: ServiceAccount name: {{ include "openshell.serviceAccountName" . }} namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml new file mode 100644 index 0000000000..afecada9b6 --- /dev/null +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: ClusterRole RBAC +templates: + - templates/clusterrole.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: grants managed namespace NetworkPolicy apply permissions + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create", "patch", "update"] + + - it: omits managed NetworkPolicy permissions when isolation is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + networkPolicy.enabled: false + asserts: + - notContains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + any: true + + - it: grants broad secret access when credential driver is enabled (operator) + set: + server.drivers.kubernetes.workspaceMode: operator + server.credentialDrivers.kubernetesSecrets.enabled: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "patch", "delete"] + + - it: restricts operator TLS sync to the configured secret + set: + server.drivers.kubernetes.workspaceMode: operator + server.tls.clientTlsSecretName: custom-client-tls + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["custom-client-tls"] + verbs: ["get", "patch"] + + - it: omits operator secret access when TLS and credential storage are disabled + set: + server.drivers.kubernetes.workspaceMode: operator + server.disableTls: true + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + any: true + + - it: restricts managed copies to TLS and configured image-pull secrets + set: + server.drivers.kubernetes.workspaceMode: managed + server.tls.clientTlsSecretName: custom-client-tls + server.sandboxImagePullSecrets: + - name: registry-one + - name: registry-two + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["custom-client-tls", "registry-one", "registry-two"] + verbs: ["get", "patch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + - it: restricts managed copies to image-pull secrets when TLS is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + server.disableTls: true + server.sandboxImagePullSecrets: + - name: registry-one + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + resourceNames: ["registry-one"] + verbs: ["get", "patch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + - it: omits secrets rule entirely in shared mode + set: + server.drivers.kubernetes.workspaceMode: shared + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + any: true diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 784180fcb3..afacd01eb4 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -208,6 +208,13 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' + - it: configures managed SSH isolation with the gateway peer + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.managed_ssh_ingress\].*?enabled\s*=\s*true.*?gateway_namespace\s*=\s*"my-namespace".*?gateway_pod_selector\s*=.*?app\.kubernetes\.io/name.*?openshell' + - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 4fbbf4915c..33337c768e 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -243,6 +243,20 @@ server: # the field, "RuntimeDefault" to force the runtime default profile, or # "Localhost/profile-name" for an operator-managed localhost profile. appArmorProfile: "Unconfined" + # Kubernetes compute driver settings. + drivers: + kubernetes: + # -- How workspaces map to Kubernetes namespaces. + # "shared" (default): all sandboxes in a single namespace. + # "managed": auto-creates per-workspace namespaces. + # "operator": uses pre-provisioned namespaces. + workspaceMode: "shared" + # -- K8s label selector for namespace discovery in operator mode. + # The driver watches namespaces matching this label. + operatorNamespaceLabel: "" + # -- Path to a JSON file containing an array of namespace names + # allowed in operator mode. Hot-reloaded on change. + operatorNamespaceFile: "" # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. disableTls: false @@ -389,7 +403,8 @@ server: # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway only. networkPolicy: - # -- Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. + # -- Restrict SSH ingress on sandbox pods to the gateway. In managed mode, + # the driver applies the equivalent policy to each workspace namespace. enabled: true # Built-in TLS PKI bootstrap via a pre-install/pre-upgrade hook Job. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 230ae77cee..9e96ce0d1e 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -423,6 +423,14 @@ client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" # external_server_names = ["gateway.example.com"] [openshell.drivers.kubernetes] +# Workspace isolation mode. "shared" renders all sandboxes into a single +# namespace. "managed" auto-creates a K8s namespace per workspace +# (openshell-{gateway_id}-{workspace}). "operator" maps each workspace to a +# pre-provisioned namespace discovered via label selector or drop-in file. +workspace_mode = "shared" +# Gateway identity used in managed-mode namespace naming. Defaults to the +# gateway JWT gateway_id. Must be a DNS-1123 label. +# gateway_id = "openshell" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" @@ -431,6 +439,7 @@ image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "IfNotPresent" + # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. supervisor_sideload_method = "image-volume" @@ -491,6 +500,18 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # back to 1000 on non-OpenShift clusters. # sandbox_uid = 1500 # sandbox_gid = 1500 +# Operator-mode namespace discovery. At least one must be set when +# workspace_mode = "operator". Both can be combined. +# operator_namespace_label discovers namespaces matching a K8s label selector. +# operator_namespace_label = "openshell.ai/workspace=true" +# operator_namespace_file reads allowed namespaces from a JSON/YAML file +# (hot-reloaded on change, e.g. via ConfigMap volume mount). +# operator_namespace_file = "/etc/openshell/workspace-namespaces.json" + +[openshell.drivers.kubernetes.managed_ssh_ingress] +enabled = true +gateway_namespace = "openshell" +gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } [openshell.drivers.kubernetes.sidecar] # UID used by relaxed long-running network sidecars. Strict process/binary-aware @@ -505,6 +526,11 @@ proxy_uid = 1337 process_binary_aware_network_policy = true ``` +In managed workspace mode, the Kubernetes driver copies each explicitly named +`image_pull_secrets` Secret from `namespace` into the managed workspace +namespace on sandbox creation. Shared and operator modes require the Secret to +already exist in the sandbox namespace. + ### Docker Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 189f685ff3..897e780c39 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -328,6 +328,17 @@ On hosts with restrictive firewalls (e.g. firewalld), the host firewall may addi Kubernetes-backed sandboxes run as pods in the configured sandbox namespace. Use Kubernetes for shared clusters, remote compute, GPU scheduling, and operator-managed environments. + +Kubernetes workspace namespaces are an administrative trust boundary. In +shared and managed modes, only the OpenShell gateway and its trusted Agent +Sandbox controller may administer Sandbox CRs, sandbox pods, or the configured +sandbox ServiceAccount in those namespaces. In operator mode, allowlist only +namespaces where the platform operator preserves that exclusive control. +Untrusted principals must not be able to create sandbox pods with fabricated +owner references or use the sandbox ServiceAccount. The operator namespace +allowlist is a trust grant, not a tenant isolation mechanism. + + Helm deployments set Kubernetes driver values through the chart. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). @@ -339,7 +350,8 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the gateway TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | -| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image pull secrets to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | +| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | +| `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | @@ -359,6 +371,13 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | +Managed-mode Secret copying requires the gateway ServiceAccount to create +Secrets. Kubernetes RBAC cannot restrict Secret `create` by resource name, so +the Helm chart grants cluster-wide Secret `create`; Secret `get` and `patch` +remain limited to the explicitly configured TLS and image-pull Secret names. +The driver creates copies only in gateway-owned managed namespaces. Do not +reuse the gateway ServiceAccount for unrelated workloads. + In `combined` topology, the agent container carries the Linux capabilities needed by the supervisor for network namespace setup, Landlock filesystem policy, process privilege changes, and network policy enforcement. In `sidecar` diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e2bc94d81d..9ff52c73ab 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -29,6 +29,8 @@ e2e-gpu = ["e2e"] e2e-docker-gpu = ["e2e-docker", "e2e-gpu"] e2e-kubernetes = ["e2e"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] +e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] +e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] @@ -139,6 +141,16 @@ name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" required-features = ["e2e-host-gateway"] +[[test]] +name = "workspace_namespace_managed" +path = "tests/workspace_namespace_managed.rs" +required-features = ["e2e-kubernetes-workspace-managed"] + +[[test]] +name = "workspace_namespace_operator" +path = "tests/workspace_namespace_operator.rs" +required-features = ["e2e-kubernetes-workspace-operator"] + [[test]] name = "gpu" path = "tests/gpu.rs" diff --git a/e2e/rust/tests/workspace_namespace_managed.rs b/e2e/rust/tests/workspace_namespace_managed.rs new file mode 100644 index 0000000000..38de1da6a3 --- /dev/null +++ b/e2e/rust/tests/workspace_namespace_managed.rs @@ -0,0 +1,815 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-workspace-managed")] + +//! E2E tests for managed workspace mode. +//! +//! The gateway is deployed with `workspace_mode = "managed"`, which +//! auto-creates a K8s namespace per workspace (`openshell-{gateway_id}-{ws}`) +//! and deletes it when the last sandbox is removed. +//! +//! Namespace cleanup after sandbox deletion is best-effort and depends on +//! controller finalization timing. These tests focus on verifiable behavior: +//! namespace creation, labels, ServiceAccount and SSH NetworkPolicy +//! provisioning, and sandbox CR placement in the correct namespace. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; +use openshell_e2e::harness::output::strip_ansi; + +fn kube_context() -> String { + std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") +} + +async fn kubectl(args: &[&str]) -> (bool, String) { + let context = kube_context(); + let output = tokio::process::Command::new("kubectl") + .arg("--context") + .arg(&context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn kubectl"); + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) +} + +fn managed_namespace(workspace: &str) -> String { + format!("openshell-openshell-{workspace}") +} + +async fn run_cli(args: &[&str]) -> (bool, String) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + let output = cmd.output().await.expect("failed to spawn openshell"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), strip_ansi(&combined)) +} + +fn unique_workspace(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + % 100_000; + format!("{prefix}-{ts}") +} + +async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; + if ok && !out.contains(sandbox) { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +struct ManagedCleanup { + workspace: String, + sandboxes: Vec, +} + +impl Drop for ManagedCleanup { + fn drop(&mut self) { + let bin = openshell_bin(); + for sb in &self.sandboxes { + let _ = std::process::Command::new(&bin) + .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = std::process::Command::new(&bin) + .args(["workspace", "delete", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); + if !context.is_empty() { + let ns = managed_namespace(&self.workspace); + let _ = std::process::Command::new("kubectl") + .args([ + "--context", + &context, + "delete", + "namespace", + &ns, + "--ignore-not-found", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} + +#[tokio::test] +async fn managed_creates_namespace_with_labels() { + let ws = unique_workspace("mgd"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["mgd-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create a sandbox — this triggers namespace creation. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "mgd-sb", + "--", + "echo", + "managed-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("managed-ok"), + "sandbox output missing expected string: {out}" + ); + + // Verify the managed namespace was created. + let (ok, out) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should exist: {out}"); + + // Verify labels on the namespace. + let (ok, label_out) = + kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; + assert!(ok, "failed to read namespace labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/managed-by"), + "namespace missing managed-by label: {label_out}" + ); + assert!( + label_out.contains("openshell.ai/gateway-id"), + "namespace missing gateway-id label: {label_out}" + ); + + // Verify the ServiceAccount was created in the managed namespace. + let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; + assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); + + // The managed driver copies only explicitly configured image-pull Secrets + // from the gateway namespace into the workspace namespace. + let (ok, copied_secret) = kubectl(&[ + "get", + "secret", + "e2e-regcred", + "-n", + &ns, + "-o", + "jsonpath={.type}", + ]) + .await; + assert!( + ok && copied_secret.contains("kubernetes.io/dockerconfigjson"), + "configured image-pull Secret should be copied into {ns}: {copied_secret}" + ); + + // Verify SSH ingress is restricted to the gateway peer. Because Kubernetes + // NetworkPolicies are allowlists, the absence of a sandbox peer here + // denies sandbox-to-sandbox TCP 2222 traffic. + let (ok, policy) = kubectl(&[ + "get", + "networkpolicy", + "openshell-sandbox-ssh", + "-n", + &ns, + "-o", + "json", + ]) + .await; + assert!( + ok, + "managed SSH NetworkPolicy should exist in {ns}: {policy}" + ); + let policy: serde_json::Value = + serde_json::from_str(&policy).expect("managed SSH NetworkPolicy should be valid JSON"); + assert_eq!( + policy["spec"]["podSelector"]["matchLabels"]["openshell.ai/managed-by"], + "openshell" + ); + assert_eq!(policy["spec"]["ingress"][0]["ports"][0]["port"], 2222); + assert_eq!( + policy["spec"]["ingress"][0]["from"][0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "openshell" + ); + assert!( + policy["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + ["app.kubernetes.io/name"] + .is_string(), + "SSH ingress peer must select gateway pods: {policy}" + ); + + // Verify sandbox CR is in the managed namespace (not the gateway namespace). + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; + assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); + assert!(out.contains("mgd-sb"), "sandbox CR name mismatch: {out}"); + + // Verify the sandbox is resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox list should find mgd-sb via control plane: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "get", "mgd-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox get failed: {out}"); + assert!( + out.contains("mgd-sb"), + "sandbox get should resolve mgd-sb via control plane: {out}" + ); + + // Verify sandbox delete works through the control plane (uses sandbox_lookup_selector). + let (ok, out) = run_cli(&["sandbox", "delete", "mgd-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox delete failed: {out}"); + + // Wait for the sandbox CR to be fully removed (deletion is asynchronous). + wait_sandbox_gone(&ws, "mgd-sb").await; +} + +#[tokio::test] +async fn managed_namespace_survives_with_remaining_sandboxes() { + let ws = unique_workspace("mgd2"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["sb-a".into(), "sb-b".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + // Create two sandboxes. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "sb-a", + "--", + "echo", + "a", + ]) + .await; + assert!(ok, "sandbox sb-a create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "sb-b", + "--", + "echo", + "b", + ]) + .await; + assert!(ok, "sandbox sb-b create failed: {out}"); + + // Delete first sandbox — namespace should survive because sb-b still exists. + let (ok, out) = run_cli(&["sandbox", "delete", "sb-a", "--workspace", &ws]).await; + assert!(ok, "sandbox sb-a delete failed: {out}"); + + // Wait for sb-a to be fully removed before checking namespace state. + wait_sandbox_gone(&ws, "sb-a").await; + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should still exist with sb-b"); + + // Verify sb-b's CR is still in the managed namespace. + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; + assert!(ok, "sandbox CRs should still exist in {ns}: {out}"); + assert!( + out.contains("sb-b"), + "sb-b CR should still be present: {out}" + ); + + // Verify sb-b is still resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("sb-b"), + "sandbox list should find sb-b via control plane: {out}" + ); +} + +#[tokio::test] +async fn managed_isolates_workspaces_into_separate_namespaces() { + let ws_a = unique_workspace("iso-a"); + let ws_b = unique_workspace("iso-b"); + let ns_a = managed_namespace(&ws_a); + let ns_b = managed_namespace(&ws_b); + let _cleanup_a = ManagedCleanup { + workspace: ws_a.clone(), + sandboxes: vec!["sb-iso-a".into()], + }; + let _cleanup_b = ManagedCleanup { + workspace: ws_b.clone(), + sandboxes: vec!["sb-iso-b".into()], + }; + + // Create two workspaces with sandboxes. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_a]).await; + assert!(ok, "workspace A create failed: {out}"); + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_b]).await; + assert!(ok, "workspace B create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws_a, + "--name", + "sb-iso-a", + "--", + "echo", + "a", + ]) + .await; + assert!(ok, "sandbox A create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws_b, + "--name", + "sb-iso-b", + "--", + "echo", + "b", + ]) + .await; + assert!(ok, "sandbox B create failed: {out}"); + + // Verify each workspace has its own namespace. + assert_ne!(ns_a, ns_b, "namespaces should differ"); + + let (ok, _) = kubectl(&["get", "namespace", &ns_a]).await; + assert!(ok, "namespace {ns_a} should exist"); + let (ok, _) = kubectl(&["get", "namespace", &ns_b]).await; + assert!(ok, "namespace {ns_b} should exist"); + + // Verify sandbox CRs are in the correct namespaces (no cross-contamination). + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_a, "-o", "name"]).await; + assert!(ok, "failed to list CRs in {ns_a}: {out}"); + assert!(out.contains("sb-iso-a"), "sb-iso-a should be in {ns_a}"); + assert!( + !out.contains("sb-iso-b"), + "sb-iso-b should NOT be in {ns_a}" + ); + + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_b, "-o", "name"]).await; + assert!(ok, "failed to list CRs in {ns_b}: {out}"); + assert!(out.contains("sb-iso-b"), "sb-iso-b should be in {ns_b}"); + assert!( + !out.contains("sb-iso-a"), + "sb-iso-a should NOT be in {ns_b}" + ); + + // Verify workspace isolation through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_a]).await; + assert!(ok, "sandbox list ws_a failed: {out}"); + assert!( + out.contains("sb-iso-a"), + "sandbox list ws_a should find sb-iso-a: {out}" + ); + assert!( + !out.contains("sb-iso-b"), + "sandbox list ws_a should NOT find sb-iso-b: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_b]).await; + assert!(ok, "sandbox list ws_b failed: {out}"); + assert!( + out.contains("sb-iso-b"), + "sandbox list ws_b should find sb-iso-b: {out}" + ); + assert!( + !out.contains("sb-iso-a"), + "sandbox list ws_b should NOT find sb-iso-a: {out}" + ); +} + +#[tokio::test] +async fn managed_workspace_delete_removes_namespace() { + let ws = unique_workspace("mgddel"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["del-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "del-sb", + "--", + "echo", + "del-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("del-ok"), + "sandbox output missing expected string: {out}" + ); + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "managed namespace {ns} should exist after sandbox create" + ); + + let (ok, out) = run_cli(&["sandbox", "delete", "del-sb", "--workspace", &ws]).await; + assert!(ok, "sandbox delete failed: {out}"); + + wait_sandbox_gone(&ws, "del-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; + assert!(ok, "workspace delete failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (exists, _) = kubectl(&["get", "namespace", &ns]).await; + if !exists { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("managed namespace {ns} still exists 30s after workspace delete"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn managed_tls_secret_copied_to_namespace() { + let (ok, config_out) = kubectl(&[ + "get", + "configmap", + "openshell-config", + "-n", + "openshell", + "-o", + "jsonpath={.data.gateway\\.toml}", + ]) + .await; + if !ok || !config_out.contains("client_tls_secret_name") { + eprintln!("SKIP: client_tls_secret_name not configured; TLS secret copying disabled"); + return; + } + + let ws = unique_workspace("mgdtls"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["tls-sb".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "tls-sb", + "--", + "echo", + "tls-ok", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + assert!( + out.contains("tls-ok"), + "sandbox output missing expected string: {out}" + ); + + let (ok, out) = kubectl(&["get", "secret", "openshell-client-tls", "-n", &ns]).await; + assert!( + ok, + "TLS secret openshell-client-tls should be copied to managed namespace {ns}: {out}" + ); + + let (ok, label_out) = kubectl(&[ + "get", + "secret", + "openshell-client-tls", + "-n", + &ns, + "-o", + "jsonpath={.metadata.labels}", + ]) + .await; + assert!(ok, "failed to read TLS secret labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/managed-by"), + "copied TLS secret missing managed-by label: {label_out}" + ); +} + +#[tokio::test] +async fn managed_rejects_namespace_owned_by_different_gateway() { + let ws = unique_workspace("mgdown"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![], + }; + + let (ok, out) = kubectl(&["create", "namespace", &ns]).await; + assert!(ok, "failed to pre-create namespace {ns}: {out}"); + + let (ok, out) = kubectl(&[ + "label", + "namespace", + &ns, + "openshell.ai/managed-by=openshell", + "openshell.ai/gateway-id=wrong-gateway", + ]) + .await; + assert!(ok, "failed to label namespace: {out}"); + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "conflict-sb", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for namespace owned by different gateway, but succeeded: {out}" + ); +} + +#[tokio::test] +async fn managed_full_lifecycle_with_multiple_sandboxes() { + let ws = unique_workspace("mgdlc"); + let ns = managed_namespace(&ws); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec!["lc-a".into(), "lc-b".into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "lc-a", + "--", + "echo", + "a", + ]) + .await; + assert!(ok, "sandbox lc-a create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "lc-b", + "--", + "echo", + "b", + ]) + .await; + assert!(ok, "sandbox lc-b create failed: {out}"); + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!(ok, "managed namespace {ns} should exist"); + + let (ok, out) = run_cli(&["sandbox", "delete", "lc-a", "--workspace", &ws]).await; + assert!(ok, "sandbox lc-a delete failed: {out}"); + + wait_sandbox_gone(&ws, "lc-a").await; + + let (ok, _) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "managed namespace {ns} should still exist with lc-b remaining" + ); + + let (ok, out) = run_cli(&["sandbox", "delete", "lc-b", "--workspace", &ws]).await; + assert!(ok, "sandbox lc-b delete failed: {out}"); + + wait_sandbox_gone(&ws, "lc-b").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; + assert!(ok, "workspace delete failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (exists, _) = kubectl(&["get", "namespace", &ns]).await; + if !exists { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("managed namespace {ns} still exists 30s after full lifecycle cleanup"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn managed_stop_waits_for_workspace_pod_to_disappear() { + let (ok, sandbox_api_version) = kubectl(&[ + "get", + "crd", + "sandboxes.agents.x-k8s.io", + "-o", + "jsonpath={.spec.versions[?(@.storage==true)].name}", + ]) + .await; + assert!( + ok, + "failed to resolve Sandbox API version: {sandbox_api_version}" + ); + if sandbox_api_version.trim() != "v1alpha1" { + eprintln!("SKIP: legacy pod-disappearance fallback applies only to Sandbox API v1alpha1"); + return; + } + + let ws = unique_workspace("mgdstop"); + let ns = managed_namespace(&ws); + let sandbox = "stop-sb"; + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![sandbox.into()], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + sandbox, + "--", + "echo", + "ready", + ]) + .await; + assert!(ok, "sandbox create failed: {out}"); + + let (ok, pod_name) = kubectl(&[ + "get", + "sandbox", + sandbox, + "-n", + &ns, + "-o", + "jsonpath={.metadata.annotations.agents\\.x-k8s\\.io/pod-name}", + ]) + .await; + assert!(ok, "failed to resolve sandbox pod name: {pod_name}"); + let pod_name = pod_name.trim(); + assert!(!pod_name.is_empty(), "sandbox pod annotation was empty"); + + let (ok, out) = run_cli(&["sandbox", "stop", sandbox, "--workspace", &ws]).await; + assert!(ok, "sandbox stop failed: {out}"); + + let (exists, out) = kubectl(&["get", "pod", pod_name, "-n", &ns]).await; + assert!( + !exists, + "sandbox stop returned before workspace pod {pod_name} disappeared: {out}" + ); +} + +#[tokio::test] +async fn managed_rejects_invalid_dns1123_sandbox_name() { + let ws = unique_workspace("mgddns"); + let _cleanup = ManagedCleanup { + workspace: ws.clone(), + sandboxes: vec![], + }; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; + assert!(ok, "workspace create failed: {out}"); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "my_bad_name", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox with underscore name should be rejected: {out}" + ); + assert!( + out.contains("lowercase alphanumeric"), + "error should mention character constraint: {out}" + ); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "MyBadName", + "--", + "echo", + "nope", + ]) + .await; + assert!(!ok, "sandbox with uppercase name should be rejected: {out}"); + assert!( + out.contains("lowercase alphanumeric"), + "error should mention character constraint: {out}" + ); + + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ws, + "--name", + "trailing-", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox with trailing hyphen should be rejected: {out}" + ); + let normalized: String = out + .chars() + .filter(|c| *c != '│') + .collect::() + .split_whitespace() + .collect::>() + .join(" "); + assert!( + normalized.contains("must not start or end with a hyphen"), + "error should mention hyphen constraint: {out}" + ); +} diff --git a/e2e/rust/tests/workspace_namespace_operator.rs b/e2e/rust/tests/workspace_namespace_operator.rs new file mode 100644 index 0000000000..f421aa6ada --- /dev/null +++ b/e2e/rust/tests/workspace_namespace_operator.rs @@ -0,0 +1,450 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-workspace-operator")] + +//! E2E tests for operator workspace mode. +//! +//! The gateway is deployed with `workspace_mode = "operator"` and +//! `operator_namespace_label = "openshell.ai/e2e-operator-workspace=true"`. +//! Namespaces must be pre-provisioned and labeled before sandbox creation. +//! The gateway discovers valid namespaces via the label selector. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; +use openshell_e2e::harness::output::strip_ansi; + +const OPERATOR_LABEL: &str = "openshell.ai/e2e-operator-workspace=true"; +const SA_NAME: &str = "openshell-sandbox"; + +fn kube_context() -> String { + std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") +} + +async fn kubectl(args: &[&str]) -> (bool, String) { + let context = kube_context(); + let output = tokio::process::Command::new("kubectl") + .arg("--context") + .arg(&context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("failed to spawn kubectl"); + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), combined) +} + +async fn run_cli(args: &[&str]) -> (bool, String) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + let output = cmd.output().await.expect("failed to spawn openshell"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + (output.status.success(), strip_ansi(&combined)) +} + +fn unique_namespace(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + % 100_000; + format!("{prefix}-{ts}") +} + +async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; + if ok && !out.contains(sandbox) { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +async fn provision_operator_namespace(name: &str) { + let (ok, out) = kubectl(&["create", "namespace", name]).await; + assert!(ok, "failed to create namespace {name}: {out}"); + + let (ok, out) = kubectl(&["label", "namespace", name, OPERATOR_LABEL]).await; + assert!(ok, "failed to label namespace {name}: {out}"); + + let (ok, out) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", name]).await; + assert!(ok, "failed to create SA in {name}: {out}"); +} + +async fn delete_namespace(name: &str) { + let _ = kubectl(&[ + "delete", + "namespace", + name, + "--ignore-not-found", + "--wait=false", + ]) + .await; +} + +struct OperatorCleanup { + workspace: String, + namespace: String, + sandboxes: Vec, +} + +impl Drop for OperatorCleanup { + fn drop(&mut self) { + let bin = openshell_bin(); + for sb in &self.sandboxes { + let _ = std::process::Command::new(&bin) + .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = std::process::Command::new(&bin) + .args(["workspace", "delete", &self.workspace]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); + if !context.is_empty() { + let _ = std::process::Command::new("kubectl") + .args([ + "--context", + &context, + "delete", + "namespace", + &self.namespace, + "--ignore-not-found", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} + +#[tokio::test] +async fn operator_sandbox_in_labeled_namespace() { + let ns = unique_namespace("op"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["op-sb".into()], + }; + + // Pre-provision the namespace with the operator label and ServiceAccount. + provision_operator_namespace(&ns).await; + + // Create a workspace matching the namespace name (operator mode: 1:1 mapping). + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Poll until the gateway's namespace watcher discovers the labeled namespace + // and sandbox creation succeeds (up to 30s). + let mut sandbox_out = String::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "op-sb", + "--", + "echo", + "operator-ok", + ]) + .await; + if ok { + sandbox_out = out; + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + assert!( + sandbox_out.contains("operator-ok"), + "sandbox output missing expected string: {sandbox_out}" + ); + + // Verify the sandbox CR lives in the pre-provisioned namespace. + let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; + assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); + assert!( + out.contains("op-sb"), + "sandbox CR name should be bare 'op-sb', got: {out}" + ); + + // Verify sandbox is resolvable through the OpenShell control plane. + let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ns]).await; + assert!(ok, "sandbox list failed: {out}"); + assert!( + out.contains("op-sb"), + "sandbox list should find op-sb via control plane: {out}" + ); + + let (ok, out) = run_cli(&["sandbox", "get", "op-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox get failed: {out}"); + assert!( + out.contains("op-sb"), + "sandbox get should resolve op-sb via control plane: {out}" + ); + + // Verify sandbox delete works through the control plane. + let (ok, out) = run_cli(&["sandbox", "delete", "op-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox delete failed: {out}"); + + // Wait for the sandbox CR to be fully removed (deletion is asynchronous). + wait_sandbox_gone(&ns, "op-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; + assert!(ok, "workspace delete failed: {out}"); + + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_rejects_unlabeled_namespace() { + let ns = unique_namespace("opun"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec![], + }; + + // Create namespace WITHOUT the operator label. + let (ok, out) = kubectl(&["create", "namespace", &ns]).await; + assert!(ok, "failed to create namespace: {out}"); + + // Create the ServiceAccount (not the label — that's the point). + let (ok, _) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", &ns]).await; + assert!(ok, "failed to create SA"); + + // Create workspace. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Attempt sandbox creation — should fail because namespace is not in the allowlist. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "should-fail", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for unlabeled namespace, but succeeded: {out}" + ); + + // Clean up. + let _ = run_cli(&["workspace", "delete", &ns]).await; + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_rejects_nonexistent_namespace() { + let ns = unique_namespace("opne"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec![], + }; + + // Create workspace with no matching namespace at all. + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + // Attempt sandbox creation — should fail. + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "should-fail", + "--", + "echo", + "nope", + ]) + .await; + assert!( + !ok, + "sandbox create should fail for nonexistent namespace, but succeeded: {out}" + ); + + // Clean up. + let _ = run_cli(&["workspace", "delete", &ns]).await; +} + +#[tokio::test] +async fn operator_workspace_delete_preserves_namespace() { + let ns = unique_namespace("opdel"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["opdel-sb".into()], + }; + + provision_operator_namespace(&ns).await; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "opdel-sb", + "--", + "echo", + "opdel-ok", + ]) + .await; + if ok { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let (ok, out) = run_cli(&["sandbox", "delete", "opdel-sb", "--workspace", &ns]).await; + assert!(ok, "sandbox delete failed: {out}"); + + wait_sandbox_gone(&ns, "opdel-sb").await; + + let (ok, out) = run_cli(&["workspace", "delete", &ns]).await; + assert!(ok, "workspace delete failed: {out}"); + + let (ok, out) = kubectl(&["get", "namespace", &ns]).await; + assert!( + ok, + "operator namespace {ns} should still exist after workspace delete: {out}" + ); + + let (ok, label_out) = + kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; + assert!(ok, "failed to read namespace labels: {label_out}"); + assert!( + label_out.contains("openshell.ai/e2e-operator-workspace"), + "operator label should be intact after workspace delete: {label_out}" + ); + + delete_namespace(&ns).await; +} + +#[tokio::test] +async fn operator_label_removal_blocks_sandbox_creation() { + let ns = unique_namespace("oplbl"); + let _cleanup = OperatorCleanup { + workspace: ns.clone(), + namespace: ns.clone(), + sandboxes: vec!["lbl-sb1".into()], + }; + + provision_operator_namespace(&ns).await; + + let (ok, out) = run_cli(&["workspace", "create", "--name", &ns]).await; + assert!(ok, "workspace create failed: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "lbl-sb1", + "--", + "echo", + "lbl-ok", + ]) + .await; + if ok { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!("sandbox create did not succeed within 30s: {out}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let (ok, out) = run_cli(&["sandbox", "delete", "lbl-sb1", "--workspace", &ns]).await; + assert!(ok, "sandbox lbl-sb1 delete failed: {out}"); + + wait_sandbox_gone(&ns, "lbl-sb1").await; + + let (ok, out) = kubectl(&[ + "label", + "namespace", + &ns, + "openshell.ai/e2e-operator-workspace-", + ]) + .await; + assert!(ok, "failed to remove operator label: {out}"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let (ok, _out) = run_cli(&[ + "sandbox", + "create", + "--workspace", + &ns, + "--name", + "lbl-sb2", + "--", + "echo", + "should-fail", + ]) + .await; + if !ok { + break; + } + // Sandbox was created despite label removal — clean it up and retry. + let _ = run_cli(&["sandbox", "delete", "lbl-sb2", "--workspace", &ns]).await; + if tokio::time::Instant::now() >= deadline { + panic!( + "sandbox creation still succeeds 30s after operator label removal; \ + watcher did not remove namespace from allowlist" + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + delete_namespace(&ns).await; +} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index cfc886ffcd..f6d0efc4ec 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -277,6 +277,22 @@ cleanup() { --ignore-not-found >/dev/null 2>&1 || true fi + # Sweep managed-mode and operator-mode workspace namespaces before + # uninstalling the Helm release (ClusterRole still needed for deletion). + if command -v kubectl >/dev/null 2>&1 && [ -n "${KUBE_CONTEXT}" ]; then + for label in "openshell.ai/managed-by=openshell" \ + "openshell.ai/e2e-operator-workspace=true"; do + ns_list="$(kctl get namespaces -l "${label}" -o name 2>/dev/null || true)" + if [ -n "${ns_list}" ]; then + echo "Cleaning up namespaces with label ${label}..." + echo "${ns_list}" | while read -r ns_ref; do + kctl delete "${ns_ref}" --wait=false --ignore-not-found \ + 2>/dev/null || true + done + fi + done + fi + if [ "${HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then if command -v helm >/dev/null 2>&1; then helmctl uninstall "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait \ @@ -798,6 +814,14 @@ else --wait --timeout 5m HELM_INSTALLED=1 + if [ -n "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET:-}" ]; then + kctl -n "${NAMESPACE}" create secret docker-registry \ + "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET}" \ + --docker-server=registry.example.test \ + --docker-username=e2e-user \ + --docker-password=e2e-password + fi + LOCAL_PORT="$(e2e_pick_port)" echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 3a0b7609ab..0ce4f61539 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -51,6 +51,13 @@ service ComputeDriver { // Stream sandbox observations from the platform. rpc WatchSandboxes(WatchSandboxesRequest) returns (stream WatchSandboxesEvent); + + // Ensure platform resources for a workspace exist (e.g. namespace). + // Idempotent: succeeds if resources already exist. + rpc EnsureWorkspace(EnsureWorkspaceRequest) returns (EnsureWorkspaceResponse); + + // Tear down platform resources for a workspace. + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); } message GetCapabilitiesRequest {} @@ -339,3 +346,17 @@ message WatchSandboxesEvent { WatchSandboxesPlatformEvent platform_event = 3; } } + +message EnsureWorkspaceRequest { + // Workspace identifier used by the gateway. + string workspace = 1; +} + +message EnsureWorkspaceResponse {} + +message DeleteWorkspaceRequest { + // Workspace identifier used by the gateway. + string workspace = 1; +} + +message DeleteWorkspaceResponse {} diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index f9ac186f52..82b8d5cfc8 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -230,11 +230,13 @@ preload_sandbox_image() { docker pull --platform "${platform}" "${PRELOAD_SANDBOX_IMAGE}" fi + # Save without --platform: the platform-specific pull already constrained the + # local image, and --platform fails on OCI index (multi-arch) manifests. tmp="$(mktemp "${TMPDIR:-/tmp}/openshell-sandbox-image.XXXXXX")" - if ! docker image save --platform "${platform}" -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}"; then + if ! docker image save -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}"; then echo "Pulling sandbox image for ${platform}..." docker pull --platform "${platform}" "${PRELOAD_SANDBOX_IMAGE}" - docker image save --platform "${platform}" -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}" + docker image save -o "${tmp}" "${PRELOAD_SANDBOX_IMAGE}" fi if ! k3d image import "${tmp}" --cluster "${CLUSTER_NAME}"; then diff --git a/tasks/test.toml b/tasks/test.toml index d50750e9cb..bef8baf2b5 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -170,6 +170,16 @@ description = "Run Kubernetes e2e for provider credential storage backed by Kube env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:workspace-managed"] +description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +run = "e2e/rust/e2e-kubernetes.sh" + +["e2e:kubernetes:workspace-operator"] +description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh"