Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions .github/workflows/branch-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,34 @@ jobs:
extra-helm-values: ${{ matrix.extra_helm_values }}
cli-artifact-prefix: rust-binary-cli

kubernetes-workspace-managed-e2e:
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-kubernetes-test.yml
with:
image-tag: ${{ github.sha }}
job-name: Kubernetes E2E (workspace managed mode)
e2e-task: e2e:kubernetes:workspace-managed
cli-artifact-prefix: rust-binary-cli

kubernetes-workspace-operator-e2e:
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-kubernetes-test.yml
with:
image-tag: ${{ github.sha }}
job-name: Kubernetes E2E (workspace operator mode)
e2e-task: e2e:kubernetes:workspace-operator
cli-artifact-prefix: rust-binary-cli

kubernetes-ha-e2e:
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
Expand Down Expand Up @@ -212,7 +240,7 @@ jobs:

core-e2e-result:
name: Core E2E result
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e]
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
runs-on: ubuntu-latest
steps:
Expand All @@ -224,6 +252,8 @@ jobs:
BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }}
E2E_RESULT: ${{ needs.e2e.result }}
KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }}
KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }}
KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }}
run: |
set -euo pipefail
failed=0
Expand All @@ -233,7 +263,9 @@ jobs:
"build-cli:$BUILD_CLI_RESULT" \
"build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \
"e2e:$E2E_RESULT" \
"kubernetes-e2e:$KUBERNETES_E2E_RESULT"; do
"kubernetes-e2e:$KUBERNETES_E2E_RESULT" \
"kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \
"kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do
name="${item%%:*}"
result="${item#*:}"
if [ "$result" != "success" ]; then
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

77 changes: 77 additions & 0 deletions architecture/compute-runtimes.md
Original file line number Diff line number Diff line change
Expand Up @@ -286,5 +286,82 @@ Standalone local deployments start the gateway with a selected runtime such as
Docker, Podman, or VM. The CLI can register multiple gateways and switch between
them without changing the sandbox architecture.

## Workspace Namespace Modes (Kubernetes)

The Kubernetes driver maps workspaces to namespaces through the `workspace_mode`
configuration field (`WorkspaceMode` in `crates/openshell-driver-kubernetes/src/config.rs`).
The mode controls namespace resolution, resource naming, sandbox CR watching, SA
token authentication, and RBAC requirements.

| Mode | Namespace resolution | Resource name | Namespace lifecycle |
|---|---|---|---|
| **Shared** (default) | Single static namespace from config | `{workspace}--{name}` | None |
| **Managed** | `openshell-{gateway_id}-{workspace}` | bare sandbox name | Driver creates and deletes |
| **Operator** | Workspace resolves to a pre-provisioned namespace | bare sandbox name | External (platform team) |

**Shared** renders all sandboxes into one configured namespace. Resource names
embed the workspace prefix for collision avoidance. No namespace lifecycle
management. RBAC uses a namespace-scoped Role.

**Managed** auto-creates a K8s namespace per workspace on first sandbox create.
Each new namespace receives a ServiceAccount and the configured gateway-only
SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the
driver's source namespace on every sandbox create so registry credential
rotations propagate. The namespace also copies OpenShift SCC UID-range and
supplemental-group annotations from the gateway namespace when present. The
driver deletes the namespace during workspace deletion. The workspace remains
durably `Terminating` until the Kubernetes API accepts namespace cleanup, so a
transient failure can be retried. Namespace deletion uses the fetched UID as a
precondition to avoid deleting a replacement namespace. Requires a non-empty
`gateway_id` (validated as a
DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character
limit. RBAC promotes sandbox CRD permissions to a ClusterRole and adds namespace
`create`/`delete` and ServiceAccount `create`/`get` permissions.

Operator mode does not create NetworkPolicies or copy image-pull Secrets.
Platform teams must apply the gateway ingress boundary and provision configured
image-pull Secrets in every operator-managed namespace.

**Operator** uses pre-provisioned namespaces selected through a K8s label
selector (`operator_namespace_label`), a drop-in allowlist file
(`operator_namespace_file`), or an explicit workspace-to-namespace map
(`operator_workspace_namespaces`). Exactly one must be configured. Label and
file sources preserve the 1:1 workspace/namespace convention; the map allows
platform operators to keep application-facing workspace names independent of
cluster namespace naming conventions.
The `OperatorNamespaceAllowlist` (`Arc<RwLock<BTreeSet<String>>>`) is populated
from the configured source and read by the namespace resolver. Sandbox creation
fails closed if the workspace does not resolve to an allowed namespace. Platform
teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as
managed mode but without namespace `create`/`delete` or ServiceAccount
permissions.

### Watching and Querying

Managed and operator modes set `is_multi_namespace() == true`, which switches
sandbox CR watchers from namespace-scoped `Api::namespaced` to cluster-wide
`Api::all_with`. In managed mode the driver scopes cluster-wide queries with a
`LABEL_GATEWAY_ID` label selector to support multiple gateways on the same
cluster. K8s Events are not watched in cluster-wide mode — the cluster-wide
watcher emits only sandbox CR changes, not platform events.

### SA Token Authentication

The gateway's `K8sServiceAccountAuthenticator` adapts its `NamespaceValidator`
per mode (`crates/openshell-server/src/auth/k8s_sa.rs`):

- **Shared:** `Exact` — accepts only the single configured namespace.
- **Managed:** `Prefix` — accepts any namespace starting with `openshell-{gateway_id}-`.
- **Operator:** `Allowlist` — accepts namespaces present in the dynamic
`BTreeSet` populated by the label/file watchers. Starts empty (fail-closed)
until the first watcher update.

### Credential Driver Integration

The Kubernetes Secrets credential driver (`openshell-driver-kubernetes-secrets`)
stores secrets in workspace-specific namespaces when `workspace_mode` is managed
or operator. In shared mode, all secrets render into the single configured
namespace.

When runtime infrastructure changes, validate the relevant sandbox e2e path and
update the matching driver README if a maintainer-facing constraint changes.
3 changes: 3 additions & 0 deletions crates/openshell-core/src/driver_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ pub const LABEL_SANDBOX_NAMESPACE: &str = "openshell.ai/sandbox-namespace";
/// Container/pod label carrying the sandbox workspace.
pub const LABEL_SANDBOX_WORKSPACE: &str = "openshell.ai/sandbox-workspace";

/// Label carrying the gateway identity on managed namespaces.
pub const LABEL_GATEWAY_ID: &str = "openshell.ai/gateway-id";

/// Label selector that matches all OpenShell-managed resources which carry a
/// sandbox ID label. Used by list and watch operations to exclude foreign
/// resources from the same namespace.
Expand Down
4 changes: 4 additions & 0 deletions crates/openshell-core/src/grpc_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,10 @@ async fn build_plain_channel(endpoint: &str) -> Result<Channel> {

let tls_enabled = endpoint.starts_with("https://");

// TODO: TLS certs are loaded once here and never re-read. The gateway
// server side supports hot-reload (ArcSwap + notify in tls.rs). The
// supervisor should do the same so that cert-manager rotations take
// effect without restarting the sandbox.
if tls_enabled {
let ca_path = std::env::var(sandbox_env::TLS_CA)
.into_diagnostic()
Expand Down
19 changes: 17 additions & 2 deletions crates/openshell-driver-docker/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ use openshell_core::progress::{
};
use openshell_core::proto::compute::v1::{
CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse,
DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus,
DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest,
DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent,
DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest,
EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest,
GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest,
GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse,
GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest,
Expand Down Expand Up @@ -1550,6 +1551,20 @@ impl ComputeDriver for DockerComputeDriver {

Ok(Response::new(Box::pin(ReceiverStream::new(out_rx))))
}

async fn ensure_workspace(
&self,
_request: Request<EnsureWorkspaceRequest>,
) -> Result<Response<EnsureWorkspaceResponse>, Status> {
Ok(Response::new(EnsureWorkspaceResponse {}))
}

async fn delete_workspace(
&self,
_request: Request<DeleteWorkspaceRequest>,
) -> Result<Response<DeleteWorkspaceResponse>, Status> {
Ok(Response::new(DeleteWorkspaceResponse {}))
}
}

impl DockerProvisioningFailure {
Expand Down
116 changes: 114 additions & 2 deletions crates/openshell-driver-kubernetes-secrets/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,53 @@ impl CredentialDriverService {
}
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
enum WorkspaceMode {
#[default]
Shared,
Managed,
Operator,
}

#[derive(Debug, Clone, PartialEq, Eq)]
struct KubernetesSecretsDriverSettings {
namespace: String,
allow_reference_namespace: bool,
workspace_mode: WorkspaceMode,
gateway_id: String,
operator_workspace_namespaces: BTreeMap<String, String>,
}

impl KubernetesSecretsDriverSettings {
fn target_namespace(&self, workspace: &str) -> Result<String, Status> {
match self.workspace_mode {
WorkspaceMode::Shared => Ok(self.namespace.clone()),
WorkspaceMode::Managed => Ok(format!("openshell-{}-{}", self.gateway_id, workspace)),
WorkspaceMode::Operator if self.operator_workspace_namespaces.is_empty() => {
Ok(workspace.to_string())
}
WorkspaceMode::Operator => self
.operator_workspace_namespaces
.get(workspace)
.cloned()
.ok_or_else(|| {
Status::failed_precondition(format!(
"workspace '{workspace}' has no configured operator namespace"
))
}),
}
}
}

#[derive(Debug, Clone, Default, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
struct KubernetesSecretsDriverConfig {
namespace: Option<String>,
allow_reference_namespace: bool,
workspace_mode: WorkspaceMode,
gateway_id: Option<String>,
operator_workspace_namespaces: BTreeMap<String, String>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
Expand Down Expand Up @@ -134,7 +170,10 @@ impl KubernetesSecretsCredentialDriver {
credential_key: &str,
) -> Result<KubernetesSecretReference, Status> {
let reference = Self::parse_handle(handle, credential_key)?;
if reference.namespace != self.settings.namespace
// In managed/operator modes secrets live in workspace-specific
// namespaces so cross-namespace handles are expected.
if self.settings.workspace_mode == WorkspaceMode::Shared
&& reference.namespace != self.settings.namespace
&& !self.settings.allow_reference_namespace
{
return Err(Status::permission_denied(format!(
Expand Down Expand Up @@ -175,7 +214,7 @@ impl KubernetesSecretsCredentialDriver {
reference
} else {
KubernetesSecretReference {
namespace: self.settings.namespace.clone(),
namespace: self.settings.target_namespace(&request.workspace)?,
secret_name: managed_secret_name(
&request.workspace,
&request.provider_id,
Expand Down Expand Up @@ -508,6 +547,9 @@ impl KubernetesSecretsDriverSettings {
Ok(Self {
namespace,
allow_reference_namespace: config.allow_reference_namespace,
workspace_mode: config.workspace_mode,
gateway_id: config.gateway_id.unwrap_or_default(),
operator_workspace_namespaces: config.operator_workspace_namespaces,
})
}
}
Expand Down Expand Up @@ -842,6 +884,9 @@ mod tests {
let settings = KubernetesSecretsDriverSettings {
namespace: "openshell".to_string(),
allow_reference_namespace: false,
workspace_mode: WorkspaceMode::Shared,
gateway_id: String::new(),
operator_workspace_namespaces: BTreeMap::new(),
};
let reference = KubernetesSecretsCredentialDriver::parse_handle(
&handle("v1:other-namespace:provider-secret"),
Expand All @@ -865,6 +910,9 @@ mod tests {
let settings = KubernetesSecretsDriverSettings {
namespace: "openshell".to_string(),
allow_reference_namespace: true,
workspace_mode: WorkspaceMode::Shared,
gateway_id: String::new(),
operator_workspace_namespaces: BTreeMap::new(),
};
let reference = KubernetesSecretsCredentialDriver::parse_handle(
&handle("v1:other-namespace:provider-secret"),
Expand Down Expand Up @@ -1065,4 +1113,68 @@ mod tests {
assert_eq!(err.code(), Code::FailedPrecondition);
assert!(err.message().contains("is not managed by OpenShell"));
}

#[test]
fn target_namespace_shared_returns_static_namespace() {
let settings = KubernetesSecretsDriverSettings {
namespace: "openshell".to_string(),
allow_reference_namespace: false,
workspace_mode: WorkspaceMode::Shared,
gateway_id: String::new(),
operator_workspace_namespaces: BTreeMap::new(),
};
assert_eq!(settings.target_namespace("team-a").unwrap(), "openshell");
assert_eq!(settings.target_namespace("team-b").unwrap(), "openshell");
}

#[test]
fn target_namespace_managed_computes_from_workspace() {
let settings = KubernetesSecretsDriverSettings {
namespace: "openshell".to_string(),
allow_reference_namespace: false,
workspace_mode: WorkspaceMode::Managed,
gateway_id: "gw1".to_string(),
operator_workspace_namespaces: BTreeMap::new(),
};
assert_eq!(
settings.target_namespace("team-a").unwrap(),
"openshell-gw1-team-a"
);
assert_eq!(
settings.target_namespace("team-b").unwrap(),
"openshell-gw1-team-b"
);
}

#[test]
fn target_namespace_operator_uses_workspace_name() {
let settings = KubernetesSecretsDriverSettings {
namespace: "openshell".to_string(),
allow_reference_namespace: false,
workspace_mode: WorkspaceMode::Operator,
gateway_id: String::new(),
operator_workspace_namespaces: BTreeMap::new(),
};
assert_eq!(settings.target_namespace("team-a").unwrap(), "team-a");
assert_eq!(settings.target_namespace("prod-ns").unwrap(), "prod-ns");
}

#[test]
fn target_namespace_operator_uses_explicit_mapping() {
let settings = KubernetesSecretsDriverSettings {
namespace: "openshell".to_string(),
allow_reference_namespace: false,
workspace_mode: WorkspaceMode::Operator,
gateway_id: String::new(),
operator_workspace_namespaces: BTreeMap::from([(
"team-a".to_string(),
"platform-team-a".to_string(),
)]),
};
assert_eq!(
settings.target_namespace("team-a").unwrap(),
"platform-team-a"
);
assert!(settings.target_namespace("unknown").is_err());
}
}
1 change: 1 addition & 0 deletions crates/openshell-driver-kubernetes/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true }
thiserror = { workspace = true }
miette = { workspace = true }
notify = "8"

[dev-dependencies]
temp-env = "0.3"
Expand Down
Loading