From d61cef2be2cafd4ec5235b9dfcc5dcbae14ada65 Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Mon, 29 Jun 2026 12:14:32 +0300 Subject: [PATCH 1/2] feat(api-rs): pin sandbox pods via SESSION_SANDBOX_NODE_SELECTOR / _TOLERATIONS api-rs inlines the sandbox pod template when it creates the agents.x-k8s.io Sandbox, so the pod previously had no nodeSelector/tolerations and could not be confined to a specific node pool (e.g. a cost-saving spot pool). Add two optional knobs, mirroring the existing SESSION_SANDBOX_* env family: - SESSION_SANDBOX_NODE_SELECTOR key=value[,key=value...] -> pod nodeSelector - SESSION_SANDBOX_TOLERATIONS JSON array of toleration objects -> pod tolerations Both default to empty, so the rendered pod is unchanged when unset. Wired through AgentSandboxConfig into the pod spec via the same insert_optional path used for imagePullSecrets. Unit tests cover arg parsing and that the fields land on (and stay absent from) the built Sandbox pod template. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../crates/centaur-api-server/src/args.rs | 53 +++++++++++++++++ .../centaur-sandbox-agent-k8s/src/lib.rs | 59 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/services/api-rs/crates/centaur-api-server/src/args.rs b/services/api-rs/crates/centaur-api-server/src/args.rs index 2ffd7ed33..623429197 100644 --- a/services/api-rs/crates/centaur-api-server/src/args.rs +++ b/services/api-rs/crates/centaur-api-server/src/args.rs @@ -509,6 +509,22 @@ struct SandboxArgs { value_delimiter = ',' )] image_pull_secrets: Vec, + /// `key=value` pairs set as the `nodeSelector` on every sandbox pod, e.g. + /// to pin sandboxes to a dedicated (spot) node pool. + #[arg( + long = "session-sandbox-node-selector", + env = "SESSION_SANDBOX_NODE_SELECTOR", + value_delimiter = ',' + )] + node_selector: Vec, + /// JSON array of toleration objects applied to every sandbox pod, so they + /// can schedule onto a tainted node pool, e.g. + /// `[{"key":"centaur","operator":"Equal","value":"true","effect":"NoSchedule"}]`. + #[arg( + long = "session-sandbox-tolerations", + env = "SESSION_SANDBOX_TOLERATIONS" + )] + tolerations: Option, #[arg( long = "session-sandbox-ready-timeout-secs", alias = "kubernetes-sandbox-ready-timeout-s", @@ -1326,6 +1342,25 @@ impl TryFrom<&SandboxArgs> for AgentSandboxConfig { .map(str::to_owned) .collect(); config.ready_timeout = Duration::from_secs(args.ready_timeout_secs); + config.node_selector = args + .node_selector + .iter() + .filter_map(|entry| entry.split_once('=')) + .map(|(key, value)| (key.trim().to_owned(), value.trim().to_owned())) + .filter(|(key, _)| !key.is_empty()) + .collect(); + if let Some(raw) = args + .tolerations + .as_deref() + .map(str::trim) + .filter(|raw| !raw.is_empty()) + { + config.tolerations = serde_json::from_str(raw).map_err(|err| { + ServerError::UnsupportedConfig(format!( + "SESSION_SANDBOX_TOLERATIONS must be a JSON array of toleration objects: {err}" + )) + })?; + } config.iron_proxy = args.iron_proxy.to_config()?; if let Some(proxy) = config.iron_proxy.as_mut() { // `to_config` only ships the harness fragment, so add infra and @@ -2067,6 +2102,10 @@ mod tests { "github-access-token-read-packages, extra-secret ", "--session-sandbox-ready-timeout-secs", "42", + "--session-sandbox-node-selector", + "centaur-pool=true, cloud.google.com/gke-spot=true ", + "--session-sandbox-tolerations", + r#"[{"key":"centaur","operator":"Equal","value":"true","effect":"NoSchedule"}]"#, "--kubernetes-sandbox-iron-proxy-mode", "disabled", ]) @@ -2080,6 +2119,20 @@ mod tests { vec!["github-access-token-read-packages", "extra-secret"] ); assert_eq!(config.ready_timeout, Duration::from_secs(42)); + assert_eq!( + config.node_selector.get("centaur-pool").map(String::as_str), + Some("true") + ); + assert_eq!( + config + .node_selector + .get("cloud.google.com/gke-spot") + .map(String::as_str), + Some("true") + ); + assert_eq!(config.tolerations.len(), 1); + assert_eq!(config.tolerations[0]["key"], "centaur"); + assert_eq!(config.tolerations[0]["effect"], "NoSchedule"); assert!(config.iron_proxy.is_none()); } diff --git a/services/api-rs/crates/centaur-sandbox-agent-k8s/src/lib.rs b/services/api-rs/crates/centaur-sandbox-agent-k8s/src/lib.rs index e3c3674f5..cb5301d85 100644 --- a/services/api-rs/crates/centaur-sandbox-agent-k8s/src/lib.rs +++ b/services/api-rs/crates/centaur-sandbox-agent-k8s/src/lib.rs @@ -72,6 +72,12 @@ pub struct AgentSandboxConfig { /// harness's usage/cost spans never leave the pod. pub otlp_egress: Option, pub ready_timeout: Duration, + /// `nodeSelector` applied to every sandbox pod. Empty = none. Lets the + /// control plane confine sandboxes to a dedicated node pool (e.g. spot). + pub node_selector: BTreeMap, + /// Toleration objects applied verbatim to every sandbox pod, so sandboxes + /// can schedule onto a tainted node pool. Empty = none. + pub tolerations: Vec, } /// Destination of the sandbox's direct OTLP export, expressed as the target @@ -113,6 +119,8 @@ impl AgentSandboxConfig { tools: None, otlp_egress: None, ready_timeout: Duration::from_secs(60), + node_selector: BTreeMap::new(), + tolerations: Vec::new(), } } @@ -711,6 +719,16 @@ fn build_agent_sandbox( .collect::>() }), ); + insert_optional( + &mut pod_spec, + "nodeSelector", + (!config.node_selector.is_empty()).then(|| config.node_selector.clone()), + ); + insert_optional( + &mut pod_spec, + "tolerations", + (!config.tolerations.is_empty()).then(|| config.tolerations.clone()), + ); let mut agent_spec = json!({ "replicas": 1, @@ -905,6 +923,47 @@ mod tests { assert!(container.resources.as_ref().unwrap().limits.is_some()); } + #[test] + fn node_selector_and_tolerations_land_on_the_pod() { + let spec = SandboxSpec::new("centaur-agent:latest"); + let mut config = AgentSandboxConfig::new("centaur"); + config + .node_selector + .insert("centaur-pool".to_owned(), "true".to_owned()); + config.tolerations.push(json!({ + "key": "centaur", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + })); + + let sandbox = build_agent_sandbox(&SandboxId::new("asbx-test"), &spec, &config).unwrap(); + let pod_spec = &sandbox.spec.pod_template.spec; + + assert_eq!( + pod_spec + .node_selector + .as_ref() + .and_then(|selector| selector.get("centaur-pool")) + .map(String::as_str), + Some("true") + ); + let tolerations = pod_spec.tolerations.as_ref().unwrap(); + assert_eq!(tolerations.len(), 1); + assert_eq!(tolerations[0].key.as_deref(), Some("centaur")); + assert_eq!(tolerations[0].effect.as_deref(), Some("NoSchedule")); + + // Unset by default: no nodeSelector/tolerations on the pod. + let bare = build_agent_sandbox( + &SandboxId::new("asbx-bare"), + &spec, + &AgentSandboxConfig::new("centaur"), + ) + .unwrap(); + assert!(bare.spec.pod_template.spec.node_selector.is_none()); + assert!(bare.spec.pod_template.spec.tolerations.is_none()); + } + #[test] fn tools_clone_rides_iron_proxy_when_enabled() { // apply_proxy_env runs before build_agent_sandbox in create(), so the From d7c5c96aa5d76d6e2679172b4c52075264005bc0 Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Mon, 29 Jun 2026 15:09:23 +0300 Subject: [PATCH 2/2] feat(api-rs): pin per-sandbox iron-proxy pod to the sandbox's node selection The per-sandbox iron-proxy pod was built with no nodeSelector/tolerations, so with sandboxes pinned to a tainted pool the proxy pods spilled onto general nodes. Carry the sandbox node_selector/tolerations into IronProxyConfig and set them on the proxy PodSpec, so the proxy follows its sandbox onto the same pool. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../crates/centaur-api-server/src/args.rs | 5 ++ .../src/iron_proxy.rs | 63 ++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/services/api-rs/crates/centaur-api-server/src/args.rs b/services/api-rs/crates/centaur-api-server/src/args.rs index 623429197..cf63de186 100644 --- a/services/api-rs/crates/centaur-api-server/src/args.rs +++ b/services/api-rs/crates/centaur-api-server/src/args.rs @@ -1362,7 +1362,12 @@ impl TryFrom<&SandboxArgs> for AgentSandboxConfig { })?; } config.iron_proxy = args.iron_proxy.to_config()?; + // The per-sandbox proxy pod follows the sandbox onto the same node pool. + let sandbox_node_selector = config.node_selector.clone(); + let sandbox_tolerations = config.tolerations.clone(); if let Some(proxy) = config.iron_proxy.as_mut() { + proxy.node_selector = sandbox_node_selector; + proxy.tolerations = sandbox_tolerations; // `to_config` only ships the harness fragment, so add infra and // discovered tool fragments for any static proxy placeholder // metadata the backend needs. diff --git a/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs b/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs index 0be0c87dd..f39438a4c 100644 --- a/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs +++ b/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs @@ -6,7 +6,7 @@ use centaur_sandbox_core::{SandboxError, SandboxId, SandboxResult, SandboxSpec}; use k8s_openapi::api::core::v1::{ Capabilities, Container, ContainerPort, EmptyDirVolumeSource, EnvFromSource, EnvVar as K8sEnvVar, HTTPGetAction, Pod, PodSpec, Probe, SecretEnvSource, SecretVolumeSource, - SecurityContext, Service, ServicePort, ServiceSpec, Volume, VolumeMount, + SecurityContext, Service, ServicePort, ServiceSpec, Toleration, Volume, VolumeMount, }; use k8s_openapi::api::networking::v1::{ NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicyIngressRule, NetworkPolicyPeer, @@ -83,6 +83,11 @@ pub struct IronProxyConfig { pub op_connect_app_name: String, pub op_connect_port: u16, pub api_pod_labels: BTreeMap, + /// `nodeSelector` for the per-sandbox proxy pod. Mirrors the sandbox's so the + /// proxy lands on the same (e.g. spot) pool. Empty = none. + pub node_selector: BTreeMap, + /// Tolerations for the per-sandbox proxy pod (verbatim JSON). Empty = none. + pub tolerations: Vec, } impl IronProxyConfig { @@ -106,6 +111,8 @@ impl IronProxyConfig { "app.kubernetes.io/component".to_owned(), "api".to_owned(), )]), + node_selector: BTreeMap::new(), + tolerations: Vec::new(), } } } @@ -1091,6 +1098,15 @@ fn build_iron_proxy_pod( restart_policy: Some("Never".to_owned()), containers: vec![iron_proxy_container(iron_proxy, resolved, sync)], volumes: Some(iron_proxy_volumes(iron_proxy)), + node_selector: (!iron_proxy.node_selector.is_empty()) + .then(|| iron_proxy.node_selector.clone()), + tolerations: (!iron_proxy.tolerations.is_empty()).then(|| { + iron_proxy + .tolerations + .iter() + .filter_map(|toleration| serde_json::from_value::(toleration.clone()).ok()) + .collect() + }), ..Default::default() }), ..Default::default() @@ -1761,6 +1777,51 @@ mod tests { } } + #[test] + fn iron_proxy_pod_carries_node_selector_and_tolerations() { + let id = SandboxId::new("asbx-test"); + let sync = ProxySyncEnv { + proxy_id: "proxy-1".to_owned(), + control_url: "http://console:3000".to_owned(), + token: "iprx-token".to_owned(), + }; + let mut iron_proxy = IronProxyConfig::new("proxy:test", "ca-cert", "ca-key"); + iron_proxy + .node_selector + .insert("centaur-pool".to_owned(), "true".to_owned()); + iron_proxy.tolerations.push(json!({ + "key": "centaur", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + })); + + let pod = build_iron_proxy_pod(&id, &iron_proxy, &resolved(), &sync); + let spec = pod.spec.as_ref().unwrap(); + assert_eq!( + spec.node_selector + .as_ref() + .and_then(|selector| selector.get("centaur-pool")) + .map(String::as_str), + Some("true") + ); + let tolerations = spec.tolerations.as_ref().unwrap(); + assert_eq!(tolerations.len(), 1); + assert_eq!(tolerations[0].key.as_deref(), Some("centaur")); + assert_eq!(tolerations[0].effect.as_deref(), Some("NoSchedule")); + + // Absent by default: proxy pod stays schedulable anywhere when unset. + let bare = build_iron_proxy_pod( + &id, + &IronProxyConfig::new("proxy:test", "ca-cert", "ca-key"), + &resolved(), + &sync, + ); + let bare_spec = bare.spec.as_ref().unwrap(); + assert!(bare_spec.node_selector.is_none()); + assert!(bare_spec.tolerations.is_none()); + } + fn rule_allows_namespace_port( rule: &NetworkPolicyEgressRule, namespace: &str,