From ef006cf7039ed04bf80a84416c98ab0e1a95d27a Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 19 Aug 2026 16:28:23 -0400 Subject: [PATCH 1/6] refactor(config): normalize compute driver field names Signed-off-by: Jesse Jaggars --- crates/openshell-driver-docker/src/lib.rs | 33 ++++---- crates/openshell-driver-docker/src/tests.rs | 34 +++++++- crates/openshell-driver-podman/README.md | 14 ++-- crates/openshell-driver-podman/src/config.rs | 38 ++++++++- .../openshell-driver-podman/src/container.rs | 6 +- crates/openshell-driver-podman/src/main.rs | 2 +- crates/openshell-driver-vm/src/driver.rs | 84 +++++++++++++++---- crates/openshell-driver-vm/src/main.rs | 34 +++++++- .../src/compute/driver_config/builtin.rs | 56 +++++++++++++ crates/openshell-server/src/compute/vm.rs | 4 +- crates/openshell-server/src/config_file.rs | 66 ++++++++++++++- deploy/docker/gateway.toml | 4 +- docs/reference/gateway-config.mdx | 13 ++- docs/reference/sandbox-compute-drivers.mdx | 4 +- e2e/configs/gateway/docker.toml | 2 +- e2e/configs/gateway/podman.toml | 1 + e2e/with-docker-gateway.sh | 4 +- rfc/0003-gateway-configuration/README.md | 2 +- rfc/0011-multi-player-design/README.md | 15 ++-- tasks/scripts/gateway-docker.sh | 2 +- 20 files changed, 337 insertions(+), 81 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 3941819c61..ca447c122d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -125,8 +125,9 @@ pub struct DockerComputeConfig { /// Image pull policy for sandbox images. pub image_pull_policy: String, - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, + /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. + #[serde(alias = "sandbox_namespace")] + pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, @@ -174,7 +175,7 @@ impl Default for DockerComputeConfig { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + sandbox_label: "default".to_string(), grpc_endpoint: String::new(), supervisor_bin: None, supervisor_image: None, @@ -201,7 +202,7 @@ pub(crate) struct DockerGuestTlsPaths { struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: String, - sandbox_namespace: String, + sandbox_label: String, grpc_endpoint: String, network_name: String, gateway_route: DockerGatewayRoute, @@ -596,7 +597,7 @@ impl DockerComputeDriver { config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), + sandbox_label: docker_config.sandbox_label.clone(), grpc_endpoint, network_name, gateway_route, @@ -901,7 +902,7 @@ impl DockerComputeDriver { ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, )); @@ -1302,7 +1303,7 @@ impl DockerComputeDriver { PendingSandboxRecord { sandbox: pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, ), @@ -1357,7 +1358,7 @@ impl DockerComputeDriver { cleanup_sandbox_token_file(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, error_condition(failure.reason, &failure.message), false, ); @@ -1563,7 +1564,7 @@ impl DockerComputeDriver { } async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); + let filters = managed_container_label_filters(&self.config.sandbox_label, []); self.docker .list_containers(Some( ListContainersOptionsBuilder::default() @@ -1588,7 +1589,7 @@ impl DockerComputeDriver { } let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); + managed_container_label_filters(&self.config.sandbox_label, label_filter_values); let containers = self .docker .list_containers(Some( @@ -1606,7 +1607,7 @@ impl DockerComputeDriver { }; let namespace_matches = labels .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); + .is_some_and(|value| value == &self.config.sandbox_label); let id_matches = sandbox_id.is_empty() || labels .get(LABEL_SANDBOX_ID) @@ -2701,7 +2702,7 @@ fn sandbox_token_host_path_by_id( ) -> Result { openshell_core::driver_utils::sandbox_token_path( "docker-sandbox-tokens", - Some(&config.sandbox_namespace), + Some(&config.sandbox_label), sandbox_id, ) .map_err(|err| { @@ -3058,13 +3059,13 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_WORKSPACE.to_string(), sandbox.workspace.clone(), ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use + // The list/get/find paths filter by `config.sandbox_label`, so use // the same value here. `DriverSandbox.namespace` is unset on the request // path (the gateway elides it), and using it would produce containers // that the driver itself cannot find afterwards. labels.insert( LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), + config.sandbox_label.clone(), ); Ok(ContainerCreateBody { @@ -3666,12 +3667,12 @@ fn label_filters(values: impl IntoIterator) -> HashMap, ) -> HashMap> { let mut values = vec![ format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_label}"), ]; values.extend(extra_values); label_filters(values) diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index fd81c9b638..5257070883 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -95,7 +95,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + sandbox_label: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { @@ -126,6 +126,34 @@ fn runtime_config() -> DockerDriverRuntimeConfig { } } +#[test] +fn docker_config_uses_canonical_sandbox_label_name() { + let config: DockerComputeConfig = + serde_json::from_value(serde_json::json!({ "sandbox_label": "tenant-a" })).unwrap(); + assert_eq!(config.sandbox_label, "tenant-a"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["sandbox_label"], "tenant-a"); + assert!(serialized.get("sandbox_namespace").is_none()); +} + +#[test] +fn docker_config_accepts_legacy_sandbox_namespace_alias() { + let config: DockerComputeConfig = + serde_json::from_value(serde_json::json!({ "sandbox_namespace": "tenant-a" })).unwrap(); + assert_eq!(config.sandbox_label, "tenant-a"); +} + +#[test] +fn docker_config_rejects_canonical_and_legacy_sandbox_label_names_together() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_label": "tenant-a", + "sandbox_namespace": "tenant-b" + })) + .expect_err("canonical and legacy names must not both be accepted"); + assert!(error.to_string().contains("duplicate field")); +} + fn json_struct(value: serde_json::Value) -> prost_types::Struct { let serde_json::Value::Object(object) = value else { panic!("expected JSON object"); @@ -2531,10 +2559,10 @@ fn build_container_create_body_uses_runtime_namespace_label() { // runtime config, not from `DriverSandbox.namespace`. The gateway // does not populate `DriverSandbox.namespace`, so a container created // with that empty value would not match subsequent list/get/find - // queries (which filter on `config.sandbox_namespace`), leaking + // queries (which filter on `config.sandbox_label`), leaking // sandboxes that the driver itself cannot observe. let mut config = runtime_config(); - config.sandbox_namespace = "tenant-a".to_string(); + config.sandbox_label = "tenant-a".to_string(); let mut sandbox = test_sandbox(); sandbox.namespace = "ignored-by-driver".to_string(); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index ccd4413b3f..1d7299564d 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -275,9 +275,9 @@ Podman follows the same end-to-end contract as the Kubernetes and VM drivers for the in-container SSH relay: gateway config to `PodmanComputeConfig` to sandbox environment to supervisor session registration on that path. -1. `openshell-core` `Config::sandbox_ssh_socket_path` is copied into - `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the - in-process driver. +1. `[openshell.drivers.podman].ssh_socket_path` is deserialized into + `PodmanComputeConfig::ssh_socket_path` when the gateway builds the in-process + driver. The field defaults to `/run/openshell/ssh.sock` when omitted. 2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that value, alongside required vars such as `OPENSHELL_ENDPOINT` and `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template @@ -459,11 +459,9 @@ matter compared to cluster or rootful runtimes: - Gateway integration: `crates/openshell-server/src/compute/mod.rs` (`new_podman` and `PodmanComputeDriver` wiring). -- Server configuration: `crates/openshell-server/src/lib.rs` - (`ComputeDriverKind::Podman` builds `PodmanComputeConfig` including - `sandbox_ssh_socket_path` from gateway `Config`). -- Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in - `crates/openshell-core/src/config.rs`. +- Server configuration: + `crates/openshell-server/src/compute/driver_config/builtin.rs` builds + `PodmanComputeConfig` from `[openshell.drivers.podman]`. - SSRF mitigation: `crates/openshell-core/src/net.rs`, `crates/openshell-sandbox/src/proxy.rs`, and `crates/openshell-server/src/grpc/policy.rs`. diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 42571f00f1..04ae3fe5e3 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -90,7 +90,8 @@ pub struct PodmanComputeConfig { /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. - pub sandbox_ssh_socket_path: String, + #[serde(alias = "sandbox_ssh_socket_path")] + pub ssh_socket_path: String, /// Name of the Podman bridge network. /// Created automatically if it does not exist. pub network_name: String, @@ -536,7 +537,7 @@ impl Default for PodmanComputeConfig { image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, - sandbox_ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), + ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS, @@ -569,7 +570,7 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("image_pull_policy", &self.image_pull_policy.as_str()) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) - .field("sandbox_ssh_socket_path", &self.sandbox_ssh_socket_path) + .field("ssh_socket_path", &self.ssh_socket_path) .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) .field("stop_timeout_secs", &self.stop_timeout_secs) @@ -605,6 +606,37 @@ impl std::fmt::Debug for PodmanComputeConfig { mod tests { use super::*; + #[test] + fn config_uses_canonical_ssh_socket_path_name() { + let config: PodmanComputeConfig = + serde_json::from_value(serde_json::json!({ "ssh_socket_path": "/run/test.sock" })) + .unwrap(); + assert_eq!(config.ssh_socket_path, "/run/test.sock"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["ssh_socket_path"], "/run/test.sock"); + assert!(serialized.get("sandbox_ssh_socket_path").is_none()); + } + + #[test] + fn config_accepts_legacy_sandbox_ssh_socket_path_alias() { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_ssh_socket_path": "/run/test.sock" + })) + .unwrap(); + assert_eq!(config.ssh_socket_path, "/run/test.sock"); + } + + #[test] + fn config_rejects_canonical_and_legacy_ssh_socket_path_names_together() { + let error = serde_json::from_value::(serde_json::json!({ + "ssh_socket_path": "/run/canonical.sock", + "sandbox_ssh_socket_path": "/run/legacy.sock" + })) + .expect_err("canonical and legacy names must not both be accepted"); + assert!(error.to_string().contains("duplicate field")); + } + #[test] fn default_config_sets_health_check_interval() { let cfg = PodmanComputeConfig::default(); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a81ee13e1d..1743015e26 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -526,7 +526,7 @@ fn build_env( ); env.insert( openshell_core::sandbox_env::SSH_SOCKET_PATH.into(), - config.sandbox_ssh_socket_path.clone(), + config.ssh_socket_path.clone(), ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) @@ -1181,7 +1181,7 @@ pub fn build_container_spec_for_image( "CMD-SHELL".into(), format!( "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", - config.sandbox_ssh_socket_path, + config.ssh_socket_path, openshell_core::config::DEFAULT_SSH_PORT ), ], @@ -2407,7 +2407,7 @@ mod tests { default_image: "test-image:latest".to_string(), grpc_endpoint: "http://localhost:50051".to_string(), host_gateway_ip: String::new(), - sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), ..PodmanComputeConfig::default() } } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index a0fa85d018..c8aa4066a8 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -205,7 +205,7 @@ async fn main() -> Result<()> { host_gateway_ip: args .host_gateway_ip .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, supervisor_image: args diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 2de65c3add..004344a5f9 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -219,7 +219,8 @@ enum GuestImagePayloadSource { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct VmDriverConfig { - pub openshell_endpoint: String, + #[serde(alias = "openshell_endpoint")] + pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, pub default_image: String, @@ -251,7 +252,7 @@ pub const DEFAULT_SANDBOX_UID: u32 = 10001; impl Default for VmDriverConfig { fn default() -> Self { Self { - openshell_endpoint: String::new(), + grpc_endpoint: String::new(), state_dir: PathBuf::from("target/openshell-vm-driver"), launcher_bin: None, default_image: String::new(), @@ -308,7 +309,7 @@ impl VmDriverConfig { } fn requires_tls_materials(&self) -> bool { - self.openshell_endpoint.starts_with("https://") + self.grpc_endpoint.starts_with("https://") } fn tls_paths(&self) -> Result, String> { @@ -447,10 +448,10 @@ impl VmDriver { .validate() .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; - if config.openshell_endpoint.trim().is_empty() { + if config.grpc_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } - validate_openshell_endpoint(&config.openshell_endpoint)?; + validate_openshell_endpoint(&config.grpc_endpoint)?; let _ = config.tls_paths()?; #[cfg(target_os = "linux")] @@ -909,7 +910,7 @@ impl VmDriver { let endpoint_override = if plan.backend == VmBackend::Qemu { plan.host_ip.as_deref().map(|host_ip| { - guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) + guest_visible_openshell_endpoint_for_tap(&self.config.grpc_endpoint, host_ip) }) } else { None @@ -1712,7 +1713,7 @@ impl VmDriver { "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] )); - plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + plan.gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); Ok(()) } @@ -1844,7 +1845,7 @@ impl VmDriver { mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] ); let tap = tap_device_name(sandbox_id); - let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + let gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); let (vcpus, mem_mib) = if is_gpu { (self.config.gpu_vcpus, self.config.gpu_mem_mib) @@ -4397,7 +4398,7 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { /// Rewrites loopback host references in a gateway URL to a hostname the guest /// can reach via gvproxy. /// -/// The driver receives the gateway endpoint from `--openshell-endpoint`, which +/// The driver receives the gateway endpoint from `--grpc-endpoint`, which /// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is /// useless inside the guest because the guest's loopback interface is its own, /// not the host's. Inside the guest we need a name that gvproxy will translate @@ -4461,7 +4462,7 @@ fn build_guest_environment( endpoint_override: Option<&str>, ) -> Vec { let openshell_endpoint = endpoint_override.map_or_else( - || guest_visible_openshell_endpoint(&config.openshell_endpoint), + || guest_visible_openshell_endpoint(&config.grpc_endpoint), String::from, ); // 1. User-supplied environment (lowest priority). @@ -5602,6 +5603,53 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + + #[test] + fn vm_config_uses_canonical_grpc_endpoint_name() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let serialized = serde_json::to_value(&config).unwrap(); + assert_eq!(serialized["grpc_endpoint"], "http://127.0.0.1:8080"); + assert!(serialized.get("openshell_endpoint").is_none()); + + let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); + } + + #[test] + fn vm_config_accepts_legacy_openshell_endpoint_alias() { + let config = VmDriverConfig::default(); + let mut serialized = serde_json::to_value(config).unwrap(); + let fields = serialized.as_object_mut().unwrap(); + fields.remove("grpc_endpoint"); + fields.insert( + "openshell_endpoint".to_string(), + serde_json::json!("http://127.0.0.1:8080"), + ); + + let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); + } + + #[test] + fn vm_config_rejects_canonical_and_legacy_endpoint_names_together() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let mut serialized = serde_json::to_value(config).unwrap(); + serialized.as_object_mut().unwrap().insert( + "openshell_endpoint".to_string(), + serde_json::json!("http://127.0.0.1:9090"), + ); + + let error = serde_json::from_value::(serialized) + .expect_err("canonical and legacy names must not both be accepted"); + assert!(error.to_string().contains("duplicate field")); + } + struct TestTracing { exporter: opentelemetry_sdk::trace::InMemorySpanExporter, _provider: opentelemetry_sdk::trace::SdkTracerProvider, @@ -7046,7 +7094,7 @@ mod tests { #[test] fn build_guest_environment_sets_supervisor_defaults() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7152,7 +7200,7 @@ mod tests { #[test] fn build_guest_environment_uses_token_file_without_raw_token_env() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7184,7 +7232,7 @@ mod tests { #[test] fn build_guest_environment_strips_gateway_tls_server_name() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7221,7 +7269,7 @@ mod tests { )], || { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7290,7 +7338,7 @@ mod tests { #[test] fn build_guest_environment_uses_endpoint_override_for_tap() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7492,7 +7540,7 @@ mod tests { #[test] fn build_guest_environment_includes_tls_paths_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), guest_tls_cert: Some(PathBuf::from("/host/tls.crt")), guest_tls_key: Some(PathBuf::from("/host/tls.key")), @@ -7514,7 +7562,7 @@ mod tests { #[test] fn vm_driver_config_requires_tls_materials_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), ..Default::default() }; let err = config @@ -8049,7 +8097,7 @@ mod tests { let (events, _) = broadcast::channel(WATCH_BUFFER); VmDriver { config: VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), vcpus: 2, mem_mib: 2048, gpu_vcpus: 8, diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 95ebf0f8b2..bd5a9c04ae 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -94,8 +94,12 @@ struct Args { #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, + #[arg( + long = "grpc-endpoint", + alias = "openshell-endpoint", + env = "OPENSHELL_GRPC_ENDPOINT" + )] + grpc_endpoint: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] default_image: String, @@ -223,8 +227,8 @@ async fn main() -> Result<()> { } let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint + grpc_endpoint: args + .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, state_dir: args.state_dir.clone(), launcher_bin: None, @@ -695,6 +699,28 @@ mod tests { assert!(err.contains("--bind-socket is required")); } + #[test] + fn accepts_canonical_grpc_endpoint_flag() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--grpc-endpoint", + "http://127.0.0.1:8080", + ]) + .unwrap(); + assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + } + + #[test] + fn accepts_legacy_openshell_endpoint_flag_alias() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--openshell-endpoint", + "http://127.0.0.1:8080", + ]) + .unwrap(); + assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs index dea867237d..22725b9062 100644 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ b/crates/openshell-server/src/compute/driver_config/builtin.rs @@ -163,6 +163,62 @@ enable_bind_mounts = true assert!(cfg.enable_bind_mounts); } + #[test] + fn docker_config_reads_canonical_sandbox_label_override() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "gateway-default" + +[openshell.drivers.docker] +sandbox_label = "driver-specific" +"#, + ) + .expect("valid config"); + + let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); + + assert_eq!(cfg.sandbox_label, "driver-specific"); + } + + #[test] + fn docker_config_reads_legacy_sandbox_namespace_override() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "gateway-default" + +[openshell.drivers.docker] +sandbox_namespace = "driver-specific" +"#, + ) + .expect("valid config"); + + let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); + + assert_eq!(cfg.sandbox_label, "driver-specific"); + } + + #[test] + fn docker_config_rejects_canonical_and_legacy_sandbox_label_names_together() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "gateway-default" + +[openshell.drivers.docker] +sandbox_label = "canonical" +sandbox_namespace = "legacy" +"#, + ) + .expect("valid config file structure"); + + let error = docker_config_from_context(test_context(Some(&file))) + .expect_err("canonical and legacy names must not both be accepted"); + + assert!(error.to_string().contains("duplicate field")); + } + #[test] fn docker_config_reads_bind_mount_opt_in_from_driver_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index 6a66fc8aa5..8377809262 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -479,9 +479,7 @@ pub async fn spawn( .arg(std::process::id().to_string()); command.arg("--log-level").arg(&config.log_level); append_otlp_args(&mut command, otlp_config, &config.name); - command - .arg("--openshell-endpoint") - .arg(&vm_config.grpc_endpoint); + command.arg("--grpc-endpoint").arg(&vm_config.grpc_endpoint); command.arg("--state-dir").arg(&vm_config.state_dir); if !vm_config.default_image.trim().is_empty() { command.arg("--default-image").arg(&vm_config.default_image); diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 74b6aad01b..6e7228a694 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -439,7 +439,7 @@ pub fn driver_table( }; for key in inheritable_keys(driver_name) { - if merged.contains_key(*key) { + if driver_field_is_present(&merged, driver_name, key) { continue; } if let Some(value) = gateway_inherited_value(gateway, key) { @@ -466,7 +466,7 @@ fn inheritable_keys(driver_name: &str) -> &'static [&'static str] { "sa_token_ttl_secs", ], Some(ComputeDriverKind::Docker) => &[ - "sandbox_namespace", + "sandbox_label", "default_image", "supervisor_image", "host_gateway_ip", @@ -494,9 +494,21 @@ fn inheritable_keys(driver_name: &str) -> &'static [&'static str] { } } +fn driver_field_is_present(table: &toml::Table, driver_name: &str, key: &str) -> bool { + if table.contains_key(key) { + return true; + } + + matches!( + driver_name.parse::().ok(), + Some(ComputeDriverKind::Docker) + ) && key == "sandbox_label" + && table.contains_key("sandbox_namespace") +} + fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { match key { - "namespace" | "sandbox_namespace" => g.sandbox_namespace.as_deref().map(string_value), + "namespace" | "sandbox_label" => g.sandbox_namespace.as_deref().map(string_value), "default_image" => g.default_image.as_deref().map(string_value), "supervisor_image" => g.supervisor_image.as_deref().map(string_value), "client_tls_secret_name" => g.client_tls_secret_name.as_deref().map(string_value), @@ -1017,7 +1029,7 @@ version = 2 let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); let table = merged.as_table().expect("table"); assert_eq!( - table.get("sandbox_namespace").and_then(|v| v.as_str()), + table.get("sandbox_label").and_then(|v| v.as_str()), Some("agents") ); assert_eq!( @@ -1030,6 +1042,52 @@ version = 2 ); } + #[test] + fn docker_driver_canonical_sandbox_label_overrides_gateway_default() { + let gateway = GatewayFileSection { + sandbox_namespace: Some("gateway-default".to_string()), + ..Default::default() + }; + let raw = toml::toml! { + sandbox_label = "driver-specific" + }; + let merged = driver_table( + ComputeDriverKind::Docker.as_str(), + &gateway, + Some(&toml::Value::Table(raw)), + ); + let table = merged.as_table().expect("table"); + assert_eq!( + table.get("sandbox_label").and_then(|value| value.as_str()), + Some("driver-specific") + ); + assert!(!table.contains_key("sandbox_namespace")); + } + + #[test] + fn docker_driver_legacy_sandbox_namespace_overrides_gateway_default() { + let gateway = GatewayFileSection { + sandbox_namespace: Some("gateway-default".to_string()), + ..Default::default() + }; + let raw = toml::toml! { + sandbox_namespace = "driver-specific" + }; + let merged = driver_table( + ComputeDriverKind::Docker.as_str(), + &gateway, + Some(&toml::Value::Table(raw)), + ); + let table = merged.as_table().expect("table"); + assert_eq!( + table + .get("sandbox_namespace") + .and_then(|value| value.as_str()), + Some("driver-specific") + ); + assert!(!table.contains_key("sandbox_label")); + } + #[test] fn podman_driver_table_inherits_gateway_host_gateway_ip() { let gateway = GatewayFileSection { diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index 4fe84d633a..b2b649e0ff 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -41,8 +41,8 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # Only pull images that are not already cached locally. image_pull_policy = "IfNotPresent" -# Prefix applied to sandbox container names. -sandbox_namespace = "openshell" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "openshell" # Address sandbox containers use to call back to the gateway. # The Docker driver replaces the host with host.openshell.internal and the # port with the gateway's own bind port (8080). Only the scheme survives. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 22b64e9e97..22680efb8a 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -605,7 +605,8 @@ socket_path = "/var/run/docker.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" # Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "docker-dev" # Empty auto-detects https://host.openshell.internal: when guest TLS is set. grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. @@ -627,6 +628,10 @@ enable_bind_mounts = false sandbox_pids_limit = 2048 ``` +Use `sandbox_label` for new Docker configurations. The legacy +`sandbox_namespace` key remains accepted as a compatibility alias. Do not set +both keys in the same driver table. + ### Podman Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. @@ -655,7 +660,7 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" -sandbox_ssh_socket_path = "/run/openshell/ssh.sock" +ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" @@ -761,6 +766,10 @@ health_check_interval_secs = 10 # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" ``` +Use `ssh_socket_path` for new Podman configurations. The legacy +`sandbox_ssh_socket_path` key remains accepted as a compatibility alias. Do not +set both keys in the same driver table. + ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 82aa68a956..2d5fc649bd 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -153,7 +153,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -231,7 +231,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 59baed1d7d..c498693063 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -23,5 +23,5 @@ ttl_secs = 0 [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "IfNotPresent" -sandbox_namespace = "openshell-e2e" +sandbox_label = "openshell-e2e" supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index c1549cd933..35b4005243 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -25,4 +25,5 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "missing" network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" +ssh_socket_path = "/run/openshell/ssh.sock" supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 0a767576dd..90d6dc2ba8 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -514,7 +514,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" @@ -532,7 +532,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then { - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 2b7c095065..9fb31085ad 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -130,7 +130,7 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell/sandbox:latest" image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override diff --git a/rfc/0011-multi-player-design/README.md b/rfc/0011-multi-player-design/README.md index 32a2f584e1..5fcb67fe51 100644 --- a/rfc/0011-multi-player-design/README.md +++ b/rfc/0011-multi-player-design/README.md @@ -755,13 +755,14 @@ use the workspace to select the target Kubernetes namespace instead of encoding it in the resource name. The label-based lookup and annotation patterns established here carry over unchanged. -**Docker and Podman drivers.** The Docker driver's `sandbox_namespace` label -provides a foundation for workspace mapping, but the driver currently uses a -single configured namespace rather than per-sandbox values. The driver contract -must be updated so that workspace flows through `DriverSandbox` and the driver -applies it as the container label filter. The same applies to Podman and other -local drivers — workspace isolation is enforced at the gateway level and does -not require Kubernetes. +**Docker and Podman drivers.** The Docker driver's `sandbox_label` +configuration value is stored in the `openshell.sandbox_namespace` container +label and provides a foundation for workspace mapping, but the driver currently +uses a single configured value rather than per-sandbox values. The driver +contract must be updated so that workspace flows through `DriverSandbox` and +the driver applies it as the container label filter. The same applies to Podman +and other local drivers — workspace isolation is enforced at the gateway level +and does not require Kubernetes. ### Compute Driver Trust Model diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index b38bfb7942..cd31dd1569 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -231,7 +231,7 @@ ttl_secs = 3600 [openshell.drivers.docker] default_image = "${SANDBOX_IMAGE}" image_pull_policy = "${SANDBOX_IMAGE_PULL_POLICY}" -sandbox_namespace = "${SANDBOX_NAMESPACE}" +sandbox_label = "${SANDBOX_NAMESPACE}" grpc_endpoint = "${GRPC_ENDPOINT}" supervisor_bin = "${SUPERVISOR_BIN}" EOF From 4042128005027eb665568cfb8fbd4ae46389be8b Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 20 Aug 2026 13:25:00 -0400 Subject: [PATCH 2/6] refactor(config): introduce canonical gateway fields Signed-off-by: Jesse Jaggars --- .../skills/debug-openshell-cluster/SKILL.md | 2 +- architecture/compute-runtimes.md | 2 +- architecture/gateway.md | 25 +- crates/openshell-core/src/config.rs | 43 +++- crates/openshell-driver-vm/README.md | 4 +- crates/openshell-driver-vm/src/driver.rs | 6 +- crates/openshell-server/src/cli.rs | 16 ++ .../src/compute/driver_config/builtin.rs | 39 +++ crates/openshell-server/src/config_file.rs | 235 +++++++++++++++++- crates/openshell-server/src/lib.rs | 6 +- deploy/docker/gateway.toml | 2 +- .../openshell/templates/gateway-config.yaml | 8 +- .../openshell/tests/gateway_config_test.yaml | 12 + .../tests/sandbox_namespace_test.yaml | 14 +- deploy/rpm/CONFIGURATION.md | 8 +- deploy/rpm/TROUBLESHOOTING.md | 2 +- deploy/rpm/gateway.toml.default | 2 +- docs/about/installation.mdx | 2 +- docs/reference/gateway-config.mdx | 26 +- docs/reference/sandbox-compute-drivers.mdx | 27 +- docs/security/best-practices.mdx | 2 +- e2e/configs/gateway/docker.toml | 2 +- e2e/configs/gateway/podman.toml | 2 +- e2e/run.sh | 9 +- e2e/rust/e2e-vm.sh | 2 +- e2e/with-podman-gateway.sh | 4 +- examples/aws-s3-sts.md | 2 +- .../podman/README.md | 2 +- .../spiffe-token-exchange-demo/podman/demo.sh | 2 +- .../podman/start-gateway.sh | 2 +- rfc/0003-gateway-configuration/README.md | 23 +- tasks/scripts/gateway-docker.sh | 2 +- tasks/scripts/gateway-podman.sh | 4 +- tasks/scripts/gateway-vm.sh | 2 +- tasks/scripts/gateway.sh | 2 +- tasks/scripts/vm/smoke-orphan-cleanup.sh | 2 +- 36 files changed, 450 insertions(+), 95 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index b2019652b8..b162389ca7 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -81,7 +81,7 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash -rg -n 'compute_drivers|socket_path' /etc/openshell/gateway.toml +rg -n 'compute_driver|compute_drivers|socket_path' /etc/openshell/gateway.toml stat /run/openshell/.sock journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index b21dd0dc80..ce452369c5 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -223,7 +223,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | -| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. diff --git a/architecture/gateway.md b/architecture/gateway.md index 0430d95159..c3ec92add5 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -680,9 +680,11 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ``` The TOML file is opt-in via `--config ` / `OPENSHELL_GATEWAY_CONFIG`. -Driver implementation settings live in the TOML driver tables. See -`docs/reference/gateway-config.mdx` for worked per-driver examples and RFC -0003 for the full schema. +Driver implementation settings live in the TOML driver tables. The canonical +selector is the singular `[openshell.gateway] compute_driver`; the legacy +`compute_drivers` list remains accepted and normalizes into the existing +exactly-one-driver runtime validation. See `docs/reference/gateway-config.mdx` +for worked per-driver examples and RFC 0003 for the full schema. Each installation has an operator-assigned gateway name. Configure it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. @@ -698,13 +700,16 @@ aliases, network names, and the sandbox JWT issuer. ### Driver inheritance -`[openshell.gateway]` carries a small set of values (`sandbox_namespace`, -`default_image`, -`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, -`host_gateway_ip`, `enable_user_namespaces`) that are inherited into each -driver's `[openshell.drivers.]` table when the driver-specific table -does not override them. The allowlist is per-driver so a gateway-wide -default cannot land in a driver that does not understand it (e.g. +`[openshell.gateway]` carries shared defaults such as `default_image`, +`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, and +`host_gateway_ip`. It also continues to accept the historical +`sandbox_namespace`, `service_account_name`, and `enable_user_namespaces` +locations as compatibility inputs. Canonical Kubernetes configuration places +those values in `[openshell.drivers.kubernetes]` as `namespace`, +`service_account_name`, and `enable_user_namespaces`; canonical Docker +configuration uses `sandbox_label`. Driver-table values take precedence over +compatibility inputs. The allowlist is per-driver so a gateway-wide default +cannot land in a driver that does not understand it (for example, `client_tls_secret_name` is K8s-only). `image_pull_policy` is intentionally **not** inheritable: Kubernetes uses diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index a1f1a6c84f..e1ec25c5e1 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -1145,11 +1145,31 @@ pub struct GatewayJwtConfig { #[serde(default = "default_gateway_id")] pub gateway_id: String, /// Token lifetime in seconds. A value of 0 disables expiration and is - /// intended only for local single-player deployments. - #[serde(default = "default_sandbox_token_ttl_secs")] + /// intended only for local single-player deployments. Canonical serialized + /// configuration omits the field for that non-expiring behavior; explicit + /// legacy zero remains accepted. + #[serde( + default = "default_sandbox_token_ttl_secs", + skip_serializing_if = "is_default" + )] pub ttl_secs: u64, } +impl GatewayJwtConfig { + /// Effective token lifetime. `None` preserves the established non-expiring + /// behavior represented by an omitted or explicit zero `ttl_secs` value. + pub fn sandbox_token_ttl(&self) -> Option { + (self.ttl_secs != 0).then(|| Duration::from_secs(self.ttl_secs)) + } +} + +fn is_default(value: &T) -> bool +where + T: Default + PartialEq, +{ + value == &T::default() +} + fn default_gateway_id() -> String { "openshell".to_string() } @@ -1585,6 +1605,25 @@ mod tests { .expect("gateway JWT config should deserialize with default ttl"); assert_eq!(cfg.ttl_secs, 0); + assert_eq!(cfg.sandbox_token_ttl(), None); + + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert!(serialized.get("ttl_secs").is_none()); + } + + #[test] + fn gateway_jwt_positive_ttl_serializes_and_has_effective_duration() { + let cfg: GatewayJwtConfig = serde_json::from_value(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 3600 + })) + .expect("gateway JWT config should deserialize with positive ttl"); + + assert_eq!(cfg.sandbox_token_ttl(), Some(Duration::from_secs(3600))); + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert_eq!(serialized["ttl_secs"], 3600); } #[test] diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index f455beaeb5..e3b7496818 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -116,7 +116,7 @@ cat > .cache/gateway-vm/gateway.toml <, // ── Drivers ────────────────────────────────────────────────────────── - #[serde(default)] + /// Canonical TOML uses the singular `compute_driver = "..."`. The legacy + /// `compute_drivers = ["..."]` form remains accepted and is normalized to + /// this existing vector representation so Rust callers and runtime + /// validation retain their current behavior. + #[serde( + default, + rename = "compute_driver", + alias = "compute_drivers", + deserialize_with = "deserialize_compute_drivers", + serialize_with = "serialize_compute_drivers", + skip_serializing_if = "Option::is_none" + )] pub compute_drivers: Option>, #[serde(default)] pub credential_drivers: Option>, @@ -115,6 +127,9 @@ pub struct GatewayFileSection { pub credential_storage: Option, // ── Sandbox / SSH ──────────────────────────────────────────────────── + /// Compatibility input for Kubernetes `namespace` and Docker + /// `sandbox_label`. Canonical configurations set those driver-owned + /// fields in their respective `[openshell.drivers.]` tables. #[serde(default)] pub sandbox_namespace: Option, #[serde(default)] @@ -143,10 +158,12 @@ pub struct GatewayFileSection { pub supervisor_image: Option, #[serde(default)] pub client_tls_secret_name: Option, + /// Compatibility input for Kubernetes `service_account_name`. #[serde(default)] pub service_account_name: Option, #[serde(default)] pub host_gateway_ip: Option, + /// Compatibility input for Kubernetes `enable_user_namespaces`. #[serde(default)] pub enable_user_namespaces: Option, /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet @@ -194,6 +211,62 @@ pub struct GatewayFileSection { pub database_url: Option, } +fn deserialize_compute_drivers<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + struct ComputeDriversVisitor; + + impl<'de> Visitor<'de> for ComputeDriversVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a compute driver name or an array of compute driver names") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + Ok(Some(vec![value.to_string()])) + } + + fn visit_string(self, value: String) -> Result + where + E: serde::de::Error, + { + Ok(Some(vec![value])) + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut drivers = Vec::new(); + while let Some(driver) = sequence.next_element::()? { + drivers.push(driver); + } + Ok(Some(drivers)) + } + } + + deserializer.deserialize_any(ComputeDriversVisitor) +} + +fn serialize_compute_drivers( + drivers: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + match drivers { + Some(drivers) if drivers.len() == 1 => serializer.serialize_str(&drivers[0]), + Some(drivers) => drivers.serialize(serializer), + None => serializer.serialize_none(), + } +} + /// `[openshell.gateway.otlp]` section. /// /// Presence of this table enables OTLP export; there is no `enabled` flag. @@ -499,6 +572,9 @@ fn driver_field_is_present(table: &toml::Table, driver_name: &str, key: &str) -> return true; } + // Docker's legacy alias must count as an explicit driver override. If it + // did not, gateway inheritance would inject `sandbox_label` alongside the + // alias and serde would reject the merged table as a duplicate field. matches!( driver_name.parse::().ok(), Some(ComputeDriverKind::Docker) @@ -554,6 +630,86 @@ mod tests { assert!(file.openshell.drivers.is_empty()); } + #[test] + fn canonical_compute_driver_scalar_normalizes_to_existing_vector() { + let file: ConfigFile = toml::from_str( + r#" +[openshell.gateway] +compute_driver = "docker" +"#, + ) + .expect("canonical compute driver parses"); + + assert_eq!( + file.openshell.gateway.compute_drivers, + Some(vec!["docker".to_string()]) + ); + } + + #[test] + fn legacy_compute_drivers_list_remains_accepted() { + for (input, expected) in [ + ("compute_drivers = []", Vec::::new()), + ("compute_drivers = [\"docker\"]", vec!["docker".to_string()]), + ( + "compute_drivers = [\"docker\", \"podman\"]", + vec!["docker".to_string(), "podman".to_string()], + ), + ] { + let file: ConfigFile = toml::from_str(&format!("[openshell.gateway]\n{input}\n")) + .expect("legacy compute drivers parse"); + assert_eq!(file.openshell.gateway.compute_drivers, Some(expected)); + } + } + + #[test] + fn compute_driver_rejects_non_string_values_with_a_clear_error() { + let error = toml::from_str::( + r" +[openshell.gateway] +compute_driver = 42 +", + ) + .expect_err("compute driver must be a string or string array"); + + assert!( + error + .to_string() + .contains("a compute driver name or an array of compute driver names") + ); + } + + #[test] + fn canonical_and_legacy_compute_driver_names_are_rejected_together() { + let error = toml::from_str::( + r#" +[openshell.gateway] +compute_driver = "docker" +compute_drivers = ["docker"] +"#, + ) + .expect_err("canonical and legacy names must not both be accepted"); + + assert!(error.to_string().contains("duplicate field")); + } + + #[test] + fn compute_driver_serialization_uses_canonical_scalar_name() { + let file = ConfigFile { + openshell: OpenShellRoot { + gateway: GatewayFileSection { + compute_drivers: Some(vec!["docker".to_string()]), + ..Default::default() + }, + ..Default::default() + }, + }; + + let serialized = toml::to_string(&file).expect("config serializes"); + assert!(serialized.contains("compute_driver = \"docker\"")); + assert!(!serialized.contains("compute_drivers")); + } + #[test] fn parses_full_example() { let toml = r#" @@ -564,7 +720,7 @@ version = 1 bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" credential_drivers = ["kubernetes-secrets"] sandbox_namespace = "agents" grpc_rate_limit_requests = 120 @@ -1018,6 +1174,73 @@ version = 2 ); } + #[test] + fn kubernetes_driver_fields_override_legacy_gateway_compatibility_values() { + let gateway = GatewayFileSection { + sandbox_namespace: Some("legacy-namespace".to_string()), + service_account_name: Some("legacy-service-account".to_string()), + enable_user_namespaces: Some(true), + ..Default::default() + }; + let raw = toml::toml! { + namespace = "canonical-namespace" + service_account_name = "canonical-service-account" + enable_user_namespaces = false + }; + let merged = driver_table( + ComputeDriverKind::Kubernetes.as_str(), + &gateway, + Some(&toml::Value::Table(raw)), + ); + let table = merged.as_table().expect("table"); + + assert_eq!( + table.get("namespace").and_then(toml::Value::as_str), + Some("canonical-namespace") + ); + assert_eq!( + table + .get("service_account_name") + .and_then(toml::Value::as_str), + Some("canonical-service-account") + ); + assert_eq!( + table + .get("enable_user_namespaces") + .and_then(toml::Value::as_bool), + Some(false) + ); + } + + #[test] + fn kubernetes_driver_inherits_legacy_gateway_compatibility_values() { + let gateway = GatewayFileSection { + sandbox_namespace: Some("legacy-namespace".to_string()), + service_account_name: Some("legacy-service-account".to_string()), + enable_user_namespaces: Some(true), + ..Default::default() + }; + let merged = driver_table(ComputeDriverKind::Kubernetes.as_str(), &gateway, None); + let table = merged.as_table().expect("table"); + + assert_eq!( + table.get("namespace").and_then(toml::Value::as_str), + Some("legacy-namespace") + ); + assert_eq!( + table + .get("service_account_name") + .and_then(toml::Value::as_str), + Some("legacy-service-account") + ); + assert_eq!( + table + .get("enable_user_namespaces") + .and_then(toml::Value::as_bool), + Some(true) + ); + } + #[test] fn docker_driver_table_inherits_gateway_defaults() { let gateway = GatewayFileSection { @@ -1185,7 +1408,7 @@ version = 2 /// - template corruption or unknown fields (`deny_unknown_fields`) /// - schema drift (version bump or field renames) /// - accidental addition of a wildcard bind-address override - /// - accidental changes to the compute driver list + /// - accidental changes to the configured compute driver #[test] fn rpm_default_config_parses_and_has_podman_defaults() { let path = @@ -1204,11 +1427,11 @@ version = 2 let drivers = gw .compute_drivers .as_ref() - .expect("compute_drivers must be explicitly set in the RPM default config"); + .expect("compute_driver must be explicitly set in the RPM default config"); assert_eq!( drivers, &["podman".to_string()], - "RPM default must pin compute_drivers to [podman] to prevent unexpected \ + "RPM default must pin compute_driver to podman to prevent unexpected \ driver selection when Docker is also installed" ); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index aafc0cd369..8a9e19a0a3 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -497,7 +497,7 @@ pub(crate) async fn run_server( &signing_pem, kid.clone(), &jwt.gateway_id, - Duration::from_secs(jwt.ttl_secs), + jwt.sandbox_token_ttl().unwrap_or_default(), ) .map_err(Error::config)?, ); @@ -1668,14 +1668,14 @@ fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { config .gateway_jwt .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) + .is_some_and(|jwt| jwt.sandbox_token_ttl().is_none()) } #[cfg(feature = "in-tree-compute-drivers")] fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config) { if kubernetes_sandbox_jwt_expiry_disabled(config) { warn!( - "Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs = 0); set ttl_secs > 0 for shared Kubernetes deployments" + "Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs is omitted or zero); set ttl_secs > 0 for shared Kubernetes deployments" ); } } diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index b2b649e0ff..da8ef72873 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -30,7 +30,7 @@ version = 1 bind_address = "127.0.0.1:8080" health_bind_address = "127.0.0.1:8081" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.drivers.docker] diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index d7f3cb9a83..981b2acb26 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -47,7 +47,6 @@ data: {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} - sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} @@ -60,9 +59,6 @@ data: {{- if .Values.server.hostGatewayIP }} host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} {{- end }} - {{- if .Values.server.enableUserNamespaces }} - enable_user_namespaces = true - {{- end }} {{- if .Values.server.disableTls }} disable_tls = true {{- else }} @@ -146,10 +142,14 @@ data: {{- end }} [openshell.drivers.kubernetes] + namespace = {{ include "openshell.sandboxNamespace" . | quote }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.enableUserNamespaces }} + enable_user_namespaces = true + {{- end }} {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index b54b036c0f..c9a14c29a9 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -145,6 +145,18 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' + - it: renders user namespace enablement under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + server.enableUserNamespaces: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?enable_user_namespaces\s*=\s*true' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces' + - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index ee89fce53d..0576bbec46 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -13,21 +13,27 @@ release: namespace: my-namespace tests: - - it: defaults sandbox_namespace to release namespace in the TOML config + - it: defaults the Kubernetes driver namespace to release namespace template: templates/gateway-config.yaml asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"my-namespace"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - - it: uses explicit sandboxNamespace when set + - it: uses explicit sandboxNamespace for the Kubernetes driver template: templates/gateway-config.yaml set: server.sandboxNamespace: other-ns asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"other-ns"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"other-ns"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - it: defaults NetworkPolicy namespace to release namespace template: templates/networkpolicy.yaml diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 4fc18e6215..aaa97d08d0 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -20,7 +20,7 @@ The defaults are tuned for rootless Podman use: version = 1 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" ``` The RPM does not override `bind_address`. The primary listener uses the @@ -28,7 +28,7 @@ built-in `127.0.0.1:17670` default. The Podman driver reports the callback interface it needs, and the gateway adds a separate listener scoped to that interface. This keeps the general API off unrelated host interfaces. -`compute_drivers = ["podman"]` pins the compute driver to Podman. Without +`compute_driver = "podman"` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver selection if Docker is also installed on the host. @@ -215,7 +215,7 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | -| `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | +| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. The legacy `compute_drivers` list remains accepted. | | `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | | `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | @@ -235,7 +235,7 @@ settings: version = 1 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" [openshell.drivers.podman] diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 103ce3bf9d..f67b69149b 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -255,7 +255,7 @@ and map the relevant variables: | Environment variable | TOML equivalent | |---|---| | `OPENSHELL_BIND_ADDRESS=A` + `OPENSHELL_SERVER_PORT=P` | `bind_address = "A:P"` under `[openshell.gateway]` | -| `OPENSHELL_DRIVERS=podman` | `compute_drivers = ["podman"]` under `[openshell.gateway]` | +| `OPENSHELL_DRIVERS=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | | `OPENSHELL_DISABLE_TLS=true` | `disable_tls = true` under `[openshell.gateway]` | | `OPENSHELL_TLS_CERT=PATH` | `cert_path = "PATH"` under `[openshell.gateway.tls]` | | `OPENSHELL_TLS_KEY=PATH` | `key_path = "PATH"` under `[openshell.gateway.tls]` | diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index cd7e0d99c3..ba76f873b2 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -25,4 +25,4 @@ version = 1 # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. -compute_drivers = ["podman"] +compute_driver = "podman" diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index a733a1b881..9026f939a6 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -30,7 +30,7 @@ Use `openshell status` to confirm the CLI can reach the gateway. ## Supported Compute Drivers -OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_drivers` in the gateway TOML when you need to pin a specific driver. +OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_driver` in the gateway TOML when you need to pin a specific driver. | Compute Driver | How It Is Configured | System Requirements | |---|---|---| diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 22680efb8a..82413ff2d6 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -59,6 +59,8 @@ version = 1 # ... credential-driver-specific settings ... ``` +The canonical gateway selector is `compute_driver = ""`. The legacy `compute_drivers = [""]` list remains accepted for compatibility. An omitted selector or an empty legacy list retains auto-detection; a legacy list with multiple entries retains the existing startup error because only one compute driver can be active. + ## Full Example A complete gateway configuration covering every section. Trim to the fields you need. @@ -78,15 +80,14 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" -# When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. +# When omitted, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" # Optional external provider credential storage backend. Omit this key to use # the gateway's default encrypted database credential storage. credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 # Reject invalid policy generations securely by default. Set @@ -109,9 +110,7 @@ default_image = "ghcr.io/nvidia/openshell/sandbox:latest" # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" client_tls_secret_name = "openshell-client-tls" -service_account_name = "openshell-sandbox" host_gateway_ip = "10.0.0.1" -enable_user_namespaces = false sa_token_ttl_secs = 3600 guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" @@ -198,6 +197,11 @@ failure_policy = "fail_closed" rpc = "openshell.v1.OpenShell/UpdateConfig" phases = ["validate"] +[openshell.drivers.kubernetes] +namespace = "openshell" +service_account_name = "openshell-sandbox" +enable_user_namespaces = false + [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" allow_reference_namespace = false @@ -444,6 +448,8 @@ args = [ Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Canonical Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical gateway-level locations remain accepted as compatibility inputs and retain the same lower precedence. Gateway-level `sandbox_namespace` also remains a compatibility default for Docker `sandbox_label`. + ### Kubernetes The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS material for sandboxes is delivered through a Kubernetes Secret rather than host-side file paths. @@ -457,7 +463,7 @@ bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" @@ -598,7 +604,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" @@ -643,7 +649,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -782,7 +788,7 @@ version = 1 bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. -compute_drivers = ["vm"] +compute_driver = "vm" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -820,7 +826,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 2d5fc649bd..5a45bf0354 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -35,24 +35,26 @@ with the exact exit code. Driver and supervisor failures remain `Error`. ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set the singular `compute_driver` key in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["docker"] +compute_driver = "docker" ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, and `vm`. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Local container runtimes must respond to an API probe before the gateway selects them. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Local container runtimes must respond to an API probe before the gateway selects them. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_DRIVERS=vm` in the launch environment. + +The legacy `compute_drivers = [""]` list remains accepted for compatibility. Empty legacy lists retain auto-detection, and lists with more than one entry retain the existing startup error because a gateway supports exactly one active compute driver. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. @@ -62,7 +64,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -153,7 +155,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -231,7 +233,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -325,11 +327,11 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: +Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" ``` For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. @@ -367,15 +369,16 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical `[openshell.gateway]` locations remain accepted as lower-precedence compatibility inputs. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | +| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | -| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 1c541f8f8e..53a68364e5 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -71,7 +71,7 @@ This provides defense-in-depth: even if a container escape vulnerability exists, | Aspect | Detail | |---|---| -| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in the gateway config to enable cluster-wide. | +| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in `[openshell.drivers.kubernetes]` to enable cluster-wide. | | What you can change | Enable cluster-wide through Helm or gateway config. Override per-sandbox through the `user_namespaces` field on `SandboxTemplate` in the API. | | Prerequisites | Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), a container runtime that supports user namespaces (containerd 2.0+, CRI-O 1.25+), and Linux 5.12+ for ID-mapped mounts. | | Risk if enabled with GPU | NVIDIA device plugin compatibility with user namespaces is unverified. OpenShell logs a warning when both GPU and user namespaces are active on the same sandbox. | diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index c498693063..878aee677c 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -7,7 +7,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.gateway.auth] diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 35b4005243..2064a081f5 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -7,7 +7,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] diff --git a/e2e/run.sh b/e2e/run.sh index 0505730f05..b904469730 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -142,7 +142,14 @@ if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then fi gateway_driver="$(python3 -c ' import sys, tomllib -print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) +gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] +driver = gateway.get("compute_driver") +if driver is None: + drivers = gateway.get("compute_drivers", []) + driver = drivers[0] if drivers else None +if not driver: + raise SystemExit("gateway config must explicitly select a compute driver") +print(driver) ' "${gateway_config}")" if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index de7e3ad09d..6c8e202abc 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -260,7 +260,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:${HOST_PORT}" -compute_drivers = ["vm"] +compute_driver = "vm" [openshell.gateway.tls] cert_path = "${PKI_DIR}/server/tls.crt" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fc3419e182..089b9923fd 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -452,7 +452,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # Start from the RPM default template so this e2e test exercises the same TOML # config path that RPM users get on first start. The template leaves -# bind_address unset and sets compute_drivers = ["podman"]. On Podman Machine, +# bind_address unset and sets compute_driver = "podman". On Podman Machine, # the driver reserves IPv4 loopback for its callback-only listener, so the # primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default. # @@ -519,7 +519,7 @@ fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # compute_drivers comes from the RPM template. Override the loopback address + # compute_driver comes from the RPM template. Override the loopback address # and port so Podman Machine can keep its IPv4 callback listener distinct. --bind-address "${PRIMARY_BIND_IP}" --port "${HOST_PORT}" diff --git a/examples/aws-s3-sts.md b/examples/aws-s3-sts.md index e622e71117..5a7acfe5f1 100644 --- a/examples/aws-s3-sts.md +++ b/examples/aws-s3-sts.md @@ -90,7 +90,7 @@ if your gateway cache directory differs): version = 1 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" disable_tls = true supervisor_image = "localhost/openshell/supervisor:dev" diff --git a/examples/spiffe-token-exchange-demo/podman/README.md b/examples/spiffe-token-exchange-demo/podman/README.md index d8fdde4b4b..43278a3570 100644 --- a/examples/spiffe-token-exchange-demo/podman/README.md +++ b/examples/spiffe-token-exchange-demo/podman/README.md @@ -62,7 +62,7 @@ to the same Podman network. `START_GATEWAY=1` automates that same-network gateway setup. It mounts the host Podman socket into the gateway container, writes a temporary gateway config with -`compute_drivers = ["podman"]`, and mounts the SPIRE Workload API socket at the +`compute_driver = "podman"`, and mounts the SPIRE Workload API socket at the same absolute host path so the gateway can pass that path to sibling sandbox containers. diff --git a/examples/spiffe-token-exchange-demo/podman/demo.sh b/examples/spiffe-token-exchange-demo/podman/demo.sh index 0c943c15f6..c482405553 100755 --- a/examples/spiffe-token-exchange-demo/podman/demo.sh +++ b/examples/spiffe-token-exchange-demo/podman/demo.sh @@ -398,7 +398,7 @@ version = 1 bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] diff --git a/examples/spiffe-token-exchange-demo/podman/start-gateway.sh b/examples/spiffe-token-exchange-demo/podman/start-gateway.sh index fea89a117a..c6f275009f 100755 --- a/examples/spiffe-token-exchange-demo/podman/start-gateway.sh +++ b/examples/spiffe-token-exchange-demo/podman/start-gateway.sh @@ -138,7 +138,7 @@ version = 1 bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 9fb31085ad..df71cc1f5b 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -72,10 +72,9 @@ metrics_bind_address = "0.0.0.0:9090" # optional; omit to disable # Logging log_level = "info" -# Compute drivers — list of driver names whose [openshell.drivers.] -# tables should be activated. When empty, the gateway auto-detects a driver -# (kubernetes → podman → docker). VM is never auto-detected. -compute_drivers = ["kubernetes"] +# Compute driver — exactly one driver may be active. When omitted, the gateway +# auto-detects a driver (kubernetes → podman → docker). VM is never auto-detected. +compute_driver = "kubernetes" # Note: database_url is a secret and must be supplied via OPENSHELL_DB_URL # (or --db-url) — it is NOT permitted in the file. @@ -113,7 +112,7 @@ scopes_claim = "" # empty disables scope enforcement # ────────────────────────────────────────────────────────────────────────────── # Compute drivers — each table is owned and parsed by its driver crate. -# Only tables for drivers listed in compute_drivers are activated. +# Only the selected or auto-detected driver's table is activated. # ────────────────────────────────────────────────────────────────────────────── [openshell.drivers.kubernetes] @@ -172,7 +171,7 @@ Each `[openshell.drivers.]` table is extracted from the parsed file and ha Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. -`[openshell.drivers.]` tables for drivers not listed in `compute_drivers` (and not the auto-detected driver) are parsed for syntax but not activated. +`[openshell.drivers.]` tables for drivers other than the selected or auto-detected driver are parsed for syntax but not activated. ### Merge semantics @@ -209,11 +208,11 @@ The following cross-field validations are applied after merging file + env + CLI - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. - When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_drivers` may be empty; in that case the gateway falls back to auto-detection. If the list contains a driver name with no matching `[openshell.drivers.]` table, the driver runs with its built-in defaults. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list remains accepted: an empty list auto-detects, a singleton selects that driver, and multiple entries retain the existing startup error. ### Backwards compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). +The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). Legacy `compute_drivers = [""]` TOML remains accepted, while canonical configurations use the singular `compute_driver = ""`. ### Example: minimal Kubernetes deployment @@ -222,8 +221,8 @@ The existing CLI interface is fully preserved. All flags continue to work exactl version = 1 [openshell.gateway] -bind_address = "0.0.0.0:8080" -compute_drivers = ["kubernetes"] +bind_address = "0.0.0.0:8080" +compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). # No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). @@ -250,7 +249,7 @@ gateway: bind_address: "0.0.0.0:8080" health_bind_address: "0.0.0.0:8081" metrics_bind_address: "0.0.0.0:9090" - compute_drivers: ["kubernetes"] + compute_driver: "kubernetes" drivers: kubernetes: namespace: agents @@ -298,5 +297,5 @@ No part of this RFC has shipped yet. The work breaks down as: 1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. 2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (e.g. `compute_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. + Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. 3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index cd31dd1569..52f9da35ac 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -215,7 +215,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.gateway.auth] diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index ab166865ef..2b9d9bc349 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -219,8 +219,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["podman"] -default_image = "${SANDBOX_IMAGE}" +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] @@ -234,6 +233,7 @@ gateway_id = "${GATEWAY_NAME}" ttl_secs = 3600 [openshell.drivers.podman] +default_image = "${SANDBOX_IMAGE}" supervisor_image = "${SUPERVISOR_IMAGE}" image_pull_policy = "$(podman_pull_policy "${SANDBOX_IMAGE_PULL_POLICY}")" EOF diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 2e8aecc5d0..5b647522d2 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -340,7 +340,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["vm"] +compute_driver = "vm" disable_tls = ${DISABLE_TLS} [openshell.gateway.auth] diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 019d1b1b63..cffad5ae2b 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -248,7 +248,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["${DRIVER}"] +compute_driver = "${DRIVER}" default_image = "${SANDBOX_IMAGE}" disable_tls = true diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh index 6da48919d1..71ac9597b3 100755 --- a/tasks/scripts/vm/smoke-orphan-cleanup.sh +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -57,7 +57,7 @@ start_gateway() { version = 1 [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" disable_tls = true [openshell.drivers.vm] From cc060a8ad4bd41f592fd8a239450d01e45955d99 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Tue, 1 Sep 2026 16:54:17 -0400 Subject: [PATCH 3/6] refactor(config): enforce gateway schema version 2 Signed-off-by: Jesse Jaggars --- .../skills/debug-openshell-cluster/SKILL.md | 10 +- .agents/skills/test-release-canary/SKILL.md | 2 +- .github/workflows/release-canary.yml | 8 +- architecture/compute-runtimes.md | 28 +- architecture/gateway.md | 52 +- crates/openshell-core/src/config.rs | 319 +++++++-- crates/openshell-core/src/container_paths.rs | 2 + crates/openshell-core/src/driver_utils.rs | 151 ++++ crates/openshell-core/src/lib.rs | 9 +- crates/openshell-driver-docker/README.md | 21 +- crates/openshell-driver-docker/src/lib.rs | 254 +++++-- crates/openshell-driver-docker/src/tests.rs | 132 +++- crates/openshell-driver-kubernetes/README.md | 13 +- .../openshell-driver-kubernetes/src/config.rs | 198 ++--- .../openshell-driver-kubernetes/src/driver.rs | 84 ++- .../openshell-driver-kubernetes/src/main.rs | 22 +- .../examples/run-mxc-e2e.ps1 | 2 +- crates/openshell-driver-podman/README.md | 13 +- crates/openshell-driver-podman/src/client.rs | 3 + crates/openshell-driver-podman/src/config.rs | 262 +++---- .../openshell-driver-podman/src/container.rs | 45 +- crates/openshell-driver-podman/src/driver.rs | 51 +- crates/openshell-driver-podman/src/main.rs | 52 +- crates/openshell-driver-vm/README.md | 29 +- .../scripts/openshell-vm-sandbox-init.sh | 60 +- crates/openshell-driver-vm/src/driver.rs | 214 +++++- crates/openshell-driver-vm/src/main.rs | 66 +- crates/openshell-driver-vm/src/rootfs.rs | 37 +- crates/openshell-server/src/cli.rs | 188 +++-- .../src/compute/driver_config.rs | 140 +++- .../src/compute/driver_config/builtin.rs | 329 ++++++--- crates/openshell-server/src/compute/vm.rs | 57 +- crates/openshell-server/src/config_file.rs | 676 ++++-------------- crates/openshell-server/src/defaults.rs | 4 +- crates/openshell-server/src/lib.rs | 59 +- deploy/docker/gateway.toml | 7 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/ci/values-skaffold.yaml | 4 +- .../openshell/templates/gateway-config.yaml | 22 +- .../openshell/tests/gateway_config_test.yaml | 38 +- deploy/helm/openshell/values.yaml | 13 +- deploy/man/openshell-gateway.8.md | 7 +- deploy/rpm/CONFIGURATION.md | 22 +- deploy/rpm/TROUBLESHOOTING.md | 4 +- deploy/rpm/gateway.toml.default | 7 +- docs/about/container-gateway.mdx | 6 +- docs/reference/gateway-config.mdx | 178 +++-- docs/reference/sandbox-compute-drivers.mdx | 14 +- e2e/configs/gateway/docker.toml | 6 +- e2e/configs/gateway/podman.toml | 7 +- .../Dockerfile.external-kubernetes-gateway | 6 +- e2e/run.sh | 3 - e2e/rust/e2e-vm.sh | 11 +- e2e/rust/tests/podman_corporate_proxy.rs | 40 +- e2e/support/gateway-common.sh | 2 +- e2e/with-docker-gateway.sh | 20 +- e2e/with-podman-gateway.sh | 41 +- examples/aws-s3-sts.md | 11 +- examples/governance-interceptor/smoke.sh | 5 +- .../podman/README.md | 2 +- .../spiffe-token-exchange-demo/podman/demo.sh | 5 +- .../podman/start-gateway.sh | 5 +- .../smoke.sh | 5 +- python/openshell/release_formula_test.py | 6 +- rfc/0003-gateway-configuration/README.md | 36 +- tasks/scripts/gateway-docker.sh | 42 +- tasks/scripts/gateway-podman.sh | 22 +- tasks/scripts/gateway-vm.sh | 42 +- tasks/scripts/gateway.sh | 26 +- tasks/scripts/release.py | 4 +- tasks/scripts/vm/smoke-orphan-cleanup.sh | 4 +- 71 files changed, 2592 insertions(+), 1677 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index b162389ca7..75425ca23c 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -81,12 +81,20 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash -rg -n 'compute_driver|compute_drivers|socket_path' /etc/openshell/gateway.toml +rg -n '^version|compute_driver|socket_path|guest_tls_' /etc/openshell/gateway.toml stat /run/openshell/.sock journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 ``` +Gateway configuration requires `[openshell] version = 2`, a singular +`compute_driver` selector, and driver-owned settings under +`[openshell.drivers.]`. The gateway rejects legacy `compute_drivers`, +`--drivers`, and `OPENSHELL_DRIVERS` selectors rather than silently migrating +them. Guest TLS CA, certificate, and key paths are the exception: configure the +complete bundle under `[openshell.gateway]`, and the gateway injects it only +into the selected local driver. + Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. For configured gateway interceptors, inspect `[[openshell.gateway.interceptors]]`, their Unix or network endpoints, and gateway startup logs: diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 5e8bbf394c..82cf9ac5ae 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -121,7 +121,7 @@ Loopback registration auto-derives the gateway name to `openshell` if `--name` i | Symptom | Likely cause | Where to look | |---|---|---| | `macos`/`ubuntu`/`fedora` job fails on `install.sh` | Latest tagged release missing an asset, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. | -| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_DRIVERS` env in the "Ensure …" step. | +| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_COMPUTE_DRIVER` env in the "Ensure …" step. | | `kubernetes` job fails on `helm install --wait` | Chart did not deploy in 5 min — usually image pull failure or readiness probe failing. | "Diagnostics on failure" step dumps `helm status`, manifest, pod describe, pod logs. | | `kubernetes` job fails on `kubectl wait` | Gateway pod stuck `CrashLoopBackOff` or `ImagePullBackOff`. | Diagnostics dump; check `:dev` image existence at `ghcr.io/nvidia/openshell/gateway`. | | `kubernetes` job fails on `openshell gateway add` or `status` | Port-forward not reachable, or CLI/gateway proto mismatch. | `port-forward.log` and `openshell gateway list` in the diagnostics dump. | diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 937e774db7..71a2a52e93 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Ensure VM driver run: | - launchctl setenv OPENSHELL_DRIVERS vm + launchctl setenv OPENSHELL_COMPUTE_DRIVER vm launchctl setenv OPENSHELL_TELEMETRY_ENABLED "$OPENSHELL_TELEMETRY_ENABLED" - name: Install and check status @@ -48,7 +48,7 @@ jobs: fi sudo systemctl start docker || sudo service docker start mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" docker info @@ -140,7 +140,7 @@ jobs: bash -s <<'EOF' set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" podman info curl -LsSf "${INSTALL_SH_URL}" | sh @@ -276,7 +276,7 @@ jobs: run: | set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\n' > "${HOME}/.config/openshell/gateway.env" + printf 'OPENSHELL_COMPUTE_DRIVER=docker\n' > "${HOME}/.config/openshell/gateway.env" curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh - name: Register kind gateway and check status diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index ce452369c5..afb58080d1 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -116,9 +116,10 @@ defines the available implementation set, while the runtime consumes a generic registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry -to `run_cli_with_compute_drivers`; factories receive merged driver config and -finish through the same in-process runtime adapter. A configured UDS endpoint -still takes precedence over a compiled registration with the same name. +to `run_cli_with_compute_drivers`; factories receive only the selected +`[openshell.drivers.]` table and finish through the same in-process +runtime adapter. A configured UDS endpoint still takes precedence over a +compiled registration with the same name. The standard server crate groups first-party registrations behind the `in-tree-compute-drivers` feature. Protocol-only gateway builds disable that @@ -223,7 +224,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | -| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--compute-driver ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. @@ -250,10 +251,21 @@ pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise they implement and validate the same complete contract. The capability marker is driver-owned supervisor input and is removed from workload environments. -Kubernetes deployments may set an AppArmor profile on sandbox agent containers -through the driver configuration. The Helm chart defaults sandbox agents to -`Unconfined` so runtime/default AppArmor profiles do not block supervisor -network namespace setup on AppArmor-enabled nodes. +Kubernetes, Docker, and Podman share one AppArmor configuration model: +`RuntimeDefault`, `Unconfined`, or `Localhost/`. Each driver translates +that model to its native API and rejects a requested confined profile when its +backend reports AppArmor unavailable. Docker and Podman use explicit +`Unconfined` by default because their runtime-default profiles commonly block +the supervisor's namespace mount setup; the Helm chart uses the same default. + +Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, +and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, +and the explicit cleartext-Basic-auth acknowledgement without allowing +workload environment to override them. Local containers project provider SPIFFE +through a dedicated host UNIX-socket parent mount. A VM cannot safely expose +that host socket: it accepts only a separately operated, concrete TCP listener +when `provider_spiffe_allow_guest_tcp = true` explicitly acknowledges guest +access. Host-only sockets are never implicitly forwarded to VM guests. Resource requirements enter the driver layer through `SandboxSpec.resource_requirements`. This includes a set of GPU requirements, where a user can request a specific number of GPUs or the driver-specific default behaviour. diff --git a/architecture/gateway.md b/architecture/gateway.md index c3ec92add5..f7c6d8f0b1 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -246,10 +246,10 @@ controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing deployments. Supervisors renew gateway JWTs in memory before expiry only while the sandbox record still exists. Older tokens are not server-revoked; shared deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. -The config default is -`gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM -gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other -shared deployments should set a positive TTL. +Omitting `gateway_jwt.ttl_secs` selects non-expiring tokens for local +single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0`. +Kubernetes and other shared deployments should set a positive TTL. Explicit +zero is rejected. Gateway JWT signing-key rotation is currently an offline operator action. The runtime loads one active signing key and one matching public verification key @@ -680,10 +680,9 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ``` The TOML file is opt-in via `--config ` / `OPENSHELL_GATEWAY_CONFIG`. -Driver implementation settings live in the TOML driver tables. The canonical -selector is the singular `[openshell.gateway] compute_driver`; the legacy -`compute_drivers` list remains accepted and normalizes into the existing -exactly-one-driver runtime validation. See `docs/reference/gateway-config.mdx` +Driver implementation settings live exclusively in TOML driver tables. The +selector is the singular `[openshell.gateway] compute_driver`; legacy +`compute_drivers` lists are rejected. See `docs/reference/gateway-config.mdx` for worked per-driver examples and RFC 0003 for the full schema. Each installation has an operator-assigned gateway name. Configure it with @@ -698,29 +697,20 @@ aliases, network names, and the sandbox JWT issuer. `database_url` is env-only and rejected when present in the file (`OPENSHELL_DB_URL` / `--db-url`). -### Driver inheritance - -`[openshell.gateway]` carries shared defaults such as `default_image`, -`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, and -`host_gateway_ip`. It also continues to accept the historical -`sandbox_namespace`, `service_account_name`, and `enable_user_namespaces` -locations as compatibility inputs. Canonical Kubernetes configuration places -those values in `[openshell.drivers.kubernetes]` as `namespace`, -`service_account_name`, and `enable_user_namespaces`; canonical Docker -configuration uses `sandbox_label`. Driver-table values take precedence over -compatibility inputs. The allowlist is per-driver so a gateway-wide default -cannot land in a driver that does not understand it (for example, -`client_tls_secret_name` is K8s-only). - -`image_pull_policy` is intentionally **not** inheritable: Kubernetes uses -`Always | IfNotPresent | Never` (passed verbatim to the K8s API) while -Podman uses the lowercase enum `always | missing | never | newer`. No -value means the same thing in both, so the key lives only under each -driver's own table. - -Driver-specific values that are not part of the inheritance allowlist -(e.g. Podman `socket_path`, VM `vcpus`) only come from the driver's own -table. +### Driver ownership + +`[openshell.gateway]` contains gateway process settings only. Each selected +driver reads its own configuration exclusively from +`[openshell.drivers.]`; values are never inherited from gateway scope. +Kubernetes owns `namespace`, `default_image`, `supervisor_image`, +`client_tls_secret_name`, `service_account_name`, `host_gateway_ip`, +`enable_user_namespaces`, and `sa_token_ttl_secs`. Docker uses +`sandbox_label` instead of the legacy `sandbox_namespace` name. Podman and VM +likewise own their image, endpoint, and runtime settings in their tables. + +`image_pull_policy` uses the shared canonical vocabulary +`always | if_not_present | never | newer`. Drivers translate it to their runtime +APIs; `newer` is supported only by Podman and rejected by Docker and Kubernetes. ### OTLP export diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index e1ec25c5e1..22636d78d1 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -10,6 +10,7 @@ use std::fmt; #[cfg(unix)] use std::io::{Read, Write}; use std::net::SocketAddr; +use std::num::NonZeroU64; #[cfg(unix)] use std::os::unix::fs::FileTypeExt; use std::path::{Path, PathBuf}; @@ -116,11 +117,6 @@ pub fn resolve_supervisor_image_tag(candidates: &[&str]) -> String { /// CDI device identifier for requesting all NVIDIA GPUs. pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; -/// Default maximum number of processes (PIDs) allowed inside a sandbox container. -/// -/// Shared by the Docker and Podman drivers; override via driver config. -pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; - /// Compute backends the gateway can orchestrate sandboxes through. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -193,7 +189,7 @@ impl FromStr for ComputeDriverKind { /// Auto-detect the appropriate compute driver based on the runtime environment. /// /// Priority order: Kubernetes → Podman → Docker. -/// VM is never auto-detected (requires explicit `--drivers vm`). +/// VM is never auto-detected (requires explicit `--compute-driver vm`). /// /// Returns the first driver where the environment check passes. /// Returns `None` if no compatible driver is found. @@ -816,12 +812,9 @@ pub struct Config { /// Database URL for persistence. pub database_url: String, - /// Compute drivers configured for the gateway. - /// - /// The config shape allows multiple drivers so the gateway can evolve - /// toward multi-backend routing. Current releases require exactly one - /// configured driver. - pub compute_drivers: Vec, + /// Explicit compute driver configured for the gateway. + /// `None` enables runtime auto-detection. + pub compute_driver: Option, /// Operator-provided endpoints for named remote compute drivers. /// @@ -1125,6 +1118,225 @@ const fn default_jwks_ttl_secs() -> u64 { 3600 } +/// Canonical policy controlling when a driver pulls a sandbox image. +/// +/// Backends translate this shared vocabulary to their runtime API. `newer` is +/// supported only by Podman; other backends reject it during configuration. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImagePullPolicy { + /// Always pull, even if a local image is available. + Always, + /// Pull only when a local image is unavailable. + #[default] + IfNotPresent, + /// Never pull; fail when a local image is unavailable. + Never, + /// Pull only when the registry image is newer than the local copy. + Newer, +} + +impl ImagePullPolicy { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Always => "always", + Self::IfNotPresent => "if_not_present", + Self::Never => "never", + Self::Newer => "newer", + } + } +} + +impl fmt::Display for ImagePullPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for ImagePullPolicy { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "always" => Ok(Self::Always), + "if_not_present" => Ok(Self::IfNotPresent), + "never" => Ok(Self::Never), + "newer" => Ok(Self::Newer), + other => Err(format!( + "invalid image pull policy '{other}'; expected one of: always, if_not_present, never, newer" + )), + } + } +} + +/// Canonical `AppArmor` confinement requested for a sandbox container. +/// +/// Drivers translate this model to their runtime API. An omitted value leaves +/// the runtime default unchanged; `Unconfined` is explicit because the +/// supervisor needs mount operations that the default Docker/Podman profile +/// commonly denies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppArmorProfile { + RuntimeDefault, + Unconfined, + Localhost(String), +} + +impl AppArmorProfile { + #[must_use] + pub const fn kubernetes_type(&self) -> &'static str { + match self { + Self::RuntimeDefault => "RuntimeDefault", + Self::Unconfined => "Unconfined", + Self::Localhost(_) => "Localhost", + } + } + + #[must_use] + pub fn localhost_profile(&self) -> Option<&str> { + match self { + Self::Localhost(profile) => Some(profile), + Self::RuntimeDefault | Self::Unconfined => None, + } + } + + /// Translate to the OCI `apparmor=` security option. + /// + /// `RuntimeDefault` deliberately returns `None`: omitting an OCI option + /// asks Docker/Podman to apply their runtime default profile. + #[must_use] + pub fn oci_security_opt(&self) -> Option { + match self { + Self::RuntimeDefault => None, + Self::Unconfined => Some("apparmor=unconfined".to_string()), + Self::Localhost(profile) => Some(format!("apparmor={profile}")), + } + } +} + +impl fmt::Display for AppArmorProfile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RuntimeDefault => f.write_str("RuntimeDefault"), + Self::Unconfined => f.write_str("Unconfined"), + Self::Localhost(profile) => write!(f, "Localhost/{profile}"), + } + } +} + +impl FromStr for AppArmorProfile { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "RuntimeDefault" => Ok(Self::RuntimeDefault), + "Unconfined" => Ok(Self::Unconfined), + other => match other.strip_prefix("Localhost/") { + Some("") => Err( + "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" + .to_string(), + ), + Some(profile) if !profile.contains(char::is_whitespace) => { + Ok(Self::Localhost(profile.to_string())) + } + Some(_) => { + Err("invalid AppArmor localhost profile; whitespace is not allowed".to_string()) + } + None => Err(format!( + "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" + )), + }, + } + } +} + +impl Serialize for AppArmorProfile { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for AppArmorProfile { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +/// Common local-driver corporate forward-proxy settings. +/// +/// This type is `flatten`ed by local compute-driver tables, preserving the +/// established TOML field names while keeping their safety contract shared. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct UpstreamProxyConfig { + pub https_proxy: Option, + pub no_proxy: Option, + pub proxy_auth_file: Option, + pub proxy_auth_allow_insecure: Option, + pub proxy_connect_by_hostname: Option, +} + +impl UpstreamProxyConfig { + /// Validate relationships that are independent of the container backend. + /// Credential contents are intentionally not read here and are never put + /// in an error message; drivers validate and stage them per sandbox. + pub fn validate(&self) -> Result<(), String> { + use crate::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; + + let proxy_secure = if let Some(url) = self.https_proxy.as_deref() { + parse_upstream_proxy_url(url) + .map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials; supply them with proxy_auth_file so they are not stored in configuration or runtime metadata".to_string(), + err => format!("https_proxy {err}"), + })? + .secure + } else { + false + }; + + if self + .no_proxy + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + return Err("no_proxy must not be empty when set; omit it instead".to_string()); + } + if self.no_proxy.is_some() && self.https_proxy.is_none() { + return Err("no_proxy is set but no https_proxy is configured".to_string()); + } + if let Some(path) = self.proxy_auth_file.as_ref() { + if path.as_os_str().is_empty() { + return Err("proxy_auth_file must not be empty when set".to_string()); + } + if self.https_proxy.is_none() { + return Err("proxy_auth_file is set but no https_proxy is configured".to_string()); + } + if !proxy_secure && self.proxy_auth_allow_insecure != Some(true) { + return Err("proxy_auth_file sends a cleartext Basic credential to an http:// proxy; set proxy_auth_allow_insecure = true to acknowledge that exposure".to_string()); + } + } else if self.proxy_auth_allow_insecure.is_some() { + return Err( + "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), + ); + } + if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { + return Err( + "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), + ); + } + Ok(()) + } +} + /// Gateway-minted sandbox JWT configuration. /// /// Points the gateway at the Ed25519 signing key (produced by `certgen`) @@ -1144,40 +1356,23 @@ pub struct GatewayJwtConfig { /// `openshell`. #[serde(default = "default_gateway_id")] pub gateway_id: String, - /// Token lifetime in seconds. A value of 0 disables expiration and is - /// intended only for local single-player deployments. Canonical serialized - /// configuration omits the field for that non-expiring behavior; explicit - /// legacy zero remains accepted. - #[serde( - default = "default_sandbox_token_ttl_secs", - skip_serializing_if = "is_default" - )] - pub ttl_secs: u64, + /// Token lifetime in seconds. Omit the field for a non-expiring token. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl_secs: Option, } impl GatewayJwtConfig { - /// Effective token lifetime. `None` preserves the established non-expiring - /// behavior represented by an omitted or explicit zero `ttl_secs` value. + /// Effective token lifetime. `None` represents a non-expiring token. pub fn sandbox_token_ttl(&self) -> Option { - (self.ttl_secs != 0).then(|| Duration::from_secs(self.ttl_secs)) + self.ttl_secs.map(|ttl| Duration::from_secs(ttl.get())) } } -fn is_default(value: &T) -> bool -where - T: Default + PartialEq, -{ - value == &T::default() -} - fn default_gateway_id() -> String { "openshell".to_string() } -const fn default_sandbox_token_ttl_secs() -> u64 { - 0 -} - fn default_roles_claim() -> String { "realm_access.roles".to_string() } @@ -1211,7 +1406,7 @@ impl Config { mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, database_url: String::new(), - compute_drivers: vec![], + compute_driver: None, compute_driver_endpoints: BTreeMap::new(), credential_drivers: Vec::new(), default_credential_driver: None, @@ -1262,17 +1457,10 @@ impl Config { self } - /// Create a new configuration with the configured compute drivers. + /// Create a new configuration with an explicit compute driver. #[must_use] - pub fn with_compute_drivers(mut self, drivers: I) -> Self - where - I: IntoIterator, - D: ToString, - { - self.compute_drivers = drivers - .into_iter() - .map(|driver| driver.to_string()) - .collect(); + pub fn with_compute_driver(mut self, driver: impl ToString) -> Self { + self.compute_driver = Some(driver.to_string()); self } @@ -1469,10 +1657,11 @@ mod tests { use super::{ ActiveMachine, ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, - GatewayJwtConfig, GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, - detect_docker_socket_from_candidates, detect_driver, detect_podman_socket_from_candidates, - docker_host_unix_socket_path, docker_socket_responds, explicit_unix_container_host, - normalize_compute_driver_name, parse_default_podman_connection, parse_podman_info_socket, + GatewayJwtConfig, GatewayProviderProfileSourceConfig, ImagePullPolicy, + PolicyValidationFailureMode, detect_docker_socket_from_candidates, detect_driver, + detect_podman_socket_from_candidates, docker_host_unix_socket_path, docker_socket_responds, + explicit_unix_container_host, normalize_compute_driver_name, + parse_default_podman_connection, parse_podman_info_socket, parse_podman_machine_inspect_socket, podman_connection_name_for_uri, podman_machine_inspect_targets, podman_socket_candidates_from_env, podman_socket_responds, resolve_active_podman_machine, run_bounded_command, unix_url_socket_path, @@ -1604,7 +1793,7 @@ mod tests { })) .expect("gateway JWT config should deserialize with default ttl"); - assert_eq!(cfg.ttl_secs, 0); + assert_eq!(cfg.ttl_secs, None); assert_eq!(cfg.sandbox_token_ttl(), None); let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); @@ -1626,6 +1815,34 @@ mod tests { assert_eq!(serialized["ttl_secs"], 3600); } + #[test] + fn gateway_jwt_ttl_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 0 + })) + .expect_err("zero TTL must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); + } + + #[test] + fn image_pull_policy_uses_canonical_vocabulary() { + for (value, expected) in [ + ("always", ImagePullPolicy::Always), + ("if_not_present", ImagePullPolicy::IfNotPresent), + ("never", ImagePullPolicy::Never), + ("newer", ImagePullPolicy::Newer), + ] { + assert_eq!(value.parse::(), Ok(expected)); + assert_eq!(expected.to_string(), value); + assert_eq!(serde_json::to_value(expected).unwrap(), value); + } + assert!("missing".parse::().is_err()); + assert!("IfNotPresent".parse::().is_err()); + } + #[test] fn name_defaults_and_can_be_overridden() { assert_eq!(Config::new(None).name, "openshell"); diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index c63e4bcdd8..97c9a0220d 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -63,6 +63,7 @@ pub const VM_GUEST_TLS_CA_PATH: &str = "/opt/openshell/tls/ca.crt"; pub const VM_GUEST_TLS_CERT_PATH: &str = "/opt/openshell/tls/tls.crt"; pub const VM_GUEST_TLS_KEY_PATH: &str = "/opt/openshell/tls/tls.key"; pub const VM_GUEST_SANDBOX_TOKEN_PATH: &str = "/opt/openshell/auth/sandbox.jwt"; +pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy"; pub const VM_GUEST_INIT_DROPIN_DIR: &str = "/opt/openshell/init.d"; pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest"; pub const VM_UMOCI_PATH: &str = "/opt/openshell/bin/umoci"; @@ -101,6 +102,7 @@ mod tests { VM_GUEST_TLS_CERT_PATH, VM_GUEST_TLS_KEY_PATH, VM_GUEST_SANDBOX_TOKEN_PATH, + VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, VM_GUEST_INIT_DROPIN_DIR, VM_GUEST_INIT_DROPIN_MANIFEST, VM_UMOCI_PATH, diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index f3279d8105..d668dac944 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -7,6 +7,73 @@ use std::path::{Path, PathBuf}; use crate::proto::compute::v1::DriverSandbox; +/// Built-in sandbox network topologies used to derive a callback endpoint +/// when an operator does not configure a per-driver `grpc_endpoint` override. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GatewayCallbackTopology<'a> { + /// A sandbox pod reaches the gateway through its Kubernetes service. + Kubernetes { namespace: &'a str }, + /// A Docker container reaches the host through Docker's gateway alias. + Docker, + /// A Podman container reaches the host through Podman's gateway alias. + Podman, + /// A libkrun guest reaches the host through gvproxy's gateway alias. + Vm, +} + +/// Build the endpoint a sandbox uses to call its gateway for a known topology. +/// +/// The result is deliberately derived by the gateway rather than baked into +/// individual driver defaults. A configured `grpc_endpoint` remains an +/// operator override for remote or non-standard deployments. +#[must_use] +pub fn gateway_callback_endpoint( + topology: GatewayCallbackTopology<'_>, + gateway_port: u16, + gateway_tls_enabled: bool, +) -> String { + let scheme = if gateway_tls_enabled { "https" } else { "http" }; + let host = match topology { + GatewayCallbackTopology::Kubernetes { namespace } => { + return format!("{scheme}://openshell-gateway.{namespace}.svc:{gateway_port}"); + } + GatewayCallbackTopology::Docker | GatewayCallbackTopology::Vm => "host.openshell.internal", + GatewayCallbackTopology::Podman => "host.containers.internal", + }; + format!("{scheme}://{host}:{gateway_port}") +} + +#[cfg(test)] +mod callback_endpoint_tests { + use super::{GatewayCallbackTopology, gateway_callback_endpoint}; + + #[test] + fn derives_endpoint_for_each_builtin_topology() { + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Docker, 17670, false), + "http://host.openshell.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Podman, 17670, true), + "https://host.containers.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Vm, 17670, true), + "https://host.openshell.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint( + GatewayCallbackTopology::Kubernetes { + namespace: "agents" + }, + 8080, + true, + ), + "https://openshell-gateway.agents.svc:8080" + ); + } +} + // --------------------------------------------------------------------------- // Sandbox container/pod label keys (openshell.ai/ namespace) // --------------------------------------------------------------------------- @@ -433,6 +500,68 @@ pub fn read_upstream_proxy_credential_file(path: &str) -> Result /// Container-side directory where the provider SPIFFE Workload API socket is mounted. pub const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = "/spiffe-workload-api"; +/// Validate a host UNIX socket selected for provider SPIFFE projection. +/// +/// Local container drivers bind-mount the socket's dedicated parent directory, +/// not a broad host root. TCP endpoints are deliberately rejected here: a +/// container projection must be a filesystem socket, while VM guest TCP +/// exposure has its own explicit acknowledgement contract. +pub fn validate_provider_spiffe_unix_socket(path: &Path) -> Result<(), String> { + let raw = path + .to_str() + .ok_or_else(|| "provider_spiffe_workload_api_socket must be valid UTF-8".to_string())?; + if raw.trim() != raw || raw.is_empty() { + return Err("provider_spiffe_workload_api_socket must not be empty or contain surrounding whitespace".to_string()); + } + if raw.starts_with("tcp:") || raw.starts_with("unix:") { + return Err("provider_spiffe_workload_api_socket must be an absolute host UNIX socket path, not a URI".to_string()); + } + if !path.is_absolute() || path.parent().is_none_or(|parent| parent == Path::new("/")) { + return Err("provider_spiffe_workload_api_socket must be an absolute UNIX socket path below a dedicated parent directory".to_string()); + } + Ok(()) +} + +/// Return the guest/container path for a projected provider SPIFFE socket. +pub fn projected_provider_spiffe_socket_path(path: &Path) -> Result { + validate_provider_spiffe_unix_socket(path)?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .ok_or_else(|| "provider_spiffe_workload_api_socket must name a socket file".to_string())?; + Ok(format!( + "{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}/{file_name}" + )) +} + +/// Validate an explicitly operator-acknowledged guest-reachable SPIFFE TCP endpoint. +/// +/// The `tcp:` spelling is the SPIFFE Workload API endpoint grammar accepted by +/// the client. The address must be concrete; wildcard and host-only UNIX +/// sockets are never silently exposed to VM guests. +pub fn validate_guest_spiffe_tcp_endpoint( + endpoint: &str, + acknowledged: bool, +) -> Result<(), String> { + if endpoint.trim() != endpoint || endpoint.is_empty() { + return Err("provider_spiffe_workload_api_tcp_endpoint must not be empty or contain surrounding whitespace".to_string()); + } + if !acknowledged { + return Err("provider_spiffe_workload_api_tcp_endpoint exposes a Workload API to VM guests; set provider_spiffe_allow_guest_tcp = true only after explicitly acknowledging that exposure".to_string()); + } + let address = endpoint.strip_prefix("tcp:").ok_or_else(|| { + "provider_spiffe_workload_api_tcp_endpoint must use tcp:host:port (for example tcp:192.0.2.10:8081)".to_string() + })?; + let address: std::net::SocketAddr = address.parse().map_err(|_| { + "provider_spiffe_workload_api_tcp_endpoint must use a concrete IP address and non-zero port".to_string() + })?; + if address.ip().is_unspecified() || address.port() == 0 { + return Err("provider_spiffe_workload_api_tcp_endpoint must not use an unspecified address or port 0".to_string()); + } + Ok(()) +} + /// Return the XDG state path for a driver's sandbox JWT token file. /// /// The resulting path is `$XDG_STATE_HOME/openshell/[/]//sandbox.jwt`. @@ -872,6 +1001,28 @@ mod tests { } #[cfg(unix)] + #[test] + fn projected_spiffe_socket_requires_dedicated_absolute_unix_path() { + assert_eq!( + projected_provider_spiffe_socket_path(Path::new("/run/spire/agent.sock")).unwrap(), + "/spiffe-workload-api/agent.sock" + ); + for path in ["relative.sock", "/agent.sock", "tcp:127.0.0.1:8081"] { + assert!( + validate_provider_spiffe_unix_socket(Path::new(path)).is_err(), + "{path}" + ); + } + } + + #[test] + fn guest_spiffe_tcp_requires_acknowledgement_and_concrete_endpoint() { + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", true).is_ok()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", false).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:0.0.0.0:8081", true).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("unix:/run/spire/agent.sock", true).is_err()); + } + #[test] fn credential_file_rejects_fifo_without_hanging() { // A FIFO with no writer would block a blocking open() forever. The diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 315bb319bb..7c2f5e8b97 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -50,10 +50,11 @@ pub mod time; pub mod transport_errors; pub use config::{ - ComputeDriverKind, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, - GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, - GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, - MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, + AppArmorProfile, ComputeDriverKind, Config, GatewayAuthConfig, + GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, + GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, GatewayJwtConfig, + GatewayProviderProfileSourceConfig, ImagePullPolicy, MtlsAuthConfig, OidcConfig, + PolicyValidationFailureMode, TlsConfig, UpstreamProxyConfig, }; pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{ diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..31c59bc6f3 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -104,7 +104,7 @@ contract: | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | +| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Omit `[openshell.drivers.docker].sandbox_pids_limit` to inherit the Docker/runtime default; explicit `0` is invalid. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | | `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | @@ -187,6 +187,25 @@ mounted into the container and exposed with: HTTP endpoints reject TLS material because the supervisor would not use it. +## Corporate proxy, SPIFFE, and AppArmor + +`https_proxy`, `no_proxy`, and `proxy_auth_file` in +`[openshell.drivers.docker]` are operator-owned supervisor settings. Docker +passes the proxy URL and bypass list on the supervisor command line and mounts +an optional `user:pass` auth file read-only at a root-only path. Credentials +never appear in container environment or Docker labels. An auth file used with +an `http://` proxy requires `proxy_auth_allow_insecure = true`; an `https://` +proxy protects the Basic-auth header in its TLS session. + +Set `provider_spiffe_workload_api_socket` to an absolute host UNIX socket to +project its dedicated parent directory into the supervisor and set +`OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to the guest path. TCP URIs are +rejected for this projection. `app_armor_profile` uses the shared +`RuntimeDefault`, `Unconfined`, or `Localhost/` vocabulary. Docker +uses explicit `Unconfined` by default because the supervisor's namespace mount +setup is incompatible with `docker-default`; requested confined profiles fail +at startup if Docker does not report AppArmor support. + ## Environment Ownership The driver merges template environment and sandbox spec environment first, then diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index ca447c122d..3bf289256e 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -21,15 +21,14 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; +use openshell_core::config::{DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_STOP_TIMEOUT_SECS}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackTopology, LABEL_MANAGED_BY, + LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -56,6 +55,7 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; +use openshell_core::{AppArmorProfile, ImagePullPolicy, UpstreamProxyConfig}; use openshell_core::{Config, Error, Result as CoreResult}; use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; @@ -83,6 +83,10 @@ const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = + openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH; +const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = + openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; @@ -123,10 +127,9 @@ pub struct DockerComputeConfig { pub default_image: String, /// Image pull policy for sandbox images. - pub image_pull_policy: String, + pub image_pull_policy: ImagePullPolicy, /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. - #[serde(alias = "sandbox_namespace")] pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. @@ -160,13 +163,29 @@ pub struct DockerComputeConfig { /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to leave Docker's runtime/default PID limit unchanged. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] pub enable_bind_mounts: bool, + + /// Corporate forward-proxy settings supplied to the supervisor on argv. + /// The flattened fields retain the common `https_proxy`, `no_proxy`, and + /// `proxy_auth_*` gateway TOML contract. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Host UNIX socket to project into sandbox supervisors for provider + /// SPIFFE token exchange. + pub provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` confinement requested for sandbox containers. The explicit + /// default preserves the prior supervisor-compatible Docker behavior. + pub app_armor_profile: Option, } impl Default for DockerComputeConfig { @@ -174,7 +193,7 @@ impl Default for DockerComputeConfig { Self { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), + image_pull_policy: ImagePullPolicy::default(), sandbox_label: "default".to_string(), grpc_endpoint: String::new(), supervisor_bin: None, @@ -185,8 +204,11 @@ impl Default for DockerComputeConfig { network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } } @@ -201,7 +223,7 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { default_image: String, - image_pull_policy: String, + image_pull_policy: ImagePullPolicy, sandbox_label: String, grpc_endpoint: String, network_name: String, @@ -215,8 +237,11 @@ struct DockerDriverRuntimeConfig { daemon_version: String, supports_gpu: bool, allow_all_default_gpu: bool, - sandbox_pids_limit: i64, + sandbox_pids_limit: Option, enable_bind_mounts: bool, + upstream_proxy: UpstreamProxyConfig, + provider_spiffe_workload_api_socket: Option, + app_armor_profile: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -560,6 +585,17 @@ impl DockerComputeDriver { let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + validate_image_pull_policy(docker_config.image_pull_policy)?; + docker_config + .upstream_proxy + .validate() + .map_err(Error::config)?; + validate_docker_proxy_auth_file(&docker_config.upstream_proxy)?; + if let Some(socket) = docker_config.provider_spiffe_workload_api_socket.as_deref() { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(Error::config)?; + } + validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = config.bind_address.port(); if gateway_port == 0 { return Err(Error::config( @@ -575,13 +611,11 @@ impl DockerComputeDriver { docker_gateway_callback_bind_address(&gateway_route, config.bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); + docker_config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Docker, + gateway_port, + config.tls.is_some(), + ); } let grpc_endpoint = docker_container_openshell_endpoint( &docker_config.grpc_endpoint, @@ -596,7 +630,7 @@ impl DockerComputeDriver { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), + image_pull_policy: docker_config.image_pull_policy, sandbox_label: docker_config.sandbox_label.clone(), grpc_endpoint, network_name, @@ -612,6 +646,11 @@ impl DockerComputeDriver { allow_all_default_gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + upstream_proxy: docker_config.upstream_proxy.clone(), + provider_spiffe_workload_api_socket: docker_config + .provider_spiffe_workload_api_socket + .clone(), + app_armor_profile: docker_config.app_armor_profile.clone(), }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -1625,9 +1664,8 @@ impl DockerComputeDriver { sandbox_id: &str, image: &str, ) -> Result { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - let inspect = match policy.as_str() { - "" | "ifnotpresent" => { + let inspect = match self.config.image_pull_policy { + ImagePullPolicy::IfNotPresent => { if let Ok(inspect) = self.docker.inspect_image(image).await { self.publish_docker_progress( sandbox_id, @@ -1644,14 +1682,14 @@ impl DockerComputeDriver { .map_err(|err| internal_status("inspect Docker image after pull", err))? } } - "always" => { + ImagePullPolicy::Always => { self.pull_image(sandbox_id, image).await?; self.docker .inspect_image(image) .await .map_err(|err| internal_status("inspect Docker image after pull", err))? } - "never" => match self.docker.inspect_image(image).await { + ImagePullPolicy::Never => match self.docker.inspect_image(image).await { Ok(inspect) => { self.publish_docker_progress( sandbox_id, @@ -1663,15 +1701,15 @@ impl DockerComputeDriver { } Err(err) if is_not_found_error(&err) => { return Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" + "docker image '{image}' is not present locally and image_pull_policy = \"never\"" ))); } Err(err) => return Err(internal_status("inspect Docker image", err)), }, - other => { - return Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))); + ImagePullPolicy::Newer => { + return Err(Status::failed_precondition( + "image_pull_policy = \"newer\" is supported only by the Podman compute driver", + )); } }; @@ -2657,6 +2695,49 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } +/// Verify the configured credential without exposing its contents. Docker +/// bind-mounts the root-owned file directly, unlike Podman which uses a native +/// secret object; this preflight makes a bad file fail before any sandbox is +/// created. +fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<()> { + let Some(path) = config.proxy_auth_file.as_ref() else { + return Ok(()); + }; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file( + path.to_str() + .ok_or_else(|| Error::config("proxy_auth_file must be valid UTF-8"))?, + ) + .map_err(Error::config)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map_err(|error| Error::config(format!("proxy_auth_file is invalid: {error}")))?; + Ok(()) +} + +/// Build immutable operator-owned proxy arguments. Credentials never appear on +/// argv: only the fixed in-container root-only file path is supplied. +fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { + let mut args = Vec::new(); + if let Some(url) = config.https_proxy.as_ref() { + args.extend(["--upstream-proxy".to_string(), url.clone()]); + } + if let Some(no_proxy) = config.no_proxy.as_ref() { + args.extend(["--upstream-no-proxy".to_string(), no_proxy.clone()]); + } + if config.proxy_auth_file.is_some() { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if config.proxy_auth_allow_insecure == Some(true) { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if config.proxy_connect_by_hostname == Some(true) { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args +} + fn build_binds( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -2686,6 +2767,23 @@ fn build_binds( SANDBOX_TOKEN_MOUNT_PATH )); } + if let Some(path) = config.upstream_proxy.proxy_auth_file.as_ref() { + binds.push(format!( + "{}:{}:ro,z", + path.display(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH + )); + } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { + let parent = socket.parent().ok_or_else(|| { + Status::failed_precondition("provider SPIFFE socket has no parent directory") + })?; + binds.push(format!( + "{}:{}:ro,rbind", + parent.display(), + PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + )); + } Ok(binds) } @@ -2868,6 +2966,15 @@ fn build_environment_for_oci_user( TLS_KEY_MOUNT_PATH.to_string(), ); } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() + && let Ok(path) = + openshell_core::driver_utils::projected_provider_spiffe_socket_path(socket) + { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + path, + ); + } environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); @@ -3078,7 +3185,11 @@ fn build_container_create_body_for_image( entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Replace the image CMD with the supervisor's resolved workspace // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + cmd: { + let mut args = vec!["--workdir".to_string(), workspace_root]; + args.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); + Some(args) + }, labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3100,17 +3211,13 @@ fn build_container_create_body_for_image( "SYS_PTRACE".to_string(), "SYSLOG".to_string(), ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), + // The default is explicitly Unconfined because the supervisor + // needs mount operations commonly denied by docker-default. + security_opt: config + .app_armor_profile + .as_ref() + .and_then(AppArmorProfile::oci_security_opt) + .map(|option| vec![option]), network_mode: Some(config.network_name.clone()), extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), ..Default::default() @@ -3395,26 +3502,55 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { +fn validate_sandbox_pids_limit(value: Option) -> CoreResult<()> { + if value.is_some_and(|limit| limit.get() < 0) { return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", + "docker sandbox_pids_limit must be positive when set", )); } Ok(()) } -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", +fn validate_image_pull_policy(policy: ImagePullPolicy) -> CoreResult<()> { + if policy == ImagePullPolicy::Newer { + return Err(Error::config( + "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", )); } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) + Ok(()) +} + +fn validate_docker_app_armor_profile( + profile: Option<&AppArmorProfile>, + info: &SystemInfo, +) -> CoreResult<()> { + let requires_apparmor = matches!( + profile, + Some(AppArmorProfile::RuntimeDefault | AppArmorProfile::Localhost(_)) + ); + if !requires_apparmor { + return Ok(()); + } + let available = info.security_options.as_ref().is_some_and(|options| { + options + .iter() + .any(|option| option.to_ascii_lowercase().contains("apparmor")) + }); + if !available { + return Err(Error::config( + "app_armor_profile requires AppArmor, but Docker reports it is unavailable; enable AppArmor on the daemon host or set app_armor_profile = \"Unconfined\" explicitly", + )); } + Ok(()) +} + +fn docker_pids_limit(value: Option) -> Result, Status> { + if value.is_some_and(|limit| limit.get() < 0) { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be positive when set", + )); + } + Ok(value.map(std::num::NonZeroI64::get)) } #[allow(clippy::cast_possible_truncation)] @@ -4034,12 +4170,6 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult bool { - docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() -} - pub(crate) fn docker_guest_tls_paths( docker_config: &DockerComputeConfig, ) -> CoreResult> { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 5257070883..1659c1f422 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -94,7 +94,7 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), - image_pull_policy: String::new(), + image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_label: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), @@ -121,8 +121,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { daemon_version: "28.0.0".to_string(), supports_gpu: false, allow_all_default_gpu: false, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } @@ -138,20 +141,69 @@ fn docker_config_uses_canonical_sandbox_label_name() { } #[test] -fn docker_config_accepts_legacy_sandbox_namespace_alias() { - let config: DockerComputeConfig = - serde_json::from_value(serde_json::json!({ "sandbox_namespace": "tenant-a" })).unwrap(); - assert_eq!(config.sandbox_label, "tenant-a"); +fn docker_config_rejects_legacy_sandbox_namespace() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_namespace": "tenant-a" + })) + .expect_err("legacy sandbox_namespace must be rejected"); + assert!(error.to_string().contains("sandbox_namespace")); } #[test] -fn docker_config_rejects_canonical_and_legacy_sandbox_label_names_together() { - let error = serde_json::from_value::(serde_json::json!({ - "sandbox_label": "tenant-a", - "sandbox_namespace": "tenant-b" +fn docker_config_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 })) - .expect_err("canonical and legacy names must not both be accepted"); - assert!(error.to_string().contains("duplicate field")); + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: DockerComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = validate_sandbox_pids_limit(negative.sandbox_pids_limit).unwrap_err(); + assert!(error.to_string().contains("must be positive")); +} + +#[test] +fn docker_rejects_newer_image_pull_policy() { + let error = validate_image_pull_policy(ImagePullPolicy::Newer).unwrap_err(); + assert!(error.to_string().contains("supported only by the Podman")); +} + +#[test] +fn docker_config_uses_shared_proxy_contract_and_explicit_apparmor_default() { + let config: DockerComputeConfig = toml::from_str( + r#" +https_proxy = "http://proxy.example:8080" +no_proxy = ".svc" +proxy_auth_file = "/run/secrets/proxy-auth" +proxy_auth_allow_insecure = true +app_armor_profile = "Localhost/openshell-supervisor" +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +"#, + ) + .unwrap(); + assert_eq!( + config.upstream_proxy.https_proxy.as_deref(), + Some("http://proxy.example:8080") + ); + assert_eq!( + config.app_armor_profile, + Some(AppArmorProfile::Localhost( + "openshell-supervisor".to_string() + )) + ); + assert!(config.upstream_proxy.validate().is_ok()); + assert!( + openshell_core::driver_utils::validate_provider_spiffe_unix_socket( + config + .provider_spiffe_workload_api_socket + .as_deref() + .unwrap() + ) + .is_ok() + ); } fn json_struct(value: serde_json::Value) -> prost_types::Struct { @@ -416,7 +468,7 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { .build(); let subscriber = tracing_subscriber::registry().with(otel_tracing::layer(&provider)); let mut config = runtime_config(); - config.image_pull_policy = "unsupported".to_string(); + config.image_pull_policy = ImagePullPolicy::Newer; let driver = test_driver_with_config(config); driver @@ -1075,13 +1127,13 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { } #[test] -fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { +fn docker_pids_limit_uses_runtime_default_when_omitted() { assert_eq!( - docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), - Some(DEFAULT_SANDBOX_PIDS_LIMIT) + docker_pids_limit(std::num::NonZeroI64::new(2048)).unwrap(), + Some(2048) ); - assert_eq!(docker_pids_limit(0).unwrap(), None); - assert!(docker_pids_limit(-1).is_err()); + assert_eq!(docker_pids_limit(None).unwrap(), None); + assert!(docker_pids_limit(std::num::NonZeroI64::new(-1)).is_err()); } #[test] @@ -1091,10 +1143,10 @@ fn docker_compute_config_disables_bind_mounts_by_default() { } #[test] -fn container_create_body_sets_driver_owned_pids_limit() { +fn container_create_body_omits_pids_limit_by_default() { let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); + assert_eq!(host_config.pids_limit, None); } #[test] @@ -2028,6 +2080,46 @@ fn build_environment_uses_token_file_without_raw_token_env() { ))); } +#[test] +fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { + let mut config = runtime_config(); + config.upstream_proxy = UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }; + config.provider_spiffe_workload_api_socket = Some(PathBuf::from("/run/spire/agent.sock")); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + let command = body.cmd.unwrap(); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy", "https://proxy.example:8443"]) + ); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy-auth-file", UPSTREAM_PROXY_AUTH_MOUNT_PATH]) + ); + let binds = body.host_config.unwrap().binds.unwrap(); + assert!( + binds + .iter() + .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) + ); + assert!( + binds + .iter() + .any(|bind| bind.contains(PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR)) + ); + let env = body.env.unwrap(); + assert!(env.iter().any(|entry| entry + == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); + assert!(!env.iter().any(|entry| entry.contains("proxy-auth"))); +} + #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 02dcfe5e87..5d6154bd17 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -152,14 +152,13 @@ abstract socket whose peer PID must match that authenticated supervisor. Both supervisors exit if the control connection closes, coupling their container restart lifecycle before a new authoritative client can be established. -The driver can request a Kubernetes AppArmor profile through -`app_armor_profile`. - +The driver uses the shared AppArmor model through `app_armor_profile`. Supported values are `Unconfined`, `RuntimeDefault`, and -`Localhost/`. An empty or unset value omits -`securityContext.appArmorProfile`. Helm deployments default sandbox agent -containers to `Unconfined` because runtime/default AppArmor profiles can block -the supervisor's network namespace mount setup on AppArmor-enabled nodes. +`Localhost/`; an empty or unset value omits +`securityContext.appArmorProfile`. Docker and Podman translate the same values +to OCI security options. Helm deployments default sandbox agent containers to +`Unconfined` because runtime/default AppArmor profiles can block the +supervisor's network namespace mount setup on AppArmor-enabled nodes. ## GPU Support diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index aadcb1342a..8c35ba001c 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub use openshell_core::OperatorNamespaceAllowlist; -use openshell_core::config; +pub use openshell_core::{AppArmorProfile, OperatorNamespaceAllowlist}; +use openshell_core::{ImagePullPolicy, config}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; #[cfg(test)] @@ -177,83 +177,6 @@ impl KubernetesSidecarConfig { } } -/// Kubernetes `AppArmor` profile requested for the sandbox agent container. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AppArmorProfile { - RuntimeDefault, - Unconfined, - Localhost(String), -} - -impl AppArmorProfile { - #[must_use] - pub fn to_k8s_type(&self) -> &'static str { - match self { - Self::RuntimeDefault => "RuntimeDefault", - Self::Unconfined => "Unconfined", - Self::Localhost(_) => "Localhost", - } - } - - #[must_use] - pub fn localhost_profile(&self) -> Option<&str> { - match self { - Self::Localhost(profile) => Some(profile), - Self::RuntimeDefault | Self::Unconfined => None, - } - } -} - -impl std::fmt::Display for AppArmorProfile { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::RuntimeDefault => f.write_str("RuntimeDefault"), - Self::Unconfined => f.write_str("Unconfined"), - Self::Localhost(profile) => write!(f, "Localhost/{profile}"), - } - } -} - -impl FromStr for AppArmorProfile { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "RuntimeDefault" => Ok(Self::RuntimeDefault), - "Unconfined" => Ok(Self::Unconfined), - other => match other.strip_prefix("Localhost/") { - Some("") => Err( - "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" - .to_string(), - ), - Some(profile) => Ok(Self::Localhost(profile.to_string())), - None => Err(format!( - "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" - )), - }, - } - } -} - -impl Serialize for AppArmorProfile { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> Deserialize<'de> for AppArmorProfile { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::from_str(&value).map_err(serde::de::Error::custom) - } -} - fn deserialize_optional_app_armor_profile<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -263,7 +186,8 @@ where let value = Option::::deserialize(deserializer)?; match value.as_deref() { None | Some("") => Ok(None), - Some(value) => AppArmorProfile::from_str(value) + Some(value) => value + .parse::() .map(Some) .map_err(serde::de::Error::custom), } @@ -306,7 +230,9 @@ pub struct KubernetesComputeConfig { /// the driver's `TokenReview` bootstrap authenticator. pub service_account_name: String, pub default_image: String, - pub image_pull_policy: String, + /// Pull policy for sandbox images. Omit to use Kubernetes's image default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_pull_policy: Option, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, /// Managed-mode SSH ingress isolation. When enabled, the driver creates a @@ -317,9 +243,10 @@ pub struct KubernetesComputeConfig { /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. pub supervisor_image: String, - /// Kubernetes `imagePullPolicy` for the supervisor image. - /// Empty string delegates to the Kubernetes default. - pub supervisor_image_pull_policy: String, + /// Pull policy for the supervisor image. Omit to use Kubernetes's image + /// default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supervisor_image_pull_policy: Option, /// How the supervisor binary is delivered into sandbox pods. pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. @@ -439,15 +366,13 @@ impl Default for KubernetesComputeConfig { operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), - // Default empty so the gateway omits `imagePullPolicy` from pod - // specs and Kubernetes applies its own default (Always for `latest`, - // IfNotPresent otherwise). `DEFAULT_IMAGE_PULL_POLICY` ("missing") - // is Podman vocabulary and is not a valid Kubernetes value. - image_pull_policy: String::new(), + // Omit the field so Kubernetes applies its own default (Always for + // `latest`, IfNotPresent otherwise). + image_pull_policy: None, image_pull_secrets: Vec::new(), managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), - supervisor_image_pull_policy: String::new(), + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), @@ -506,17 +431,52 @@ impl KubernetesComputeConfig { self.sidecar.validate_proxy_uid() } + /// Reject pull policies Kubernetes cannot express before creating pods. + pub fn validate_image_pull_policies(&self) -> Result<(), String> { + for (field, policy) in [ + ("image_pull_policy", self.image_pull_policy), + ( + "supervisor_image_pull_policy", + self.supervisor_image_pull_policy, + ), + ] { + if policy == Some(ImagePullPolicy::Newer) { + return Err(format!( + "{field} = \"newer\" is supported only by the Podman compute driver" + )); + } + } + Ok(()) + } + + /// Translate a validated shared policy to Kubernetes's API vocabulary. + #[must_use] + pub fn image_pull_policy_value(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "Always", + ImagePullPolicy::IfNotPresent => "IfNotPresent", + ImagePullPolicy::Never => "Never", + ImagePullPolicy::Newer => unreachable!("newer must be rejected during validation"), + } + } + /// Validate the operator-owned corporate upstream proxy configuration. pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - if let Some(url) = &self.https_proxy { - parse_upstream_proxy_url(url).map_err(|err| match err { - UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), - UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), - err => format!("https_proxy {err}"), - })?; - } + let proxy_addr = self + .https_proxy + .as_deref() + .map(|url| { + parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => { + "https_proxy must not be empty when set".to_string() + } + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), + err => format!("https_proxy {err}"), + }) + }) + .transpose()?; if let Some(list) = self.no_proxy.as_deref() { if list.trim().is_empty() { @@ -575,7 +535,9 @@ impl KubernetesComputeConfig { .to_string(), ); } - if self.proxy_auth_allow_insecure != Some(true) { + if proxy_addr.as_ref().is_some_and(|proxy| !proxy.secure) + && self.proxy_auth_allow_insecure != Some(true) + { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } if self.topology == SupervisorTopology::Combined { @@ -928,6 +890,34 @@ mod tests { assert!(cfg.sidecar.process_binary_aware_network_policy); } + #[test] + fn image_pull_policy_uses_shared_canonical_values() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "image_pull_policy": "if_not_present", + "supervisor_image_pull_policy": "never" + })) + .unwrap(); + assert_eq!(cfg.image_pull_policy, Some(ImagePullPolicy::IfNotPresent)); + assert_eq!( + cfg.supervisor_image_pull_policy, + Some(ImagePullPolicy::Never) + ); + assert_eq!( + KubernetesComputeConfig::image_pull_policy_value(ImagePullPolicy::IfNotPresent), + "IfNotPresent" + ); + } + + #[test] + fn image_pull_policy_rejects_newer() { + let cfg = KubernetesComputeConfig { + image_pull_policy: Some(ImagePullPolicy::Newer), + ..KubernetesComputeConfig::default() + }; + let error = cfg.validate_image_pull_policies().unwrap_err(); + assert!(error.contains("supported only by the Podman")); + } + #[test] fn serde_override_topology_sidecar() { let json = serde_json::json!({ @@ -1387,6 +1377,18 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } + #[test] + fn upstream_proxy_config_accepts_tls_protected_secret_credentials() { + let cfg = KubernetesComputeConfig { + topology: SupervisorTopology::Sidecar, + https_proxy: Some("https://proxy.corp.example:8443".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + #[test] fn toml_deserializes_sidecar_upstream_proxy_settings() { let cfg: KubernetesComputeConfig = toml::from_str( diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 484cd07708..2194588737 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -508,6 +508,9 @@ impl KubernetesComputeDriver { config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_image_pull_policies() + .map_err(KubernetesDriverError::Precondition)?; config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; @@ -1504,10 +1507,16 @@ impl KubernetesComputeDriver { let params = SandboxPodParams { default_image: &self.config.default_image, - image_pull_policy: &self.config.image_pull_policy, + image_pull_policy: self + .config + .image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), image_pull_secrets: &self.config.image_pull_secrets, supervisor_image: &self.config.supervisor_image, - supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, + supervisor_image_pull_policy: self + .config + .supervisor_image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, proxy_uid: self.config.sidecar.proxy_uid, @@ -2752,13 +2761,13 @@ fn supervisor_volume_mount() -> serde_json::Value { /// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. fn supervisor_image_volume( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let mut image_spec = serde_json::json!({ "reference": supervisor_image, }); - if !supervisor_image_pull_policy.is_empty() { - image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + image_spec["pullPolicy"] = serde_json::json!(policy); } serde_json::json!({ "name": SUPERVISOR_VOLUME_NAME, @@ -2776,7 +2785,7 @@ fn supervisor_image_volume( /// emissary executor. fn supervisor_init_container( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); let mut spec = serde_json::json!({ @@ -2794,8 +2803,8 @@ fn supervisor_init_container( "readOnly": false }] }); - if !supervisor_image_pull_policy.is_empty() { - spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + spec["imagePullPolicy"] = serde_json::json!(policy); } spec } @@ -2803,7 +2812,7 @@ fn supervisor_init_container( fn apply_supervisor_binary_source( spec: &mut serde_json::Map, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, ) { let volumes = spec @@ -2937,7 +2946,7 @@ fn apply_supervisor_sideload_with_params( fn apply_supervisor_sideload( pod_template: &mut serde_json::Value, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, sandbox_uid: u32, sandbox_gid: u32, @@ -3147,8 +3156,8 @@ fn supervisor_sidecar_container( .into_iter() .map(serde_json::Value::String), ); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if params.provider_spiffe_enabled { container["volumeMounts"] @@ -3210,8 +3219,8 @@ fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_jso sidecar_tls_volume_mount(), ] }); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if !params.client_tls_secret_name.is_empty() { container["volumeMounts"] @@ -3411,7 +3420,7 @@ fn apply_supervisor_sidecar_topology( fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, - image_pull_policy: &str, + image_pull_policy: Option<&str>, sandbox_gid: u32, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { @@ -3501,8 +3510,8 @@ fn apply_workspace_persistence( "mountPath": WORKSPACE_INIT_MOUNT_PATH }] }); - if !image_pull_policy.is_empty() { - init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + if let Some(policy) = image_pull_policy { + init_spec["imagePullPolicy"] = serde_json::json!(policy); } init_containers.push(init_spec); } @@ -3549,10 +3558,10 @@ fn default_workspace_volume_claim_templates( #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { default_image: &'a str, - image_pull_policy: &'a str, + image_pull_policy: Option<&'a str>, image_pull_secrets: &'a [String], supervisor_image: &'a str, - supervisor_image_pull_policy: &'a str, + supervisor_image_pull_policy: Option<&'a str>, supervisor_sideload_method: SupervisorSideloadMethod, topology: SupervisorTopology, proxy_uid: u32, @@ -3590,10 +3599,10 @@ impl Default for SandboxPodParams<'_> { fn default() -> Self { Self { default_image: "", - image_pull_policy: "", + image_pull_policy: None, image_pull_secrets: &[], supervisor_image: "", - supervisor_image_pull_policy: "", + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, @@ -3913,11 +3922,8 @@ fn sandbox_template_to_k8s_with_validated_config( }; if !image.is_empty() { container.insert("image".to_string(), serde_json::json!(image)); - if !params.image_pull_policy.is_empty() { - container.insert( - "imagePullPolicy".to_string(), - serde_json::json!(params.image_pull_policy), - ); + if let Some(policy) = params.image_pull_policy { + container.insert("imagePullPolicy".to_string(), serde_json::json!(policy)); } } @@ -4219,7 +4225,7 @@ fn image_pull_secret_refs(secrets: &[String]) -> Vec { fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { let mut value = serde_json::json!({ - "type": profile.to_k8s_type() + "type": profile.kubernetes_type() }); if let Some(localhost_profile) = profile.localhost_profile() { value["localhostProfile"] = serde_json::json!(localhost_profile); @@ -6194,7 +6200,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "custom-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, // sandbox_uid 1500, // sandbox_gid @@ -6231,7 +6237,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, 1600, @@ -6278,7 +6284,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6305,7 +6311,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6392,7 +6398,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6435,7 +6441,7 @@ mod tests { } #[test] - fn supervisor_image_volume_omits_pull_policy_when_empty() { + fn supervisor_image_volume_omits_pull_policy_when_unspecified() { let mut pod_template = serde_json::json!({ "spec": { "containers": [{ @@ -6448,7 +6454,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "", + None, SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6458,7 +6464,7 @@ mod tests { assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); assert!( volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when empty" + "pullPolicy should be omitted when unspecified" ); } @@ -6468,7 +6474,7 @@ mod tests { topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: "IfNotPresent", + supervisor_image_pull_policy: Some("IfNotPresent"), grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", client_tls_secret_name: "openshell-client-tls", proxy_uid: 2200, @@ -7346,7 +7352,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "openshell/sandbox:latest", - "IfNotPresent", + Some("IfNotPresent"), 1000, // sandbox_gid ); @@ -7405,7 +7411,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "my-custom-image:v2", - "IfNotPresent", + Some("IfNotPresent"), 1000, ); @@ -7429,7 +7435,7 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence(&mut pod_template, "img:latest", Some("Always"), 1000); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 7690ccee85..2cf6dfcefe 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,8 +10,9 @@ use tracing::info; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; -use openshell_core::VERSION; +use openshell_core::driver_utils::{GatewayCallbackTopology, gateway_callback_endpoint}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::otel_tracing::compute_driver_rpc_layer; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, @@ -75,7 +76,7 @@ struct Args { sandbox_image: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, + sandbox_image_pull_policy: Option, #[arg( long, @@ -117,7 +118,7 @@ struct Args { supervisor_image: Option, #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, + supervisor_image_pull_policy: Option, #[arg( long, @@ -251,6 +252,15 @@ async fn main() -> Result<()> { .collect::>>()?; let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let grpc_endpoint = args.grpc_endpoint.unwrap_or_else(|| { + gateway_callback_endpoint( + GatewayCallbackTopology::Kubernetes { + namespace: &args.sandbox_namespace, + }, + openshell_core::config::DEFAULT_SERVER_PORT, + false, + ) + }); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { workspace_mode: args.workspace_mode, @@ -260,7 +270,7 @@ async fn main() -> Result<()> { operator_namespace_file: args.operator_namespace_file, service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, image_pull_secrets: args.sandbox_image_pull_secrets, managed_ssh_ingress: ManagedSshIngressConfig { enabled: args.managed_ssh_ingress_enabled, @@ -270,7 +280,7 @@ async fn main() -> Result<()> { supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_image_pull_policy: args.supervisor_image_pull_policy, supervisor_sideload_method: args.supervisor_sideload_method, topology: args.topology, sidecar: KubernetesSidecarConfig { @@ -284,7 +294,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + grpc_endpoint, ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 9f8baa3446..837913ed1f 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -240,7 +240,7 @@ Step "Prepare DemoDir $DemoDir" New-Item -ItemType Directory -Force $DemoDir | Out-Null Ok "DemoDir ready" -$env:OPENSHELL_DRIVERS = "mxc" +$env:OPENSHELL_COMPUTE_DRIVER = "mxc" $env:OPENSHELL_MXC_SHARE_DIR = $DemoDir # ── Start gateway ───────────────────────────────────────────────────────────── diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 1d7299564d..67583b04b2 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -380,14 +380,14 @@ Podman resources after out-of-band container removal or label drift. |---|---|---|---| | `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | | `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, or `newer`. | +| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `if_not_present` | Pull policy: `always`, `if_not_present`, `never`, or `newer`. | | `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | | `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | | `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | +| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | unset | Podman cgroup PID limit for sandbox containers. Omit it to inherit Podman's runtime/default PID limit; explicit `0` is invalid. | | `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | | `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | | `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | @@ -405,6 +405,15 @@ Through the gateway, the same settings are the `https_proxy`, `no_proxy`, and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see `docs/reference/gateway-config.mdx`. +`provider_spiffe_workload_api_socket` accepts either an absolute host UNIX +Workload API socket, projected through a dedicated read-only mount, or an +explicit container-reachable `tcp:IP:port` endpoint. The driver sets the +supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. +`app_armor_profile` shares the canonical +`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and +Kubernetes. Podman defaults to explicit `Unconfined` for the supervisor mount +setup; confined choices fail early when Podman reports AppArmor unavailable. + This is an operator-owned egress boundary: the driver passes the settings on the supervisor's command line, so sandbox and template environment — and any `ENV` baked into the sandbox image — cannot override them, and the diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 508b604ce7..8088a50418 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -275,6 +275,9 @@ pub struct HostInfo { pub struct SecurityInfo { #[serde(default)] pub rootless: bool, + /// Whether the Podman host has `AppArmor` support enabled. + #[serde(default)] + pub apparmor_enabled: bool, } // ── Client ─────────────────────────────────────────────────────────────── diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 04ae3fe5e3..196c1fa571 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::net::IpAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; -use std::str::FromStr; + +use openshell_core::{AppArmorProfile, ImagePullPolicy}; /// Default Podman bridge network name. pub const DEFAULT_NETWORK_NAME: &str = "openshell"; @@ -11,59 +13,14 @@ pub const MACOS_PODMAN_MACHINE_HOST_GATEWAY_IP: &str = "192.168.127.254"; /// Default Podman stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_PODMAN_STOP_TIMEOUT_SECS: u32 = 45; -// Re-export the shared default so existing imports inside this crate keep working. -pub use openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT; - -/// Image pull policy for sandbox and supervisor images. -/// -/// Controls when the Podman driver fetches a newer copy of an OCI image -/// from the registry. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ImagePullPolicy { - /// Always pull, even if a local copy exists. - Always, - /// Pull only when no local copy exists (default). - #[default] - Missing, - /// Never pull; fail if not available locally. - Never, - /// Pull only if the remote image is newer. - Newer, -} - -impl ImagePullPolicy { - /// Return the policy string expected by the Podman libpod API. - #[must_use] - pub fn as_str(&self) -> &'static str { - match self { - Self::Always => "always", - Self::Missing => "missing", - Self::Never => "never", - Self::Newer => "newer", - } - } -} - -impl std::fmt::Display for ImagePullPolicy { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ImagePullPolicy { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "always" => Ok(Self::Always), - "missing" => Ok(Self::Missing), - "never" => Ok(Self::Never), - "newer" => Ok(Self::Newer), - other => Err(format!( - "invalid pull policy '{other}'; expected one of: always, missing, never, newer" - )), - } +/// Translate the shared pull-policy vocabulary to the Podman libpod API. +#[must_use] +pub const fn podman_image_pull_policy(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "always", + ImagePullPolicy::IfNotPresent => "missing", + ImagePullPolicy::Never => "never", + ImagePullPolicy::Newer => "newer", } } @@ -90,7 +47,6 @@ pub struct PodmanComputeConfig { /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. - #[serde(alias = "sandbox_ssh_socket_path")] pub ssh_socket_path: String, /// Name of the Podman bridge network. /// Created automatically if it does not exist. @@ -121,8 +77,10 @@ pub struct PodmanComputeConfig { pub guest_tls_key: Option, /// Container cgroup PID limit for Podman-managed sandboxes. /// - /// Set to `0` to leave Podman's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to leave Podman's runtime/default PID limit unchanged. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] @@ -130,14 +88,19 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, + /// `AppArmor` confinement requested for sandbox containers. The default + /// explicitly opts out because the supervisor needs mount operations that + /// the runtime default profile denies. + pub app_armor_profile: Option, /// Health check interval in seconds for sandbox containers. /// /// Podman runs the health check command at this interval to determine /// container readiness. Lower values detect readiness faster but /// increase process churn (each check spawns a conmon subprocess). - /// Set to `0` to disable health checks entirely. - /// Defaults to [`DEFAULT_HEALTH_CHECK_INTERVAL_SECS`] (10 seconds). - pub health_check_interval_secs: u64, + /// Omit the field to disable health checks entirely. Explicit zero is + /// invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub health_check_interval_secs: Option, /// Corporate forward proxy URL passed to the in-container supervisor /// (e.g. `http://proxy.corp.com:8080` or `https://proxy.corp.com:3130`). /// @@ -218,8 +181,6 @@ pub struct PodmanComputeConfig { pub gidmap: Vec, } -pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; - /// Parse a single `"container_id:host_id:size"` mapping entry. /// /// Returns `(container_id, host_id, size)` on success. @@ -299,9 +260,9 @@ impl PodmanComputeConfig { /// Validate runtime resource-limit configuration. pub fn validate_runtime_limits(&self) -> Result<(), crate::client::PodmanApiError> { - if self.sandbox_pids_limit < 0 { + if self.sandbox_pids_limit.is_some_and(|limit| limit.get() < 0) { return Err(crate::client::PodmanApiError::InvalidInput( - "sandbox_pids_limit must be zero or greater".to_string(), + "sandbox_pids_limit must be positive when set".to_string(), )); } Ok(()) @@ -319,91 +280,19 @@ impl PodmanComputeConfig { /// the URL is rejected because it would otherwise be stored in /// `gateway.toml` and exposed in container metadata. pub fn validate_proxy_config(&self) -> Result<(), crate::client::PodmanApiError> { - use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - let proxy_secure = if let Some(url) = &self.https_proxy { - let addr = parse_upstream_proxy_url(url).map_err(|err| { - crate::client::PodmanApiError::InvalidInput(match err { - UpstreamProxyUrlError::Empty => { - "https_proxy must not be empty when set".to_string() - } - UpstreamProxyUrlError::InlineCredentials => { - "https_proxy must not embed credentials in the URL; supply them via \ - proxy_auth_file so they are not stored in config or container metadata" - .to_string() - } - err => format!("https_proxy {err}"), - }) - })?; - addr.secure - } else { - false - }; - - // The supervisor treats a present-but-empty driver-supplied argument - // as a fatal misconfiguration, so never accept (and later pass) one. - if let Some(list) = self.no_proxy.as_deref() { - if list.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy must not be empty when set; omit it instead".to_string(), - )); - } - // A bypass list only makes sense relative to a proxy boundary. An - // operator who set one believed proxying was in effect, so accepting - // it while all egress dials directly would hide a fail-open state. - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy is set but no https_proxy is configured".to_string(), - )); - } - } - - if let Some(path) = self.proxy_auth_file.as_deref() { - if path.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file must not be empty when set".to_string(), - )); - } - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file is set but no https_proxy is configured".to_string(), - )); - } - // Basic auth over the plain-TCP proxy connection is readable by - // anyone on the network path; sending it requires an explicit - // operator acknowledgement rather than being an implicit side - // effect of configuring credentials. For an https:// proxy the - // credential is inside the verified TLS session, so the - // acknowledgement is unnecessary (but tolerated). - if self.proxy_auth_allow_insecure != Some(true) && !proxy_secure { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file sends the credential as cleartext Basic auth over the \ - plain-TCP connection to the http:// proxy; set proxy_auth_allow_insecure \ - = true to accept that exposure, or remove proxy_auth_file" - .to_string(), - )); - } - } else if self.proxy_auth_allow_insecure.is_some() { - // The acknowledgement without credentials means the operator - // believed an auth file was configured; surface the mismatch. - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), - )); - } - - // The CONNECT-target mode only means something relative to a proxy - // boundary the operator believed was in effect. - if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), - )); + // Keep the Podman-only CA-bundle behaviour below, but delegate the + // shared URL, bypass-list, credential-file, and acknowledgement + // contract to openshell-core so Docker and VM cannot drift. + openshell_core::UpstreamProxyConfig { + https_proxy: self.https_proxy.clone(), + no_proxy: self.no_proxy.clone(), + proxy_auth_file: self.proxy_auth_file.as_ref().map(PathBuf::from), + proxy_auth_allow_insecure: self.proxy_auth_allow_insecure, + proxy_connect_by_hostname: self.proxy_connect_by_hostname, } + .validate() + .map_err(crate::client::PodmanApiError::InvalidInput)?; - // A CA bundle only makes sense relative to a proxy boundary (an - // https:// proxy handshake, or a TLS-intercepting proxy's re-sign CA). - // Mirror the proxy_auth_file pairing so a stray setting cannot hide a - // fail-open state. The file's readability and certificate content are - // checked at sandbox-create time (see the driver) and fail closed in - // the supervisor. if let Some(path) = self.proxy_ca_bundle.as_deref() { if path.trim().is_empty() { return Err(crate::client::PodmanApiError::InvalidInput( @@ -502,6 +391,18 @@ impl PodmanComputeConfig { } /// Validate optional host gateway override. + /// Validate `AppArmor` syntax before contacting the runtime. Drivers check + /// runtime availability after querying their backend. + pub fn validate_app_armor_profile(&self) -> Result<(), crate::client::PodmanApiError> { + if matches!(self.app_armor_profile, Some(AppArmorProfile::Localhost(ref name)) if name.is_empty()) + { + return Err(crate::client::PodmanApiError::InvalidInput( + "app_armor_profile Localhost profile must not be empty".to_string(), + )); + } + Ok(()) + } + pub fn validate_host_gateway_ip(&self) -> Result<(), crate::client::PodmanApiError> { let trimmed = self.host_gateway_ip.trim(); if trimmed.is_empty() { @@ -545,10 +446,11 @@ impl Default for PodmanComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, - health_check_interval_secs: DEFAULT_HEALTH_CHECK_INTERVAL_SECS, + app_armor_profile: Some(AppArmorProfile::Unconfined), + health_check_interval_secs: None, https_proxy: None, no_proxy: None, proxy_auth_file: None, @@ -567,7 +469,7 @@ impl std::fmt::Debug for PodmanComputeConfig { f.debug_struct("PodmanComputeConfig") .field("socket_path", &self.socket_path) .field("default_image", &self.default_image) - .field("image_pull_policy", &self.image_pull_policy.as_str()) + .field("image_pull_policy", &self.image_pull_policy) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) .field("ssh_socket_path", &self.ssh_socket_path) @@ -584,6 +486,7 @@ impl std::fmt::Debug for PodmanComputeConfig { "provider_spiffe_workload_api_socket", &self.provider_spiffe_workload_api_socket, ) + .field("app_armor_profile", &self.app_armor_profile) .field( "health_check_interval_secs", &self.health_check_interval_secs, @@ -619,30 +522,19 @@ mod tests { } #[test] - fn config_accepts_legacy_sandbox_ssh_socket_path_alias() { - let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ - "sandbox_ssh_socket_path": "/run/test.sock" - })) - .unwrap(); - assert_eq!(config.ssh_socket_path, "/run/test.sock"); - } - - #[test] - fn config_rejects_canonical_and_legacy_ssh_socket_path_names_together() { + fn config_rejects_legacy_sandbox_ssh_socket_path() { let error = serde_json::from_value::(serde_json::json!({ - "ssh_socket_path": "/run/canonical.sock", - "sandbox_ssh_socket_path": "/run/legacy.sock" + "sandbox_ssh_socket_path": "/run/test.sock" })) - .expect_err("canonical and legacy names must not both be accepted"); - assert!(error.to_string().contains("duplicate field")); + .expect_err("legacy sandbox_ssh_socket_path must be rejected"); + assert!(error.to_string().contains("sandbox_ssh_socket_path")); } #[test] - fn default_config_sets_health_check_interval() { - let cfg = PodmanComputeConfig::default(); + fn default_config_disables_health_checks() { assert_eq!( - cfg.health_check_interval_secs, - DEFAULT_HEALTH_CHECK_INTERVAL_SECS + PodmanComputeConfig::default().health_check_interval_secs, + None ); } @@ -653,11 +545,10 @@ mod tests { } #[test] - fn default_config_sets_driver_owned_pids_limit() { + fn default_config_uses_runtime_pids_limit() { let cfg = PodmanComputeConfig::default(); - assert_eq!(cfg.sandbox_pids_limit, DEFAULT_SANDBOX_PIDS_LIMIT); + assert_eq!(cfg.sandbox_pids_limit, None); assert!(!cfg.enable_bind_mounts); - assert!(cfg.validate_runtime_limits().is_ok()); } #[test] @@ -687,13 +578,28 @@ mod tests { } #[test] - fn runtime_limit_validation_rejects_negative_pids_limit() { - let cfg = PodmanComputeConfig { - sandbox_pids_limit: -1, - ..PodmanComputeConfig::default() - }; - let err = cfg.validate_runtime_limits().unwrap_err(); - assert!(err.to_string().contains("sandbox_pids_limit")); + fn runtime_limit_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 + })) + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = negative.validate_runtime_limits().unwrap_err(); + assert!(error.to_string().contains("must be positive")); + } + + #[test] + fn health_check_interval_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "health_check_interval_secs": 0 + })) + .expect_err("zero health-check interval must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); } // ── Proxy config validation ─────────────────────────────────────── diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1743015e26..1d0fce6f2a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -216,8 +216,11 @@ struct ContainerSpec { cap_add: Vec, no_new_privileges: bool, seccomp_profile_path: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + security_opt: Vec, image_pull_policy: String, - healthconfig: HealthConfig, + #[serde(skip_serializing_if = "Option::is_none")] + healthconfig: Option, resource_limits: ResourceLimits, /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. /// Podman's libpod `SpecGenerator` uses `secret_env` (a flat map) for @@ -652,14 +655,10 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids_limit: podman_pids_limit(config.sandbox_pids_limit), + pids_limit: config.sandbox_pids_limit.map(std::num::NonZeroI64::get), } } -fn podman_pids_limit(value: i64) -> Option { - if value > 0 { Some(value) } else { None } -} - pub fn podman_driver_volume_mount_sources( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -1175,8 +1174,14 @@ pub fn build_container_spec_for_image( // locks itself down. no_new_privileges: true, seccomp_profile_path: "unconfined".into(), + security_opt: config + .app_armor_profile + .as_ref() + .and_then(openshell_core::AppArmorProfile::oci_security_opt) + .into_iter() + .collect(), image_pull_policy: "never".to_string(), - healthconfig: HealthConfig { + healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { test: vec![ "CMD-SHELL".into(), format!( @@ -1185,11 +1190,11 @@ pub fn build_container_spec_for_image( openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: config.health_check_interval_secs * 1_000_000_000, + interval: interval_secs.get() * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, - }, + }), resource_limits, secret_env: BTreeMap::new(), secrets: { @@ -1561,7 +1566,8 @@ mod tests { }), ..Default::default() }); - let config = test_config(); + let mut config = test_config(); + config.sandbox_pids_limit = std::num::NonZeroI64::new(2048); let spec = build_container_spec(&sandbox, &config); assert_eq!( @@ -1572,17 +1578,14 @@ mod tests { spec["resource_limits"]["memory"]["limit"].as_u64(), Some(2 * 1024 * 1024 * 1024) ); - assert_eq!( - spec["resource_limits"]["PidsLimit"].as_i64(), - Some(crate::config::DEFAULT_SANDBOX_PIDS_LIMIT) - ); + assert_eq!(spec["resource_limits"]["PidsLimit"].as_i64(), Some(2048)); } #[test] fn container_spec_can_inherit_runtime_pids_limit() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.sandbox_pids_limit = 0; + config.sandbox_pids_limit = None; let spec = build_container_spec(&sandbox, &config); assert!(spec["resource_limits"].get("PidsLimit").is_none()); @@ -1961,7 +1964,8 @@ mod tests { #[test] fn container_spec_healthcheck_accepts_supervisor_socket() { let sandbox = test_sandbox("test-id", "test-name"); - let config = test_config(); + let mut config = test_config(); + config.health_check_interval_secs = std::num::NonZeroU64::new(10); let spec = build_container_spec(&sandbox, &config); let healthcheck = spec["healthconfig"]["test"] @@ -1977,11 +1981,18 @@ mod tests { ); } + #[test] + fn container_spec_omits_healthcheck_when_disabled() { + let sandbox = test_sandbox("test-id", "test-name"); + let spec = build_container_spec(&sandbox, &test_config()); + assert!(spec.get("healthconfig").is_none()); + } + #[test] fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.health_check_interval_secs = 30; + config.health_check_interval_secs = std::num::NonZeroU64::new(30); let spec = build_container_spec(&sandbox, &config); let interval = spec["healthconfig"]["Interval"] diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 8f7c0d32f6..e621894d77 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -4,7 +4,7 @@ //! Podman compute driver. use crate::client::{ContainerListEntry, PodmanApiError, PodmanClient, VolumeInspect}; -use crate::config::PodmanComputeConfig; +use crate::config::{PodmanComputeConfig, podman_image_pull_policy}; use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, PodmanSandboxDriverConfig}; use crate::watcher::{ self, LifecycleEventFences, WatchStream, driver_sandbox_from_inspect, @@ -13,8 +13,9 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + GatewayCallbackTopology, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -366,6 +367,22 @@ impl PodmanComputeDriver { config.validate_runtime_limits()?; config.validate_host_gateway_ip()?; config.validate_proxy_config()?; + config.validate_app_armor_profile()?; + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_deref() { + let raw = socket.to_str().ok_or_else(|| { + PodmanApiError::InvalidInput( + "provider_spiffe_workload_api_socket must be valid UTF-8".to_string(), + ) + })?; + // Preserve Podman's established pass-through support for an + // explicitly configured container-reachable Workload API TCP + // endpoint. The Workload API client validates its endpoint grammar + // when it connects. + if !raw.starts_with("tcp:") { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(PodmanApiError::InvalidInput)?; + } + } config.canonicalize_userns()?; config.validate_userns_mappings()?; @@ -404,11 +421,25 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } + if matches!( + config.app_armor_profile, + Some( + openshell_core::AppArmorProfile::RuntimeDefault + | openshell_core::AppArmorProfile::Localhost(_) + ) + ) && !info.host.security.apparmor_enabled + { + return Err(PodmanApiError::InvalidInput( + "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" + .to_string(), + )); + } info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, rootless = info.host.security.rootless, rootless_network_cmd = %info.host.rootless_network_cmd, + apparmor_enabled = info.host.security.apparmor_enabled, "Connected to Podman" ); (info.host.security.rootless, info.host.rootless_network_cmd) @@ -430,14 +461,10 @@ impl PodmanComputeDriver { // Auto-detect the gRPC callback endpoint before deciding whether this // topology needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = format!( - "{scheme}://host.containers.internal:{}", - config.gateway_port + config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Podman, + config.gateway_port, + config.tls_enabled(), ); info!( grpc_endpoint = %config.grpc_endpoint, @@ -783,7 +810,7 @@ impl PodmanComputeDriver { .to_string(), )); } - let pull_policy = self.config.image_pull_policy.as_str(); + let pull_policy = podman_image_pull_policy(self.config.image_pull_policy); info!(image = %image, policy = %pull_policy, "Ensuring sandbox image"); self.client .pull_image(image, pull_policy) diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index c8aa4066a8..07ec3cb155 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -5,17 +5,15 @@ use clap::Parser; use miette::{IntoDiagnostic, Result}; use std::future::Future; use std::net::SocketAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use tracing::info; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; -use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; +use openshell_core::{AppArmorProfile, ImagePullPolicy, VERSION}; +use openshell_driver_podman::config::{DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS}; use openshell_driver_podman::otel_tracing::compute_driver_rpc_layer; use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; @@ -53,7 +51,7 @@ struct Args { #[arg( long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing + default_value_t = ImagePullPolicy::IfNotPresent )] sandbox_image_pull_policy: ImagePullPolicy, @@ -92,14 +90,19 @@ struct Args { #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] stop_timeout: u32, - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit + /// Container cgroup PID limit for sandbox containers. Omit to inherit /// Podman's runtime/default PID limit. + #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT")] + sandbox_pids_limit: Option, + + /// Health check interval in seconds. Omit it in gateway TOML to disable + /// health checks; the standalone driver keeps its prior 10-second default. #[arg( long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + env = "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS", + default_value = "10" )] - sandbox_pids_limit: i64, + health_check_interval_secs: Option, /// OCI image containing the openshell-sandbox supervisor binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] @@ -117,6 +120,14 @@ struct Args { #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] podman_tls_key: Option, + /// Host UNIX socket projected into supervisors for provider SPIFFE token exchange. + #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] + provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` model: `RuntimeDefault`, `Unconfined`, or `Localhost/`. + #[arg(long, env = "OPENSHELL_APP_ARMOR_PROFILE")] + app_armor_profile: Option, + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, /// in explicit `http://host:port` form (scheme and port required). /// Credentials must not be embedded in the URL; use @@ -214,7 +225,10 @@ async fn main() -> Result<()> { guest_tls_ca: args.podman_tls_ca, guest_tls_cert: args.podman_tls_cert, guest_tls_key: args.podman_tls_key, + provider_spiffe_workload_api_socket: args.provider_spiffe_workload_api_socket, + app_armor_profile: args.app_armor_profile, sandbox_pids_limit: args.sandbox_pids_limit, + health_check_interval_secs: args.health_check_interval_secs, https_proxy: args.sandbox_https_proxy, no_proxy: args.sandbox_no_proxy, proxy_auth_file: args.sandbox_proxy_auth_file, @@ -225,7 +239,6 @@ async fn main() -> Result<()> { uidmap: args.uidmap, gidmap: args.gidmap, enable_bind_mounts: args.enable_bind_mounts, - ..PodmanComputeConfig::default() }) .await .into_diagnostic()?; @@ -327,4 +340,21 @@ mod tests { ); assert_eq!(args.gateway_name.as_deref(), Some("production-us-west")); } + + #[test] + fn standalone_defaults_preserve_health_checks_and_reject_zero_limits() { + let defaults = Args::try_parse_from(["openshell-driver-podman"]) + .expect("standalone driver defaults should parse"); + assert_eq!( + defaults.health_check_interval_secs.map(NonZeroU64::get), + Some(10) + ); + + for flag in ["--sandbox-pids-limit", "--health-check-interval-secs"] { + let result = Args::try_parse_from(["openshell-driver-podman", flag, "0"]); + assert!(result.is_err(), "zero must be rejected for {flag}"); + let error = result.err().expect("error was asserted above"); + assert!(error.to_string().contains("invalid value"), "flag: {flag}"); + } + } } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index e3b7496818..1865b2c46c 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -113,7 +113,7 @@ codesign \ mkdir -p /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm cat > .cache/gateway-vm/gateway.toml <` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | +| `grpc_endpoint` | topology-derived | Optional override for the URL the sandbox guest dials to reach the gateway. The gateway derives `http(s)://host.openshell.internal:` when absent. Use `host.containers.internal`, `host.docker.internal`, or another routable host only for a non-standard topology. Loopback URLs are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | | `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | @@ -151,9 +152,15 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | -| `guest_tls_cert` | unset | Guest client certificate. | -| `guest_tls_key` | unset | Guest client private key. | +| `sandbox_uid` / `sandbox_gid` | `1000` / UID | Identity written into newly prepared guest rootfs images. Existing persisted rootfs and overlays with the former `10001:10001` account are detected by guest init and retain their legacy ownership. | +| `https_proxy`, `no_proxy`, `proxy_auth_file` | unset | Operator-owned corporate TLS proxy settings. The driver injects only URL/list/path controls into protected guest startup; it copies a validated `user:pass` auth file into the private overlay, never into logs or process arguments. An `http://` proxy with credentials requires `proxy_auth_allow_insecure = true`. | +| `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | + +For gateway-managed VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and +`guest_tls_key` together under `[openshell.gateway]`; the gateway validates and +injects that bundle into only the selected local driver. The standalone +`openshell-driver-vm` CLI retains its `--guest-tls-*` inputs for independent +operation. See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. @@ -274,8 +281,8 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical On Debian-family Linux amd64 and arm64 systems, `install.sh` installs the Debian package from the selected `OPENSHELL_VERSION` release tag. That package includes `openshell-gateway` and `openshell-driver-vm`, but leaves -`OPENSHELL_DRIVERS` unset so the gateway uses its normal runtime -auto-detection. Set `OPENSHELL_DRIVERS=vm` to force the VM driver. +`OPENSHELL_COMPUTE_DRIVER` unset so the gateway uses its normal runtime +auto-detection. Set `OPENSHELL_COMPUTE_DRIVER=vm` to force the VM driver. On RPM-family Linux x86_64 and aarch64 systems, `install.sh` installs the `openshell` and `openshell-gateway` RPM packages from the selected release tag. @@ -286,7 +293,7 @@ formula from the selected release in the `nvidia/openshell` Homebrew tap. Homebrew installs `openshell`, `openshell-gateway`, and `openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement in `post_install`, and owns the `brew services` gateway lifecycle. The service -also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless +also leaves `OPENSHELL_COMPUTE_DRIVER` unset so driver choice remains automatic unless the user explicitly overrides it. ## TODOs diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 14dbc0466b..3748b43059 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -90,7 +90,9 @@ sandbox_owner_from_passwd() { done < "$passwd_path" fi - printf '10001:10001\n' + # New images use the conventional sandbox UID. Existing images retain the + # sandbox account read above, including the legacy 10001:10001 identity. + printf '1000:1000\n' } source_overlay_env_if_present() { @@ -103,6 +105,8 @@ source_overlay_env_if_present() { ensure_target_runtime() { local image_root="$1" + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-1000}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-$sandbox_uid}" mkdir -p \ "$image_root/srv" \ @@ -119,14 +123,22 @@ ensure_target_runtime() { fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" - if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then - printf 'sandbox:x:10001:\n' >> "$image_root/etc/group" + # This is a newly prepared target image, so replace a baked-in legacy + # sandbox account with the identity selected by the driver. Persisted + # overlays do not take this path; setup_sandbox_workdir preserves their + # existing 10001:10001 account instead. + if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" fi if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" fi - if ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then - printf 'sandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n' >> "$image_root/etc/passwd" + if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" fi if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" @@ -136,7 +148,7 @@ ensure_target_runtime() { owner="$(sandbox_owner_for_root "$image_root")" if chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then owner_normalized=1 - elif chown -R 10001:10001 "$image_root/sandbox" 2>/dev/null; then + elif chown -R 1000:1000 "$image_root/sandbox" 2>/dev/null; then owner_normalized=1 fi chmod 0755 "$image_root/sandbox" @@ -214,14 +226,14 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" "$@" fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$supervisor" "$@" fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox --workdir /sandbox + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox "$@" fi done @@ -554,10 +566,13 @@ setup_sandbox_workdir() { owner="$(sandbox_owner)" mkdir -p "$sandbox_dir" current_owner="$(stat -c '%u:%g' "$sandbox_dir" 2>/dev/null || true)" + if [ "$owner" = "10001:10001" ]; then + ts "preserving legacy sandbox ownership (10001:10001)" + fi if [ "$current_owner" != "$owner" ] \ || [ ! -f "$(root_path "$SANDBOX_OWNER_NORMALIZED_MARKER")" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then - chown -R 10001:10001 "$sandbox_dir" + chown -R 1000:1000 "$sandbox_dir" fi fi chmod 0755 "$sandbox_dir" @@ -833,11 +848,32 @@ if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}" fi +# Proxy settings are driver-owned environment values. Turn them into +# supervisor argv only after all user/template values have been overwritten; +# the credential itself stays in the root-only overlay file and is never put +# in argv or logged. +set -- --workdir /sandbox +if [ -n "${OPENSHELL_VM_UPSTREAM_PROXY:-}" ]; then + set -- "$@" --upstream-proxy "$OPENSHELL_VM_UPSTREAM_PROXY" +fi +if [ -n "${OPENSHELL_VM_UPSTREAM_NO_PROXY:-}" ]; then + set -- "$@" --upstream-no-proxy "$OPENSHELL_VM_UPSTREAM_NO_PROXY" +fi +if [ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE:-}" ]; then + set -- "$@" --upstream-proxy-auth-file "$OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE" +fi +if [ "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE:-}" = "true" ]; then + set -- "$@" --upstream-proxy-auth-allow-insecure +fi +if [ "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME:-}" = "true" ]; then + set -- "$@" --upstream-proxy-connect-by-hostname +fi + ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then - exec_supervisor_in_newroot + exec_supervisor_in_newroot "$@" fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox +exec /opt/openshell/bin/openshell-sandbox "$@" } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 1b4ea058b9..1b00ba62ce 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -29,6 +29,7 @@ use oci_client::manifest::{ }; use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; +use openshell_core::UpstreamProxyConfig; use openshell_core::gpu::{ driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; @@ -152,6 +153,8 @@ const GUEST_TLS_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CA const GUEST_TLS_CERT_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CERT_PATH; const GUEST_TLS_KEY_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_KEY_PATH; const GUEST_SANDBOX_TOKEN_PATH: &str = openshell_core::container_paths::VM_GUEST_SANDBOX_TOKEN_PATH; +const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = + openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH; const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_DIR; /// Guest path of the driver-authored manifest enumerating which /// `init.d` drop-ins the guest init script is allowed to execute. @@ -218,8 +221,8 @@ enum GuestImagePayloadSource { } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct VmDriverConfig { - #[serde(alias = "openshell_endpoint")] pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, @@ -233,11 +236,21 @@ pub struct VmDriverConfig { pub guest_tls_ca: Option, pub guest_tls_cert: Option, pub guest_tls_key: Option, + /// Corporate forward proxy settings delivered to the guest init script. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + /// Guest-reachable SPIFFE Workload API TCP endpoint. A VM cannot safely + /// project a host UNIX socket; this must be a deliberately exposed TCP + /// listener and requires `provider_spiffe_allow_guest_tcp`. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - /// Resolved sandbox UID for rootfs `/etc/passwd` entry. - /// When empty, defaults to 10001 (the legacy hardcoded value). + /// Resolved sandbox UID for newly prepared rootfs `/etc/passwd` entries. + /// When empty, new sandboxes use 1000. Existing rootfs and overlays retain + /// their recorded sandbox account for legacy 10001 compatibility. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, /// Resolved sandbox GID for rootfs `/etc/passwd` and `/etc/group` entries. @@ -246,8 +259,12 @@ pub struct VmDriverConfig { pub sandbox_gid: Option, } -/// Default sandbox UID used by the VM driver when no config value is set. -pub const DEFAULT_SANDBOX_UID: u32 = 10001; +/// Default sandbox UID used when preparing new VM rootfs images. +/// +/// The guest-init script detects an existing `sandbox` account and preserves +/// its UID/GID, so persisted rootfs and overlays prepared with legacy UID 10001 +/// continue to start without an ownership migration. +pub const DEFAULT_SANDBOX_UID: u32 = 1000; impl Default for VmDriverConfig { fn default() -> Self { @@ -265,6 +282,9 @@ impl Default for VmDriverConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, gpu_enabled: false, gpu_mem_mib: 8192, gpu_vcpus: 4, @@ -285,6 +305,19 @@ impl VmDriverConfig { self.sandbox_gid.unwrap_or(resolved_uid) } + pub fn validate_runtime_security_config(&self) -> Result<(), String> { + self.upstream_proxy.validate()?; + if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + self.provider_spiffe_allow_guest_tcp, + )?; + } else if self.provider_spiffe_allow_guest_tcp { + return Err("provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured".to_string()); + } + Ok(()) + } + pub fn validate_sandbox_identity(&self) -> Result<(), String> { let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; if let Some(uid) = self.sandbox_uid @@ -448,6 +481,7 @@ impl VmDriver { .validate() .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; + config.validate_runtime_security_config()?; if config.grpc_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } @@ -2053,6 +2087,7 @@ impl VmDriver { None => None, }; let sandbox_token = sandbox_token.map(str::to_string); + let proxy_auth = self.read_proxy_auth_credential().await?; let overlay_disk = overlay_disk.to_path_buf(); let overlay_size_bytes = self .config @@ -2082,6 +2117,7 @@ impl VmDriver { &overlay_disk, tls_materials.as_ref(), sandbox_token.as_deref(), + proxy_auth.as_deref(), preparation, overlay_size_bytes, ) @@ -2091,6 +2127,26 @@ impl VmDriver { span_status.finish(result) } + async fn read_proxy_auth_credential(&self) -> Result, String> { + let Some(path) = self.config.upstream_proxy.proxy_auth_file.as_ref() else { + return Ok(None); + }; + let path = path.clone(); + Ok(Some( + tokio::task::spawn_blocking(move || { + let path = path + .to_str() + .ok_or_else(|| "proxy_auth_file must be valid UTF-8".to_string())?; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file(path)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map(str::to_owned) + .map_err(|error| format!("proxy_auth_file is invalid: {error}")) + }) + .await + .map_err(|error| format!("proxy_auth_file read task failed: {error}"))??, + )) + } + fn resolved_sandbox_image(&self, sandbox: &Sandbox) -> Option { requested_sandbox_image(sandbox) .map(ToOwned::to_owned) @@ -2774,6 +2830,7 @@ impl VmDriver { Ok(()) } + #[allow(clippy::similar_names)] async fn run_image_prep_vm( &self, bootstrap_root_disk: &Path, @@ -2802,6 +2859,14 @@ impl VmDriver { command .arg("--vm-env") .arg(format!("OPENSHELL_VM_INIT_MODE={IMAGE_PREP_INIT_MODE}")); + let resolved_uid = self.config.resolve_sandbox_uid(); + let resolved_gid = self.config.resolve_sandbox_gid(resolved_uid); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_UID={resolved_uid}")); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_GID={resolved_gid}")); let mut child = command .spawn() @@ -4531,6 +4596,39 @@ fn build_guest_environment( GUEST_TLS_KEY_PATH.to_string(), ); } + if let Some(url) = config.upstream_proxy.https_proxy.as_ref() { + environment.insert("OPENSHELL_VM_UPSTREAM_PROXY".to_string(), url.clone()); + } + if let Some(no_proxy) = config.upstream_proxy.no_proxy.as_ref() { + environment.insert( + "OPENSHELL_VM_UPSTREAM_NO_PROXY".to_string(), + no_proxy.clone(), + ); + } + if config.upstream_proxy.proxy_auth_file.is_some() { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE".to_string(), + GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string(), + ); + } + if config.upstream_proxy.proxy_auth_allow_insecure == Some(true) { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE".to_string(), + "true".to_string(), + ); + } + if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME".to_string(), + "true".to_string(), + ); + } + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + endpoint.clone(), + ); + } environment.insert( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), @@ -4992,6 +5090,7 @@ fn create_sandbox_overlay_image_from_template( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, ) -> Result<(), String> { clone_or_copy_sparse_file(template_path, overlay_disk)?; if let Some(tls) = tls_materials { @@ -5000,6 +5099,9 @@ fn create_sandbox_overlay_image_from_template( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } Ok(()) } @@ -5008,6 +5110,7 @@ fn prepare_sandbox_overlay_image( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, preparation: OverlayPreparation, expected_size_bytes: u64, ) -> Result<(), String> { @@ -5020,6 +5123,9 @@ fn prepare_sandbox_overlay_image( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } return Ok(()); } Ok(metadata) if metadata.is_file() => { @@ -5051,6 +5157,7 @@ fn prepare_sandbox_overlay_image( overlay_disk, tls_materials, sandbox_token, + proxy_auth, ) } @@ -5079,6 +5186,12 @@ fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), St set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600) } +fn inject_guest_proxy_auth(overlay_disk: &Path, credential: &str) -> Result<(), String> { + let path = overlay_upper_path(GUEST_UPSTREAM_PROXY_AUTH_PATH); + write_rootfs_image_file(overlay_disk, &path, format!("{credential}\n").as_bytes())?; + set_rootfs_image_file_mode(overlay_disk, &path, 0o600) +} + #[allow(clippy::result_large_err)] #[tracing::instrument( name = "vm.prepare_guest", @@ -5619,7 +5732,7 @@ mod tests { } #[test] - fn vm_config_accepts_legacy_openshell_endpoint_alias() { + fn vm_config_rejects_legacy_openshell_endpoint() { let config = VmDriverConfig::default(); let mut serialized = serde_json::to_value(config).unwrap(); let fields = serialized.as_object_mut().unwrap(); @@ -5629,25 +5742,9 @@ mod tests { serde_json::json!("http://127.0.0.1:8080"), ); - let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); - assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); - } - - #[test] - fn vm_config_rejects_canonical_and_legacy_endpoint_names_together() { - let config = VmDriverConfig { - grpc_endpoint: "http://127.0.0.1:8080".to_string(), - ..Default::default() - }; - let mut serialized = serde_json::to_value(config).unwrap(); - serialized.as_object_mut().unwrap().insert( - "openshell_endpoint".to_string(), - serde_json::json!("http://127.0.0.1:9090"), - ); - let error = serde_json::from_value::(serialized) - .expect_err("canonical and legacy names must not both be accepted"); - assert!(error.to_string().contains("duplicate field")); + .expect_err("legacy openshell_endpoint must be rejected"); + assert!(!error.to_string().is_empty()); } struct TestTracing { @@ -6841,6 +6938,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "saved-overlay".len() as u64, ) @@ -6864,6 +6962,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "fresh-overlay".len() as u64, ) @@ -7116,6 +7215,16 @@ mod tests { ))); } + #[test] + fn new_vm_sandbox_identity_defaults_to_1000() { + let config = VmDriverConfig::default(); + assert_eq!(config.resolve_sandbox_uid(), 1000); + assert_eq!( + config.resolve_sandbox_gid(config.resolve_sandbox_uid()), + 1000 + ); + } + #[test] fn validate_sandbox_identity_accepts_non_root_system_ids() { let config = VmDriverConfig { @@ -7537,6 +7646,63 @@ mod tests { ); } + #[test] + fn vm_proxy_and_spiffe_config_require_explicit_safe_acknowledgements() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: None, + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: false, + ..Default::default() + }; + let error = config.validate_runtime_security_config().unwrap_err(); + assert!(error.contains("provider_spiffe_allow_guest_tcp")); + + let config = VmDriverConfig { + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + assert!(config.validate_runtime_security_config().is_ok()); + } + + #[test] + fn build_guest_environment_projects_operator_proxy_and_spiffe_endpoint() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + let sandbox = Sandbox { + id: "vm-spiffe".to_string(), + name: "vm-spiffe".to_string(), + ..Default::default() + }; + let env = build_guest_environment(&sandbox, &config, None); + assert!( + env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY=https://proxy.example:8443".to_string()) + ); + assert!(env.contains(&format!( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE={GUEST_UPSTREAM_PROXY_AUTH_PATH}" + ))); + assert!(env.contains( + &"OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=tcp:192.0.2.10:8081".to_string() + )); + assert!(!env.iter().any(|value| value.contains("user:pass"))); + } + #[test] fn build_guest_environment_includes_tls_paths_for_https_endpoint() { let config = VmDriverConfig { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index bd5a9c04ae..b9468afd2b 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -94,11 +94,7 @@ struct Args { #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, - #[arg( - long = "grpc-endpoint", - alias = "openshell-endpoint", - env = "OPENSHELL_GRPC_ENDPOINT" - )] + #[arg(long = "grpc-endpoint", env = "OPENSHELL_GRPC_ENDPOINT")] grpc_endpoint: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] @@ -123,6 +119,47 @@ struct Args { #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] guest_tls_key: Option, + /// Corporate forward proxy for supervisor TLS egress. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY")] + upstream_proxy: Option, + + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_NO_PROXY")] + upstream_no_proxy: Option, + + /// Root-owned gateway-host file containing `user:pass` proxy credentials. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE")] + upstream_proxy_auth_file: Option, + + /// Explicitly acknowledge cleartext Basic authentication to an http proxy. + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE", + default_value_t = false + )] + upstream_proxy_auth_allow_insecure: bool, + + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", + default_value_t = false + )] + upstream_proxy_connect_by_hostname: bool, + + /// Guest-reachable SPIFFE Workload API endpoint (`tcp:IP:port`). + #[arg( + long = "provider-spiffe-workload-api-tcp-endpoint", + env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT" + )] + provider_spiffe_workload_api_tcp_endpoint: Option, + + /// Explicit acknowledgement that the configured Workload API listener is exposed to VM guests. + #[arg( + long, + env = "OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP", + default_value_t = false + )] + provider_spiffe_allow_guest_tcp: bool, + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] krun_log_level: u32, @@ -242,6 +279,17 @@ async fn main() -> Result<()> { guest_tls_ca: args.guest_tls_ca.clone(), guest_tls_cert: args.guest_tls_cert.clone(), guest_tls_key: args.guest_tls_key.clone(), + upstream_proxy: openshell_core::UpstreamProxyConfig { + https_proxy: args.upstream_proxy.clone(), + no_proxy: args.upstream_no_proxy.clone(), + proxy_auth_file: args.upstream_proxy_auth_file.clone(), + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname.then_some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: args + .provider_spiffe_workload_api_tcp_endpoint + .clone(), + provider_spiffe_allow_guest_tcp: args.provider_spiffe_allow_guest_tcp, gpu_enabled: args.gpu, gpu_mem_mib: args.gpu_mem_mib, gpu_vcpus: args.gpu_vcpus, @@ -711,14 +759,14 @@ mod tests { } #[test] - fn accepts_legacy_openshell_endpoint_flag_alias() { - let args = Args::try_parse_from([ + fn rejects_legacy_openshell_endpoint_flag() { + let error = Args::try_parse_from([ "openshell-driver-vm", "--openshell-endpoint", "http://127.0.0.1:8080", ]) - .unwrap(); - assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + .expect_err("legacy --openshell-endpoint must be rejected"); + assert!(error.to_string().contains("--openshell-endpoint")); } #[test] diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..81f68f6c44 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -822,23 +822,32 @@ fn ensure_line_in_file( line: &str, exists: impl Fn(&str) -> bool, ) -> Result<(), String> { - let mut contents = if path.exists() { + let contents = if path.exists() { fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))? } else { String::new() }; - if contents.lines().any(exists) { - return Ok(()); + let mut replaced = false; + let mut updated = String::new(); + for existing in contents.lines() { + if exists(existing) { + if !replaced { + updated.push_str(line); + updated.push('\n'); + replaced = true; + } + } else { + updated.push_str(existing); + updated.push('\n'); + } } - - if !contents.is_empty() && !contents.ends_with('\n') { - contents.push('\n'); + if !replaced { + updated.push_str(line); + updated.push('\n'); } - contents.push_str(line); - contents.push('\n'); - fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display())) + fs::write(path, updated).map_err(|e| format!("write {}: {e}", path.display())) } fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { @@ -959,10 +968,10 @@ mod tests { write_fake_runtime_binaries(&rootfs); fs::write( rootfs.join("etc/passwd"), - "root:x:0:0:root:/root:/bin/bash\n", + "root:x:0:0:root:/root:/bin/bash\nsandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n", ) .expect("write passwd"); - fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group"); + fs::write(rootfs.join("etc/group"), "root:x:0:\nsandbox:x:10001:\n").expect("write group"); fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts"); fs::create_dir_all(rootfs.join("bin")).expect("create bin"); fs::create_dir_all(rootfs.join("sbin")).expect("create sbin"); @@ -1002,6 +1011,12 @@ mod tests { .expect("read group") .contains(&format!("sandbox:x:{uid}:")) ); + assert!( + !fs::read_to_string(rootfs.join("etc/passwd")) + .expect("read passwd") + .contains("sandbox:x:10001:"), + "newly prepared rootfs must replace the legacy sandbox account" + ); assert_eq!( fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"), "127.0.0.1 localhost\n" diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 03486c82ca..73cfefd140 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -105,29 +105,24 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_DB_URL")] db_url: Option, - /// Compute drivers configured for this gateway. + /// Compute driver configured for this gateway. /// - /// Accepts a comma-delimited list such as `kubernetes` or - /// `kubernetes,podman`. The configuration format is future-proofed for - /// multiple drivers, but the gateway currently requires exactly one. /// When unset, the gateway auto-detects the driver based on the runtime - /// environment (Kubernetes → Podman → Docker). VM is never - /// auto-detected and requires explicit configuration. + /// environment (Kubernetes → Podman → Docker). VM is never auto-detected + /// and requires explicit configuration. #[arg( - long, - alias = "driver", - env = "OPENSHELL_DRIVERS", - value_delimiter = ',', + long = "compute-driver", + env = "OPENSHELL_COMPUTE_DRIVER", value_parser = parse_compute_driver )] - drivers: Vec, + compute_driver: Option, /// Path to a Unix domain socket served by a remote compute driver /// implementing `compute_driver.proto`. /// - /// When set, the socket is associated with the single driver name supplied - /// by `--drivers` or `OPENSHELL_DRIVERS` and replaces normal construction - /// for that selected name, including canonical built-in names. The gateway + /// When set, the socket is associated with the driver name supplied by + /// `--compute-driver` or `OPENSHELL_COMPUTE_DRIVER` and replaces normal + /// construction for that selected name, including canonical built-in names. The gateway /// connects to this operator-provided endpoint; it does not provision the /// remote driver. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] @@ -268,12 +263,17 @@ fn prepare_server_config( } normalize_compute_driver_socket_args(args, matches)?; let compute_driver = compute_drivers - .select(&args.drivers) + .select(args.compute_driver.as_deref()) .map_err(|error| miette::miette!("{error}"))?; let compute_driver_kind = compute_driver.name().parse::().ok(); let local_tls = apply_runtime_defaults(args)?; - let guest_tls = local_tls.as_ref().map(GuestTlsPaths::from); + let guest_tls = GuestTlsPaths::resolve( + file.as_ref().map(|file| &file.openshell.gateway), + local_tls.as_ref(), + args.disable_tls, + ) + .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; let local_jwt = defaults::complete_local_jwt_config()?; let bind = SocketAddr::new(args.bind_address, args.port); @@ -408,9 +408,11 @@ fn prepare_server_config( config = config.with_metrics_bind_address(addr); } + config = config.with_database_url(db_url); + if let Some(driver) = &args.compute_driver { + config = config.with_compute_driver(driver); + } config = config - .with_database_url(db_url) - .with_compute_drivers(args.drivers.clone()) .with_grpc_rate_limit( args.grpc_rate_limit_requests, args.grpc_rate_limit_window_seconds, @@ -443,8 +445,8 @@ fn prepare_server_config( )?; if let Some(socket) = args.compute_driver_socket.clone() { let driver = args - .drivers - .first() + .compute_driver + .as_ref() .expect("normalize_compute_driver_socket_args sets a driver for socket endpoints"); config = config.with_compute_driver_endpoint(driver.clone(), socket); } @@ -693,10 +695,10 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.log_level.clone_from(level); } - if let Some(drivers) = &file.compute_drivers - && arg_defaulted(matches, "drivers") + if let Some(driver) = &file.compute_driver + && arg_defaulted(matches, "compute_driver") { - args.drivers.clone_from(drivers); + args.compute_driver = Some(driver.clone()); } if let Some(sans) = &file.server_sans && args.server_sans.is_empty() @@ -797,24 +799,21 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches "--compute-driver-socket must not be an empty path" )); } - if arg_defaulted(matches, "drivers") { + if arg_defaulted(matches, "compute_driver") { return Err(miette::miette!( - "--compute-driver-socket requires --drivers or OPENSHELL_DRIVERS= to select a compute driver name" + "--compute-driver-socket requires --compute-driver or OPENSHELL_COMPUTE_DRIVER=" )); } - match args.drivers.as_slice() { - [driver] => { - let driver = openshell_core::config::normalize_compute_driver_name(driver) - .map_err(|err| miette::miette!("{err}"))?; - args.drivers[0] = driver; - Ok(()) - } - drivers => Err(miette::miette!( - "--compute-driver-socket requires exactly one compute driver name, got: {}", - drivers.join(",") - )), - } + let driver = args + .compute_driver + .as_deref() + .expect("explicit compute driver is required for socket endpoints"); + args.compute_driver = Some( + openshell_core::config::normalize_compute_driver_name(driver) + .map_err(|err| miette::miette!("{err}"))?, + ); + Ok(()) } fn is_singleplayer_driver(driver: Option) -> bool { @@ -1241,6 +1240,14 @@ mod tests { toml::from_str(toml).expect("valid TOML in test fixture") } + #[test] + fn rejects_legacy_drivers_flag() { + let error = command() + .try_get_matches_from(["openshell-gateway", "--drivers", "docker"]) + .expect_err("legacy --drivers flag must be rejected"); + assert!(error.to_string().contains("--drivers")); + } + #[test] fn default_config_path_is_loaded_only_when_present() { let _lock = ENV_LOCK @@ -1255,7 +1262,7 @@ mod tests { let config = tmp.path().join("openshell").join("gateway.toml"); std::fs::create_dir_all(config.parent().unwrap()).unwrap(); - std::fs::write(&config, "[openshell]\nversion = 1\n").unwrap(); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); assert_eq!(super::resolve_config_path(&args).unwrap(), Some(config)); } @@ -1349,7 +1356,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "docker", "--tls-cert", "/tmp/server.crt", @@ -1378,7 +1385,7 @@ mod tests { let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config.path().to_str().unwrap()); let _kubernetes = EnvVarGuard::set("KUBERNETES_SERVICE_HOST", "10.0.0.1"); let _mtls = EnvVarGuard::remove("OPENSHELL_ENABLE_MTLS_AUTH"); - let _drivers = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _drivers = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); let (mut args, matches) = parse_with_args(&[ @@ -1397,7 +1404,7 @@ mod tests { let prepared = super::prepare_server_config(&mut args, &matches, ®istry).unwrap(); assert_eq!(prepared.compute_driver.name(), "docker"); - assert!(prepared.config.compute_drivers.is_empty()); + assert!(prepared.config.compute_driver.is_none()); assert!(prepared.config.mtls_auth.enabled); assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 1); } @@ -1413,7 +1420,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "kubernetes", "--tls-cert", "/tmp/server.crt", @@ -1442,7 +1449,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "docker", "--tls-cert", "/tmp/server.crt", @@ -1727,13 +1734,13 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "Kyma", "--compute-driver-socket", "/run/openshell/kyma.sock", @@ -1743,9 +1750,11 @@ ssh_session_ttl_secs = 1234 args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); assert!( - args.drivers[0] + args.compute_driver + .as_deref() + .unwrap() .parse::() .is_err() ); @@ -1757,7 +1766,7 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", @@ -1769,7 +1778,7 @@ ssh_session_ttl_secs = 1234 let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); assert!( - err.to_string().contains("requires --drivers "), + err.to_string().contains("requires --compute-driver "), "unexpected error: {err}" ); } @@ -1780,19 +1789,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "docker", "--compute-driver-socket", "/run/openshell/extension.sock", ]); super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["docker"]); + assert_eq!(args.compute_driver.as_deref(), Some("docker")); } #[test] @@ -1801,19 +1810,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "vm", "--compute-driver-socket", "/run/openshell/vm.sock", ]); super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["vm"]); + assert_eq!(args.compute_driver.as_deref(), Some("vm")); } #[test] @@ -1825,7 +1834,7 @@ ssh_session_ttl_secs = 1234 "OPENSHELL_COMPUTE_DRIVER_SOCKET", "/var/run/openshell/kyma.sock", ); - let _g2 = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + let _g2 = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "kyma"); let (mut args, matches) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); @@ -1834,7 +1843,7 @@ ssh_session_ttl_secs = 1234 args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/var/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1890,19 +1899,14 @@ enable_loopback_service_http = false } #[test] - fn canonical_and_legacy_file_driver_selectors_merge_equivalently() { - for input in [ - "[openshell.gateway]\ncompute_driver = \"podman\"\n", - "[openshell.gateway]\ncompute_drivers = [\"podman\"]\n", - ] { - let (mut args, matches) = - parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); - let file = config_file_from_toml(input); + fn canonical_file_driver_selector_populates_cli_args() { + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let file = config_file_from_toml("[openshell.gateway]\ncompute_driver = \"podman\"\n"); - merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); - assert_eq!(args.drivers, vec!["podman".to_string()]); - } + assert_eq!(args.compute_driver.as_deref(), Some("podman")); } #[test] @@ -1921,6 +1925,9 @@ enable_loopback_service_http = false std::fs::write( &config_path, r#" +[openshell] +version = 2 + [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" @@ -1939,7 +1946,7 @@ mem_mib = "not-a-number" config_path.to_str().unwrap(), "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "podman", "--disable-tls", ]); @@ -1951,7 +1958,7 @@ mem_mib = "not-a-number" ) .expect("server config is prepared"); - assert_eq!(prepared.config.compute_drivers, vec!["podman".to_string()]); + assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman")); assert_eq!( prepared.config.policy_validation_failure_mode, openshell_core::PolicyValidationFailureMode::RetainLastValid @@ -1963,50 +1970,19 @@ mem_mib = "not-a-number" #[test] #[cfg(not(target_os = "windows"))] - fn driver_inherits_shared_image_from_gateway_section() { - // [openshell.gateway].default_image inherits into the K8s driver - // table when the driver-specific table does not set it. + fn driver_reads_image_from_driver_owned_table() { let file = config_file_from_toml( r#" -[openshell.gateway] -default_image = "ghcr.io/nvidia/openshell/sandbox:1.0" - [openshell.drivers.kubernetes] namespace = "agents" -"#, - ); - let merged = crate::config_file::driver_table( - super::ComputeDriverKind::Kubernetes.as_str(), - &file.openshell.gateway, - file.openshell.drivers.get("kubernetes"), - ); - let parsed = merged - .try_into::() - .expect("merged table deserializes"); - assert_eq!(parsed.default_image, "ghcr.io/nvidia/openshell/sandbox:1.0"); - assert_eq!(parsed.namespace, "agents"); - } - - #[test] - #[cfg(not(target_os = "windows"))] - fn driver_specific_value_overrides_gateway_inheritance() { - let file = config_file_from_toml( - r#" -[openshell.gateway] -default_image = "gateway-default:1.0" - -[openshell.drivers.kubernetes] default_image = "k8s-specific:1.0" "#, ); - let merged = crate::config_file::driver_table( - super::ComputeDriverKind::Kubernetes.as_str(), - &file.openshell.gateway, - file.openshell.drivers.get("kubernetes"), - ); - let parsed = merged + let table = crate::config_file::driver_table(file.openshell.drivers.get("kubernetes")); + let parsed = table .try_into::() - .expect("deserializes"); + .expect("driver table deserializes"); assert_eq!(parsed.default_image, "k8s-specific:1.0"); + assert_eq!(parsed.namespace, "agents"); } } diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index c25c63ded2..e44a3f9666 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -34,13 +34,68 @@ impl GuestTlsPaths { } } -impl From<&LocalTlsPaths> for GuestTlsPaths { - fn from(paths: &LocalTlsPaths) -> Self { - Self { +impl GuestTlsPaths { + /// Resolve gateway-owned guest TLS inputs. Explicit TOML values take + /// precedence over the package-managed local bundle; partial bundles are + /// rejected before any driver is deserialized or constructed. + pub(crate) fn resolve( + gateway: Option<&config_file::GatewayFileSection>, + local: Option<&LocalTlsPaths>, + tls_disabled: bool, + ) -> std::result::Result, String> { + let configured = gateway.map(|gateway| { + ( + gateway.guest_tls_ca.as_ref(), + gateway.guest_tls_cert.as_ref(), + gateway.guest_tls_key.as_ref(), + ) + }); + let provided = configured + .is_some_and(|(ca, cert, key)| ca.is_some() || cert.is_some() || key.is_some()); + + if tls_disabled { + if provided { + return Err( + "guest_tls_ca, guest_tls_cert, and guest_tls_key require gateway TLS; remove them or omit --disable-tls" + .to_string(), + ); + } + return Ok(None); + } + + if let Some((ca, cert, key)) = configured + && (ca.is_some() || cert.is_some() || key.is_some()) + { + let (Some(ca), Some(cert), Some(key)) = (ca, cert, key) else { + return Err( + "guest TLS requires one complete bundle: guest_tls_ca, guest_tls_cert, and guest_tls_key" + .to_string(), + ); + }; + for (field, path) in [ + ("guest_tls_ca", ca), + ("guest_tls_cert", cert), + ("guest_tls_key", key), + ] { + if !path.is_file() { + return Err(format!( + "{field} '{}' does not exist or is not a file", + path.display() + )); + } + } + return Ok(Some(Self { + ca: ca.clone(), + cert: cert.clone(), + key: key.clone(), + })); + } + + Ok(local.map(|paths| Self { ca: paths.ca.clone(), cert: paths.client_cert.clone(), key: paths.client_key.clone(), - } + })) } } @@ -68,11 +123,8 @@ pub fn remote_driver_config_from_context( ) -> Result { let mut cfg = RemoteDriverConfig::default(); if let Some(file) = context.file { - let merged = config_file::driver_table( - name, - &file.openshell.gateway, - file.openshell.drivers.get(name), - ); + let merged = config_file::driver_table(file.openshell.drivers.get(name)); + reject_driver_owned_guest_tls_fields(&merged)?; if let Some(socket_path) = merged.get("socket_path").and_then(toml::Value::as_str) { cfg.socket_path = PathBuf::from(socket_path); } @@ -105,14 +157,11 @@ fn driver_config_from_file( where T: Default + serde::de::DeserializeOwned, { - let Some(file) = file else { - return Ok(T::default()); - }; - let merged = config_file::driver_table( - driver_name, - &file.openshell.gateway, - file.openshell.drivers.get(driver_name), + let merged = file.map_or_else( + || config_file::driver_table(None), + |file| config_file::driver_table(file.openshell.drivers.get(driver_name)), ); + reject_driver_owned_guest_tls_fields(&merged)?; merged.try_into().map_err(|e| { Error::config(format!( "invalid [openshell.drivers.{driver_name}] table: {e}" @@ -120,6 +169,36 @@ where }) } +/// Reject TLS paths in gateway driver tables. These credentials are gateway +/// inputs and are injected only into the selected local driver after the +/// gateway has validated the complete bundle. +fn reject_driver_owned_guest_tls_fields(table: &toml::Value) -> Result<()> { + let Some(table) = table.as_table() else { + return Ok(()); + }; + for field in ["guest_tls_ca", "guest_tls_cert", "guest_tls_key"] { + if table.contains_key(field) { + return Err(Error::config(format!( + "{field} belongs in [openshell.gateway], not a [openshell.drivers.*] table" + ))); + } + } + Ok(()) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn driver_table_from_context( + context: DriverStartupContext<'_>, + driver_name: &str, +) -> Result { + let table = context.file.map_or_else( + || config_file::driver_table(None), + |file| config_file::driver_table(file.openshell.drivers.get(driver_name)), + ); + reject_driver_owned_guest_tls_fields(&table)?; + Ok(table) +} + fn apply_remote_driver_overrides( cfg: &mut RemoteDriverConfig, context: DriverStartupContext<'_>, @@ -163,6 +242,30 @@ mod tests { } } + #[test] + fn gateway_guest_tls_requires_complete_bundle() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("partial guest TLS must fail"); + assert!(error.contains("one complete bundle")); + } + + #[test] + fn gateway_guest_tls_rejects_plaintext_gateway() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: Some(PathBuf::from("/tmp/cert.pem")), + guest_tls_key: Some(PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, true) + .expect_err("guest TLS and plaintext gateway conflict"); + assert!(error.contains("require gateway TLS")); + } + #[test] fn remote_driver_config_reads_socket_path_from_named_table() { let file: config_file::ConfigFile = toml::from_str( @@ -180,12 +283,9 @@ socket_path = "/run/openshell/kyma.sock" } #[test] - fn remote_driver_config_ignores_in_process_driver_fields() { + fn remote_driver_config_reads_only_socket_path() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "sandboxes" - [openshell.drivers.kubernetes] socket_path = "/run/openshell/kubernetes.sock" workspace_mode = "shared" diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs index 2f9706a315..193ecc0389 100644 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ b/crates/openshell-server/src/compute/driver_config/builtin.rs @@ -3,11 +3,12 @@ //! Configuration construction for built-in compute drivers. -use super::{DriverStartupContext, GuestTlsPaths, driver_config_from_context}; +use super::{DriverStartupContext, driver_table_from_context}; use crate::compute::VmComputeConfig; #[cfg(test)] use crate::config_file; -use openshell_core::{ComputeDriverKind, Result}; +use openshell_core::driver_utils::{GatewayCallbackTopology, gateway_callback_endpoint}; +use openshell_core::{ComputeDriverKind, Error, Result}; use openshell_driver_docker::DockerComputeConfig; use openshell_driver_kubernetes::KubernetesComputeConfig; use openshell_driver_podman::PodmanComputeConfig; @@ -17,7 +18,14 @@ use std::path::PathBuf; pub fn kubernetes_config_from_context( context: DriverStartupContext<'_>, ) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Kubernetes.as_str())?; + let mut cfg = local_driver_config_from_context( + context, + ComputeDriverKind::Kubernetes.as_str(), + GatewayCallbackTopology::Kubernetes { + namespace: driver_namespace(context), + }, + false, + )?; apply_kubernetes_runtime_defaults(&mut cfg); Ok(cfg) } @@ -26,7 +34,12 @@ pub fn kubernetes_config_from_context( pub fn podman_config_from_context( context: DriverStartupContext<'_>, ) -> Result { - let mut podman = driver_config_from_context(context, ComputeDriverKind::Podman.as_str())?; + let mut podman = local_driver_config_from_context( + context, + ComputeDriverKind::Podman.as_str(), + GatewayCallbackTopology::Podman, + true, + )?; apply_podman_runtime_defaults(&mut podman, context); Ok(podman) } @@ -35,18 +48,103 @@ pub fn podman_config_from_context( pub fn docker_config_from_context( context: DriverStartupContext<'_>, ) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Docker.as_str())?; - apply_docker_runtime_defaults(&mut cfg, context); + let mut cfg = local_driver_config_from_context( + context, + ComputeDriverKind::Docker.as_str(), + GatewayCallbackTopology::Docker, + true, + )?; + apply_docker_runtime_defaults(&mut cfg); Ok(cfg) } /// Build the selected VM config from TOML plus runtime defaults. pub fn vm_config_from_context(context: DriverStartupContext<'_>) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Vm.as_str())?; - apply_vm_runtime_defaults(&mut cfg, context); + let mut cfg = local_driver_config_from_context( + context, + ComputeDriverKind::Vm.as_str(), + GatewayCallbackTopology::Vm, + true, + )?; + apply_vm_runtime_defaults(&mut cfg); Ok(cfg) } +fn driver_namespace(context: DriverStartupContext<'_>) -> &str { + context + .file + .and_then(|file| { + file.openshell + .drivers + .get(ComputeDriverKind::Kubernetes.as_str()) + }) + .and_then(toml::Value::as_table) + .and_then(|table| table.get("namespace")) + .and_then(toml::Value::as_str) + .filter(|namespace| !namespace.trim().is_empty()) + .unwrap_or("openshell") +} + +fn local_driver_config_from_context( + context: DriverStartupContext<'_>, + driver_name: &str, + topology: GatewayCallbackTopology<'_>, + requires_guest_tls: bool, +) -> Result +where + T: Default + serde::de::DeserializeOwned, +{ + let mut table = driver_table_from_context(context, driver_name)?; + let table = table + .as_table_mut() + .expect("driver_table_from_context always returns a TOML table"); + + let endpoint_is_absent = table + .get("grpc_endpoint") + .and_then(toml::Value::as_str) + .is_none_or(|endpoint| endpoint.trim().is_empty()); + if endpoint_is_absent { + table.insert( + "grpc_endpoint".to_string(), + toml::Value::String(gateway_callback_endpoint( + topology, + context.gateway_port, + context.gateway_tls_enabled, + )), + ); + } + + if requires_guest_tls { + if context.gateway_tls_enabled && context.guest_tls.is_none() { + return Err(Error::config(format!( + "gateway TLS requires guest_tls_ca, guest_tls_cert, and guest_tls_key in [openshell.gateway] when using the {driver_name} compute driver" + ))); + } + if let Some(tls) = context.guest_tls { + table.insert( + "guest_tls_ca".to_string(), + toml::Value::String(tls.ca.display().to_string()), + ); + table.insert( + "guest_tls_cert".to_string(), + toml::Value::String(tls.cert.display().to_string()), + ); + table.insert( + "guest_tls_key".to_string(), + toml::Value::String(tls.key.display().to_string()), + ); + } + } + + toml::Value::Table(table.clone()) + .try_into() + .map_err(|error| { + Error::config(format!( + "invalid [openshell.drivers.{driver_name}] table: {error}" + )) + }) +} + fn apply_kubernetes_runtime_defaults(k8s: &mut KubernetesComputeConfig) { if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { k8s.workspace_default_storage_size = size; @@ -62,61 +160,14 @@ fn apply_podman_runtime_defaults( ) { podman.gateway_port = context.gateway_port; apply_podman_env_overrides(podman); - apply_guest_tls_defaults_to_split_fields( - &mut podman.guest_tls_ca, - &mut podman.guest_tls_cert, - &mut podman.guest_tls_key, - context.guest_tls, - ); } -fn apply_docker_runtime_defaults(cfg: &mut DockerComputeConfig, context: DriverStartupContext<'_>) { - apply_guest_tls_defaults_to_split_fields( - &mut cfg.guest_tls_ca, - &mut cfg.guest_tls_cert, - &mut cfg.guest_tls_key, - context.guest_tls, - ); -} +fn apply_docker_runtime_defaults(_cfg: &mut DockerComputeConfig) {} -fn apply_vm_runtime_defaults(cfg: &mut VmComputeConfig, context: DriverStartupContext<'_>) { +fn apply_vm_runtime_defaults(cfg: &mut VmComputeConfig) { if cfg.state_dir.as_os_str().is_empty() { cfg.state_dir = VmComputeConfig::default_state_dir(); } - if cfg.grpc_endpoint.trim().is_empty() - && (!context.gateway_tls_enabled || context.guest_tls.is_some()) - { - let scheme = if context.gateway_tls_enabled { - "https" - } else { - "http" - }; - cfg.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port); - } - - apply_guest_tls_defaults_to_split_fields( - &mut cfg.guest_tls_ca, - &mut cfg.guest_tls_cert, - &mut cfg.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_guest_tls_defaults_to_split_fields( - ca: &mut Option, - cert: &mut Option, - key: &mut Option, - defaults: Option<&GuestTlsPaths>, -) { - if ca.is_none() - && cert.is_none() - && key.is_none() - && let Some(paths) = defaults - { - *ca = Some(paths.ca.clone()); - *cert = Some(paths.cert.clone()); - *key = Some(paths.key.clone()); - } } fn apply_podman_env_overrides(podman: &mut PodmanComputeConfig) { @@ -149,19 +200,8 @@ mod tests { } #[test] - fn kubernetes_canonical_and_legacy_field_locations_are_equivalent() { - let legacy: config_file::ConfigFile = toml::from_str( - r#" -[openshell.gateway] -sandbox_namespace = "sandboxes" -service_account_name = "sandbox-sa" -enable_user_namespaces = true - -[openshell.drivers.kubernetes] -"#, - ) - .expect("legacy config"); - let canonical: config_file::ConfigFile = toml::from_str( + fn kubernetes_config_reads_driver_owned_fields() { + let file: config_file::ConfigFile = toml::from_str( r#" [openshell.drivers.kubernetes] namespace = "sandboxes" @@ -169,22 +209,14 @@ service_account_name = "sandbox-sa" enable_user_namespaces = true "#, ) - .expect("canonical config"); + .expect("valid config"); - let legacy_cfg = - kubernetes_config_from_context(test_context(Some(&legacy))).expect("legacy config"); - let canonical_cfg = kubernetes_config_from_context(test_context(Some(&canonical))) - .expect("canonical config"); + let cfg = + kubernetes_config_from_context(test_context(Some(&file))).expect("kubernetes config"); - assert_eq!(legacy_cfg.namespace, canonical_cfg.namespace); - assert_eq!( - legacy_cfg.service_account_name, - canonical_cfg.service_account_name - ); - assert_eq!( - legacy_cfg.enable_user_namespaces, - canonical_cfg.enable_user_namespaces - ); + assert_eq!(cfg.namespace, "sandboxes"); + assert_eq!(cfg.service_account_name, "sandbox-sa"); + assert!(cfg.enable_user_namespaces); } #[test] @@ -203,12 +235,9 @@ enable_bind_mounts = true } #[test] - fn docker_config_reads_canonical_sandbox_label_override() { + fn docker_config_reads_driver_owned_sandbox_label() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "gateway-default" - [openshell.drivers.docker] sandbox_label = "driver-specific" "#, @@ -221,41 +250,19 @@ sandbox_label = "driver-specific" } #[test] - fn docker_config_reads_legacy_sandbox_namespace_override() { + fn docker_config_rejects_legacy_sandbox_namespace() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "gateway-default" - [openshell.drivers.docker] -sandbox_namespace = "driver-specific" -"#, - ) - .expect("valid config"); - - let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); - - assert_eq!(cfg.sandbox_label, "driver-specific"); - } - - #[test] - fn docker_config_rejects_canonical_and_legacy_sandbox_label_names_together() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.gateway] -sandbox_namespace = "gateway-default" - -[openshell.drivers.docker] -sandbox_label = "canonical" sandbox_namespace = "legacy" "#, ) .expect("valid config file structure"); let error = docker_config_from_context(test_context(Some(&file))) - .expect_err("canonical and legacy names must not both be accepted"); + .expect_err("legacy sandbox_namespace must be rejected"); - assert!(error.to_string().contains("duplicate field")); + assert!(error.to_string().contains("sandbox_namespace")); } #[test] @@ -306,6 +313,102 @@ unknown_docker_key = true ); } + #[test] + fn local_drivers_receive_derived_endpoints_and_gateway_owned_tls() { + let guest_tls = super::super::GuestTlsPaths { + ca: PathBuf::from("/gateway/ca.pem"), + cert: PathBuf::from("/gateway/client.pem"), + key: PathBuf::from("/gateway/client-key.pem"), + }; + let endpoint_overrides = BTreeMap::new(); + let context = DriverStartupContext { + file: None, + guest_tls: Some(&guest_tls), + gateway_port: 17670, + gateway_tls_enabled: true, + endpoint_overrides: &endpoint_overrides, + }; + + let docker = docker_config_from_context(context).expect("docker config"); + assert_eq!( + docker.grpc_endpoint, + "https://host.openshell.internal:17670" + ); + assert_eq!(docker.guest_tls_ca, Some(PathBuf::from("/gateway/ca.pem"))); + + let podman = podman_config_from_context(context).expect("podman config"); + assert_eq!( + podman.grpc_endpoint, + "https://host.containers.internal:17670" + ); + assert_eq!( + podman.guest_tls_cert, + Some(PathBuf::from("/gateway/client.pem")) + ); + + let vm = vm_config_from_context(context).expect("VM config"); + assert_eq!(vm.grpc_endpoint, "https://host.openshell.internal:17670"); + assert_eq!( + vm.guest_tls_key, + Some(PathBuf::from("/gateway/client-key.pem")) + ); + } + + #[test] + fn kubernetes_derives_service_endpoint_and_preserves_explicit_override() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.drivers.kubernetes] +namespace = "agents" +"#, + ) + .expect("valid config"); + let mut context = test_context(Some(&file)); + context.gateway_port = 8443; + context.gateway_tls_enabled = true; + let derived = kubernetes_config_from_context(context).expect("kubernetes config"); + assert_eq!( + derived.grpc_endpoint, + "https://openshell-gateway.agents.svc:8443" + ); + + let override_file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.drivers.kubernetes] +grpc_endpoint = "https://remote-gateway.example:9443" +"#, + ) + .expect("valid config"); + let overridden = kubernetes_config_from_context(test_context(Some(&override_file))) + .expect("kubernetes config"); + assert_eq!( + overridden.grpc_endpoint, + "https://remote-gateway.example:9443" + ); + } + + #[test] + fn local_tls_requires_a_gateway_owned_bundle() { + let mut context = test_context(None); + context.gateway_tls_enabled = true; + let error = docker_config_from_context(context).expect_err("TLS bundle is required"); + assert!(error.to_string().contains("[openshell.gateway]")); + } + + #[test] + fn gateway_rejects_guest_tls_in_driver_tables() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.drivers.vm] +guest_tls_ca = "/wrong/place.pem" +"#, + ) + .expect("valid TOML structure"); + let error = vm_config_from_context(test_context(Some(&file))) + .expect_err("driver table must not own guest TLS"); + assert!(error.to_string().contains("belongs in [openshell.gateway]")); + } + #[test] fn vm_config_reports_selected_invalid_driver_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index 8377809262..1f86e4764b 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -41,7 +41,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{ComputeDriverKind, Config, Error, Result}; +use openshell_core::{ComputeDriverKind, Config, Error, Result, UpstreamProxyConfig}; #[cfg(unix)] use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; #[cfg(unix)] @@ -103,6 +103,17 @@ pub struct VmComputeConfig { /// Host-side private key for the guest's mTLS client bundle. pub guest_tls_key: Option, + + /// Corporate forward-proxy settings passed to the VM driver. Flattening + /// preserves the shared local-driver TOML field names. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Explicit guest-reachable SPIFFE Workload API TCP listener. VM guests + /// cannot receive a host UNIX socket, so this requires acknowledgement. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, } impl VmComputeConfig { @@ -167,6 +178,9 @@ impl Default for VmComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, } } } @@ -463,6 +477,21 @@ pub async fn spawn( )); } + vm_config.upstream_proxy.validate().map_err(Error::config)?; + if let Some(endpoint) = vm_config + .provider_spiffe_workload_api_tcp_endpoint + .as_deref() + { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + vm_config.provider_spiffe_allow_guest_tcp, + ) + .map_err(Error::config)?; + } else if vm_config.provider_spiffe_allow_guest_tcp { + return Err(Error::config( + "provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured", + )); + } let driver_bin = resolve_compute_driver_bin(vm_config)?; let socket_path = compute_driver_socket_path(vm_config); let guest_tls_paths = compute_driver_guest_tls_paths(vm_config)?; @@ -502,6 +531,7 @@ pub async fn spawn( command.arg("--guest-tls-cert").arg(tls.cert); command.arg("--guest-tls-key").arg(tls.key); } + append_vm_proxy_and_spiffe_args(&mut command, vm_config); let mut child = command.spawn().map_err(|e| { Error::execution(format!( @@ -519,6 +549,31 @@ pub async fn spawn( } #[cfg(unix)] +fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConfig) { + let proxy = &config.upstream_proxy; + if let Some(url) = proxy.https_proxy.as_ref() { + command.arg("--upstream-proxy").arg(url); + } + if let Some(no_proxy) = proxy.no_proxy.as_ref() { + command.arg("--upstream-no-proxy").arg(no_proxy); + } + if let Some(auth_file) = proxy.proxy_auth_file.as_ref() { + command.arg("--upstream-proxy-auth-file").arg(auth_file); + } + if proxy.proxy_auth_allow_insecure == Some(true) { + command.arg("--upstream-proxy-auth-allow-insecure"); + } + if proxy.proxy_connect_by_hostname == Some(true) { + command.arg("--upstream-proxy-connect-by-hostname"); + } + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + command + .arg("--provider-spiffe-workload-api-tcp-endpoint") + .arg(endpoint); + command.arg("--provider-spiffe-allow-guest-tcp"); + } +} + fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { if let Some(config) = otlp_config { command.arg("--otlp-endpoint").arg(&config.endpoint); diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 8fe12cb251..edfebbd915 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -6,9 +6,8 @@ //! See `rfc/0003-gateway-configuration/README.md` for the file format. This //! module parses the file into [`ConfigFile`], rejects fields that must be //! supplied via env/CLI (database URL), and provides -//! [`driver_table`] which overlays shared `[openshell.gateway]` defaults onto -//! a `[openshell.drivers.]` table so each driver crate's -//! `Deserialize` impl sees a fully-populated table. +//! [`driver_table`] which returns a driver-owned +//! `[openshell.drivers.]` table without gateway-level inheritance. //! //! The merge precedence for gateway process settings is: //! ```text @@ -26,17 +25,15 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use base64::Engine as _; -use openshell_core::config::ComputeDriverKind; use openshell_core::proto::SupervisorMiddlewareService; use openshell_core::{ GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, TlsConfig, }; -use serde::de::{SeqAccess, Visitor}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::{Deserialize, Serialize}; -/// Latest schema version this build understands. -pub const SCHEMA_VERSION: u32 = 1; +/// Gateway configuration schema version supported by this build. +pub const SCHEMA_VERSION: u32 = 2; /// Root of the gateway TOML config file. /// @@ -54,8 +51,8 @@ pub struct ConfigFile { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct OpenShellRoot { - /// Reserved for future schema migrations. Versions greater than - /// [`SCHEMA_VERSION`] are rejected at load time. + /// Gateway configuration schema version. Loaded files must set this to + /// [`SCHEMA_VERSION`]. #[serde(default)] pub version: Option, @@ -66,8 +63,8 @@ pub struct OpenShellRoot { pub supervisor: SupervisorFileSection, /// `[openshell.drivers.]` tables — passed verbatim to each driver - /// crate's `Deserialize` impl after the gateway-side inheritance merge. - /// Stored as raw [`toml::Value`] so each driver can evolve its schema + /// crate's `Deserialize` impl. Stored as raw [`toml::Value`] so each + /// driver can evolve its schema /// independently of this crate. #[serde(default)] pub drivers: BTreeMap, @@ -82,9 +79,8 @@ pub struct OpenShellRoot { /// /// All fields are `Option` so the loader can tell whether a key was set /// in the file (`Some`) or not (`None` — value is taken from CLI/env/default). -/// -/// The fields under "Shared driver defaults" are inherited into -/// `[openshell.drivers.]` tables per [`inheritable_keys`]. +/// Driver-specific settings belong exclusively in +/// `[openshell.drivers.]` tables. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GatewayFileSection { @@ -106,19 +102,9 @@ pub struct GatewayFileSection { pub log_level: Option, // ── Drivers ────────────────────────────────────────────────────────── - /// Canonical TOML uses the singular `compute_driver = "..."`. The legacy - /// `compute_drivers = ["..."]` form remains accepted and is normalized to - /// this existing vector representation so Rust callers and runtime - /// validation retain their current behavior. - #[serde( - default, - rename = "compute_driver", - alias = "compute_drivers", - deserialize_with = "deserialize_compute_drivers", - serialize_with = "serialize_compute_drivers", - skip_serializing_if = "Option::is_none" - )] - pub compute_drivers: Option>, + /// Explicit compute driver selection. `None` enables auto-detection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compute_driver: Option, #[serde(default)] pub credential_drivers: Option>, #[serde(default)] @@ -127,11 +113,6 @@ pub struct GatewayFileSection { pub credential_storage: Option, // ── Sandbox / SSH ──────────────────────────────────────────────────── - /// Compatibility input for Kubernetes `namespace` and Docker - /// `sandbox_label`. Canonical configurations set those driver-owned - /// fields in their respective `[openshell.drivers.]` tables. - #[serde(default)] - pub sandbox_namespace: Option, #[serde(default)] pub ssh_session_ttl_secs: Option, #[serde(default)] @@ -151,26 +132,7 @@ pub struct GatewayFileSection { #[serde(default)] pub enable_loopback_service_http: Option, - // ── Shared driver defaults (inherited into [openshell.drivers.]) ─ - #[serde(default)] - pub default_image: Option, - #[serde(default)] - pub supervisor_image: Option, - #[serde(default)] - pub client_tls_secret_name: Option, - /// Compatibility input for Kubernetes `service_account_name`. - #[serde(default)] - pub service_account_name: Option, - #[serde(default)] - pub host_gateway_ip: Option, - /// Compatibility input for Kubernetes `enable_user_namespaces`. - #[serde(default)] - pub enable_user_namespaces: Option, - /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet - /// writes for the `IssueSandboxToken` bootstrap exchange. Driver - /// clamps to `[600, 86400]`. - #[serde(default)] - pub sa_token_ttl_secs: Option, + // ── Sandbox client TLS ─────────────────────────────────────────────── #[serde(default)] pub guest_tls_ca: Option, #[serde(default)] @@ -211,62 +173,6 @@ pub struct GatewayFileSection { pub database_url: Option, } -fn deserialize_compute_drivers<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - struct ComputeDriversVisitor; - - impl<'de> Visitor<'de> for ComputeDriversVisitor { - type Value = Option>; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("a compute driver name or an array of compute driver names") - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - Ok(Some(vec![value.to_string()])) - } - - fn visit_string(self, value: String) -> Result - where - E: serde::de::Error, - { - Ok(Some(vec![value])) - } - - fn visit_seq(self, mut sequence: A) -> Result - where - A: SeqAccess<'de>, - { - let mut drivers = Vec::new(); - while let Some(driver) = sequence.next_element::()? { - drivers.push(driver); - } - Ok(Some(drivers)) - } - } - - deserializer.deserialize_any(ComputeDriversVisitor) -} - -fn serialize_compute_drivers( - drivers: &Option>, - serializer: S, -) -> Result -where - S: Serializer, -{ - match drivers { - Some(drivers) if drivers.len() == 1 => serializer.serialize_str(&drivers[0]), - Some(drivers) => drivers.serialize(serializer), - None => serializer.serialize_none(), - } -} - /// `[openshell.gateway.otlp]` section. /// /// Presence of this table enables OTLP export; there is no `enabled` flag. @@ -409,7 +315,11 @@ pub enum ConfigFileError { source: toml::de::Error, }, #[error( - "unsupported gateway config version {version}; this build only supports version {SCHEMA_VERSION}" + "gateway config schema version is required; add `[openshell]` and `version = {SCHEMA_VERSION}`" + )] + MissingVersion, + #[error( + "unsupported gateway config version {version}; this build requires version {SCHEMA_VERSION}; migrate legacy fields to the version {SCHEMA_VERSION} schema" )] UnsupportedVersion { version: u32 }, #[error( @@ -425,6 +335,8 @@ pub enum ConfigFileError { field: &'static str, message: &'static str, }, + #[error("invalid gateway config field `openshell.drivers.{name}`: expected a TOML table")] + InvalidDriverTable { name: String }, #[error( "failed to read TLS CA certificate for supervisor middleware '{name}' from '{}': {source}", path.display() @@ -448,8 +360,8 @@ pub enum ConfigFileError { /// Load and validate a TOML config file. /// -/// Returns `Ok(ConfigFile::default())` for an empty file (the gateway then -/// falls back entirely to CLI/env/built-in defaults). +/// Configuration files must declare exactly [`SCHEMA_VERSION`]. Running +/// without a config file still uses CLI, environment, and built-in defaults. #[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] pub fn load(path: &Path) -> Result { let contents = std::fs::read_to_string(path).map_err(|source| ConfigFileError::Io { @@ -457,17 +369,17 @@ pub fn load(path: &Path) -> Result { source, })?; if contents.trim().is_empty() { - return Ok(ConfigFile::default()); + return Err(ConfigFileError::MissingVersion); } let file: ConfigFile = toml::from_str(&contents).map_err(|source| ConfigFileError::Parse { path: path.to_path_buf(), source, })?; - if let Some(version) = file.openshell.version - && version > SCHEMA_VERSION - { - return Err(ConfigFileError::UnsupportedVersion { version }); + match file.openshell.version { + Some(SCHEMA_VERSION) => {} + Some(version) => return Err(ConfigFileError::UnsupportedVersion { version }), + None => return Err(ConfigFileError::MissingVersion), } if file.openshell.gateway.database_url.is_some() { @@ -489,130 +401,34 @@ pub fn load(path: &Path) -> Result { message: "omit the field to use default encrypted gateway credential storage, or specify exactly one external credential driver", }); } - - Ok(file) -} - -/// Build the merged TOML table for `driver` by overlaying inheritable -/// `[openshell.gateway]` defaults onto `[openshell.drivers.]`. -/// -/// The returned [`toml::Value`] is a Table ready to feed into the driver's -/// `Deserialize` impl — keys present in `raw` win over the gateway defaults. -/// Keys outside [`inheritable_keys`] for this driver are never copied from -/// the gateway section, which keeps each driver's `deny_unknown_fields` -/// invariant intact. -pub fn driver_table( - driver_name: &str, - gateway: &GatewayFileSection, - raw: Option<&toml::Value>, -) -> toml::Value { - let mut merged = match raw { - Some(toml::Value::Table(table)) => table.clone(), - _ => toml::Table::new(), - }; - - for key in inheritable_keys(driver_name) { - if driver_field_is_present(&merged, driver_name, key) { - continue; - } - if let Some(value) = gateway_inherited_value(gateway, key) { - merged.insert((*key).to_string(), value); - } - } - - toml::Value::Table(merged) -} - -/// Inheritance allowlist (the Q4 "high-overlap set"). Each driver opts in -/// to a specific subset so a gateway-wide default does not accidentally land -/// in a driver table that does not understand the field. -fn inheritable_keys(driver_name: &str) -> &'static [&'static str] { - match driver_name.parse::().ok() { - Some(ComputeDriverKind::Kubernetes) => &[ - "namespace", - "default_image", - "supervisor_image", - "client_tls_secret_name", - "service_account_name", - "host_gateway_ip", - "enable_user_namespaces", - "sa_token_ttl_secs", - ], - Some(ComputeDriverKind::Docker) => &[ - "sandbox_label", - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - Some(ComputeDriverKind::Podman) => &[ - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - Some(ComputeDriverKind::Vm) => &[ - "default_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - // MXC reads its own settings from the driver config table and has no - // gateway-inherited required fields. - Some(ComputeDriverKind::Mxc) | None => &[], - } -} - -fn driver_field_is_present(table: &toml::Table, driver_name: &str, key: &str) -> bool { - if table.contains_key(key) { - return true; + if let Some((name, _)) = file + .openshell + .drivers + .iter() + .find(|(_, value)| !value.is_table()) + { + return Err(ConfigFileError::InvalidDriverTable { name: name.clone() }); } - // Docker's legacy alias must count as an explicit driver override. If it - // did not, gateway inheritance would inject `sandbox_label` alongside the - // alias and serde would reject the merged table as a duplicate field. - matches!( - driver_name.parse::().ok(), - Some(ComputeDriverKind::Docker) - ) && key == "sandbox_label" - && table.contains_key("sandbox_namespace") + Ok(file) } -fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { - match key { - "namespace" | "sandbox_label" => g.sandbox_namespace.as_deref().map(string_value), - "default_image" => g.default_image.as_deref().map(string_value), - "supervisor_image" => g.supervisor_image.as_deref().map(string_value), - "client_tls_secret_name" => g.client_tls_secret_name.as_deref().map(string_value), - "service_account_name" => g.service_account_name.as_deref().map(string_value), - "host_gateway_ip" => g.host_gateway_ip.as_deref().map(string_value), - "enable_user_namespaces" => g.enable_user_namespaces.map(toml::Value::Boolean), - "sa_token_ttl_secs" => g.sa_token_ttl_secs.map(toml::Value::Integer), - "guest_tls_ca" => g.guest_tls_ca.as_deref().map(path_value), - "guest_tls_cert" => g.guest_tls_cert.as_deref().map(path_value), - "guest_tls_key" => g.guest_tls_key.as_deref().map(path_value), - _ => None, +/// Return a driver's table without gateway-level inheritance. +/// Driver-specific configuration belongs exclusively to +/// `[openshell.drivers.]` in schema version 2. +pub fn driver_table(raw: Option<&toml::Value>) -> toml::Value { + match raw { + Some(toml::Value::Table(table)) => toml::Value::Table(table.clone()), + _ => toml::Value::Table(toml::Table::new()), } } -fn string_value(s: &str) -> toml::Value { - toml::Value::String(s.to_owned()) -} - -fn path_value(p: &Path) -> toml::Value { - toml::Value::String(p.display().to_string()) -} - #[cfg(test)] mod tests { use super::*; use std::io::Write; - fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + fn write_raw_tmp(contents: &str) -> tempfile::NamedTempFile { let mut tmp = tempfile::Builder::new() .suffix(".toml") .tempfile() @@ -621,17 +437,38 @@ mod tests { tmp } + fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + if contents.contains("[openshell]") { + write_raw_tmp(contents) + } else { + write_raw_tmp(&format!("[openshell]\nversion = 2\n\n{contents}")) + } + } + #[test] - fn empty_file_yields_default_config() { - let tmp = write_tmp(""); - let file = load(tmp.path()).expect("empty file parses"); - assert!(file.openshell.version.is_none()); - assert!(file.openshell.gateway.bind_address.is_none()); - assert!(file.openshell.drivers.is_empty()); + fn empty_file_requires_schema_version() { + let tmp = write_raw_tmp(""); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); + } + + #[test] + fn compute_driver_entries_must_be_tables() { + for value in ["\"not-a-table\"", "[\"also\", \"not-a-table\"]", "42"] { + let tmp = write_raw_tmp(&format!( + "[openshell]\nversion = 2\n\n[openshell.drivers]\ndocker = {value}\n" + )); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::InvalidDriverTable { ref name }) if name == "docker" + )); + } } #[test] - fn canonical_compute_driver_scalar_normalizes_to_existing_vector() { + fn canonical_compute_driver_is_singular() { let file: ConfigFile = toml::from_str( r#" [openshell.gateway] @@ -641,64 +478,37 @@ compute_driver = "docker" .expect("canonical compute driver parses"); assert_eq!( - file.openshell.gateway.compute_drivers, - Some(vec!["docker".to_string()]) + file.openshell.gateway.compute_driver.as_deref(), + Some("docker") ); } #[test] - fn legacy_compute_drivers_list_remains_accepted() { - for (input, expected) in [ - ("compute_drivers = []", Vec::::new()), - ("compute_drivers = [\"docker\"]", vec!["docker".to_string()]), - ( - "compute_drivers = [\"docker\", \"podman\"]", - vec!["docker".to_string(), "podman".to_string()], - ), - ] { - let file: ConfigFile = toml::from_str(&format!("[openshell.gateway]\n{input}\n")) - .expect("legacy compute drivers parse"); - assert_eq!(file.openshell.gateway.compute_drivers, Some(expected)); - } + fn legacy_compute_drivers_list_is_rejected() { + let error = + toml::from_str::("[openshell.gateway]\ncompute_drivers = [\"docker\"]\n") + .expect_err("legacy compute_drivers must be rejected"); + assert!(error.to_string().contains("compute_drivers")); } #[test] - fn compute_driver_rejects_non_string_values_with_a_clear_error() { + fn compute_driver_rejects_non_string_values() { let error = toml::from_str::( r" [openshell.gateway] compute_driver = 42 ", ) - .expect_err("compute driver must be a string or string array"); - - assert!( - error - .to_string() - .contains("a compute driver name or an array of compute driver names") - ); + .expect_err("compute driver must be a string"); + assert!(error.to_string().contains("invalid type")); } #[test] - fn canonical_and_legacy_compute_driver_names_are_rejected_together() { - let error = toml::from_str::( - r#" -[openshell.gateway] -compute_driver = "docker" -compute_drivers = ["docker"] -"#, - ) - .expect_err("canonical and legacy names must not both be accepted"); - - assert!(error.to_string().contains("duplicate field")); - } - - #[test] - fn compute_driver_serialization_uses_canonical_scalar_name() { + fn compute_driver_serialization_uses_scalar_name() { let file = ConfigFile { openshell: OpenShellRoot { gateway: GatewayFileSection { - compute_drivers: Some(vec!["docker".to_string()]), + compute_driver: Some("docker".to_string()), ..Default::default() }, ..Default::default() @@ -707,14 +517,13 @@ compute_drivers = ["docker"] let serialized = toml::to_string(&file).expect("config serializes"); assert!(serialized.contains("compute_driver = \"docker\"")); - assert!(!serialized.contains("compute_drivers")); } #[test] fn parses_full_example() { let toml = r#" [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" @@ -722,14 +531,9 @@ health_bind_address = "0.0.0.0:8081" log_level = "info" compute_driver = "kubernetes" credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "agents" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 policy_validation_failure_mode = "retain_last_valid" -default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -client_tls_secret_name = "openshell-sandbox-tls" -service_account_name = "openshell-sandbox" [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" @@ -742,6 +546,10 @@ audience = "openshell-cli" [openshell.drivers.kubernetes] namespace = "agents" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +client_tls_secret_name = "openshell-sandbox-tls" +service_account_name = "openshell-sandbox" grpc_endpoint = "https://openshell-gateway.agents.svc:8080" [openshell.credential_drivers.kubernetes-secrets] @@ -751,10 +559,6 @@ namespace = "agents" let file = load(tmp.path()).expect("valid file parses"); let gw = &file.openshell.gateway; assert_eq!(gw.log_level.as_deref(), Some("info")); - assert_eq!( - gw.default_image.as_deref(), - Some("ghcr.io/nvidia/openshell/sandbox:latest") - ); assert_eq!(gw.grpc_rate_limit_requests, Some(120)); assert_eq!(gw.grpc_rate_limit_window_seconds, Some(60)); assert_eq!( @@ -1099,6 +903,27 @@ nonsense = true assert!(matches!(err, ConfigFileError::Parse { .. })); } + #[test] + fn rejects_removed_driver_fields_at_gateway_scope() { + for field in [ + "sandbox_namespace = \"agents\"", + "default_image = \"sandbox:latest\"", + "supervisor_image = \"supervisor:latest\"", + "client_tls_secret_name = \"sandbox-tls\"", + "service_account_name = \"sandbox-sa\"", + "host_gateway_ip = \"10.0.0.1\"", + "enable_user_namespaces = true", + "sa_token_ttl_secs = 3600", + ] { + let tmp = write_tmp(&format!("[openshell.gateway]\n{field}\n")); + let err = load(tmp.path()).expect_err("gateway-scoped driver field must be rejected"); + assert!( + matches!(err, ConfigFileError::Parse { .. }), + "field: {field}" + ); + } + } + #[test] fn rejects_unknown_field_in_nested_gateway_jwt_table() { // Regression guard for the class of silent-misconfig bug fixed in @@ -1131,266 +956,37 @@ ssh_gateway_port = 8080 } #[test] - fn rejects_unsupported_version() { - let toml = r" -[openshell] -version = 2 -"; - let tmp = write_tmp(toml); - let err = load(tmp.path()).expect_err("version > 1 must be rejected"); + fn rejects_legacy_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 1\n"); + let err = load(tmp.path()).expect_err("version 1 must be rejected"); assert!(matches!( err, - ConfigFileError::UnsupportedVersion { version: 2 } + ConfigFileError::UnsupportedVersion { version: 1 } )); } #[test] - fn driver_table_inherits_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), - supervisor_image: Some("ghcr.io/nvidia/openshell/supervisor:0.9".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - namespace = "agents" - }; - let merged = driver_table( - ComputeDriverKind::Kubernetes.as_str(), - &gateway, - Some(&toml::Value::Table(raw)), - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("namespace").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/sandbox:0.9") - ); - assert_eq!( - table.get("supervisor_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/supervisor:0.9") - ); - } - - #[test] - fn kubernetes_driver_fields_override_legacy_gateway_compatibility_values() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("legacy-namespace".to_string()), - service_account_name: Some("legacy-service-account".to_string()), - enable_user_namespaces: Some(true), - ..Default::default() - }; - let raw = toml::toml! { - namespace = "canonical-namespace" - service_account_name = "canonical-service-account" - enable_user_namespaces = false - }; - let merged = driver_table( - ComputeDriverKind::Kubernetes.as_str(), - &gateway, - Some(&toml::Value::Table(raw)), - ); - let table = merged.as_table().expect("table"); - - assert_eq!( - table.get("namespace").and_then(toml::Value::as_str), - Some("canonical-namespace") - ); - assert_eq!( - table - .get("service_account_name") - .and_then(toml::Value::as_str), - Some("canonical-service-account") - ); - assert_eq!( - table - .get("enable_user_namespaces") - .and_then(toml::Value::as_bool), - Some(false) - ); - } - - #[test] - fn kubernetes_driver_inherits_legacy_gateway_compatibility_values() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("legacy-namespace".to_string()), - service_account_name: Some("legacy-service-account".to_string()), - enable_user_namespaces: Some(true), - ..Default::default() - }; - let merged = driver_table(ComputeDriverKind::Kubernetes.as_str(), &gateway, None); - let table = merged.as_table().expect("table"); - - assert_eq!( - table.get("namespace").and_then(toml::Value::as_str), - Some("legacy-namespace") - ); - assert_eq!( - table - .get("service_account_name") - .and_then(toml::Value::as_str), - Some("legacy-service-account") - ); - assert_eq!( - table - .get("enable_user_namespaces") - .and_then(toml::Value::as_bool), - Some(true) - ); - } - - #[test] - fn docker_driver_table_inherits_gateway_defaults() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("agents".to_string()), - default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("sandbox_label").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/sandbox:0.9") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("10.0.0.1") - ); - } - - #[test] - fn docker_driver_canonical_sandbox_label_overrides_gateway_default() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("gateway-default".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - sandbox_label = "driver-specific" - }; - let merged = driver_table( - ComputeDriverKind::Docker.as_str(), - &gateway, - Some(&toml::Value::Table(raw)), - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("sandbox_label").and_then(|value| value.as_str()), - Some("driver-specific") - ); - assert!(!table.contains_key("sandbox_namespace")); - } - - #[test] - fn docker_driver_legacy_sandbox_namespace_overrides_gateway_default() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("gateway-default".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - sandbox_namespace = "driver-specific" - }; - let merged = driver_table( - ComputeDriverKind::Docker.as_str(), - &gateway, - Some(&toml::Value::Table(raw)), - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table - .get("sandbox_namespace") - .and_then(|value| value.as_str()), - Some("driver-specific") - ); - assert!(!table.contains_key("sandbox_label")); - } - - #[test] - fn podman_driver_table_inherits_gateway_host_gateway_ip() { - let gateway = GatewayFileSection { - default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), - host_gateway_ip: Some("192.168.127.254".to_string()), - ..Default::default() - }; - let merged = driver_table(ComputeDriverKind::Podman.as_str(), &gateway, None); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/sandbox:0.9") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("192.168.127.254") - ); - } - - #[test] - fn driver_table_specific_value_overrides_gateway_default() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default".to_string()), - ..Default::default() - }; + fn driver_table_uses_only_driver_owned_values() { let raw = toml::toml! { default_image = "driver-specific" + socket_path = "/run/openshell/driver.sock" }; - let merged = driver_table( - ComputeDriverKind::Podman.as_str(), - &gateway, - Some(&toml::Value::Table(raw)), - ); + let table = driver_table(Some(&toml::Value::Table(raw))); + let table = table.as_table().expect("driver table"); assert_eq!( - merged - .as_table() - .unwrap() - .get("default_image") - .and_then(|v| v.as_str()), + table.get("default_image").and_then(toml::Value::as_str), Some("driver-specific") ); - } - - #[test] - fn driver_table_does_not_leak_keys_outside_allowlist() { - // `client_tls_secret_name` is K8s-only; Docker must not receive it - // even when set at gateway scope. - let gateway = GatewayFileSection { - client_tls_secret_name: Some("openshell-sandbox-tls".to_string()), - ..Default::default() - }; - let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); - assert!( - !merged - .as_table() - .unwrap() - .contains_key("client_tls_secret_name") + assert_eq!( + table.get("socket_path").and_then(toml::Value::as_str), + Some("/run/openshell/driver.sock") ); } #[test] - fn remote_driver_table_does_not_inherit_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default:1.0".to_string()), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - socket_path = "/run/openshell/kyma.sock" - }; - - let merged = driver_table("kyma", &gateway, Some(&toml::Value::Table(raw))); - let table = merged.as_table().expect("table"); - - assert_eq!( - table.get("socket_path").and_then(|v| v.as_str()), - Some("/run/openshell/kyma.sock") - ); - assert!(!table.contains_key("default_image")); - assert!(!table.contains_key("host_gateway_ip")); + fn driver_table_does_not_inject_gateway_values() { + let table = driver_table(None); + assert!(table.as_table().expect("driver table").is_empty()); } #[test] @@ -1424,15 +1020,23 @@ version = 2 ); } - let drivers = gw - .compute_drivers - .as_ref() - .expect("compute_driver must be explicitly set in the RPM default config"); assert_eq!( - drivers, - &["podman".to_string()], + gw.compute_driver.as_deref(), + Some("podman"), "RPM default must pin compute_driver to podman to prevent unexpected \ driver selection when Docker is also installed" ); + + let podman: openshell_driver_podman::PodmanComputeConfig = + driver_table(config.openshell.drivers.get("podman")) + .try_into() + .expect("RPM Podman settings must deserialize"); + assert_eq!( + podman + .health_check_interval_secs + .map(std::num::NonZeroU64::get), + Some(10), + "RPM defaults must retain Podman's readiness health check" + ); } } diff --git a/crates/openshell-server/src/defaults.rs b/crates/openshell-server/src/defaults.rs index b5a5a5e924..21e66b02bd 100644 --- a/crates/openshell-server/src/defaults.rs +++ b/crates/openshell-server/src/defaults.rs @@ -104,7 +104,7 @@ pub fn complete_local_jwt_config() -> Result> { public_key_path: paths.public_key, kid_path: paths.kid, gateway_id: "openshell".to_string(), - ttl_secs: 0, + ttl_secs: None, })), _ => Err(miette::miette!( "partial local sandbox JWT state in {}: expected jwt/signing.pem, jwt/public.pem, and jwt/kid", @@ -237,6 +237,6 @@ mod tests { assert_eq!(config.public_key_path, tmp.path().join("jwt/public.pem")); assert_eq!(config.kid_path, tmp.path().join("jwt/kid")); assert_eq!(config.gateway_id, "openshell"); - assert_eq!(config.ttl_secs, 0); + assert_eq!(config.ttl_secs, None); } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 8a9e19a0a3..65b1304a15 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -507,7 +507,7 @@ pub(crate) async fn run_server( ); info!( gateway_id = %jwt.gateway_id, - ttl_secs = jwt.ttl_secs, + ttl_secs = jwt.ttl_secs.map(std::num::NonZeroU64::get), "gateway-minted sandbox JWT enabled" ); (Some(issuer), Some(authenticator)) @@ -1180,27 +1180,23 @@ impl ComputeDriverRegistry { ComputeDriverDetection { available } } - pub(crate) fn select(&self, configured_drivers: &[String]) -> Result { - match configured_drivers { - [] => { + pub(crate) fn select(&self, configured_driver: Option<&str>) -> Result { + match configured_driver { + None => { let detection = self.detect(); if detection.selected().is_none() { return Err(Error::config( "no compute driver configured and auto-detection found no suitable installed \ - driver; set --drivers or OPENSHELL_DRIVERS=", + driver; set --compute-driver or OPENSHELL_COMPUTE_DRIVER=", )); } Ok(ComputeDriverSelection::AutoDetected(detection)) } - [driver] => { + Some(driver) => { let name = openshell_core::config::normalize_compute_driver_name(driver) .map_err(Error::config)?; Ok(ComputeDriverSelection::Configured { name }) } - drivers => Err(Error::config(format!( - "multiple compute drivers are not supported yet; configured drivers: {}", - drivers.join(",") - ))), } } } @@ -1675,7 +1671,7 @@ fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config) { if kubernetes_sandbox_jwt_expiry_disabled(config) { warn!( - "Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs is omitted or zero); set ttl_secs > 0 for shared Kubernetes deployments" + "Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs is omitted); set ttl_secs to a positive value for shared Kubernetes deployments" ); } } @@ -1880,7 +1876,7 @@ mod tests { config: &Config, driver_startup: crate::compute::driver_config::DriverStartupContext<'_>, ) -> openshell_core::Result { - let selection = registry.select(&config.compute_drivers)?; + let selection = registry.select(config.compute_driver.as_deref())?; super::resolve_configured_compute_driver(registry, selection.name(), driver_startup) } @@ -2226,7 +2222,7 @@ mod tests { #[test] fn configured_compute_driver_triggers_auto_detection_when_empty() { - let config = Config::new(None).with_compute_drivers(std::iter::empty::()); + let config = Config::new(None); // Empty drivers triggers auto-detection, which may return Some or None // depending on the environment. This test verifies the auto-detection path // is taken rather than immediately returning an error. @@ -2318,26 +2314,9 @@ mod tests { ); } - #[test] - fn configured_compute_driver_rejects_multiple_entries() { - let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Kubernetes, ComputeDriverKind::Podman]); - let err = select_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap_err(); - assert!( - err.to_string() - .contains("multiple compute drivers are not supported yet") - ); - assert!(err.to_string().contains("kubernetes,podman")); - } - #[test] fn configured_compute_driver_accepts_podman() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); + let config = Config::new(None).with_compute_driver(ComputeDriverKind::Podman); let driver = select_compute_driver( &test_compute_drivers(), &config, @@ -2352,7 +2331,7 @@ mod tests { #[test] fn configured_compute_driver_accepts_vm() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); + let config = Config::new(None).with_compute_driver(ComputeDriverKind::Vm); let driver = select_compute_driver( &test_compute_drivers(), &config, @@ -2367,7 +2346,7 @@ mod tests { #[test] fn configured_compute_driver_accepts_docker() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Docker]); + let config = Config::new(None).with_compute_driver(ComputeDriverKind::Docker); let driver = select_compute_driver( &test_compute_drivers(), &config, @@ -2382,7 +2361,7 @@ mod tests { #[test] fn configured_compute_driver_resolves_named_remote() { - let config = Config::new(None).with_compute_drivers(["kyma"]); + let config = Config::new(None).with_compute_driver("kyma"); let driver = select_compute_driver( &test_compute_drivers(), @@ -2407,7 +2386,7 @@ mod tests { #[test] fn configured_compute_driver_uses_vm_endpoint_override() { let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Vm]) + .with_compute_driver(ComputeDriverKind::Vm) .with_compute_driver_endpoint("vm", "/run/openshell/vm.sock"); let driver = select_compute_driver( @@ -2425,7 +2404,7 @@ mod tests { #[test] fn configured_compute_driver_uses_builtin_endpoint_override() { let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Docker]) + .with_compute_driver(ComputeDriverKind::Docker) .with_compute_driver_endpoint("docker", "/run/openshell/docker.sock"); let driver = select_compute_driver( @@ -2441,8 +2420,8 @@ mod tests { } #[test] - fn kubernetes_sandbox_jwt_expiry_disabled_warns_for_zero_ttl() { - fn config_with_jwt_ttl(ttl_secs: u64) -> Config { + fn kubernetes_sandbox_jwt_expiry_disabled_warns_for_omitted_ttl() { + fn config_with_jwt_ttl(ttl_secs: Option) -> Config { let mut config = Config::new(None); config.gateway_jwt = Some(openshell_core::GatewayJwtConfig { signing_key_path: "/tmp/signing.pem".into(), @@ -2455,10 +2434,10 @@ mod tests { } assert!(kubernetes_sandbox_jwt_expiry_disabled( - &config_with_jwt_ttl(0) + &config_with_jwt_ttl(None) )); assert!(!kubernetes_sandbox_jwt_expiry_disabled( - &config_with_jwt_ttl(3600) + &config_with_jwt_ttl(std::num::NonZeroU64::new(3600)) )); assert!(!kubernetes_sandbox_jwt_expiry_disabled(&Config::new(None))); } diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index da8ef72873..9fbd574035 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -22,7 +22,7 @@ # - "host.openshell.internal:host-gateway" [openshell] -version = 1 +version = 2 [openshell.gateway] # Bind to loopback only. The Docker driver adds an extra listener on the @@ -40,7 +40,7 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # first start. The binary is cached to XDG_DATA_HOME and reused on restart. supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # Only pull images that are not already cached locally. -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" # Value assigned to the openshell.sandbox_namespace label on sandbox containers. sandbox_label = "openshell" # Address sandbox containers use to call back to the gateway. @@ -49,3 +49,6 @@ sandbox_label = "openshell" # The gateway must be published on port 8080 on the Docker host so that # host.openshell.internal:8080 resolves to the gateway container. grpc_endpoint = "http://host.openshell.internal:8080" +# Explicit supervisor-compatible Docker default. Set RuntimeDefault or +# Localhost/ only when the daemon host has AppArmor available. +app_armor_profile = "Unconfined" diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 1786a46f63..9539ca4349 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -268,7 +268,7 @@ discovery endpoint or its TLS CA. | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `""` | Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev clusters so new images are picked up without manual eviction. | +| server.sandboxImagePullPolicy | string | `nil` | Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Use always, if_not_present, or never; newer is supported only by Podman. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -289,7 +289,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | +| supervisor.image.pullPolicy | string | `nil` | Canonical sandbox supervisor pull policy. Leave unset to use the Kubernetes image default; use always, if_not_present, or never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 15ff87554e..706df3eca5 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -3,7 +3,7 @@ # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). server: - sandboxImagePullPolicy: IfNotPresent + sandboxImagePullPolicy: if_not_present otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). @@ -13,4 +13,4 @@ server: supervisor: image: - pullPolicy: IfNotPresent + pullPolicy: if_not_present diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 981b2acb26..0ec13328b8 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -32,7 +32,7 @@ metadata: data: gateway.toml: | [openshell] - version = 1 + version = 2 [openshell.gateway] name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} @@ -44,6 +44,7 @@ data: metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" {{- end }} log_level = {{ .Values.server.logLevel | quote }} + compute_driver = "kubernetes" {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} @@ -52,17 +53,8 @@ data: {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} {{- end }} policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} - default_image = {{ .Values.server.sandboxImage | quote }} - {{- if include "openshell.supervisorImageOverrideEnabled" . }} - supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- end }} - {{- if .Values.server.hostGatewayIP }} - host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} - {{- end }} {{- if .Values.server.disableTls }} disable_tls = true - {{- else }} - client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} {{- end }} enable_loopback_service_http = {{ .Values.server.enableLoopbackServiceHttp }} {{- $sans := list -}} @@ -143,6 +135,16 @@ data: [openshell.drivers.kubernetes] namespace = {{ include "openshell.sandboxNamespace" . | quote }} + default_image = {{ .Values.server.sandboxImage | quote }} + {{- if include "openshell.supervisorImageOverrideEnabled" . }} + supervisor_image = {{ include "openshell.supervisorImage" . | quote }} + {{- end }} + {{- if .Values.server.hostGatewayIP }} + host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} + {{- end }} + {{- if not .Values.server.disableTls }} + client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} + {{- end }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index c9a14c29a9..e42fc345f0 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -27,6 +27,16 @@ tests: path: data["gateway.toml"] pattern: '(?m)^name\s*=\s*"production-us-west"$' + - it: renders schema version 2 and the Kubernetes compute driver selector + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\]\s*version\s*=\s*2' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?compute_driver\s*=\s*"kubernetes"' + # Regression for Drew's P2: a ConfigMap-only mutation in `helm upgrade` # must roll the StatefulSet, otherwise pods keep running with stale config. - it: annotates the StatefulSet pod template with a ConfigMap checksum @@ -138,12 +148,36 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?grpc_endpoint' - - it: renders the sandbox service account name under [openshell.drivers.kubernetes] + - it: omits pull policies by default so Kubernetes applies its own defaults + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^\s*(image_pull_policy|supervisor_image_pull_policy)\s*=' + + - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: if_not_present + supervisor.image.pullPolicy: never asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' + + - it: renders driver-owned Kubernetes settings only in its driver table + template: templates/gateway-config.yaml + set: + server.hostGatewayIP: 10.0.0.1 + server.enableUserNamespaces: true + supervisor.image.tag: test + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace".*?default_image\s*=.*?supervisor_image\s*=.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?client_tls_secret_name\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?enable_user_namespaces\s*=\s*true.*?sa_token_ttl_secs\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' - it: renders user namespace enablement under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 6b0b6242b0..b0aa01adb5 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -33,8 +33,9 @@ supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor - # -- Supervisor image pull policy. Defaults to the gateway image pull policy when empty. - pullPolicy: "" + # -- Canonical sandbox supervisor pull policy. Leave unset to use the + # Kubernetes image default; use always, if_not_present, or never. + pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" # -- How the supervisor binary is delivered into sandbox pods. @@ -216,10 +217,10 @@ server: externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default - # (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev - # clusters so new images are picked up without manual eviction. - sandboxImagePullPolicy: "" + # -- Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes + # image default (Always for :latest, IfNotPresent otherwise). Use always, + # if_not_present, or never; newer is supported only by Podman. + sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. sandboxImagePullSecrets: [] diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 2d584c4ba1..68439ea596 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -58,12 +58,11 @@ TLS. stores SQLite state under *~/.local/state/openshell/gateway/*. Environment: **OPENSHELL_DB_URL**. -**--drivers** *DRIVER*\[,*DRIVER*\] -: Compute driver. Accepts a comma-delimited list. The gateway - currently requires exactly one driver. Options: **podman**, +**--compute-driver** *DRIVER* +: Compute driver. Selects exactly one driver. Options: **podman**, **docker**, **kubernetes**, **vm**. When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. VM is opt-in. - Environment: **OPENSHELL_DRIVERS**. + Environment: **OPENSHELL_COMPUTE_DRIVER**. **--tls-cert** *PATH* : Path to server TLS certificate file. Defaults to the local generated diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index aaa97d08d0..45a813d0e8 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -17,7 +17,7 @@ The defaults are tuned for rootless Podman use: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] compute_driver = "podman" @@ -215,9 +215,9 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | -| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. The legacy `compute_drivers` list remains accepted. | -| `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | +| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | +| `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | | `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | @@ -232,14 +232,15 @@ settings: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] compute_driver = "podman" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell" stop_timeout_secs = 10 ``` @@ -247,7 +248,7 @@ stop_timeout_secs = 10 ### Image management The gateway pulls container images automatically on first sandbox -creation. The default pull policy is `missing`, which means images are +creation. The default pull policy is `if_not_present`, which means images are pulled once and then cached by Podman. To update cached images: @@ -260,9 +261,10 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest Or set `image_pull_policy = "always"` in `[openshell.drivers.podman]` to pull on every sandbox creation. -To pin specific image versions instead of `:latest`: +To pin specific image versions instead of `:latest`, set these values in +`[openshell.drivers.podman]`: -```shell +```toml supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" ``` diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index f67b69149b..a8460a473e 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -182,7 +182,7 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ### Images not updating -The default image pull policy is `missing` -- images are pulled once +The default image pull policy is `if_not_present` -- images are pulled once and cached. To update: ```shell @@ -255,7 +255,7 @@ and map the relevant variables: | Environment variable | TOML equivalent | |---|---| | `OPENSHELL_BIND_ADDRESS=A` + `OPENSHELL_SERVER_PORT=P` | `bind_address = "A:P"` under `[openshell.gateway]` | -| `OPENSHELL_DRIVERS=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | +| `OPENSHELL_COMPUTE_DRIVER=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | | `OPENSHELL_DISABLE_TLS=true` | `disable_tls = true` under `[openshell.gateway]` | | `OPENSHELL_TLS_CERT=PATH` | `cert_path = "PATH"` under `[openshell.gateway.tls]` | | `OPENSHELL_TLS_KEY=PATH` | `key_path = "PATH"` under `[openshell.gateway.tls]` | diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index ba76f873b2..a0a6e296f0 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -15,7 +15,7 @@ # systemctl --user edit openshell-gateway [openshell] -version = 1 +version = 2 [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The @@ -26,3 +26,8 @@ version = 1 # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. compute_driver = "podman" + +[openshell.drivers.podman] +# Keep the packaged local-gateway readiness behavior after health checks became +# opt-in in the Podman driver. Omit this setting only to disable health checks. +health_check_interval_secs = 10 diff --git a/docs/about/container-gateway.mdx b/docs/about/container-gateway.mdx index 42fe2c7858..57be7d0142 100644 --- a/docs/about/container-gateway.mdx +++ b/docs/about/container-gateway.mdx @@ -59,7 +59,7 @@ docker run -d \ -v openshell-state:/var/openshell \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=http://host.openshell.internal:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ @@ -128,7 +128,7 @@ docker run -d \ -v "$HOME/.local/state/openshell:/home/openshell/.local/state/openshell" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=https://127.0.0.1:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/home/openshell/.local/state/openshell/openshell.db \ @@ -188,7 +188,7 @@ podman run -d \ -p 127.0.0.1:8080:8080 \ -v openshell-state:/var/openshell \ -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/podman.sock" \ - -e OPENSHELL_DRIVERS=podman \ + -e OPENSHELL_COMPUTE_DRIVER=podman \ -e OPENSHELL_PODMAN_SOCKET=/var/run/podman.sock \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ -e OPENSHELL_DISABLE_TLS=true \ diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 82413ff2d6..fea9c111be 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -37,11 +37,11 @@ The Homebrew formula creates its prefix config without setting `bind_address`, s ## Layout -The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Shared compute-driver keys set at gateway scope are inherited into compute driver tables when not overridden. +The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Driver-specific values are never inherited from gateway scope. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] # ... gateway-wide settings ... @@ -59,7 +59,39 @@ version = 1 # ... credential-driver-specific settings ... ``` -The canonical gateway selector is `compute_driver = ""`. The legacy `compute_drivers = [""]` list remains accepted for compatibility. An omitted selector or an empty legacy list retains auto-detection; a legacy list with multiple entries retains the existing startup error because only one compute driver can be active. +The gateway selector is `compute_driver = ""`. It accepts one scalar driver name. Omit it to retain auto-detection. + +## Migrate to schema version 2 + +Schema version 2 is an intentional breaking cutover. The gateway rejects files +that omit `[openshell] version`, declare version 1, or declare an unsupported +future version. To migrate an existing file: + +1. Set `[openshell] version = 2`. +2. Replace `compute_drivers = [""]` with the scalar + `compute_driver = ""`. Replace `--drivers` and `OPENSHELL_DRIVERS` + with `--compute-driver` and `OPENSHELL_COMPUTE_DRIVER`. +3. Move every compute-driver option into `[openshell.drivers.]`. Schema + version 2 does not inherit driver defaults from `[openshell.gateway]`. + Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway + scope; set all three or omit all three when TLS is disabled. +4. Rename Docker `sandbox_namespace` to `sandbox_label`, Podman + `sandbox_ssh_socket_path` to `ssh_socket_path`, and VM + `openshell_endpoint` to `grpc_endpoint`. +5. Use canonical image pull policies: `always`, `if_not_present`, `never`, or + Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` + spelling are rejected. +6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, + omit Docker or Podman `sandbox_pids_limit` for the runtime default, and omit + Podman `health_check_interval_secs` to disable health checks. Explicit zero + values are invalid. +7. Remove `grpc_endpoint` when the topology-derived callback is correct, or + retain it as an explicit override. New VM root filesystems use UID/GID 1000; + existing persisted VM state using 10001 remains compatible. + +Unknown fields and non-table `[openshell.drivers.]` values fail startup. +This strict validation prevents misspelled or misplaced security-sensitive +settings from being silently ignored. ## Full Example @@ -70,7 +102,7 @@ A complete gateway configuration covering every section. Trim to the fields you # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] name = "production-us-west" @@ -104,14 +136,8 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Shared driver defaults. These inherit into [openshell.drivers.] tables -# when the driver-specific table does not override them. -default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -# Defaults to the gateway version; override to pin a specific build. -# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -client_tls_secret_name = "openshell-client-tls" -host_gateway_ip = "10.0.0.1" -sa_token_ttl_secs = 3600 +# Guest TLS paths remain gateway settings. Set all three for TLS, or omit all +# three only when TLS is disabled. Driver tables must not repeat these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -154,7 +180,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit or set to 0 only for local single-player Docker, Podman, or VM gateways. +# Omit only for local single-player Docker, Podman, or VM gateways. ttl_secs = 3600 [openshell.gateway.auth] @@ -199,8 +225,14 @@ phases = ["validate"] [openshell.drivers.kubernetes] namespace = "openshell" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +client_tls_secret_name = "openshell-client-tls" service_account_name = "openshell-sandbox" +host_gateway_ip = "10.0.0.1" enable_user_namespaces = false +sa_token_ttl_secs = 3600 [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" @@ -213,7 +245,7 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. +`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token `exp` claim and `expires_at_ms` response field become `0`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -357,7 +389,7 @@ The gateway validates snapshot structure and provider-profile semantics. It trea `failure_policy` accepts `fail_closed` or `fail_open`. `timeout` accepts `ms` and `s` suffixes. In `dynamic` mode, binding overrides may select a manifest binding by `id`, `rpc`, or `service` plus `method`; they can disable a binding, narrow its phases, or override its failure policy. -`image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. +`image_pull_policy` is a shared driver setting with the canonical values `always`, `if_not_present`, `never`, and `newer`. Set it inside the relevant driver table. Drivers translate these values to their runtime APIs; `newer` is supported only by Podman and is rejected at Docker and Kubernetes startup. ## Credential Drivers @@ -446,9 +478,9 @@ args = [ ## Driver References -Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. -Canonical Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical gateway-level locations remain accepted as compatibility inputs and retain the same lower precedence. Gateway-level `sandbox_namespace` also remains a compatibility default for Docker `sandbox_label`. +Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. ### Kubernetes @@ -456,7 +488,7 @@ The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS ma ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" @@ -486,11 +518,11 @@ workspace_mode = "shared" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. @@ -525,6 +557,8 @@ topology = "combined" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true +# Optional override. When omitted, the gateway derives +# https://openshell-gateway..svc:. grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" @@ -595,34 +629,36 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required). Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" compute_driver = "docker" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -# Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. -image_pull_policy = "IfNotPresent" +# Canonical values: always | if_not_present | never. `newer` is Podman-only. +image_pull_policy = "if_not_present" # Value assigned to the openshell.sandbox_namespace label on sandbox containers. sandbox_label = "docker-dev" -# Empty auto-detects https://host.openshell.internal: when guest TLS is set. +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for this topology. grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" ssh_socket_path = "/run/openshell/ssh.sock" @@ -630,26 +666,42 @@ ssh_socket_path = "/run/openshell/ssh.sock" # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Docker's runtime default unchanged. +# Omit to leave Docker's runtime default unchanged. Explicit 0 is invalid. sandbox_pids_limit = 2048 +# Explicit supervisor-compatible default. RuntimeDefault requires Docker to +# report AppArmor support; Localhost/ requires an operator-loaded profile. +app_armor_profile = "Unconfined" +# Corporate TLS egress proxy. These are supervisor argv settings, not workload +# environment variables. Do not embed credentials in the URL. +https_proxy = "https://proxy.corp.example:8443" +no_proxy = ".svc.cluster.local,10.0.0.0/8" +# Optional root-owned host file containing user:pass. An http:// proxy also +# requires proxy_auth_allow_insecure = true as an explicit acknowledgement. +proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# Project a host Unix Workload API socket into the supervisor for provider +# token exchange. The socket parent must be a dedicated absolute directory. +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" ``` -Use `sandbox_label` for new Docker configurations. The legacy -`sandbox_namespace` key remains accepted as a compatibility alias. Do not set -both keys in the same driver table. +Use `sandbox_label` for Docker configurations. The legacy +`sandbox_namespace` key is rejected. ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" compute_driver = "podman" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -658,7 +710,8 @@ compute_driver = "podman" # one. Set this to pin a specific Podman machine instead. socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "missing" # always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer +# Optional override. When omitted, the gateway derives this endpoint. grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 @@ -670,18 +723,15 @@ ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Podman's runtime default unchanged. +# Omit to leave Podman's runtime default unchanged. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Health check interval in seconds. Lower values detect readiness faster -# but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to disable health checks entirely. Default: 10. +# Health check interval in seconds. Omit to disable health checks; explicit 0 +# is invalid. Lower values detect readiness faster but increase process churn +# (each check spawns a conmon subprocess). health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -770,11 +820,17 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# Project a host Workload API Unix socket into the supervisor, or use an +# explicit container-reachable TCP endpoint, for provider token exchange. +# provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +# provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" +# Explicit supervisor-compatible default. RuntimeDefault and Localhost/ +# require Podman to report AppArmor support. +app_armor_profile = "Unconfined" ``` -Use `ssh_socket_path` for new Podman configurations. The legacy -`sandbox_ssh_socket_path` key remains accepted as a compatibility alias. Do not -set both keys in the same driver table. +Use `ssh_socket_path` for Podman configurations. The legacy +`sandbox_ssh_socket_path` key is rejected. ### MicroVM @@ -782,33 +838,49 @@ Each sandbox runs inside its own libkrun microVM managed by the standalone `open ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. compute_driver = "vm" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -grpc_endpoint = "https://host.containers.internal:17670" +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for the VM topology. +grpc_endpoint = "https://host.openshell.internal:17670" # Empty falls back to default_image. bootstrap_image = "ghcr.io/nvidia/openshell/sandbox:latest" krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" -# Resolved sandbox UID/GID for the rootfs /etc/passwd entry. -# Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. +# Resolved sandbox UID/GID for new rootfs /etc/passwd entries. +# Defaults to 1000 when unset; matching GID is used if sandbox_gid is empty. +# Existing persisted VM rootfs/overlays with UID 10001 retain that identity. # Any non-root Linux UID/GID is valid. # sandbox_uid = 20001 +# Corporate TLS egress proxy. These settings are converted to protected +# supervisor argv inside the guest; workload environment cannot override them. +# https_proxy = "https://proxy.corp.example:8443" +# no_proxy = ".svc.cluster.local,10.0.0.0/8" +# proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# An http:// proxy with proxy_auth_file requires this explicit acknowledgement: +# proxy_auth_allow_insecure = true +# VM guests cannot mount a host Workload API Unix socket. Configure only a +# separately operated guest-reachable TCP listener and explicitly acknowledge +# the exposure; host-only sockets are never exposed automatically. +# provider_spiffe_workload_api_tcp_endpoint = "tcp:192.0.2.10:8081" +# provider_spiffe_allow_guest_tcp = true ``` ### Extension Driver @@ -821,7 +893,7 @@ key used for driver-owned sandbox config such as `template.driver_config.` ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 5a45bf0354..5aea22c976 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -46,9 +46,9 @@ Reserved built-in values are `docker`, `podman`, `kubernetes`, and `vm`. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Local container runtimes must respond to an API probe before the gateway selects them. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Local container runtimes must respond to an API probe before the gateway selects them. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. -The legacy `compute_drivers = [""]` list remains accepted for compatibility. Empty legacy lists retain auto-detection, and lists with more than one entry retain the existing startup error because a gateway supports exactly one active compute driver. +`compute_driver` accepts exactly one scalar driver name. The legacy `compute_drivers` list is rejected by schema version 2. Common gateway options: @@ -75,8 +75,8 @@ socket path. The endpoint replaces normal driver construction for that name, including canonical built-in names: ```shell -openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock -openshell-gateway --drivers docker --compute-driver-socket /run/openshell/docker.sock +openshell-gateway --compute-driver kyma --compute-driver-socket /run/openshell/kyma.sock +openshell-gateway --compute-driver docker --compute-driver-socket /run/openshell/docker.sock ``` The gateway connects to the operator-provided endpoint; it does not provision @@ -334,7 +334,7 @@ Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: compute_driver = "vm" ``` -For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. @@ -380,13 +380,13 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | +| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | +| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 878aee677c..63e587ef62 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" @@ -18,10 +18,10 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" sandbox_label = "openshell-e2e" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 2064a081f5..6eff3b5615 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" @@ -18,12 +18,13 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.podman] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" ssh_socket_path = "/run/openshell/ssh.sock" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 5d650bae89..4bda320440 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -11,16 +11,16 @@ ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes -ENV OPENSHELL_DRIVERS=kubernetes \ +ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/var/run/openshell-compute/driver/driver.sock \ OPENSHELL_GATEWAY_ID=openshell \ OPENSHELL_SANDBOX_NAMESPACE=openshell \ OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT=openshell-sandbox \ OPENSHELL_SANDBOX_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ - OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ OPENSHELL_K8S_TOPOLOGY=combined diff --git a/e2e/run.sh b/e2e/run.sh index b904469730..a8c97260b6 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -144,9 +144,6 @@ gateway_driver="$(python3 -c ' import sys, tomllib gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] driver = gateway.get("compute_driver") -if driver is None: - drivers = gateway.get("compute_drivers", []) - driver = drivers[0] if drivers else None if not driver: raise SystemExit("gateway config must explicitly select a compute driver") print(driver) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 6c8e202abc..9961e6e36d 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -256,11 +256,14 @@ e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" cat >"${GATEWAY_CONFIG}" <&2 exit 2 @@ -501,8 +501,11 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" { - printf '[openshell]\nversion = 1\n\n' - printf '[openshell.gateway]\nlog_level = "info"\n\n' + printf '[openshell]\nversion = 2\n\n' + printf '[openshell.gateway]\nlog_level = "info"\n' + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-docker-${HOST_PORT}" if [ "${OIDC_MODE}" != "1" ]; then e2e_write_gateway_mtls_auth_config @@ -519,9 +522,6 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then @@ -560,7 +560,7 @@ GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" - --drivers docker + --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 089b9923fd..efc829cb62 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -459,15 +459,22 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # We append the driver-specific table and override the port via CLI flag # (CLI > TOML in the merge precedence) so the test can use an ephemeral port. cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" -{ - e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" - if [ "${OIDC_MODE}" != "1" ]; then - e2e_write_gateway_mtls_auth_config - if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then - e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" - fi +# The TLS listener credentials are supplied by CLI below. Schema v2 keeps the +# supervisor client bundle gateway-owned, so add it to [openshell.gateway] +# before the RPM template opens the Podman driver table. +GATEWAY_CONFIG_WITH_TLS="${GATEWAY_CONFIG}.tls" +while IFS= read -r line; do + if [ "${line}" = "[openshell.drivers.podman]" ]; then + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" fi - printf '\n[openshell.drivers.podman]\n' + printf '%s\n' "${line}" +done <"${GATEWAY_CONFIG}" >"${GATEWAY_CONFIG_WITH_TLS}" +mv "${GATEWAY_CONFIG_WITH_TLS}" "${GATEWAY_CONFIG}" +{ + # The RPM template ends in [openshell.drivers.podman]. Append driver-owned + # overrides before opening any nested gateway tables below. if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else @@ -475,14 +482,12 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" printf 'gateway_port = %s\n' "${HOST_PORT}" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = "missing"\n' + printf 'image_pull_policy = "if_not_present"\n' + # The RPM template already opts into the 10-second Podman health check. # Keep CI teardown bounded while the production Podman driver default stays # conservative for real user workloads. printf 'stop_timeout_secs = %s\n' "${PODMAN_STOP_TIMEOUT_SECS}" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' if [ -n "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" ]; then printf 'provider_spiffe_workload_api_socket = %s\n' "$(toml_string "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" @@ -496,13 +501,22 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" fi fi + + e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" + if [ "${OIDC_MODE}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then + e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" + fi + fi } >> "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="missing" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="if_not_present" \ + OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ @@ -549,6 +563,7 @@ e2e_export_gateway_restart_metadata \ "${GATEWAY_LOG}" \ "${GATEWAY_PID_FILE}" +OPENSHELL_LOCAL_TLS_DIR="${PKI_DIR}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}" >"${GATEWAY_LOG}" 2>&1 & diff --git a/examples/aws-s3-sts.md b/examples/aws-s3-sts.md index 5a7acfe5f1..53e2030580 100644 --- a/examples/aws-s3-sts.md +++ b/examples/aws-s3-sts.md @@ -87,13 +87,11 @@ if your gateway cache directory differs): ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] compute_driver = "podman" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" disable_tls = true -supervisor_image = "localhost/openshell/supervisor:dev" [openshell.gateway.auth] allow_unauthenticated_users = true @@ -106,7 +104,10 @@ gateway_id = "podman-dev" ttl_secs = 3600 [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +supervisor_image = "localhost/openshell/supervisor:dev" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 ``` If the JWT key files do not exist yet, run `mise run gateway` once to generate @@ -118,7 +119,7 @@ Start the gateway: eval "$(aws configure export-credentials --format env)" ./target/debug/openshell-gateway \ --config .cache/gateway-podman/gateway.toml \ - --port 18080 --log-level info --drivers podman --disable-tls \ + --port 18080 --log-level info --compute-driver podman --disable-tls \ --db-url "sqlite:.cache/gateway-podman/gateway.db?mode=rwc" ``` diff --git a/examples/governance-interceptor/smoke.sh b/examples/governance-interceptor/smoke.sh index 88610cf1ee..0e8000c8d5 100755 --- a/examples/governance-interceptor/smoke.sh +++ b/examples/governance-interceptor/smoke.sh @@ -325,7 +325,7 @@ generate_gateway_jwt_bundle() { write_gateway_config() { cat >"$GATEWAY_CONFIG" <"$config_path" <"$config_path" <"$GATEWAY_CONFIG" < None: ) assert "EnvironmentFile=-%%E/openshell/gateway.env" in spec assert "%%S/openshell/tls" not in spec - assert "Environment=OPENSHELL_DRIVERS" not in spec + assert "Environment=OPENSHELL_COMPUTE_DRIVER" not in spec assert "Environment=OPENSHELL_BIND_ADDRESS" not in spec assert "Environment=OPENSHELL_PODMAN_TLS_CA" not in spec assert "ExecStart=/usr/bin/openshell-gateway" in spec diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index df71cc1f5b..57d20b30d8 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -48,7 +48,7 @@ The file path is provided via: OPENSHELL_GATEWAY_CONFIG=/path/to/gateway.toml ``` -The file must have a `.toml` extension. A missing path is a hard error; an empty existing file is treated as "no configuration" — the gateway falls back to defaults and to whatever the CLI/env supply. +The file must have a `.toml` extension. A missing path is a hard error. A configured file must declare the exact supported schema version; an empty existing file is rejected. ### TOML schema @@ -58,7 +58,7 @@ The file is rooted at an `[openshell]` table. This namespacing reserves room for ```toml [openshell] -version = 1 # optional; reserved for future schema migrations +version = 2 # required schema version # ────────────────────────────────────────────────────────────────────────────── # Gateway-wide settings @@ -93,6 +93,11 @@ enable_loopback_service_http = true # ignores the [openshell.gateway.tls] table below. disable_tls = false +# Gateway-owned TLS bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" + [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" @@ -118,9 +123,9 @@ scopes_claim = "" # empty disables scope enforcement [openshell.drivers.kubernetes] namespace = "openshell" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" grpc_endpoint = "https://host.openshell.internal:8080" client_tls_secret_name = "openshell-sandbox-tls" host_gateway_ip = "10.0.0.1" @@ -128,26 +133,20 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] socket_path = "/run/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "missing" # Podman vocabulary: always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" network_name = "openshell" stop_timeout_secs = 10 -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -156,9 +155,6 @@ grpc_endpoint = "https://host.containers.internal:8080" vcpus = 2 mem_mib = 2048 krun_log_level = 1 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` ### Driver configuration @@ -166,7 +162,7 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" Each `[openshell.drivers.]` table is extracted from the parsed file and handed to the driver's initialization function as a raw TOML value. The driver is then responsible for: 1. **Parsing** — deserializing the table into its own typed config struct (e.g. `KubernetesComputeConfig`, `DockerComputeConfig`, `PodmanComputeConfig`, `VmComputeConfig`). -2. **Validation** — applying cross-field checks specific to that driver (e.g. requiring TLS triplets when sandbox-side mTLS is enabled). +2. **Validation** — applying cross-field checks specific to that driver. Gateway-owned guest TLS paths are validated as one bundle and injected only into the selected local driver before this step. 3. **Consumption** — using the resulting struct to initialize internal state. Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. @@ -208,17 +204,17 @@ The following cross-field validations are applied after merging file + env + CLI - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. - When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list remains accepted: an empty list auto-detects, a singleton selects that driver, and multiple entries retain the existing startup error. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list is rejected. -### Backwards compatibility +### Schema compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). Legacy `compute_drivers = [""]` TOML remains accepted, while canonical configurations use the singular `compute_driver = ""`. +Schema version 2 requires `version = 2`, a singular `compute_driver` when a driver is selected, and driver-owned fields under `[openshell.drivers.]`. Legacy schema versions and `compute_drivers` lists are rejected. `OPENSHELL_DB_URL` remains a required process input and is not accepted from the file. ### Example: minimal Kubernetes deployment ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 52f9da35ac..773b98329b 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -29,7 +29,7 @@ GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" @@ -52,6 +52,18 @@ linux_target_triple() { esac } +# Escape a value for a TOML basic string before copying an operator-provided +# proxy path or URL into the generated local configuration. +toml_escape() { + local s=$1 + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/\\n} + s=${s//$'\r'/\\r} + s=${s//$'\t'/\\t} + printf '%s' "${s}" +} + port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then @@ -211,7 +223,7 @@ mkdir -p "${STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" < only on a Docker host with AppArmor enabled. +app_armor_profile = "Unconfined" EOF +# Keep the local task's proxy inputs aligned with [openshell.drivers.docker]. +# Credentials stay in the referenced root-owned file; do not echo their value. +if [[ -n "${OPENSHELL_SANDBOX_HTTPS_PROXY+x}" ]]; then + printf 'https_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_HTTPS_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then + printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -256,6 +292,6 @@ exec "${GATEWAY_BIN}" \ --config "${CONFIG_PATH}" \ --port "${PORT}" \ --log-level "${LOG_LEVEL}" \ - --drivers docker \ + --compute-driver docker \ --disable-tls \ --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index 2b9d9bc349..d1d86ad4a2 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -26,7 +26,7 @@ GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -90,19 +90,6 @@ ensure_podman_supervisor_image() { fi } -podman_pull_policy() { - case "$1" in - Always|always) echo "always" ;; - IfNotPresent|ifnotpresent|missing|"") echo "missing" ;; - Never|never) echo "never" ;; - Newer|newer) echo "newer" ;; - *) - echo "ERROR: unsupported Podman image pull policy '$1'" >&2 - exit 2 - ;; - esac -} - # Escape a value for embedding in a double-quoted TOML basic string, so # quotes, backslashes, or control characters in an environment value cannot # corrupt gateway.toml or inject extra configuration keys. @@ -215,7 +202,7 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" </dev/null 2>&1; then @@ -336,7 +347,7 @@ chmod 700 "${VM_DRIVER_STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT+x}" ]]; then + printf 'provider_spiffe_workload_api_tcp_endpoint = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP+x}" ]]; then + printf 'provider_spiffe_allow_guest_tcp = %s\n' "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP}" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -386,7 +422,7 @@ GATEWAY_ARGS=( --config "${CONFIG_PATH}" --port "${PORT}" --log-level "${LOG_LEVEL}" - --drivers vm + --compute-driver vm --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" ) diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index cffad5ae2b..da7f91fb68 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -10,7 +10,7 @@ # # VM/MicroVM is intentionally explicit-only because it requires runtime setup. # Use either: -# OPENSHELL_DRIVERS=vm mise run gateway +# OPENSHELL_COMPUTE_DRIVER=vm mise run gateway # mise run gateway:vm set -euo pipefail @@ -33,7 +33,7 @@ Options: -h, --help Show this help. Environment: - OPENSHELL_DRIVERS Driver override used by openshell-gateway. + OPENSHELL_COMPUTE_DRIVER Driver override used by openshell-gateway. OPENSHELL_GATEWAY_NAME Gateway name for delegated or Kubernetes runs. OPENSHELL_BIND_ADDRESS Gateway listener address. Defaults to 127.0.0.1, or ::1 for Podman Machine on macOS. @@ -104,7 +104,7 @@ detect_driver() { fi echo "ERROR: no compute driver detected." >&2 - echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_DRIVERS." >&2 + echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_COMPUTE_DRIVER." >&2 exit 2 } @@ -171,17 +171,17 @@ while [[ "$#" -gt 0 ]]; do esac done -if [[ -n "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - echo "ERROR: use either --driver or OPENSHELL_DRIVERS, not both" >&2 +if [[ -n "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + echo "ERROR: use either --driver or OPENSHELL_COMPUTE_DRIVER, not both" >&2 exit 2 fi -if [[ -z "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - if [[ "${OPENSHELL_DRIVERS}" == *,* ]]; then - echo "ERROR: mise run gateway supports one driver; got OPENSHELL_DRIVERS=${OPENSHELL_DRIVERS}" >&2 +if [[ -z "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + if [[ "${OPENSHELL_COMPUTE_DRIVER}" == *,* ]]; then + echo "ERROR: mise run gateway supports one driver; got OPENSHELL_COMPUTE_DRIVER=${OPENSHELL_COMPUTE_DRIVER}" >&2 exit 2 fi - explicit_driver="$(normalize_driver "${OPENSHELL_DRIVERS}")" + explicit_driver="$(normalize_driver "${OPENSHELL_COMPUTE_DRIVER}")" fi DRIVER="${explicit_driver:-$(detect_driver)}" @@ -206,7 +206,7 @@ GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -244,12 +244,11 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" <"$config" < Date: Tue, 1 Sep 2026 18:13:51 -0400 Subject: [PATCH 4/6] fix(config): preserve compute driver runtime guarantees Signed-off-by: Jesse Jaggars --- architecture/gateway.md | 5 +- crates/openshell-core/src/config.rs | 11 +- crates/openshell-core/src/driver_utils.rs | 19 +- crates/openshell-driver-docker/README.md | 2 +- crates/openshell-driver-docker/src/lib.rs | 11 +- crates/openshell-driver-docker/src/tests.rs | 10 + crates/openshell-driver-kubernetes/README.md | 7 +- .../openshell-driver-kubernetes/src/main.rs | 25 +- crates/openshell-driver-podman/README.md | 2 +- crates/openshell-driver-podman/src/config.rs | 28 +- .../openshell-driver-podman/src/container.rs | 56 +++- crates/openshell-driver-podman/src/main.rs | 10 +- crates/openshell-driver-podman/src/watcher.rs | 26 +- crates/openshell-driver-vm/README.md | 2 +- .../scripts/openshell-vm-sandbox-init.sh | 97 ++++--- crates/openshell-driver-vm/src/driver.rs | 239 ++++++++++++++---- crates/openshell-driver-vm/src/rootfs.rs | 88 ++++++- .../src/compute/driver_config/builtin.rs | 51 ++-- .../openshell/tests/grpc_endpoint_test.yaml | 26 ++ deploy/rpm/CONFIGURATION.md | 2 +- docs/reference/gateway-config.mdx | 22 +- docs/reference/sandbox-compute-drivers.mdx | 12 +- 22 files changed, 553 insertions(+), 198 deletions(-) create mode 100644 deploy/helm/openshell/tests/grpc_endpoint_test.yaml diff --git a/architecture/gateway.md b/architecture/gateway.md index f7c6d8f0b1..63dfa60d47 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -771,7 +771,10 @@ system entry instead of pretending to delete package-manager owned state. - Gateway TLS and client certificate distribution are deployment concerns owned by the operator or packaging layer. - Compute runtimes own the mechanics of starting workloads and injecting - callback configuration. + callback configuration. Local Docker, Podman, and VM callback endpoints can + be derived from their fixed host aliases. Kubernetes requires an explicit + endpoint from deployment topology; Helm renders it from the gateway Service + name and namespace rather than inferring it from sandbox placement. - Docker-backed local gateways use Docker's `host-gateway` callback alias on macOS and Docker Desktop-style runtimes. They request IPv4 loopback callback reachability and add a listener only when the primary does not cover it. diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 22636d78d1..3000c96a6c 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -10,7 +10,7 @@ use std::fmt; #[cfg(unix)] use std::io::{Read, Write}; use std::net::SocketAddr; -use std::num::NonZeroU64; +use std::num::{NonZeroI64, NonZeroU64}; #[cfg(unix)] use std::os::unix::fs::FileTypeExt; use std::path::{Path, PathBuf}; @@ -36,6 +36,15 @@ pub const DEFAULT_GATEWAY_NAME: &str = "openshell"; /// Default container stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; +/// Default cgroup PID limit for local container sandboxes. +pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; + +/// Typed default cgroup PID limit for local container sandboxes. +#[must_use] +pub fn default_sandbox_pids_limit() -> Option { + NonZeroI64::new(DEFAULT_SANDBOX_PIDS_LIMIT) +} + /// Default Docker bridge network name for local sandboxes. pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index d668dac944..f70398fcb9 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -10,9 +10,7 @@ use crate::proto::compute::v1::DriverSandbox; /// Built-in sandbox network topologies used to derive a callback endpoint /// when an operator does not configure a per-driver `grpc_endpoint` override. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GatewayCallbackTopology<'a> { - /// A sandbox pod reaches the gateway through its Kubernetes service. - Kubernetes { namespace: &'a str }, +pub enum GatewayCallbackTopology { /// A Docker container reaches the host through Docker's gateway alias. Docker, /// A Podman container reaches the host through Podman's gateway alias. @@ -28,15 +26,12 @@ pub enum GatewayCallbackTopology<'a> { /// operator override for remote or non-standard deployments. #[must_use] pub fn gateway_callback_endpoint( - topology: GatewayCallbackTopology<'_>, + topology: GatewayCallbackTopology, gateway_port: u16, gateway_tls_enabled: bool, ) -> String { let scheme = if gateway_tls_enabled { "https" } else { "http" }; let host = match topology { - GatewayCallbackTopology::Kubernetes { namespace } => { - return format!("{scheme}://openshell-gateway.{namespace}.svc:{gateway_port}"); - } GatewayCallbackTopology::Docker | GatewayCallbackTopology::Vm => "host.openshell.internal", GatewayCallbackTopology::Podman => "host.containers.internal", }; @@ -61,16 +56,6 @@ mod callback_endpoint_tests { gateway_callback_endpoint(GatewayCallbackTopology::Vm, 17670, true), "https://host.openshell.internal:17670" ); - assert_eq!( - gateway_callback_endpoint( - GatewayCallbackTopology::Kubernetes { - namespace: "agents" - }, - 8080, - true, - ), - "https://openshell-gateway.agents.svc:8080" - ); } } diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 31c59bc6f3..10e085ecf0 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -104,7 +104,7 @@ contract: | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Omit `[openshell.drivers.docker].sandbox_pids_limit` to inherit the Docker/runtime default; explicit `0` is invalid. | +| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. `[openshell.drivers.docker].sandbox_pids_limit` defaults to `2048`; explicit `0` is invalid. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | | `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 3bf289256e..d26bdd62ee 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -163,9 +163,12 @@ pub struct DockerComputeConfig { /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Omit the field to leave Docker's runtime/default PID limit unchanged. - /// Explicit zero is invalid. - #[serde(default, skip_serializing_if = "Option::is_none")] + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through @@ -204,7 +207,7 @@ impl Default for DockerComputeConfig { network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: None, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, upstream_proxy: UpstreamProxyConfig::default(), provider_spiffe_workload_api_socket: None, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 1659c1f422..902f3156c2 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -149,6 +149,16 @@ fn docker_config_rejects_legacy_sandbox_namespace() { assert!(error.to_string().contains("sandbox_namespace")); } +#[test] +fn docker_config_defaults_to_driver_owned_pids_limit() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!( + config.sandbox_pids_limit.map(std::num::NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); +} + #[test] fn docker_config_rejects_invalid_pids_limits() { let zero = serde_json::from_value::(serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 5d6154bd17..b64bf0c6e4 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -96,8 +96,11 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC. ## Credentials, TLS, and Relay The driver injects gateway callback configuration, sandbox identity, TLS client -material, and the supervisor SSH socket path into the workload. Driver-owned -values must override image-provided environment variables. +material, and the supervisor SSH socket path into the workload. The callback +endpoint is required because the sandbox namespace does not identify the +Gateway Service; Helm renders it from the release topology, while standalone +and raw TOML configurations must set it explicitly. Driver-owned values must +override image-provided environment variables. Sandbox pods run as `service_account_name` and keep `automountServiceAccountToken: false`. The only Kubernetes token exposed to the diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 2cf6dfcefe..cedc6a333f 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,7 +10,6 @@ use tracing::info; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; -use openshell_core::driver_utils::{GatewayCallbackTopology, gateway_callback_endpoint}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::otel_tracing::compute_driver_rpc_layer; @@ -98,8 +97,10 @@ struct Args { )] managed_ssh_gateway_pod_selector: Vec, + /// Gateway callback endpoint reachable from sandbox pods. Kubernetes + /// service topology cannot be inferred from the sandbox namespace. #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, + grpc_endpoint: String, #[arg( long, @@ -252,15 +253,6 @@ async fn main() -> Result<()> { .collect::>>()?; let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); - let grpc_endpoint = args.grpc_endpoint.unwrap_or_else(|| { - gateway_callback_endpoint( - GatewayCallbackTopology::Kubernetes { - namespace: &args.sandbox_namespace, - }, - openshell_core::config::DEFAULT_SERVER_PORT, - false, - ) - }); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { workspace_mode: args.workspace_mode, @@ -294,7 +286,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint, + grpc_endpoint: args.grpc_endpoint, ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), @@ -363,6 +355,13 @@ async fn main() -> Result<()> { mod tests { use super::*; + #[test] + fn requires_explicit_gateway_callback_endpoint() { + let error = Args::try_parse_from(["openshell-driver-kubernetes"]) + .expect_err("Kubernetes service topology must be explicit"); + assert!(error.to_string().contains("--grpc-endpoint")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ @@ -371,6 +370,8 @@ mod tests { "http://collector.example:4317", "--gateway-name", "kubernetes-dev", + "--grpc-endpoint", + "http://openshell.example:8080", ]) .expect("OTLP endpoint should parse"); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 67583b04b2..df1970b890 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -387,7 +387,7 @@ Podman resources after out-of-band container removal or label drift. | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | unset | Podman cgroup PID limit for sandbox containers. Omit it to inherit Podman's runtime/default PID limit; explicit `0` is invalid. | +| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Omission uses OpenShell's `2048` default; explicit `0` is invalid. | | `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | | `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | | `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 196c1fa571..c71df3a727 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -77,9 +77,12 @@ pub struct PodmanComputeConfig { pub guest_tls_key: Option, /// Container cgroup PID limit for Podman-managed sandboxes. /// - /// Omit the field to leave Podman's runtime/default PID limit unchanged. - /// Explicit zero is invalid. - #[serde(default, skip_serializing_if = "Option::is_none")] + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. @@ -446,7 +449,7 @@ impl Default for PodmanComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - sandbox_pids_limit: None, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, app_armor_profile: Some(AppArmorProfile::Unconfined), @@ -545,12 +548,25 @@ mod tests { } #[test] - fn default_config_uses_runtime_pids_limit() { + fn default_config_sets_driver_owned_pids_limit() { let cfg = PodmanComputeConfig::default(); - assert_eq!(cfg.sandbox_pids_limit, None); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); assert!(!cfg.enable_bind_mounts); } + #[test] + fn omitted_pids_limit_uses_driver_owned_default() { + let cfg: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Podman config should deserialize"); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); + } + #[test] #[cfg(target_os = "macos")] fn default_config_uses_gvproxy_host_gateway_ip_on_macos() { diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1d0fce6f2a..4918c9210a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -216,8 +216,11 @@ struct ContainerSpec { cap_add: Vec, no_new_privileges: bool, seccomp_profile_path: String, - #[serde(skip_serializing_if = "Vec::is_empty")] - security_opt: Vec, + /// Podman's container create API accepts `AppArmor` through the dedicated + /// `apparmor_profile` `SpecGenerator` field. This is not Docker's + /// `security_opt` representation. + #[serde(skip_serializing_if = "Option::is_none")] + apparmor_profile: Option, image_pull_policy: String, #[serde(skip_serializing_if = "Option::is_none")] healthconfig: Option, @@ -939,6 +942,14 @@ fn validate_tmpfs_options(options: &[String]) -> Result, String> { .collect() } +fn podman_apparmor_profile(profile: Option<&openshell_core::AppArmorProfile>) -> Option { + match profile { + None | Some(openshell_core::AppArmorProfile::RuntimeDefault) => None, + Some(openshell_core::AppArmorProfile::Unconfined) => Some("unconfined".to_string()), + Some(openshell_core::AppArmorProfile::Localhost(profile)) => Some(profile.clone()), + } +} + /// Build the Podman container creation JSON spec. #[cfg(test)] #[must_use] @@ -1174,12 +1185,7 @@ pub fn build_container_spec_for_image( // locks itself down. no_new_privileges: true, seccomp_profile_path: "unconfined".into(), - security_opt: config - .app_armor_profile - .as_ref() - .and_then(openshell_core::AppArmorProfile::oci_security_opt) - .into_iter() - .collect(), + apparmor_profile: podman_apparmor_profile(config.app_armor_profile.as_ref()), image_pull_policy: "never".to_string(), healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { test: vec![ @@ -1591,6 +1597,40 @@ mod tests { assert!(spec["resource_limits"].get("PidsLimit").is_none()); } + #[test] + fn container_spec_uses_podman_apparmor_profile_field() { + let sandbox = test_sandbox("test-id", "test-name"); + + for (profile, expected) in [ + (openshell_core::AppArmorProfile::Unconfined, "unconfined"), + ( + openshell_core::AppArmorProfile::Localhost("openshell-supervisor".to_string()), + "openshell-supervisor", + ), + ] { + let mut config = test_config(); + config.app_armor_profile = Some(profile); + let spec = build_container_spec(&sandbox, &config); + + assert_eq!(spec["apparmor_profile"].as_str(), Some(expected)); + assert!(spec.get("security_opt").is_none()); + } + } + + #[test] + fn container_spec_omits_podman_apparmor_profile_for_runtime_default() { + let sandbox = test_sandbox("test-id", "test-name"); + + for profile in [None, Some(openshell_core::AppArmorProfile::RuntimeDefault)] { + let mut config = test_config(); + config.app_armor_profile = profile; + let spec = build_container_spec(&sandbox, &config); + + assert!(spec.get("apparmor_profile").is_none()); + assert!(spec.get("security_opt").is_none()); + } + } + #[test] fn container_name_is_workspace_qualified() { assert_eq!( diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 07ec3cb155..b3be4d273a 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -90,9 +90,9 @@ struct Args { #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] stop_timeout: u32, - /// Container cgroup PID limit for sandbox containers. Omit to inherit - /// Podman's runtime/default PID limit. - #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT")] + /// Container cgroup PID limit for sandbox containers. Omit to use + /// `OpenShell`'s 2048-process default. + #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT", default_value = "2048")] sandbox_pids_limit: Option, /// Health check interval in seconds. Omit it in gateway TOML to disable @@ -349,6 +349,10 @@ mod tests { defaults.health_check_interval_secs.map(NonZeroU64::get), Some(10) ); + assert_eq!( + defaults.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); for flag in ["--sandbox-pids-limit", "--health-check-interval-secs"] { let result = Args::try_parse_from(["openshell-driver-podman", flag, "0"]); diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index 0c94bf72b2..c6d5fb3f32 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -442,7 +442,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { Some(HealthState { status }) if status == "starting" => { ("False", "HealthCheckStarting", String::new()) } - _ => ("False", CONDITION_STARTING, String::new()), + None => ( + "True", + CONDITION_RUNNING, + "Container is running".to_string(), + ), + Some(_) => ("False", CONDITION_STARTING, String::new()), }, "created" => ("False", "ContainerCreated", String::new()), "exited" | "stopped" => { @@ -581,6 +586,25 @@ mod tests { assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); } + #[test] + fn condition_running_without_healthcheck_is_ready() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: None, + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let cond = condition_from_state(&state); + assert_eq!(cond.r#type, "Ready"); + assert_eq!(cond.status, "True"); + assert_eq!(cond.reason, CONDITION_RUNNING); + assert_eq!(cond.message, "Container is running"); + assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); + } + #[test] fn condition_oom_killed() { let state = ContainerState { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 1865b2c46c..9844cbf979 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -152,7 +152,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `sandbox_uid` / `sandbox_gid` | `1000` / UID | Identity written into newly prepared guest rootfs images. Existing persisted rootfs and overlays with the former `10001:10001` account are detected by guest init and retain their legacy ownership. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Existing overlay state without the per-sandbox identity marker is restored as legacy `10001:10001`. | | `https_proxy`, `no_proxy`, `proxy_auth_file` | unset | Operator-owned corporate TLS proxy settings. The driver injects only URL/list/path controls into protected guest startup; it copies a validated `user:pass` auth file into the private overlay, never into logs or process arguments. An `http://` proxy with credentials requires `proxy_auth_allow_insecure = true`. | | `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 3748b43059..22844a15c6 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -29,7 +29,6 @@ BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) GVPROXY_GATEWAY_IP="192.168.127.1" GVPROXY_HOST_LOOPBACK_IP="192.168.127.254" GATEWAY_IP="$GVPROXY_GATEWAY_IP" -SANDBOX_OWNER_NORMALIZED_MARKER="/opt/openshell/.sandbox-owner-normalized" GPU_ENABLED="${GPU_ENABLED:-false}" VM_NET_IP="${VM_NET_IP:-}" @@ -105,8 +104,21 @@ source_overlay_env_if_present() { ensure_target_runtime() { local image_root="$1" - local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-1000}" - local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-$sandbox_uid}" + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local replace_account=0 + + # An omitted identity means the image owns its sandbox account contract. + # Fall back to 1000 only when the image has no sandbox account at all. + if [ -n "$sandbox_uid" ] || [ -n "$sandbox_gid" ]; then + sandbox_uid="${sandbox_uid:-1000}" + sandbox_gid="${sandbox_gid:-$sandbox_uid}" + replace_account=1 + elif ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sandbox_uid=1000 + sandbox_gid=1000 + replace_account=1 + fi mkdir -p \ "$image_root/srv" \ @@ -123,39 +135,28 @@ ensure_target_runtime() { fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" - # This is a newly prepared target image, so replace a baked-in legacy - # sandbox account with the identity selected by the driver. Persisted - # overlays do not take this path; setup_sandbox_workdir preserves their - # existing 10001:10001 account instead. - if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" - else - printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" - fi - if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then - printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" - fi - if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" - else - printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" - fi - if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then - printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + if [ "$replace_account" -eq 1 ]; then + if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" + fi + if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then + printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" + fi + if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" + fi + if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then + printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + fi fi local owner - local owner_normalized=0 owner="$(sandbox_owner_for_root "$image_root")" - if chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 - elif chown -R 1000:1000 "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 - fi + chown -R "$owner" "$image_root/sandbox" 2>/dev/null || chown -R 1000:1000 "$image_root/sandbox" || true chmod 0755 "$image_root/sandbox" - if [ "$owner_normalized" -eq 1 ]; then - mkdir -p "$image_root/opt/openshell" - printf '1\n' > "$image_root${SANDBOX_OWNER_NORMALIZED_MARKER}" - fi } prepare_guest_image_rootfs() { @@ -558,6 +559,34 @@ setup_gpu() { fi } +reconcile_sandbox_account() { + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local etc + + [ -n "$sandbox_uid" ] && [ -n "$sandbox_gid" ] || return 0 + [[ "$sandbox_uid" =~ ^[0-9]+$ ]] && [[ "$sandbox_gid" =~ ^[0-9]+$ ]] || { + ts "FATAL: invalid requested sandbox identity" + exit 1 + } + etc="$(root_path /etc)" + mkdir -p "$etc" + touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow" + if grep -q '^sandbox:' "$etc/group"; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group" + fi + if grep -q '^sandbox:' "$etc/passwd"; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd" + fi + grep -q '^sandbox:' "$etc/gshadow" || printf 'sandbox:!::\n' >> "$etc/gshadow" + grep -q '^sandbox:' "$etc/shadow" || printf 'sandbox:!:20123:0:99999:7:::\n' >> "$etc/shadow" + ts "reconciled sandbox account (${sandbox_uid}:${sandbox_gid})" +} + setup_sandbox_workdir() { local sandbox_dir local owner @@ -569,8 +598,7 @@ setup_sandbox_workdir() { if [ "$owner" = "10001:10001" ]; then ts "preserving legacy sandbox ownership (10001:10001)" fi - if [ "$current_owner" != "$owner" ] \ - || [ ! -f "$(root_path "$SANDBOX_OWNER_NORMALIZED_MARKER")" ]; then + if [ "$current_owner" != "$owner" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then chown -R 1000:1000 "$sandbox_dir" fi @@ -699,6 +727,7 @@ run_post_overlay_setup() { echo 1 > /proc/sys/net/netfilter/nf_log_all_netns 2>/dev/null || true fi + reconcile_sandbox_account setup_sandbox_workdir configure_hostname diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 1b00ba62ce..fb656687a9 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -170,6 +170,8 @@ const IMAGE_CACHE_ROOTFS_IMAGE: &str = "rootfs.ext4"; const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; const OVERLAY_TEMPLATE_CACHE_LAYOUT_VERSION: &str = "sandbox-overlay-ext4-v1"; const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; +const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; +const SANDBOX_OWNER_STATE_VERSION: &str = "sandbox-owner-v1"; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -248,22 +250,18 @@ pub struct VmDriverConfig { pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - /// Resolved sandbox UID for newly prepared rootfs `/etc/passwd` entries. - /// When empty, new sandboxes use 1000. Existing rootfs and overlays retain - /// their recorded sandbox account for legacy 10001 compatibility. + /// Optional UID override for the sandbox account in newly prepared rootfs images. + /// When both identity fields are empty, an image-provided sandbox account is preserved. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, - /// Resolved sandbox GID for rootfs `/etc/passwd` and `/etc/group` entries. - /// When empty, defaults to the resolved UID. + /// Optional GID override for rootfs `/etc/passwd` and `/etc/group` entries. + /// When one override is supplied, its missing counterpart defaults to the UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, } -/// Default sandbox UID used when preparing new VM rootfs images. -/// -/// The guest-init script detects an existing `sandbox` account and preserves -/// its UID/GID, so persisted rootfs and overlays prepared with legacy UID 10001 -/// continue to start without an ownership migration. +/// Fallback sandbox UID for images without a `sandbox` account and partial +/// operator identity overrides. pub const DEFAULT_SANDBOX_UID: u32 = 1000; impl Default for VmDriverConfig { @@ -295,12 +293,12 @@ impl Default for VmDriverConfig { } impl VmDriverConfig { - /// Resolve the sandbox UID, falling back to `DEFAULT_SANDBOX_UID`. + /// Resolve a fallback sandbox UID for an image that has no sandbox account. pub fn resolve_sandbox_uid(&self) -> u32 { self.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID) } - /// Resolve the sandbox GID, falling back to the resolved UID. + /// Resolve a fallback sandbox GID from the selected UID. pub fn resolve_sandbox_gid(&self, resolved_uid: u32) -> u32 { self.sandbox_gid.unwrap_or(resolved_uid) } @@ -435,6 +433,27 @@ enum OverlayPreparation { PreserveExisting, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SandboxOwnerState { + Current, + Legacy, +} + +impl SandboxOwnerState { + fn guest_environment(self) -> Option<[String; 2]> { + match self { + Self::Current => None, + // A state directory with an overlay but no state marker predates + // the 1000 migration. Its upperdir may contain 10001-owned files + // anywhere in the rootfs, not just /sandbox. + Self::Legacy => Some([ + "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), + "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), + ]), + } + } +} + fn provisioning_span( parent: &opentelemetry::Context, sandbox_id: &str, @@ -791,8 +810,9 @@ impl VmDriver { "Preparing writable VM overlay disk".to_string(), ), ); - if let Err(err) = self + let sandbox_owner_state = self .prepare_runtime_overlay( + &state_dir, &overlay_disk, tls_paths.as_ref(), sandbox @@ -803,11 +823,7 @@ impl VmDriver { overlay_preparation, ) .await - { - return Err(Status::internal(format!( - "prepare guest overlay disk failed: {err}" - ))); - } + .map_err(|err| Status::internal(format!("prepare guest overlay disk failed: {err}")))?; self.ensure_provisioning_active(&sandbox.id).await?; if let Err(err) = @@ -1008,6 +1024,11 @@ impl VmDriver { for env in &plan.env { command.arg("--vm-env").arg(env); } + if let Some(identity_env) = sandbox_owner_state.guest_environment() { + for env in identity_env { + command.arg("--vm-env").arg(env); + } + } info!( sandbox_id = %sandbox.id, @@ -2076,12 +2097,15 @@ impl VmDriver { )] async fn prepare_runtime_overlay( &self, + state_dir: &Path, overlay_disk: &Path, tls_paths: Option<&VmDriverTlsPaths>, sandbox_token: Option<&str>, preparation: OverlayPreparation, - ) -> Result<(), String> { + ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); + let (owner_state, write_owner_state) = + sandbox_owner_state_for_launch(state_dir, overlay_disk, preparation).await?; let tls_materials = match tls_paths { Some(paths) => Some(read_guest_tls_materials(paths).await?), None => None, @@ -2124,7 +2148,11 @@ impl VmDriver { }) .await .map_err(|err| format!("overlay image preparation panicked: {err}"))?; - span_status.finish(result) + result?; + if write_owner_state { + write_sandbox_owner_state(state_dir).await?; + } + span_status.finish(Ok(owner_state)) } async fn read_proxy_auth_credential(&self) -> Result, String> { @@ -2519,7 +2547,7 @@ impl VmDriver { image_identity: &str, bootstrap_root_disk: &Path, ) -> Result { - let cache_identity = prepared_image_cache_identity(image_identity); + let cache_identity = prepared_image_cache_identity(image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2628,7 +2656,7 @@ impl VmDriver { "failed to resolve vm sandbox image '{image_ref}': {err}" )) })?; - let cache_identity = prepared_image_cache_identity(&source_image_identity); + let cache_identity = prepared_image_cache_identity(&source_image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2859,14 +2887,14 @@ impl VmDriver { command .arg("--vm-env") .arg(format!("OPENSHELL_VM_INIT_MODE={IMAGE_PREP_INIT_MODE}")); - let resolved_uid = self.config.resolve_sandbox_uid(); - let resolved_gid = self.config.resolve_sandbox_gid(resolved_uid); - command - .arg("--vm-env") - .arg(format!("OPENSHELL_VM_SANDBOX_UID={resolved_uid}")); - command - .arg("--vm-env") - .arg(format!("OPENSHELL_VM_SANDBOX_GID={resolved_gid}")); + if let Some((uid, gid)) = configured_sandbox_identity(&self.config) { + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_UID={uid}")); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_GID={gid}")); + } let mut child = command .spawn() @@ -2984,19 +3012,20 @@ impl VmDriver { let image_identity_owned = image_identity.to_string(); let exported_rootfs_for_build = exported_rootfs.clone(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!( - "Preparing VM rootfs for local image \"{image_ref}\" (sandbox uid={sandbox_uid})" - ), + format!("Preparing VM rootfs for local image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "local_docker".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -3125,17 +3154,20 @@ impl VmDriver { let image_ref_owned = image_ref.to_string(); let image_identity_owned = image_identity.to_string(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!("Preparing VM rootfs for image \"{image_ref}\" (sandbox uid={sandbox_uid})"), + format!("Preparing VM rootfs for image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "registry".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -4712,6 +4744,55 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { } } +/// Select the identity the guest must use for this overlay and whether a +/// successful preparation creates the state-version marker. A missing marker +/// is legacy only when an overlay already exists; an interrupted create with +/// no overlay is safe to initialize as current. +async fn sandbox_owner_state_for_launch( + state_dir: &Path, + overlay_disk: &Path, + preparation: OverlayPreparation, +) -> Result<(SandboxOwnerState, bool), String> { + if preparation == OverlayPreparation::Fresh { + return Ok((SandboxOwnerState::Current, true)); + } + + match tokio::fs::read_to_string(state_dir.join(SANDBOX_OWNER_STATE_FILE)).await { + Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_VERSION => { + Ok((SandboxOwnerState::Current, false)) + } + Ok(_) => Err(format!( + "sandbox owner state {} has an unsupported version", + state_dir.join(SANDBOX_OWNER_STATE_FILE).display() + )), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + match tokio::fs::metadata(overlay_disk).await { + Ok(_) => Ok((SandboxOwnerState::Legacy, false)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Ok((SandboxOwnerState::Current, true)) + } + Err(err) => Err(format!( + "stat overlay disk {}: {err}", + overlay_disk.display() + )), + } + } + Err(err) => Err(format!( + "read sandbox owner state {}: {err}", + state_dir.join(SANDBOX_OWNER_STATE_FILE).display() + )), + } +} + +async fn write_sandbox_owner_state(state_dir: &Path) -> Result<(), String> { + write_private_file( + &state_dir.join(SANDBOX_OWNER_STATE_FILE), + format!("{SANDBOX_OWNER_STATE_VERSION}\n").into_bytes(), + ) + .await + .map_err(|err| format!("write sandbox owner state: {err}")) +} + #[allow(clippy::result_large_err)] fn validate_sandbox_state_dir(root: &Path, state_dir: &Path) -> Result<(), Status> { let sandboxes_root = sandboxes_root_dir(root); @@ -4869,9 +4950,20 @@ fn bootstrap_image_cache_identity(image_identity: &str) -> String { ) } -fn prepared_image_cache_identity(image_identity: &str) -> String { +fn configured_sandbox_identity(config: &VmDriverConfig) -> Option<(u32, u32)> { + (config.sandbox_uid.is_some() || config.sandbox_gid.is_some()).then(|| { + let uid = config.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + (uid, config.sandbox_gid.unwrap_or(uid)) + }) +} + +fn prepared_image_cache_identity(image_identity: &str, config: &VmDriverConfig) -> String { + let identity = configured_sandbox_identity(config).map_or_else( + || "image-account".to_string(), + |(uid, gid)| format!("configured-{uid}-{gid}"), + ); format!( - "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}", + "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{identity}:{image_identity}", openshell_core::VERSION ) } @@ -6204,7 +6296,13 @@ mod tests { let parent = tracing::info_span!("vm.provision"); let result = driver - .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh) + .prepare_runtime_overlay( + Path::new("/unused"), + Path::new("/unused"), + None, + None, + OverlayPreparation::Fresh, + ) .instrument(parent) .await; assert!(result.is_err(), "overflow should stop before disk I/O"); @@ -6790,6 +6888,52 @@ mod tests { } } + #[tokio::test] + async fn legacy_overlay_state_uses_legacy_guest_identity() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + + let (state, write_marker) = + sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) + .await + .unwrap(); + + assert_eq!(state, SandboxOwnerState::Legacy); + assert!(!write_marker); + assert_eq!( + state.guest_environment(), + Some([ + "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), + "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), + ]) + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn current_overlay_state_is_not_inferred_from_the_lower_rootfs() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"current overlay").unwrap(); + write_sandbox_owner_state(&dir).await.unwrap(); + + let (state, write_marker) = + sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) + .await + .unwrap(); + + assert_eq!(state, SandboxOwnerState::Current); + assert!(!write_marker); + assert_eq!( + std::fs::read_to_string(dir.join(SANDBOX_OWNER_STATE_FILE)).unwrap(), + "sandbox-owner-v1\n" + ); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn sandbox_state_dir_rejects_path_unsafe_ids() { let err = sandbox_state_dir(Path::new("/tmp/openshell-vm"), "../escape") @@ -7985,9 +8129,9 @@ mod tests { #[test] fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() { assert_eq!( - prepared_image_cache_identity("sha256:local-image"), + prepared_image_cache_identity("sha256:local-image", &VmDriverConfig::default()), format!( - "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:sha256:local-image", + "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:sha256:local-image", openshell_core::VERSION ) ); @@ -8023,7 +8167,10 @@ mod tests { &staging_dir, &GuestImagePayload { image_ref: "ghcr.io/example/app:latest".to_string(), - image_identity: prepared_image_cache_identity("sha256:abc"), + image_identity: prepared_image_cache_identity( + "sha256:abc", + &VmDriverConfig::default(), + ), source: GuestImagePayloadSource::RegistryOciLayout { layout_dir }, }, ) diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 81f68f6c44..2c84e547aa 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -16,8 +16,7 @@ const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; -const SANDBOX_OWNER_NORMALIZED_MARKER: &str = - openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; +const DEFAULT_SANDBOX_UID: u32 = 1000; const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024; const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024; const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024; @@ -31,8 +30,8 @@ pub const fn sandbox_guest_init_path() -> &'static str { pub fn prepare_sandbox_rootfs_from_image_root( rootfs: &Path, image_identity: &str, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { prepare_sandbox_rootfs(rootfs, sandbox_uid, sandbox_gid)?; validate_sandbox_rootfs(rootfs)?; @@ -353,7 +352,11 @@ fn append_symlink_to_archive( } #[allow(clippy::similar_names)] -fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> Result<(), String> { +fn prepare_sandbox_rootfs( + rootfs: &Path, + sandbox_uid: Option, + sandbox_gid: Option, +) -> Result<(), String> { for relative in ["opt/openshell/.initialized", "opt/openshell/.rootfs-type"] { remove_rootfs_path(rootfs, relative)?; } @@ -567,8 +570,7 @@ fn normalize_sandbox_owner_in_rootfs_image(source: &Path, image_path: &Path) -> return Ok(()); } - run_debugfs_batch(image_path, &commands)?; - write_rootfs_image_file(image_path, SANDBOX_OWNER_NORMALIZED_MARKER, b"1\n") + run_debugfs_batch(image_path, &commands) } fn collect_sandbox_owner_commands( @@ -789,9 +791,18 @@ fn temporary_injection_path(image_path: &Path) -> PathBuf { #[allow(clippy::similar_names)] fn ensure_sandbox_guest_user( rootfs: &Path, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { + // An image's sandbox account is part of its filesystem contract. Leave it + // intact unless an operator explicitly configured either side of the + // identity. A missing account still gets the OpenShell default. + if sandbox_uid.is_none() && sandbox_gid.is_none() && sandbox_guest_user_ids(rootfs)?.is_some() { + return Ok(()); + } + + let sandbox_uid = sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + let sandbox_gid = sandbox_gid.unwrap_or(sandbox_uid); let etc_dir = rootfs.join("etc"); fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?; @@ -983,7 +994,7 @@ mod tests { // Use a non-standard UID so the test doesn't collide with the default. let uid = 20001; - prepare_sandbox_rootfs(&rootfs, uid, uid).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(uid), Some(uid)).expect("prepare sandbox rootfs"); validate_sandbox_rootfs(&rootfs).expect("validate sandbox rootfs"); assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); @@ -1035,7 +1046,7 @@ mod tests { fs::create_dir_all(rootfs.join("sandbox")).expect("create sandbox workdir"); fs::write(rootfs.join("sandbox/app.py"), "print('hello')\n").expect("write app"); - prepare_sandbox_rootfs(&rootfs, 10001, 10001).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(10001), Some(10001)).expect("prepare sandbox rootfs"); assert!(rootfs.join("sandbox").is_dir()); assert_eq!( @@ -1115,6 +1126,61 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn sandbox_user_preserves_image_account_when_identity_is_omitted() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((4242, 4343))); + assert!( + fs::read_to_string(rootfs.join("etc/passwd")) + .unwrap() + .contains("Image:/image-home:/bin/false") + ); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_defaults_to_1000_when_image_has_no_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((1000, 1000))); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_explicit_identity_overrides_image_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, Some(2000), Some(3000)).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((2000, 3000))); + assert!( + fs::read_to_string(rootfs.join("etc/group")) + .unwrap() + .contains("sandbox:x:3000:") + ); + let _ = fs::remove_dir_all(dir); + } + #[test] fn sandbox_guest_user_ids_reads_existing_sandbox_user() { let dir = unique_temp_dir(); diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs index 193ecc0389..eb7e34c1f4 100644 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ b/crates/openshell-server/src/compute/driver_config/builtin.rs @@ -18,14 +18,17 @@ use std::path::PathBuf; pub fn kubernetes_config_from_context( context: DriverStartupContext<'_>, ) -> Result { - let mut cfg = local_driver_config_from_context( + let mut cfg: KubernetesComputeConfig = local_driver_config_from_context( context, ComputeDriverKind::Kubernetes.as_str(), - GatewayCallbackTopology::Kubernetes { - namespace: driver_namespace(context), - }, + None, false, )?; + if cfg.grpc_endpoint.trim().is_empty() { + return Err(Error::config( + "kubernetes compute driver requires grpc_endpoint in [openshell.drivers.kubernetes]; the gateway service location cannot be inferred from the sandbox namespace", + )); + } apply_kubernetes_runtime_defaults(&mut cfg); Ok(cfg) } @@ -37,7 +40,7 @@ pub fn podman_config_from_context( let mut podman = local_driver_config_from_context( context, ComputeDriverKind::Podman.as_str(), - GatewayCallbackTopology::Podman, + Some(GatewayCallbackTopology::Podman), true, )?; apply_podman_runtime_defaults(&mut podman, context); @@ -51,7 +54,7 @@ pub fn docker_config_from_context( let mut cfg = local_driver_config_from_context( context, ComputeDriverKind::Docker.as_str(), - GatewayCallbackTopology::Docker, + Some(GatewayCallbackTopology::Docker), true, )?; apply_docker_runtime_defaults(&mut cfg); @@ -63,32 +66,17 @@ pub fn vm_config_from_context(context: DriverStartupContext<'_>) -> Result) -> &str { - context - .file - .and_then(|file| { - file.openshell - .drivers - .get(ComputeDriverKind::Kubernetes.as_str()) - }) - .and_then(toml::Value::as_table) - .and_then(|table| table.get("namespace")) - .and_then(toml::Value::as_str) - .filter(|namespace| !namespace.trim().is_empty()) - .unwrap_or("openshell") -} - fn local_driver_config_from_context( context: DriverStartupContext<'_>, driver_name: &str, - topology: GatewayCallbackTopology<'_>, + topology: Option, requires_guest_tls: bool, ) -> Result where @@ -103,7 +91,7 @@ where .get("grpc_endpoint") .and_then(toml::Value::as_str) .is_none_or(|endpoint| endpoint.trim().is_empty()); - if endpoint_is_absent { + if endpoint_is_absent && let Some(topology) = topology { table.insert( "grpc_endpoint".to_string(), toml::Value::String(gateway_callback_endpoint( @@ -205,6 +193,7 @@ mod tests { r#" [openshell.drivers.kubernetes] namespace = "sandboxes" +grpc_endpoint = "https://gateway.example:8443" service_account_name = "sandbox-sa" enable_user_namespaces = true "#, @@ -355,7 +344,7 @@ unknown_docker_key = true } #[test] - fn kubernetes_derives_service_endpoint_and_preserves_explicit_override() { + fn kubernetes_requires_endpoint_and_preserves_explicit_override() { let file: config_file::ConfigFile = toml::from_str( r#" [openshell.drivers.kubernetes] @@ -363,14 +352,10 @@ namespace = "agents" "#, ) .expect("valid config"); - let mut context = test_context(Some(&file)); - context.gateway_port = 8443; - context.gateway_tls_enabled = true; - let derived = kubernetes_config_from_context(context).expect("kubernetes config"); - assert_eq!( - derived.grpc_endpoint, - "https://openshell-gateway.agents.svc:8443" - ); + let error = kubernetes_config_from_context(test_context(Some(&file))) + .expect_err("sandbox namespace must not imply gateway service location"); + assert!(error.to_string().contains("requires grpc_endpoint")); + assert!(error.to_string().contains("cannot be inferred")); let override_file: config_file::ConfigFile = toml::from_str( r#" diff --git a/deploy/helm/openshell/tests/grpc_endpoint_test.yaml b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml new file mode 100644 index 0000000000..f84442f605 --- /dev/null +++ b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: Kubernetes gateway callback endpoint + +templates: + - templates/gateway-config.yaml + +release: + name: team-a + namespace: gateway-system + +tests: + - it: derives callback from the gateway Service when sandbox namespace differs + set: + server.sandboxNamespace: agent-sandboxes + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^namespace\s*=\s*"agent-sandboxes"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=\s*"https://team-a-openshell\.gateway-system\.svc\.cluster\.local:8080"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=.*agent-sandboxes' diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 45a813d0e8..ce20d28e25 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -218,7 +218,7 @@ overrides that persist across package upgrades. | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | | `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | -| `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | +| `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index fea9c111be..e783665257 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -82,12 +82,15 @@ future version. To migrate an existing file: Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` spelling are rejected. 6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, - omit Docker or Podman `sandbox_pids_limit` for the runtime default, and omit - Podman `health_check_interval_secs` to disable health checks. Explicit zero - values are invalid. -7. Remove `grpc_endpoint` when the topology-derived callback is correct, or - retain it as an explicit override. New VM root filesystems use UID/GID 1000; - existing persisted VM state using 10001 remains compatible. + omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit + of 2048, and omit Podman `health_check_interval_secs` to disable health + checks. Explicit zero values are invalid. +7. Remove local Docker, Podman, or VM `grpc_endpoint` when the topology-derived + callback is correct, or retain it as an explicit override. Kubernetes raw + TOML requires an explicit endpoint; Helm derives one from the release's + gateway Service. New VM root filesystems use an image-provided `sandbox` + account when present and otherwise use UID/GID 1000. Existing persisted VM + state using 10001 remains compatible. Unknown fields and non-table `[openshell.drivers.]` values fail startup. This strict validation prevents misspelled or misplaced security-sensitive @@ -557,9 +560,10 @@ topology = "combined" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true -# Optional override. When omitted, the gateway derives -# https://openshell-gateway..svc:. -grpc_endpoint = "https://openshell-gateway.agents.svc:8080" +# Required in raw gateway TOML because `namespace` identifies sandbox +# placement, not the gateway Service. Helm renders this from the release's +# gateway Service name and namespace. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 5aea22c976..f6b3eb1ec7 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -56,7 +56,7 @@ Common gateway options: |---|---| | `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. For gateway-managed Docker, Podman, and VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` together in `[openshell.gateway]`; driver tables reject those gateway-owned fields. See the [Gateway Configuration File](./gateway-config) reference for the full schema. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -155,7 +155,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -233,7 +233,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -336,7 +336,7 @@ compute_driver = "vm" For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, and `krun_log_level` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. @@ -369,7 +369,7 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical `[openshell.gateway]` locations remain accepted as lower-precedence compatibility inputs. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`; schema version 2 rejects their historical `[openshell.gateway]` locations. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). @@ -383,7 +383,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | +| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. Raw TOML and the standalone Kubernetes driver require an explicit endpoint because the sandbox namespace does not identify the gateway Service. Helm derives it from the release's gateway Service when the value is empty. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | From 4c04d1714b85467c1cc0efe62d1b8dfcf026ac81 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Tue, 1 Sep 2026 19:14:51 -0400 Subject: [PATCH 5/6] fix(config): address schema v2 review regressions Signed-off-by: Jesse Jaggars --- architecture/compute-runtimes.md | 11 +- crates/openshell-driver-docker/src/lib.rs | 2 +- crates/openshell-driver-docker/src/tests.rs | 9 +- crates/openshell-driver-podman/README.md | 8 +- crates/openshell-driver-podman/src/client.rs | 7 +- crates/openshell-driver-podman/src/driver.rs | 55 ++- crates/openshell-driver-vm/README.md | 2 +- .../scripts/openshell-vm-sandbox-init.sh | 20 +- crates/openshell-driver-vm/src/driver.rs | 407 ++++++++++++++---- crates/openshell-driver-vm/src/rootfs.rs | 62 ++- crates/openshell-server/src/cli.rs | 27 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/templates/_helpers.tpl | 20 + .../openshell/templates/gateway-config.yaml | 4 +- .../openshell/tests/gateway_config_test.yaml | 26 ++ deploy/helm/openshell/values.yaml | 12 +- docs/reference/gateway-config.mdx | 6 +- docs/reference/sandbox-compute-drivers.mdx | 2 +- 18 files changed, 554 insertions(+), 130 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index afb58080d1..8596251c66 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -272,10 +272,13 @@ can request a specific number of GPUs or the driver-specific default behaviour. For all in-tree drivers, this is equivalent to selecting a single GPU. VM runtime state paths are derived only from driver-validated sandbox IDs -matching `[A-Za-z0-9._-]{1,128}`. The gateway-owned VM driver socket uses a -private `run/` directory plus Unix peer UID/PID checks. Standalone -unauthenticated TCP mode is disabled unless explicitly enabled for local -development. +matching `[A-Za-z0-9._-]{1,128}`. Each writable overlay records its effective +sandbox UID/GID so later rootfs cache changes cannot rewrite persisted file +ownership. Unmarked pre-migration overlays recover the account from their +persisted prepared rootfs before falling back to explicit configuration or the +legacy `10001:10001` default. The gateway-owned VM driver socket uses a private +`run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP +mode is disabled unless explicitly enabled for local development. Runtime-specific implementation notes belong in the driver crate README: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d26bdd62ee..1103d480cc 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2782,7 +2782,7 @@ fn build_binds( Status::failed_precondition("provider SPIFFE socket has no parent directory") })?; binds.push(format!( - "{}:{}:ro,rbind", + "{}:{}:ro", parent.display(), PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR )); diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 902f3156c2..53dfa99a6d 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2119,11 +2119,10 @@ fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { .iter() .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) ); - assert!( - binds - .iter() - .any(|bind| bind.contains(PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR)) - ); + assert!(binds.contains(&format!( + "/run/spire:{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}:ro" + ))); + assert!(binds.iter().all(|bind| !bind.contains("rbind"))); let env = body.env.unwrap(); assert!(env.iter().any(|entry| entry == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index df1970b890..51fab8cbf2 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -356,8 +356,12 @@ signals succeeds: - `test -S` on the configured supervisor Unix socket path. - The prior TCP check for a listener on the in-container SSH port. -The Unix socket check allows relay-only readiness when the supervisor exposes -the socket without the old marker or published-port signal. +The Unix socket check allows relay-only backend readiness when the supervisor +exposes the socket without the old marker or published-port signal. Omitting +`health_check_interval_secs` disables these Podman/conmon probes, but it does +not bypass public readiness gating: the gateway keeps a backend-ready sandbox +in `Provisioning` with `SupervisorNotConnected` until its supervisor control +session is connected. ### Deletion Flow diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 8088a50418..5e0dadfb7f 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -272,6 +272,7 @@ pub struct HostInfo { /// Podman returns `host.security.rootless: true` when the daemon is /// running without root privileges (rootless mode). #[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "camelCase")] pub struct SecurityInfo { #[serde(default)] pub rootless: bool, @@ -962,13 +963,17 @@ mod tests { "cgroupVersion": "v2", "networkBackend": "netavark", "rootlessNetworkCmd": "pasta", - "security": {"rootless": true} + "security": { + "rootless": true, + "apparmorEnabled": true + } } }"#, ) .unwrap(); assert!(info.host.security.rootless); + assert!(info.host.security.apparmor_enabled); assert_eq!(info.host.rootless_network_cmd, "pasta"); } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index e621894d77..3a7472eb79 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -421,19 +421,10 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } - if matches!( - config.app_armor_profile, - Some( - openshell_core::AppArmorProfile::RuntimeDefault - | openshell_core::AppArmorProfile::Localhost(_) - ) - ) && !info.host.security.apparmor_enabled - { - return Err(PodmanApiError::InvalidInput( - "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" - .to_string(), - )); - } + validate_apparmor_support( + config.app_armor_profile.as_ref(), + info.host.security.apparmor_enabled, + )?; info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, @@ -1406,6 +1397,26 @@ impl PodmanComputeDriver { } } +fn validate_apparmor_support( + profile: Option<&openshell_core::AppArmorProfile>, + apparmor_enabled: bool, +) -> Result<(), PodmanApiError> { + let requires_apparmor = matches!( + profile, + Some( + openshell_core::AppArmorProfile::RuntimeDefault + | openshell_core::AppArmorProfile::Localhost(_) + ) + ); + if requires_apparmor && !apparmor_enabled { + return Err(PodmanApiError::InvalidInput( + "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" + .to_string(), + )); + } + Ok(()) +} + fn supervisor_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" @@ -2217,6 +2228,24 @@ mod tests { ); } + #[test] + fn confined_apparmor_profiles_follow_podman_capability() { + use openshell_core::AppArmorProfile; + + for profile in [ + AppArmorProfile::RuntimeDefault, + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + ] { + validate_apparmor_support(Some(&profile), true) + .expect("confined profile should be accepted when Podman reports AppArmor"); + let error = validate_apparmor_support(Some(&profile), false) + .expect_err("confined profile must fail when AppArmor is unavailable"); + assert!(error.to_string().contains("AppArmor is unavailable")); + } + validate_apparmor_support(Some(&AppArmorProfile::Unconfined), false) + .expect("Unconfined does not require AppArmor support"); + } + #[test] #[cfg(target_os = "linux")] fn rootless_pasta_requests_default_route_interface() { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 9844cbf979..0ecbc18c97 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -152,7 +152,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Existing overlay state without the per-sandbox identity marker is restored as legacy `10001:10001`. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers that identity from its persisted prepared rootfs, then falls back to explicit configuration or the legacy `10001:10001` default. | | `https_proxy`, `no_proxy`, `proxy_auth_file` | unset | Operator-owned corporate TLS proxy settings. The driver injects only URL/list/path controls into protected guest startup; it copies a validated `user:pass` auth file into the private overlay, never into logs or process arguments. An `http://` proxy with credentials requires `proxy_auth_allow_insecure = true`. | | `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 22844a15c6..05d22b58da 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -573,12 +573,28 @@ reconcile_sandbox_account() { mkdir -p "$etc" touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow" if grep -q '^sandbox:' "$etc/group"; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$etc/group" + if ! awk -F: -v OFS=: -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = gid } { print }' \ + "$etc/group" >"$etc/group.openshell"; then + rm -f "$etc/group.openshell" + ts "FATAL: failed to reconcile sandbox group" + exit 1 + fi + mv "$etc/group.openshell" "$etc/group" else printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group" fi if grep -q '^sandbox:' "$etc/passwd"; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$etc/passwd" + # Preserve image-owned account metadata (home, shell, and description) + # while restoring the UID/GID contract recorded for this overlay. + if ! awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = uid; $4 = gid } { print }' \ + "$etc/passwd" >"$etc/passwd.openshell"; then + rm -f "$etc/passwd.openshell" + ts "FATAL: failed to reconcile sandbox account" + exit 1 + fi + mv "$etc/passwd.openshell" "$etc/passwd" else printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd" fi diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index fb656687a9..3fcc4f9796 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11,7 +11,7 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, - set_rootfs_image_file_mode, write_rootfs_image_file, + sandbox_guest_user_ids_from_image, set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; use bollard::Docker; @@ -171,7 +171,9 @@ const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; const OVERLAY_TEMPLATE_CACHE_LAYOUT_VERSION: &str = "sandbox-overlay-ext4-v1"; const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; -const SANDBOX_OWNER_STATE_VERSION: &str = "sandbox-owner-v1"; +const SANDBOX_OWNER_STATE_V1: &str = "sandbox-owner-v1"; +const SANDBOX_OWNER_STATE_V2: &str = "sandbox-owner-v2"; +const LEGACY_SANDBOX_UID: u32 = 10001; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -434,23 +436,21 @@ enum OverlayPreparation { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SandboxOwnerState { - Current, - Legacy, -} - -impl SandboxOwnerState { - fn guest_environment(self) -> Option<[String; 2]> { - match self { - Self::Current => None, - // A state directory with an overlay but no state marker predates - // the 1000 migration. Its upperdir may contain 10001-owned files - // anywhere in the rootfs, not just /sandbox. - Self::Legacy => Some([ - "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), - "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), - ]), - } +struct SandboxOwnerIdentity { + uid: u32, + gid: u32, +} + +impl SandboxOwnerIdentity { + fn guest_environment(self) -> [String; 2] { + [ + format!("OPENSHELL_VM_SANDBOX_UID={}", self.uid), + format!("OPENSHELL_VM_SANDBOX_GID={}", self.gid), + ] + } + + fn marker_contents(self) -> String { + format!("{SANDBOX_OWNER_STATE_V2}:{}:{}\n", self.uid, self.gid) } } @@ -799,6 +799,7 @@ impl VmDriver { let disk_paths = sandbox_runtime_disk_paths(&state_dir); let root_disk = image_plan.root_disk; let image_disk = image_plan.image_disk; + let owner_source_disk = image_disk.as_ref().unwrap_or(&root_disk).clone(); let overlay_disk = disk_paths.overlay_disk; self.publish_platform_event( @@ -814,6 +815,7 @@ impl VmDriver { .prepare_runtime_overlay( &state_dir, &overlay_disk, + &owner_source_disk, tls_paths.as_ref(), sandbox .spec @@ -1024,10 +1026,8 @@ impl VmDriver { for env in &plan.env { command.arg("--vm-env").arg(env); } - if let Some(identity_env) = sandbox_owner_state.guest_environment() { - for env in identity_env { - command.arg("--vm-env").arg(env); - } + for env in sandbox_owner_state.guest_environment() { + command.arg("--vm-env").arg(env); } info!( @@ -2099,13 +2099,12 @@ impl VmDriver { &self, state_dir: &Path, overlay_disk: &Path, + owner_source_disk: &Path, tls_paths: Option<&VmDriverTlsPaths>, sandbox_token: Option<&str>, preparation: OverlayPreparation, - ) -> Result { + ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); - let (owner_state, write_owner_state) = - sandbox_owner_state_for_launch(state_dir, overlay_disk, preparation).await?; let tls_materials = match tls_paths { Some(paths) => Some(read_guest_tls_materials(paths).await?), None => None, @@ -2123,6 +2122,14 @@ impl VmDriver { self.config.overlay_disk_mib ) })?; + let (owner_state, write_owner_state) = sandbox_owner_state_for_launch( + state_dir, + &overlay_disk, + owner_source_disk, + &self.config, + preparation, + ) + .await?; let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes); if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? { @@ -2150,7 +2157,7 @@ impl VmDriver { .map_err(|err| format!("overlay image preparation panicked: {err}"))?; result?; if write_owner_state { - write_sandbox_owner_state(state_dir).await?; + write_sandbox_owner_state(state_dir, owner_state).await?; } span_status.finish(Ok(owner_state)) } @@ -4744,50 +4751,155 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { } } -/// Select the identity the guest must use for this overlay and whether a -/// successful preparation creates the state-version marker. A missing marker -/// is legacy only when an overlay already exists; an interrupted create with -/// no overlay is safe to initialize as current. +/// Select the exact identity the guest must use for this overlay and whether a +/// successful preparation must create or upgrade its state marker. +/// +/// For an unmarked persisted overlay, inspect the prepared rootfs recorded by +/// the previous driver before falling back to the old 10001 default. This +/// preserves images and explicit configurations that used another UID/GID. async fn sandbox_owner_state_for_launch( state_dir: &Path, overlay_disk: &Path, + owner_source_disk: &Path, + config: &VmDriverConfig, preparation: OverlayPreparation, -) -> Result<(SandboxOwnerState, bool), String> { - if preparation == OverlayPreparation::Fresh { - return Ok((SandboxOwnerState::Current, true)); - } - - match tokio::fs::read_to_string(state_dir.join(SANDBOX_OWNER_STATE_FILE)).await { - Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_VERSION => { - Ok((SandboxOwnerState::Current, false)) - } - Ok(_) => Err(format!( - "sandbox owner state {} has an unsupported version", - state_dir.join(SANDBOX_OWNER_STATE_FILE).display() - )), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - match tokio::fs::metadata(overlay_disk).await { - Ok(_) => Ok((SandboxOwnerState::Legacy, false)), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - Ok((SandboxOwnerState::Current, true)) - } - Err(err) => Err(format!( - "stat overlay disk {}: {err}", - overlay_disk.display() - )), +) -> Result<(SandboxOwnerIdentity, bool), String> { + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + match tokio::fs::read_to_string(&marker_path).await { + Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_V1 => { + let identity = match persisted_sandbox_owner_identity(state_dir, config).await? { + Some(identity) => identity, + None => sandbox_owner_identity_from_image(owner_source_disk).await?, + }; + return Ok((identity, true)); + } + Ok(contents) => { + let identity = parse_sandbox_owner_state(&contents).map_err(|error| { + format!( + "invalid sandbox owner state {}: {error}", + marker_path.display() + ) + })?; + return Ok((identity, false)); + } + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!( + "read sandbox owner state {}: {error}", + marker_path.display() + )); + } + Err(_) => {} + } + + let overlay_exists = match tokio::fs::metadata(overlay_disk).await { + Ok(metadata) => metadata.is_file(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "stat overlay disk {}: {error}", + overlay_disk.display() + )); + } + }; + + if preparation == OverlayPreparation::PreserveExisting && overlay_exists { + if let Some(identity) = persisted_sandbox_owner_identity(state_dir, config).await? { + return Ok((identity, true)); + } + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + return Ok(( + SandboxOwnerIdentity { + uid: LEGACY_SANDBOX_UID, + gid: LEGACY_SANDBOX_UID, + }, + true, + )); + } + + let identity = sandbox_owner_identity_from_image(owner_source_disk) + .await + .or_else(|error| { + configured_sandbox_identity(config) + .map(|(uid, gid)| SandboxOwnerIdentity { uid, gid }) + .ok_or(error) + })?; + Ok((identity, true)) +} + +async fn persisted_sandbox_owner_identity( + state_dir: &Path, + config: &VmDriverConfig, +) -> Result, String> { + let prior_identity = match tokio::fs::read_to_string(state_dir.join(IMAGE_IDENTITY_FILE)).await + { + Ok(identity) if !identity.trim().is_empty() => identity, + Ok(_) => String::new(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(format!("read persisted VM image identity: {error}")), + }; + + if !prior_identity.is_empty() { + let prior_disk = image_cache_rootfs_image(&config.state_dir, prior_identity.trim()); + if tokio::fs::metadata(&prior_disk).await.is_ok() { + match sandbox_owner_identity_from_image(&prior_disk).await { + Ok(identity) => return Ok(Some(identity)), + Err(error) => warn!( + image_path = %prior_disk.display(), + error = %error, + "could not read sandbox identity from persisted VM rootfs; using compatibility fallback" + ), } } - Err(err) => Err(format!( - "read sandbox owner state {}: {err}", - state_dir.join(SANDBOX_OWNER_STATE_FILE).display() - )), } + + Ok(None) +} + +async fn sandbox_owner_identity_from_image( + image_path: &Path, +) -> Result { + let image_path = image_path.to_path_buf(); + let display = image_path.display().to_string(); + let identity = + tokio::task::spawn_blocking(move || sandbox_guest_user_ids_from_image(&image_path)) + .await + .map_err(|error| format!("read sandbox identity task failed: {error}"))??; + let (uid, gid) = identity.ok_or_else(|| { + format!("prepared VM rootfs {display} does not contain a sandbox account") + })?; + Ok(SandboxOwnerIdentity { uid, gid }) +} + +fn parse_sandbox_owner_state(contents: &str) -> Result { + let mut fields = contents.trim().split(':'); + if fields.next() != Some(SANDBOX_OWNER_STATE_V2) { + return Err("unsupported version".to_string()); + } + let uid = fields + .next() + .ok_or_else(|| "missing uid".to_string())? + .parse::() + .map_err(|error| format!("invalid uid: {error}"))?; + let gid = fields + .next() + .ok_or_else(|| "missing gid".to_string())? + .parse::() + .map_err(|error| format!("invalid gid: {error}"))?; + if fields.next().is_some() { + return Err("unexpected fields".to_string()); + } + Ok(SandboxOwnerIdentity { uid, gid }) } -async fn write_sandbox_owner_state(state_dir: &Path) -> Result<(), String> { +async fn write_sandbox_owner_state( + state_dir: &Path, + identity: SandboxOwnerIdentity, +) -> Result<(), String> { write_private_file( &state_dir.join(SANDBOX_OWNER_STATE_FILE), - format!("{SANDBOX_OWNER_STATE_VERSION}\n").into_bytes(), + identity.marker_contents().into_bytes(), ) .await .map_err(|err| format!("write sandbox owner state: {err}")) @@ -5825,18 +5937,19 @@ mod tests { #[test] fn vm_config_rejects_legacy_openshell_endpoint() { - let config = VmDriverConfig::default(); + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; let mut serialized = serde_json::to_value(config).unwrap(); - let fields = serialized.as_object_mut().unwrap(); - fields.remove("grpc_endpoint"); - fields.insert( + serialized.as_object_mut().unwrap().insert( "openshell_endpoint".to_string(), serde_json::json!("http://127.0.0.1:8080"), ); let error = serde_json::from_value::(serialized) - .expect_err("legacy openshell_endpoint must be rejected"); - assert!(!error.to_string().is_empty()); + .expect_err("legacy openshell_endpoint must be rejected as unknown"); + assert!(error.to_string().contains("openshell_endpoint")); } struct TestTracing { @@ -6297,6 +6410,7 @@ mod tests { let result = driver .prepare_runtime_overlay( + Path::new("/unused"), Path::new("/unused"), Path::new("/unused"), None, @@ -6889,51 +7003,176 @@ mod tests { } #[tokio::test] - async fn legacy_overlay_state_uses_legacy_guest_identity() { + async fn unmarked_legacy_overlay_falls_back_to_legacy_default_identity() { let dir = unique_temp_dir(); std::fs::create_dir_all(&dir).unwrap(); let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; - let (state, write_marker) = - sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) - .await - .unwrap(); + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); - assert_eq!(state, SandboxOwnerState::Legacy); - assert!(!write_marker); assert_eq!( - state.guest_environment(), - Some([ + identity, + SandboxOwnerIdentity { + uid: LEGACY_SANDBOX_UID, + gid: LEGACY_SANDBOX_UID, + } + ); + assert!(write_marker); + assert_eq!( + identity.guest_environment(), + [ "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), - ]) + ] + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_legacy_overlay_uses_explicit_config_when_old_rootfs_is_missing() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000, + } ); + assert!(write_marker); let _ = std::fs::remove_dir_all(dir); } + #[test] + fn sandbox_owner_marker_rejects_malformed_or_unknown_state() { + for marker in [ + "sandbox-owner-v3:1000:1000", + "sandbox-owner-v2", + "sandbox-owner-v2:nope:1000", + "sandbox-owner-v2:1000:1000:extra", + ] { + assert!( + parse_sandbox_owner_state(marker).is_err(), + "marker should be rejected: {marker}" + ); + } + } + #[tokio::test] - async fn current_overlay_state_is_not_inferred_from_the_lower_rootfs() { + async fn persisted_owner_marker_preserves_exact_identity() { let dir = unique_temp_dir(); std::fs::create_dir_all(&dir).unwrap(); let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); std::fs::write(&overlay, b"current overlay").unwrap(); - write_sandbox_owner_state(&dir).await.unwrap(); + let expected = SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + }; + write_sandbox_owner_state(&dir, expected).await.unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; - let (state, write_marker) = - sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) - .await - .unwrap(); + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); - assert_eq!(state, SandboxOwnerState::Current); + assert_eq!(identity, expected); assert!(!write_marker); assert_eq!( std::fs::read_to_string(dir.join(SANDBOX_OWNER_STATE_FILE)).unwrap(), - "sandbox-owner-v1\n" + "sandbox-owner-v2:4242:4343\n" ); let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_persisted_rootfs() { + let root = unique_temp_dir(); + let state_dir = root.join("sandboxes/sandbox-1"); + std::fs::create_dir_all(&state_dir).unwrap(); + let overlay = state_dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let prior_identity = "legacy-cache:sha256:abc"; + std::fs::write( + state_dir.join(IMAGE_IDENTITY_FILE), + format!("{prior_identity}\n"), + ) + .unwrap(); + let source = root.join("legacy-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let prior_disk = image_cache_rootfs_image(&root, prior_identity); + create_ext4_image_from_dir_with_size(&source, &prior_disk, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: root.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &state_dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn sandbox_state_dir_rejects_path_unsafe_ids() { let err = sandbox_state_dir(Path::new("/tmp/openshell-vm"), "../escape") diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 2c84e547aa..5e38c28919 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -657,6 +657,56 @@ fn debugfs_quote_argument(argument: &str) -> Option { Some(quoted) } +/// Read the sandbox account identity directly from an ext4 rootfs image. +/// +/// Persisted VM overlays may outlive the prepared-image cache layout that +/// created them. Reading the matching old lower disk lets the driver preserve +/// that overlay's real ownership contract during an upgrade. +pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result, String> { + let quoted_path = debugfs_quote_absolute_path("/etc/passwd") + .expect("the static passwd path is a valid debugfs path"); + let command = format!("cat {quoted_path}"); + let mut last_error = None; + + for candidate in e2fs_tool_candidates("debugfs") { + let label = candidate.display().to_string(); + match Command::new(&candidate) + .arg("-R") + .arg(&command) + .arg(image_path) + .output() + { + Ok(output) if output.status.success() => { + let passwd = String::from_utf8(output.stdout).map_err(|error| { + format!( + "read /etc/passwd from {} as UTF-8: {error}", + image_path.display() + ) + })?; + return parse_sandbox_guest_user_ids(&passwd, &image_path.display().to_string()); + } + Ok(output) => { + last_error = Some(format!( + "{label} failed with status {}\nstdout: {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + last_error = Some(format!("{label} not found")); + } + Err(error) => last_error = Some(format!("run {label}: {error}")), + } + } + + Err(format!( + "debugfs command '{command}' failed for {}: {}. Install e2fsprogs (debugfs) and retry", + image_path.display(), + last_error.unwrap_or_else(|| "debugfs not found".to_string()) + )) +} + fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd_path = rootfs.join("etc/passwd"); if !passwd_path.exists() { @@ -665,6 +715,10 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd = fs::read_to_string(&passwd_path) .map_err(|e| format!("read {}: {e}", passwd_path.display()))?; + parse_sandbox_guest_user_ids(&passwd, &passwd_path.display().to_string()) +} + +fn parse_sandbox_guest_user_ids(passwd: &str, source: &str) -> Result, String> { for line in passwd.lines() { let mut parts = line.split(':'); if parts.next() != Some("sandbox") { @@ -673,14 +727,14 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let _password = parts.next(); let uid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing uid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing uid"))? .parse::() - .map_err(|e| format!("sandbox uid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox uid in {source} is invalid: {e}"))?; let gid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing gid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing gid"))? .parse::() - .map_err(|e| format!("sandbox gid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox gid in {source} is invalid: {e}"))?; return Ok(Some((uid, gid))); } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 73cfefd140..d9c534b850 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -244,11 +244,22 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry } } +fn reject_legacy_driver_selector_env() -> Result<()> { + if std::env::var_os("OPENSHELL_DRIVERS").is_some() { + return Err(miette::miette!( + "OPENSHELL_DRIVERS is no longer supported; use OPENSHELL_COMPUTE_DRIVER with exactly one driver name" + )); + } + Ok(()) +} + fn prepare_server_config( args: &mut RunArgs, matches: &ArgMatches, compute_drivers: &ComputeDriverRegistry, ) -> Result { + reject_legacy_driver_selector_env()?; + // Load TOML when explicitly requested, or from the default XDG location // when that file exists. Missing default config is not an error: runtime // defaults and OPENSHELL_* env vars are enough for package-managed starts. @@ -850,7 +861,7 @@ fn resolve_mtls_auth_enabled( #[cfg(test)] mod tests { - use super::{Cli, command}; + use super::{Cli, command, reject_legacy_driver_selector_env}; use crate::TEST_ENV_LOCK as ENV_LOCK; use clap::Parser; use std::net::{IpAddr, Ipv4Addr}; @@ -1248,6 +1259,20 @@ mod tests { assert!(error.to_string().contains("--drivers")); } + #[test] + fn rejects_legacy_drivers_environment_variable() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for value in ["docker", ""] { + let _guard = EnvVarGuard::set("OPENSHELL_DRIVERS", value); + let error = reject_legacy_driver_selector_env() + .expect_err("legacy OPENSHELL_DRIVERS must be rejected when present"); + assert!(error.to_string().contains("OPENSHELL_DRIVERS")); + assert!(error.to_string().contains("OPENSHELL_COMPUTE_DRIVER")); + } + } + #[test] fn default_config_path_is_loaded_only_when_present() { let _lock = ENV_LOCK diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 9539ca4349..66f2268a2f 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -268,7 +268,7 @@ discovery endpoint or its TLS CA. | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `nil` | Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Use always, if_not_present, or never; newer is supported only by Podman. | +| server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -289,7 +289,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `nil` | Canonical sandbox supervisor pull policy. Leave unset to use the Kubernetes image default; use always, if_not_present, or never. | +| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 548418abc6..cbb91ada72 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -225,6 +225,26 @@ database requires persistent per-pod storage. {{- default "statefulset" (get $workload "kind") | lower -}} {{- end }} +{{/* +Translate chart image pull policy values to the canonical gateway vocabulary. +The Kubernetes spellings remain accepted so existing values files continue to +work across the schema-v2 chart upgrade. +*/}} +{{- define "openshell.canonicalImagePullPolicy" -}} +{{- $policy := printf "%v" . -}} +{{- if eq $policy "Always" -}} +always +{{- else if eq $policy "IfNotPresent" -}} +if_not_present +{{- else if eq $policy "Never" -}} +never +{{- else if has $policy (list "always" "if_not_present" "never") -}} +{{- $policy -}} +{{- else -}} +{{- fail (printf "image pull policy %q must be one of: always, if_not_present, never, Always, IfNotPresent, Never" $policy) -}} +{{- end -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0ec13328b8..829fa56899 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -183,7 +183,7 @@ data: provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} {{- if .Values.server.sandboxImagePullPolicy }} - image_pull_policy = {{ .Values.server.sandboxImagePullPolicy | quote }} + image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.server.sandboxImagePullPolicy | quote }} {{- end }} {{- $sandboxImagePullSecretNames := list -}} {{- range .Values.server.sandboxImagePullSecrets }} @@ -207,7 +207,7 @@ data: app_armor_profile = {{ .Values.server.appArmorProfile | quote }} {{- end }} {{- if .Values.supervisor.image.pullPolicy }} - supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} + supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} {{- end }} [openshell.drivers.kubernetes.managed_ssh_ingress] diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index e42fc345f0..ffdac8d186 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -165,6 +165,32 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' + - it: translates legacy Kubernetes pull policy values to canonical gateway values + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Always + supervisor.image.pullPolicy: IfNotPresent + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"always".*?supervisor_image_pull_policy\s*=\s*"if_not_present"' + + - it: rejects unsupported sandbox image pull policies + template: templates/statefulset.yaml + set: + server.sandboxImagePullPolicy: Sometimes + asserts: + - failedTemplate: + errorMessage: 'image pull policy "Sometimes" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + + - it: rejects unsupported supervisor image pull policies + template: templates/statefulset.yaml + set: + supervisor.image.pullPolicy: newer + asserts: + - failedTemplate: + errorMessage: 'image pull policy "newer" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + - it: renders driver-owned Kubernetes settings only in its driver table template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index b0aa01adb5..f52d8009b0 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -33,8 +33,9 @@ supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor - # -- Canonical sandbox supervisor pull policy. Leave unset to use the - # Kubernetes image default; use always, if_not_present, or never. + # -- Sandbox supervisor pull policy. Leave unset to use the Kubernetes + # image default. Prefer always, if_not_present, or never; the chart also + # accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" @@ -217,9 +218,10 @@ server: externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes - # image default (Always for :latest, IfNotPresent otherwise). Use always, - # if_not_present, or never; newer is supported only by Podman. + # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image + # default (Always for :latest, IfNotPresent otherwise). Prefer always, + # if_not_present, or never; the chart also accepts legacy Kubernetes spellings + # Always, IfNotPresent, and Never. sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e783665257..4657293bb2 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -228,6 +228,8 @@ phases = ["validate"] [openshell.drivers.kubernetes] namespace = "openshell" +# Required in raw TOML; Helm derives this from the gateway Service. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" @@ -670,7 +672,7 @@ ssh_socket_path = "/run/openshell/ssh.sock" # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Omit to leave Docker's runtime default unchanged. Explicit 0 is invalid. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 # Explicit supervisor-compatible default. RuntimeDefault requires Docker to # report AppArmor support; Localhost/ requires an operator-loaded profile. @@ -731,7 +733,7 @@ stop_timeout_secs = 45 # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Omit to leave Podman's runtime default unchanged. Explicit 0 is invalid. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 # Health check interval in seconds. Omit to disable health checks; explicit 0 # is invalid. Lower values detect readiness faster but increase process churn diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index f6b3eb1ec7..c71fd72f18 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -577,7 +577,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/etc/group`, and `/etc/gshadow` during rootfs preparation. Default UID is `10001`; configure `sandbox_uid` in `[openshell.drivers.vm]` to use a different value. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created so a driver upgrade does not rewrite their ownership contract. ### Custom Images From a4acfb1ce3b931a4cb3eec52e328f05e4be049ff Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Tue, 1 Sep 2026 22:24:56 -0400 Subject: [PATCH 6/6] fix(config): complete schema v2 migration safeguards Signed-off-by: Jesse Jaggars --- .../skills/debug-openshell-cluster/SKILL.md | 17 +- .agents/skills/test-release-canary/SKILL.md | 4 + Cargo.lock | 1 + architecture/compute-runtimes.md | 19 +- architecture/gateway.md | 14 + crates/openshell-driver-docker/src/lib.rs | 1 + crates/openshell-driver-docker/src/tests.rs | 9 + crates/openshell-driver-podman/README.md | 10 +- crates/openshell-driver-podman/src/config.rs | 34 ++- crates/openshell-driver-podman/src/driver.rs | 2 + crates/openshell-driver-vm/README.md | 4 +- .../scripts/openshell-vm-sandbox-init.sh | 8 +- crates/openshell-driver-vm/src/driver.rs | 268 +++++++++++++++--- crates/openshell-driver-vm/src/rootfs.rs | 30 +- crates/openshell-gateway/Cargo.toml | 2 + crates/openshell-gateway/src/lib.rs | 56 +++- crates/openshell-gateway/src/vm.rs | 72 ++++- .../openshell-server/src/auth/sandbox_jwt.rs | 60 ++-- crates/openshell-server/src/cli.rs | 3 + .../src/compute/driver_config.rs | 110 ++++++- crates/openshell-server/src/config_file.rs | 33 ++- crates/openshell-server/src/grpc/auth_rpc.rs | 12 +- crates/openshell-server/src/lib.rs | 28 +- .../openshell/tests/gateway_config_test.yaml | 3 + deploy/rpm/CONFIGURATION.md | 8 +- deploy/rpm/TROUBLESHOOTING.md | 22 +- deploy/rpm/gateway.toml.default.v1 | 28 ++ deploy/rpm/migrate-gateway-config.sh | 44 +++ docs/about/installation.mdx | 4 +- docs/reference/gateway-config.mdx | 46 ++- docs/reference/sandbox-compute-drivers.mdx | 10 +- openshell.spec | 19 +- python/openshell/release_formula_test.py | 26 +- .../rpm_gateway_config_migration_test.py | 69 +++++ rfc/0003-gateway-configuration/README.md | 55 ++-- tasks/scripts/gateway-docker.sh | 4 +- tasks/scripts/gateway-podman.sh | 7 +- tasks/scripts/gateway-pull-policy.sh | 28 ++ tasks/scripts/gateway.sh | 4 +- tasks/scripts/release.py | 16 +- tasks/scripts/test-gateway-pull-policy.sh | 47 +++ tasks/test.toml | 7 + 42 files changed, 1052 insertions(+), 192 deletions(-) create mode 100644 deploy/rpm/gateway.toml.default.v1 create mode 100755 deploy/rpm/migrate-gateway-config.sh create mode 100644 python/openshell/rpm_gateway_config_migration_test.py create mode 100755 tasks/scripts/gateway-pull-policy.sh create mode 100755 tasks/scripts/test-gateway-pull-policy.sh diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 623fb79641..8bcd7c5d9e 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -92,9 +92,15 @@ Gateway configuration requires `[openshell] version = 2`, a singular `compute_driver` selector, and driver-owned settings under `[openshell.drivers.]`. The gateway rejects legacy `compute_drivers`, `--drivers`, and `OPENSHELL_DRIVERS` selectors rather than silently migrating -them. Guest TLS CA, certificate, and key paths are the exception: configure the -complete bundle under `[openshell.gateway]`, and the gateway injects it only -into the selected local driver. +them. Homebrew and RPM package startup migrates only exact package-generated v1 +defaults. If an upgraded package still reports an unsupported version, inspect +the active prefix or `~/.config/openshell/gateway.toml`; an edited v1 file must +follow the published schema-v2 migration steps and must not be overwritten. +Guest TLS CA, certificate, and key paths are the exception to driver ownership: +configure the complete bundle under `[openshell.gateway]`, and the gateway +injects it only into the selected local driver. TLS-enabled Docker, Podman, and +VM drivers fail startup when neither those paths nor the package-managed local +bundle is available; Kubernetes projects its bundle through a Secret. Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. @@ -628,6 +634,11 @@ Use the VM driver logs and host diagnostics available in the user's environment. - The VM driver process is running and reachable by the gateway. - The runtime rootfs exists and matches the expected architecture. +- `mke2fs` or `mkfs.ext4` and `debugfs` from e2fsprogs are installed; explicit + `sandbox_uid`/`sandbox_gid` does not remove this prerequisite. +- A persisted overlay identity error is resolved from its owner marker, overlay + upper layer, prepared rootfs, explicit config, or current image. Do not assign + `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. - The sandbox supervisor can establish its callback connection to the gateway. diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 82cf9ac5ae..f70489ea28 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -23,6 +23,10 @@ does not contribute to product usage metrics. `install.sh` defaults to the *latest tagged* release — the canary is therefore checking that the most recent public release still installs, not the just-published `dev` build. The `kubernetes` job is the exception: it pins to `0.0.0-dev` chart + `:dev` images. +The host-package jobs exercise fresh installs, not upgrades from a persisted +schema-v1 gateway config. Validate Homebrew and RPM exact-default migration with +the release-tooling and package lifecycle tests before relying on the canary. + The canary does not install or import `@nvidia/openshell-sdk`. TypeScript SDK validation lives in the `TypeScript SDK` branch check, including a publish dry-run. The tagged release workflow publishes the package to GitHub Packages; diff --git a/Cargo.lock b/Cargo.lock index 90240d6b3a..3feba7cde5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4129,6 +4129,7 @@ dependencies = [ "openshell-driver-mxc", "openshell-driver-podman", "openshell-otel", + "openshell-policy", "openshell-server", "rustix 1.1.4", "serde", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e6b4284080..642374ba77 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -291,10 +291,12 @@ is driver-owned supervisor input and is removed from workload environments. Kubernetes, Docker, and Podman share one AppArmor configuration model: `RuntimeDefault`, `Unconfined`, or `Localhost/`. Each driver translates -that model to its native API and rejects a requested confined profile when its -backend reports AppArmor unavailable. Docker and Podman use explicit -`Unconfined` by default because their runtime-default profiles commonly block -the supervisor's namespace mount setup; the Helm chart uses the same default. +that model to its native API and rejects an explicitly requested confined +profile when its backend reports AppArmor unavailable. Docker keeps its +historical explicit `Unconfined` default. Podman sends no override when the +field is omitted, preserving the runtime-selected profile; development paths +that require the supervisor's namespace mount setup opt into `Unconfined` +explicitly. The Helm chart independently uses `Unconfined` for Kubernetes. Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, @@ -323,10 +325,11 @@ For all in-tree drivers, this is equivalent to selecting a single GPU. VM runtime state paths are derived only from driver-validated sandbox IDs matching `[A-Za-z0-9._-]{1,128}`. Each writable overlay records its effective sandbox UID/GID so later rootfs cache changes cannot rewrite persisted file -ownership. Unmarked pre-migration overlays recover the account from their -persisted prepared rootfs before falling back to explicit configuration or the -legacy `10001:10001` default. The gateway-owned VM driver socket uses a private -`run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP +ownership. Unmarked pre-migration overlays recover identity from concrete +overlay or prepared-rootfs state, an explicit operator override, or the current +image account. The driver never assumes `10001:10001`; it preserves that legacy +identity only when persisted state reports it. The gateway-owned VM driver +socket uses a private `run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP mode is disabled unless explicitly enabled for local development. Runtime-specific implementation notes belong in the driver crate README: diff --git a/architecture/gateway.md b/architecture/gateway.md index f14b3999da..3665fc9699 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -38,6 +38,20 @@ immediately without a grace period. Finalization is persisted separately from the exit result; the gateway deletes an ephemeral sandbox only after the finalized supervisor session disconnects. +## Configuration Boundary + +The gateway accepts exactly schema version 2. Missing, legacy, and future +versions fail before runtime construction, and driver settings belong only to +`[openshell.drivers.]`. The process does not migrate legacy files. +Package lifecycle code may replace an exact package-generated v1 default, but +it preserves edited configurations for explicit operator migration. + +Gateway listener TLS and sandbox callback TLS are separate inputs. A selected +local Docker, Podman, or VM driver requires a complete guest bundle whenever +the gateway listener uses TLS; package-managed local TLS can supply that bundle. +Kubernetes instead projects guest credentials through its configured Secret. +The gateway validates this requirement before constructing the selected driver. + ## Protocol and Auth The gateway listens on one service port and multiplexes gRPC and HTTP traffic. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d808916b7c..12b4d09522 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -189,6 +189,7 @@ pub struct DockerComputeConfig { /// `AppArmor` confinement requested for sandbox containers. The explicit /// default preserves the prior supervisor-compatible Docker behavior. + #[serde(skip_serializing_if = "Option::is_none")] pub app_armor_profile: Option, } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 69ed65bfb0..101cae80d6 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -150,6 +150,15 @@ fn docker_config_rejects_legacy_sandbox_namespace() { assert!(error.to_string().contains("sandbox_namespace")); } +#[test] +fn docker_config_keeps_explicit_unconfined_apparmor_default() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!(config.app_armor_profile, Some(AppArmorProfile::Unconfined)); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], "Unconfined"); +} + #[test] fn docker_config_defaults_to_driver_owned_pids_limit() { let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 0d8be045a1..5781594158 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -413,10 +413,12 @@ and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see Workload API socket, projected through a dedicated read-only mount, or an explicit container-reachable `tcp:IP:port` endpoint. The driver sets the supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. -`app_armor_profile` shares the canonical `RuntimeDefault`, `Unconfined`, or -`Localhost/` model with Docker and Kubernetes. Podman defaults to -explicit `Unconfined` for the supervisor mount setup; confined choices fail -early when Podman reports AppArmor unavailable. +`app_armor_profile` shares the canonical +`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and +Kubernetes. When omitted, the driver sends no override and preserves Podman's +runtime-selected profile. Set `Unconfined` explicitly only when the deployment +requires the supervisor's mount setup to bypass that profile. Explicit confined +choices fail early when Podman reports AppArmor unavailable. This is an operator-owned egress boundary: the driver passes the settings on the supervisor's command line, so sandbox and template environment — and any diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index c71df3a727..855b7f0e24 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -91,9 +91,9 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, - /// `AppArmor` confinement requested for sandbox containers. The default - /// explicitly opts out because the supervisor needs mount operations that - /// the runtime default profile denies. + /// `AppArmor` confinement requested for sandbox containers. Omission sends + /// no override and preserves Podman's runtime-selected profile. + #[serde(default, skip_serializing_if = "Option::is_none")] pub app_armor_profile: Option, /// Health check interval in seconds for sandbox containers. /// @@ -452,7 +452,7 @@ impl Default for PodmanComputeConfig { sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, - app_armor_profile: Some(AppArmorProfile::Unconfined), + app_armor_profile: None, health_check_interval_secs: None, https_proxy: None, no_proxy: None, @@ -541,6 +541,32 @@ mod tests { ); } + #[test] + fn omitted_apparmor_profile_preserves_runtime_default() { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("omitted AppArmor profile should deserialize"); + assert_eq!(config.app_armor_profile, None); + + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert!(serialized.get("app_armor_profile").is_none()); + } + + #[test] + fn explicit_apparmor_profiles_round_trip() { + for value in [ + "RuntimeDefault", + "Unconfined", + "Localhost/openshell-supervisor", + ] { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "app_armor_profile": value, + })) + .expect("explicit AppArmor profile should deserialize"); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], value); + } + } + #[test] fn default_config_sets_podman_stop_timeout() { let cfg = PodmanComputeConfig::default(); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 9127576643..04322236f5 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -2252,6 +2252,8 @@ mod tests { } validate_apparmor_support(Some(&AppArmorProfile::Unconfined), false) .expect("Unconfined does not require AppArmor support"); + validate_apparmor_support(None, false) + .expect("an omitted profile preserves Podman's runtime behavior"); } #[test] diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index b03d05d963..6e7013e215 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -152,7 +152,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers that identity from its persisted prepared rootfs, then falls back to explicit configuration or the legacy `10001:10001` default. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers identity from its upper layer or prepared rootfs, an explicit override, or the current image. Legacy `10001:10001` is retained only when persisted state reports it. | | `https_proxy`, `no_proxy`, `proxy_auth_file` | unset | Operator-owned corporate TLS proxy settings. The driver injects only URL/list/path controls into protected guest startup; it copies a validated `user:pass` auth file into the private overlay, never into logs or process arguments. An `http://` proxy with credentials requires `proxy_auth_allow_insecure = true`. | | `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | @@ -256,7 +256,7 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain -- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection +- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation, identity inspection, and QEMU environment injection. Explicit `sandbox_uid`/`sandbox_gid` values do not remove this runtime prerequisite. - Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 05d22b58da..ada2fb0a54 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -155,7 +155,10 @@ ensure_target_runtime() { fi local owner owner="$(sandbox_owner_for_root "$image_root")" - chown -R "$owner" "$image_root/sandbox" 2>/dev/null || chown -R 1000:1000 "$image_root/sandbox" || true + if ! chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then + ts "FATAL: failed to apply sandbox image ownership (${owner})" + exit 1 + fi chmod 0755 "$image_root/sandbox" } @@ -616,7 +619,8 @@ setup_sandbox_workdir() { fi if [ "$current_owner" != "$owner" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then - chown -R 1000:1000 "$sandbox_dir" + ts "FATAL: failed to apply sandbox ownership (${owner})" + exit 1 fi fi chmod 0755 "$sandbox_dir" diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 2d942c18a6..2356b107df 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11,7 +11,8 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, - sandbox_guest_user_ids_from_image, set_rootfs_image_file_mode, write_rootfs_image_file, + sandbox_guest_user_ids_from_image, sandbox_guest_user_ids_from_overlay_image, + set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; use bollard::Docker; @@ -173,7 +174,6 @@ const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; const SANDBOX_OWNER_STATE_V1: &str = "sandbox-owner-v1"; const SANDBOX_OWNER_STATE_V2: &str = "sandbox-owner-v2"; -const LEGACY_SANDBOX_UID: u32 = 10001; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -189,6 +189,7 @@ const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; const IMAGE_PREP_INIT_MODE: &str = "image-prep"; static IMAGE_CACHE_BUILD_COUNTER: AtomicU64 = AtomicU64::new(0); +static OWNER_STATE_WRITE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone)] struct VmDriverTlsPaths { @@ -2131,6 +2132,14 @@ impl VmDriver { preparation, ) .await?; + let owner_state_written_before_prepare = + write_owner_state && preparation == OverlayPreparation::Fresh; + if owner_state_written_before_prepare { + // Persist the selected identity before creating the overlay. A + // crash during preparation can then retry without misclassifying + // the partial overlay as legacy state. + write_sandbox_owner_state(state_dir, owner_state).await?; + } let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes); if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? { @@ -2157,7 +2166,7 @@ impl VmDriver { .await .map_err(|err| format!("overlay image preparation panicked: {err}"))?; result?; - if write_owner_state { + if write_owner_state && !owner_state_written_before_prepare { write_sandbox_owner_state(state_dir, owner_state).await?; } span_status.finish(Ok(owner_state)) @@ -4759,9 +4768,9 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { /// Select the exact identity the guest must use for this overlay and whether a /// successful preparation must create or upgrade its state marker. /// -/// For an unmarked persisted overlay, inspect the prepared rootfs recorded by -/// the previous driver before falling back to the old 10001 default. This -/// preserves images and explicit configurations that used another UID/GID. +/// Persisted overlays are resolved only from concrete state. In particular, +/// absence of a marker is not evidence that an overlay used the historical +/// 10001 identity: it can also mean fresh provisioning was interrupted. async fn sandbox_owner_state_for_launch( state_dir: &Path, overlay_disk: &Path, @@ -4772,11 +4781,8 @@ async fn sandbox_owner_state_for_launch( let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); match tokio::fs::read_to_string(&marker_path).await { Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_V1 => { - let identity = match persisted_sandbox_owner_identity(state_dir, config).await? { - Some(identity) => identity, - None => sandbox_owner_identity_from_image(owner_source_disk).await?, - }; - return Ok((identity, true)); + // The v1 marker recorded no identity. Resolve it through the same + // evidence-based migration path as an unmarked overlay. } Ok(contents) => { let identity = parse_sandbox_owner_state(&contents).map_err(|error| { @@ -4808,29 +4814,32 @@ async fn sandbox_owner_state_for_launch( }; if preparation == OverlayPreparation::PreserveExisting && overlay_exists { + match sandbox_owner_identity_from_overlay(overlay_disk).await { + Ok(Some(identity)) => return Ok((identity, true)), + Ok(None) => {} + Err(error) => warn!( + overlay_path = %overlay_disk.display(), + error = %error, + "could not read sandbox identity from VM overlay upper layer" + ), + } if let Some(identity) = persisted_sandbox_owner_identity(state_dir, config).await? { return Ok((identity, true)); } if let Some((uid, gid)) = configured_sandbox_identity(config) { return Ok((SandboxOwnerIdentity { uid, gid }, true)); } - return Ok(( - SandboxOwnerIdentity { - uid: LEGACY_SANDBOX_UID, - gid: LEGACY_SANDBOX_UID, - }, - true, - )); + return sandbox_owner_identity_from_image(owner_source_disk) + .await + .map(|identity| (identity, true)); } - let identity = sandbox_owner_identity_from_image(owner_source_disk) + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + sandbox_owner_identity_from_image(owner_source_disk) .await - .or_else(|error| { - configured_sandbox_identity(config) - .map(|(uid, gid)| SandboxOwnerIdentity { uid, gid }) - .ok_or(error) - })?; - Ok((identity, true)) + .map(|identity| (identity, true)) } async fn persisted_sandbox_owner_identity( @@ -4866,17 +4875,26 @@ async fn sandbox_owner_identity_from_image( image_path: &Path, ) -> Result { let image_path = image_path.to_path_buf(); - let display = image_path.display().to_string(); let identity = tokio::task::spawn_blocking(move || sandbox_guest_user_ids_from_image(&image_path)) .await .map_err(|error| format!("read sandbox identity task failed: {error}"))??; - let (uid, gid) = identity.ok_or_else(|| { - format!("prepared VM rootfs {display} does not contain a sandbox account") - })?; + let (uid, gid) = identity.unwrap_or((DEFAULT_SANDBOX_UID, DEFAULT_SANDBOX_UID)); Ok(SandboxOwnerIdentity { uid, gid }) } +async fn sandbox_owner_identity_from_overlay( + overlay_path: &Path, +) -> Result, String> { + let overlay_path = overlay_path.to_path_buf(); + let identity = tokio::task::spawn_blocking(move || { + sandbox_guest_user_ids_from_overlay_image(&overlay_path) + }) + .await + .map_err(|error| format!("read sandbox overlay identity task failed: {error}"))??; + Ok(identity.map(|(uid, gid)| SandboxOwnerIdentity { uid, gid })) +} + fn parse_sandbox_owner_state(contents: &str) -> Result { let mut fields = contents.trim().split(':'); if fields.next() != Some(SANDBOX_OWNER_STATE_V2) { @@ -4895,6 +4913,7 @@ fn parse_sandbox_owner_state(contents: &str) -> Result Result<(), String> { - write_private_file( - &state_dir.join(SANDBOX_OWNER_STATE_FILE), - identity.marker_contents().into_bytes(), - ) - .await - .map_err(|err| format!("write sandbox owner state: {err}")) + validate_sandbox_owner_identity(identity.uid, identity.gid)?; + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + let sequence = OWNER_STATE_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary_path = state_dir.join(format!( + ".{SANDBOX_OWNER_STATE_FILE}.{}.{sequence}.tmp", + std::process::id() + )); + write_private_file(&temporary_path, identity.marker_contents().into_bytes()) + .await + .map_err(|err| format!("write temporary sandbox owner state: {err}"))?; + if let Err(error) = tokio::fs::rename(&temporary_path, &marker_path).await { + let _ = tokio::fs::remove_file(&temporary_path).await; + return Err(format!("install sandbox owner state: {error}")); + } + Ok(()) +} + +fn validate_sandbox_owner_identity(uid: u32, gid: u32) -> Result<(), String> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + if !range.contains(&uid) { + return Err(format!( + "uid {uid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + if !range.contains(&gid) { + return Err(format!( + "gid {gid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + Ok(()) } #[allow(clippy::result_large_err)] @@ -7008,17 +7055,58 @@ mod tests { } #[tokio::test] - async fn unmarked_legacy_overlay_falls_back_to_legacy_default_identity() { + async fn unmarked_overlay_uses_current_image_instead_of_blind_legacy_identity() { let dir = unique_temp_dir(); std::fs::create_dir_all(&dir).unwrap(); let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); - std::fs::write(&overlay, b"legacy overlay").unwrap(); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let source = dir.join("current-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let current_rootfs = dir.join("current-rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, ¤t_rootfs, 32 * 1024 * 1024).unwrap(); let config = VmDriverConfig { state_dir: dir.clone(), ..Default::default() }; let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + ¤t_rootfs, + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_without_identity_evidence_fails_safely() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let error = sandbox_owner_state_for_launch( &dir, &overlay, Path::new("/missing-current-rootfs"), @@ -7026,22 +7114,74 @@ mod tests { OverlayPreparation::PreserveExisting, ) .await + .expect_err("ambiguous overlay must not receive a guessed identity"); + + assert!(error.contains("missing-current-rootfs")); + assert!(!error.contains("10001")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_overlay_uses_explicit_identity_without_image_inspection() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::Fresh, + ) + .await .unwrap(); assert_eq!( identity, SandboxOwnerIdentity { - uid: LEGACY_SANDBOX_UID, - gid: LEGACY_SANDBOX_UID, + uid: 2000, + gid: 3000 } ); assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_image_without_sandbox_account_uses_default_identity() { + let dir = unique_temp_dir(); + let source = dir.join("rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write(source.join("etc/passwd"), "root:x:0:0:root:/root:/bin/sh\n").unwrap(); + let rootfs = dir.join("rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, &rootfs, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, _) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + &rootfs, + &config, + OverlayPreparation::Fresh, + ) + .await + .unwrap(); + assert_eq!( - identity.guest_environment(), - [ - "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), - "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), - ] + identity, + SandboxOwnerIdentity { + uid: DEFAULT_SANDBOX_UID, + gid: DEFAULT_SANDBOX_UID, + } ); let _ = std::fs::remove_dir_all(dir); } @@ -7086,6 +7226,8 @@ mod tests { "sandbox-owner-v3:1000:1000", "sandbox-owner-v2", "sandbox-owner-v2:nope:1000", + "sandbox-owner-v2:0:1000", + "sandbox-owner-v2:1000:0", "sandbox-owner-v2:1000:1000:extra", ] { assert!( @@ -7130,6 +7272,44 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_upper_passwd() { + let dir = unique_temp_dir(); + let overlay_source = dir.join("overlay-source"); + std::fs::create_dir_all(overlay_source.join("upper/etc")).unwrap(); + std::fs::write( + overlay_source.join("upper/etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:10001:10001:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + create_ext4_image_from_dir_with_size(&overlay_source, &overlay, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 10001, + gid: 10001, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn unmarked_overlay_recovers_identity_from_persisted_rootfs() { let root = unique_temp_dir(); diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 5e38c28919..4821844a7d 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -663,7 +663,33 @@ fn debugfs_quote_argument(argument: &str) -> Option { /// created them. Reading the matching old lower disk lets the driver preserve /// that overlay's real ownership contract during an upgrade. pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result, String> { - let quoted_path = debugfs_quote_absolute_path("/etc/passwd") + sandbox_guest_user_ids_from_image_path(image_path, "/etc/passwd") +} + +/// Read a sandbox account copied into an overlay upper layer. +/// +/// An upper-layer passwd file is the most direct evidence of the identity an +/// existing overlay observed, so migration consults it before any lower image. +pub fn sandbox_guest_user_ids_from_overlay_image( + image_path: &Path, +) -> Result, String> { + sandbox_guest_user_ids_from_image_path(image_path, "/upper/etc/passwd") +} + +fn sandbox_guest_user_ids_from_image_path( + image_path: &Path, + guest_path: &str, +) -> Result, String> { + let metadata = fs::metadata(image_path) + .map_err(|error| format!("stat rootfs image {}: {error}", image_path.display()))?; + if !metadata.is_file() { + return Err(format!( + "rootfs image {} is not a regular file", + image_path.display() + )); + } + + let quoted_path = debugfs_quote_absolute_path(guest_path) .expect("the static passwd path is a valid debugfs path"); let command = format!("cat {quoted_path}"); let mut last_error = None; @@ -679,7 +705,7 @@ pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result { let passwd = String::from_utf8(output.stdout).map_err(|error| { format!( - "read /etc/passwd from {} as UTF-8: {error}", + "read {guest_path} from {} as UTF-8: {error}", image_path.display() ) })?; diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index f9d02027e0..e9685adad4 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -26,6 +26,7 @@ tokio = { workspace = true } openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +openshell-policy = { path = "../openshell-policy", optional = true } hyper-util = { workspace = true, optional = true } nix = { workspace = true, optional = true } serde = { workspace = true, optional = true } @@ -43,6 +44,7 @@ in-tree-compute-drivers = [ "dep:openshell-driver-docker", "dep:openshell-driver-kubernetes", "dep:openshell-driver-podman", + "dep:openshell-policy", "dep:openshell-otel", "dep:hyper-util", "dep:nix", diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 7fdea18943..a63a58068b 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -388,7 +388,20 @@ fn require_guest_tls_for_local_driver( context: &openshell_server::ComputeDriverBuildContext<'_>, driver_name: &str, ) -> openshell_core::Result<()> { - if context.gateway_tls_enabled() && context.guest_tls_paths().is_none() { + validate_local_driver_guest_tls( + context.gateway_tls_enabled(), + context.guest_tls_paths().is_some(), + driver_name, + ) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn validate_local_driver_guest_tls( + gateway_tls_enabled: bool, + has_guest_tls: bool, + driver_name: &str, +) -> openshell_core::Result<()> { + if gateway_tls_enabled && !has_guest_tls { return Err(openshell_core::Error::config(format!( "gateway TLS requires guest_tls_ca, guest_tls_cert, and guest_tls_key in [openshell.gateway] when using the {driver_name} compute driver" ))); @@ -414,6 +427,47 @@ fn apply_guest_tls( } } +#[cfg(all(test, not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +mod local_driver_tests { + use super::{apply_guest_tls, validate_local_driver_guest_tls}; + use std::path::{Path, PathBuf}; + + #[test] + fn tls_enabled_local_drivers_require_a_guest_bundle() { + for driver_name in ["docker", "podman", "vm"] { + let error = validate_local_driver_guest_tls(true, false, driver_name) + .expect_err("TLS-enabled local driver must require guest TLS"); + let message = error.to_string(); + assert!(message.contains(driver_name)); + assert!(message.contains("guest_tls_ca")); + } + validate_local_driver_guest_tls(true, true, "docker") + .expect("a complete guest bundle satisfies the requirement"); + validate_local_driver_guest_tls(false, false, "docker") + .expect("plaintext gateways do not require guest TLS"); + } + + #[test] + fn package_managed_guest_bundle_is_injected_when_driver_paths_are_absent() { + let mut ca = None; + let mut cert = None; + let mut key = None; + apply_guest_tls( + &mut ca, + &mut cert, + &mut key, + Some(( + Path::new("/managed/ca.pem"), + Path::new("/managed/client.pem"), + Path::new("/managed/client-key.pem"), + )), + ); + assert_eq!(ca, Some(PathBuf::from("/managed/ca.pem"))); + assert_eq!(cert, Some(PathBuf::from("/managed/client.pem"))); + assert_eq!(key, Some(PathBuf::from("/managed/client-key.pem"))); + } +} + #[cfg(all(test, target_os = "windows", feature = "in-tree-compute-drivers"))] mod windows_tests { use super::*; diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index c748e53e20..8bdec15f07 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -91,6 +91,12 @@ pub struct VmComputeConfig { /// Writable overlay disk size for each VM sandbox, in MiB. pub overlay_disk_mib: u64, + /// Optional UID override for the VM guest sandbox account. + pub sandbox_uid: Option, + + /// Optional GID override for the VM guest sandbox account. + pub sandbox_gid: Option, + /// Host-side CA certificate for the guest's mTLS client bundle. pub guest_tls_ca: Option, @@ -171,6 +177,8 @@ impl Default for VmComputeConfig { vcpus: Self::default_vcpus(), mem_mib: Self::default_mem_mib(), overlay_disk_mib: Self::default_overlay_disk_mib(), + sandbox_uid: None, + sandbox_gid: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, @@ -474,6 +482,7 @@ pub async fn spawn( )); } + validate_vm_sandbox_identity(vm_config)?; vm_config.upstream_proxy.validate().map_err(Error::config)?; if let Some(endpoint) = vm_config .provider_spiffe_workload_api_tcp_endpoint @@ -523,6 +532,7 @@ pub async fn spawn( command .arg("--overlay-disk-mib") .arg(vm_config.overlay_disk_mib.to_string()); + append_vm_identity_args(&mut command, vm_config); if let Some(tls) = guest_tls_paths { command.arg("--guest-tls-ca").arg(tls.ca); command.arg("--guest-tls-cert").arg(tls.cert); @@ -543,6 +553,35 @@ pub async fn spawn( )) } +fn validate_vm_sandbox_identity(config: &VmComputeConfig) -> Result<()> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + for (field, value) in [ + ("sandbox_uid", config.sandbox_uid), + ("sandbox_gid", config.sandbox_gid), + ] { + if let Some(value) = value + && !range.contains(&value) + { + return Err(Error::config(format!( + "{field} {value} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + ))); + } + } + Ok(()) +} + +#[cfg(unix)] +fn append_vm_identity_args(command: &mut Command, config: &VmComputeConfig) { + if let Some(uid) = config.sandbox_uid { + command.arg("--sandbox-uid").arg(uid.to_string()); + } + if let Some(gid) = config.sandbox_gid { + command.arg("--sandbox-gid").arg(gid.to_string()); + } +} + #[cfg(unix)] fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConfig) { let proxy = &config.upstream_proxy; @@ -659,9 +698,10 @@ async fn connect_compute_driver(socket_path: &Path) -> Result { #[cfg(all(test, unix))] mod tests { use super::{ - VmComputeConfig, append_otlp_args, compute_driver_guest_tls_paths, + VmComputeConfig, append_otlp_args, append_vm_identity_args, compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, + validate_vm_sandbox_identity, }; use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; @@ -697,6 +737,36 @@ mod tests { ); } + #[test] + fn vm_driver_command_includes_configured_sandbox_identity() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + let config = VmComputeConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + validate_vm_sandbox_identity(&config).expect("valid identity"); + append_vm_identity_args(&mut command, &config); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(args, ["--sandbox-uid", "2000", "--sandbox-gid", "3000"]); + } + + #[test] + fn vm_gateway_config_rejects_root_identity() { + let config = VmComputeConfig { + sandbox_uid: Some(0), + ..Default::default() + }; + let error = validate_vm_sandbox_identity(&config).expect_err("root UID must fail"); + assert!(error.to_string().contains("sandbox_uid 0")); + } + #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 9dc10b8401..0a16000999 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -84,7 +84,7 @@ pub struct SandboxJwtIssuer { kid: String, issuer: String, audience: String, - ttl: Duration, + ttl: Option, } impl std::fmt::Debug for SandboxJwtIssuer { @@ -110,10 +110,14 @@ impl SandboxJwtIssuer { signing_key_pem: &[u8], kid: String, gateway_id: &str, - ttl: Duration, + ttl: Option, ) -> Result { crate::install_jsonwebtoken_crypto_provider(); + if ttl.is_some_and(|ttl| ttl.is_zero()) { + return Err("sandbox token TTL must be positive when configured".to_string()); + } + let encoding_key = EncodingKey::from_ed_pem(signing_key_pem) .map_err(|e| format!("failed to parse Ed25519 signing key PEM: {e}"))?; let identity = format!("openshell-gateway:{gateway_id}"); @@ -132,11 +136,9 @@ impl SandboxJwtIssuer { crate::install_jsonwebtoken_crypto_provider(); let now = now_secs(); - let exp = if self.ttl.is_zero() { - 0 - } else { - now.saturating_add(i64::try_from(self.ttl.as_secs()).unwrap_or(3_600)) - }; + let exp = self.ttl.map_or(0, |ttl| { + now.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(3_600)) + }); let claims = SandboxJwtClaims { sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"), iss: self.issuer.clone(), @@ -226,7 +228,7 @@ impl SandboxJwtIssuer { }) } - pub fn ttl(&self) -> Duration { + pub fn sandbox_token_ttl(&self) -> Option { self.ttl } } @@ -417,10 +419,10 @@ mod tests { } fn pair() -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { - pair_with_ttl(Duration::from_secs(3600)) + pair_with_ttl(Some(Duration::from_secs(3600))) } - fn pair_with_ttl(ttl: Duration) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { + fn pair_with_ttl(ttl: Option) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { let mat = generate_jwt_key().expect("jwt key"); let issuer = SandboxJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), @@ -446,6 +448,7 @@ mod tests { async fn mint_and_validate_round_trip() { let (issuer, auth) = pair(); let minted = issuer.mint("sandbox-a").unwrap(); + assert!(minted.expires_at_ms > 0); let principal = auth .authenticate(&header_map_with_bearer(&minted.token), "/anything") .await @@ -472,7 +475,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); let auth = SandboxJwtAuthenticator::from_pem( @@ -514,8 +517,8 @@ mod tests { } #[tokio::test] - async fn ttl_zero_mints_non_expiring_token() { - let (issuer, auth) = pair_with_ttl(Duration::ZERO); + async fn ttl_none_mints_non_expiring_token() { + let (issuer, auth) = pair_with_ttl(None); let minted = issuer.mint("sandbox-never").unwrap(); assert_eq!(minted.expires_at_ms, 0); @@ -537,6 +540,19 @@ mod tests { assert_eq!(decoded.claims.exp, 0); } + #[test] + fn ttl_some_zero_is_rejected() { + let mat = generate_jwt_key().expect("jwt key"); + let error = SandboxJwtIssuer::from_pem( + mat.signing_key_pem.as_bytes(), + mat.kid, + "test-gateway", + Some(Duration::ZERO), + ) + .expect_err("Some(Duration::ZERO) must not reintroduce a sentinel"); + assert!(error.contains("must be positive")); + } + #[tokio::test] async fn token_signed_by_other_key_is_rejected() { let (_, auth_a) = pair(); @@ -582,7 +598,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "g", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); let auth = @@ -613,7 +629,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::ZERO, + None, ) .expect("issuer"); let decoding_key = DecodingKey::from_ed_pem(mat.public_key_pem.as_bytes()).unwrap(); @@ -662,7 +678,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); @@ -692,13 +708,9 @@ mod tests { #[test] fn extension_token_rejects_wrong_audience() { let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid, - "gateway-a", - Duration::ZERO, - ) - .expect("issuer"); + let issuer = + SandboxJwtIssuer::from_pem(mat.signing_key_pem.as_bytes(), mat.kid, "gateway-a", None) + .expect("issuer"); let minted = issuer .mint_extension_token( &extension_audience("service-a"), @@ -734,7 +746,7 @@ mod tests { #[test] fn extension_token_enforces_positive_bounded_ttl_and_caller_shape() { - let (issuer, _) = pair_with_ttl(Duration::ZERO); + let (issuer, _) = pair_with_ttl(None); for ttl in [ Duration::ZERO, MAX_EXTENSION_TOKEN_TTL + Duration::from_secs(1), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index bc5b10039d..b746fb247b 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -1910,6 +1910,9 @@ enable_loopback_service_http = false std::fs::write( &config_path, r#" +[openshell] +version = 2 + [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 082eabc253..d7488270a7 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -194,6 +194,7 @@ fn validate_remote_driver_config(cfg: &RemoteDriverConfig, name: &str) -> Result mod tests { use super::*; use std::collections::BTreeMap; + use std::path::Path; fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = @@ -214,6 +215,111 @@ mod tests { } } + #[test] + fn gateway_guest_tls_resolves_explicit_complete_bundle() { + let dir = tempfile::tempdir().expect("temp dir"); + let ca = dir.path().join("ca.pem"); + let cert = dir.path().join("cert.pem"); + let key = dir.path().join("key.pem"); + for path in [&ca, &cert, &key] { + std::fs::write(path, b"test").expect("write TLS fixture"); + } + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(ca.clone()), + guest_tls_cert: Some(cert.clone()), + guest_tls_key: Some(key.clone()), + ..Default::default() + }; + + let resolved = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect("complete guest TLS should resolve") + .expect("guest TLS bundle"); + + assert_eq!( + resolved.as_paths(), + (ca.as_path(), cert.as_path(), key.as_path()) + ); + } + + #[test] + fn gateway_guest_tls_rejects_every_partial_bundle() { + let path = PathBuf::from("/tmp/guest-tls.pem"); + for (ca, cert, key) in [ + (Some(path.clone()), None, None), + (None, Some(path.clone()), None), + (None, None, Some(path.clone())), + (Some(path.clone()), Some(path.clone()), None), + (Some(path.clone()), None, Some(path.clone())), + (None, Some(path.clone()), Some(path)), + ] { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: ca, + guest_tls_cert: cert, + guest_tls_key: key, + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("partial guest TLS must fail"); + assert!(error.contains("one complete bundle")); + } + } + + #[test] + fn gateway_guest_tls_rejects_missing_explicit_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(dir.path().join("missing-ca.pem")), + guest_tls_cert: Some(dir.path().join("missing-cert.pem")), + guest_tls_key: Some(dir.path().join("missing-key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("missing explicit file must fail"); + assert!(error.contains("guest_tls_ca")); + assert!(error.contains("does not exist")); + } + + #[test] + fn gateway_guest_tls_uses_package_managed_bundle() { + let local = LocalTlsPaths { + ca: PathBuf::from("/managed/ca.pem"), + server_cert: PathBuf::from("/managed/server-cert.pem"), + server_key: PathBuf::from("/managed/server-key.pem"), + client_cert: PathBuf::from("/managed/client-cert.pem"), + client_key: PathBuf::from("/managed/client-key.pem"), + }; + let resolved = GuestTlsPaths::resolve(None, Some(&local), false) + .expect("managed bundle should resolve") + .expect("guest TLS bundle"); + assert_eq!( + resolved.as_paths(), + ( + Path::new("/managed/ca.pem"), + Path::new("/managed/client-cert.pem"), + Path::new("/managed/client-key.pem"), + ) + ); + } + + #[test] + fn gateway_guest_tls_can_be_absent() { + assert!(GuestTlsPaths::resolve(None, None, false).unwrap().is_none()); + assert!(GuestTlsPaths::resolve(None, None, true).unwrap().is_none()); + } + + #[test] + fn gateway_guest_tls_rejects_plaintext_gateway() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: Some(PathBuf::from("/tmp/cert.pem")), + guest_tls_key: Some(PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, true) + .expect_err("guest TLS and plaintext gateway conflict"); + assert!(error.contains("require gateway TLS")); + } + #[test] fn remote_driver_config_reads_socket_path_from_named_table() { let file: config_file::ConfigFile = toml::from_str( @@ -234,8 +340,8 @@ socket_path = "/run/openshell/kyma.sock" fn remote_driver_config_ignores_in_process_driver_fields() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "sandboxes" +[openshell] +version = 2 [openshell.drivers.kubernetes] socket_path = "/run/openshell/kubernetes.sock" diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index edfebbd915..5270247b3e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -965,6 +965,30 @@ ssh_gateway_port = 8080 )); } + #[test] + fn rejects_missing_version_in_nonempty_file() { + let tmp = write_raw_tmp("[openshell]\n\n[openshell.gateway]\nname = \"test\"\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); + } + + #[test] + fn rejects_future_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 3\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::UnsupportedVersion { version: 3 }) + )); + } + + #[test] + fn accepts_current_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 2\n"); + load(tmp.path()).expect("schema version 2 must be accepted"); + } + #[test] fn driver_table_uses_only_driver_owned_values() { let raw = toml::toml! { @@ -1027,14 +1051,11 @@ ssh_gateway_port = 8080 driver selection when Docker is also installed" ); - let podman: openshell_driver_podman::PodmanComputeConfig = - driver_table(config.openshell.drivers.get("podman")) - .try_into() - .expect("RPM Podman settings must deserialize"); + let podman = driver_table(config.openshell.drivers.get("podman")); assert_eq!( podman - .health_check_interval_secs - .map(std::num::NonZeroU64::get), + .get("health_check_interval_secs") + .and_then(toml::Value::as_integer), Some(10), "RPM defaults must retain Podman's readiness health check" ); diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs index 104d639584..35325f3bbf 100644 --- a/crates/openshell-server/src/grpc/auth_rpc.rs +++ b/crates/openshell-server/src/grpc/auth_rpc.rs @@ -226,11 +226,11 @@ fn mint_extension_credentials( .iter() .map(|service| (service.name.as_str(), service)) .collect(); - let ttl = if issuer.ttl().is_zero() { - DEFAULT_EXTENSION_TOKEN_TTL - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - }; + let ttl = issuer + .sandbox_token_ttl() + .map_or(DEFAULT_EXTENSION_TOKEN_TTL, |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }); requested_names .iter() @@ -323,7 +323,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid, "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); state.sandbox_jwt_issuer = Some(Arc::new(issuer)); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index ea72b5bb07..4b64cf40a0 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -97,11 +97,11 @@ struct GatewayExtensionCredential { } fn extension_token_ttl(issuer: &auth::sandbox_jwt::SandboxJwtIssuer) -> Duration { - if issuer.ttl().is_zero() { - Duration::from_secs(15 * 60) - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - } + issuer + .sandbox_token_ttl() + .map_or(Duration::from_secs(15 * 60), |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }) } /// Mint the gateway-caller credential for one extension registration. @@ -496,7 +496,7 @@ pub(crate) async fn run_server( &signing_pem, kid.clone(), &jwt.gateway_id, - jwt.sandbox_token_ttl().unwrap_or_default(), + jwt.sandbox_token_ttl(), ) .map_err(Error::config)?, ); @@ -1605,7 +1605,7 @@ mod tests { BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, + configured_compute_driver, extension_token_ttl, is_benign_tls_handshake_failure, mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ @@ -1651,18 +1651,30 @@ mod tests { } fn extension_test_issuer() -> Arc { + extension_test_issuer_with_ttl(Some(Duration::from_secs(900))) + } + + fn extension_test_issuer_with_ttl( + ttl: Option, + ) -> Arc { let material = openshell_bootstrap::jwt::generate_jwt_key().expect("jwt key"); Arc::new( crate::auth::sandbox_jwt::SandboxJwtIssuer::from_pem( material.signing_key_pem.as_bytes(), material.kid, "gateway-a", - Duration::from_secs(900), + ttl, ) .expect("issuer"), ) } + #[test] + fn non_expiring_sandbox_tokens_use_finite_extension_ttl() { + let issuer = extension_test_issuer_with_ttl(None); + assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(15 * 60)); + } + #[test] fn plaintext_extension_endpoint_is_rejected_unless_explicitly_opted_out() { let issuer = extension_test_issuer(); diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index ebe57ae1bf..7130871f22 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -204,6 +204,9 @@ tests: - notMatchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'guest_tls_(ca|cert|key)\s*=' - it: renders user namespace enablement under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index ce20d28e25..f6889e4fb9 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -34,9 +34,11 @@ prevents unexpected driver selection if Docker is also installed on the host. ### Customizing the configuration -Edit `~/.config/openshell/gateway.toml` directly. The template at -`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime -and is not overwritten by RPM upgrades. +Edit `~/.config/openshell/gateway.toml` directly. The package-owned template at +`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime and +may change during an RPM upgrade. The active user copy is preserved. During a +schema-v2 upgrade, the service replaces only an exact package-generated v1 +copy; it never rewrites an edited configuration. To apply environment variable overrides that persist across upgrades without editing the TOML file, add them to `~/.config/openshell/gateway.env`: diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index a8460a473e..58de9e7856 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -227,9 +227,14 @@ non-functional until restarted, causing the gateway to fail with a connection error on `/run/user//podman/podman.sock`. The gateway retries briefly on startup, but a stale socket will not recover on its own. -Package upgrades do not overwrite `~/.config/openshell/gateway.toml` when you -create one. New gateway process options can be added manually by referencing -CONFIGURATION.md or running `openshell-gateway --help`. +Package upgrades preserve edited `~/.config/openshell/gateway.toml` files. On +the schema-v2 upgrade, the user service replaces only an exact copy of the v1 +file previously seeded by the RPM. If you edited that file, migrate it manually +before restarting the service; direct `dnf` or `rpm` upgrades do not use the +breaking-upgrade guard in `install.sh`. See the +[Gateway Configuration File](https://docs.nvidia.com/openshell/latest/reference/gateway-config#migrate-to-schema-version-2) +for the field-by-field migration steps. New gateway process options are listed +in CONFIGURATION.md and `openshell-gateway --help`. To pick up new container images after an upgrade: @@ -238,6 +243,17 @@ podman pull ghcr.io/nvidia/openshell/supervisor:latest podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ``` +### Migrating a TLS-enabled local driver to schema version 2 + +Docker, Podman, and VM sandboxes connect back to the gateway with a guest TLS +bundle. Package-managed installs use the complete bundle generated under +`~/.local/state/openshell/tls`, so the RPM default requires no additional TOML. +If you override the listener with custom `--tls-cert` and `--tls-key` inputs and +do not use that managed bundle, configure all three `guest_tls_ca`, +`guest_tls_cert`, and `guest_tls_key` paths under `[openshell.gateway]`. The +gateway now fails at startup instead of allowing sandboxes to fail later. Omit +all three fields when TLS is disabled. + ### Migrating from gateway.env Previous releases generated `~/.config/openshell/gateway.env` on first diff --git a/deploy/rpm/gateway.toml.default.v1 b/deploy/rpm/gateway.toml.default.v1 new file mode 100644 index 0000000000..ba76f873b2 --- /dev/null +++ b/deploy/rpm/gateway.toml.default.v1 @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Default gateway configuration for RPM installs. +# +# This file is seeded to ~/.config/openshell/gateway.toml on first start +# of the openshell-gateway.service systemd user unit. Edit that copy to +# customize. This file is not read directly at runtime. +# +# Configuration precedence (highest to lowest): +# CLI flag > OPENSHELL_* env var > TOML file > built-in default +# +# To override settings without editing this file, set OPENSHELL_* variables +# in ~/.config/openshell/gateway.env or run: +# systemctl --user edit openshell-gateway + +[openshell] +version = 1 + +[openshell.gateway] +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. + +# Pin to the Podman compute driver. Without this, the gateway auto-detects +# in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver +# selection if Docker is also installed on the host. +compute_driver = "podman" diff --git a/deploy/rpm/migrate-gateway-config.sh b/deploy/rpm/migrate-gateway-config.sh new file mode 100755 index 0000000000..5c75af1f9f --- /dev/null +++ b/deploy/rpm/migrate-gateway-config.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +if [ "$#" -ne 3 ]; then + echo "usage: $0 DESTINATION CURRENT_DEFAULT LEGACY_DEFAULT" >&2 + exit 2 +fi + +destination=$1 +current_default=$2 +legacy_default=$3 + +for source in "$current_default" "$legacy_default"; do + if [ ! -f "$source" ]; then + echo "gateway config migration source is not a regular file: $source" >&2 + exit 1 + fi +done + +if [ -L "$destination" ] || { [ -e "$destination" ] && [ ! -f "$destination" ]; }; then + echo "refusing to replace non-regular gateway config: $destination" >&2 + exit 1 +fi + +if [ ! -e "$destination" ]; then + install -Dm 0644 "$current_default" "$destination" + exit 0 +fi + +# Replace only the exact config seeded by the schema-v1 RPM. Any edit, +# including whitespace or comments, makes the operator-owned file authoritative. +if ! cmp -s "$legacy_default" "$destination"; then + exit 0 +fi + +destination_dir=$(dirname "$destination") +temporary=$(mktemp "$destination_dir/.gateway.toml.XXXXXX") +trap 'rm -f "$temporary"' EXIT HUP INT TERM +install -m 0644 "$current_default" "$temporary" +mv -f "$temporary" "$destination" +trap - EXIT HUP INT TERM diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 9026f939a6..7064fe7a56 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -44,7 +44,7 @@ For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sand On macOS, the install script uses Homebrew. The Homebrew package installs the `openshell` CLI, the gateway binary, and a Homebrew-managed gateway service. -The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew preserves user-edited prefix and user configs during upgrades; it removes the IPv6 bind only from an unchanged config generated by the affected formula. +The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew upgrades migrate exact package-generated schema-v1 prefix configs, including the affected IPv6 variant. They preserve edited prefix configs and all user configs. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) for an edited v1 file. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. @@ -63,7 +63,7 @@ On Debian and Ubuntu, the install script uses a Debian package. The Debian packa Linux packages require glibc 2.28 or newer. The installer checks libc before downloading packages and exits with an error on older glibc versions, Alpine, musl-based distributions, or unknown libc environments. -The Linux user service listens on `https://127.0.0.1:17670`, starts from built-in defaults, and generates a local mTLS bundle before the gateway starts. Create `~/.config/openshell/gateway.toml` only when you need to override those defaults. +The Linux user service listens on `https://127.0.0.1:17670` and generates a local mTLS bundle before the gateway starts. Debian uses built-in gateway defaults unless you create a config. RPM seeds `~/.config/openshell/gateway.toml` from its packaged Podman template on first start. RPM upgrades migrate only an unchanged package-generated schema-v1 file; they preserve edited files. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) when upgrading an edited v1 configuration. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 42350b03d8..7a0a4e4156 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -22,18 +22,20 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ## Package-Managed Locations -Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. +Package-managed gateways use either built-in defaults or a package-seeded TOML file. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. -| Package | Optional Gateway TOML location | +| Package | Gateway TOML location | |---|---| | Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | -| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | +| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. Upgrades preserve user-edited configs and migrate only an unchanged prefix config generated with the affected IPv6-loopback default. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. + +Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. ## Layout @@ -74,7 +76,10 @@ future version. To migrate an existing file: 3. Move every compute-driver option into `[openshell.drivers.]`. Schema version 2 does not inherit driver defaults from `[openshell.gateway]`. Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway - scope; set all three or omit all three when TLS is disabled. + scope. A TLS-enabled Docker, Podman, or VM gateway requires one complete + guest bundle. Set all three paths unless the package-managed local TLS + bundle supplies them. When TLS is disabled, omit all three. Kubernetes + projects sandbox TLS through `client_tls_secret_name` instead. 4. Rename Docker `sandbox_namespace` to `sandbox_label`, Podman `sandbox_ssh_socket_path` to `ssh_socket_path`, and VM `openshell_endpoint` to `grpc_endpoint`. @@ -90,7 +95,8 @@ future version. To migrate an existing file: TOML requires an explicit endpoint; Helm derives one from the release's gateway Service. New VM root filesystems use an image-provided `sandbox` account when present and otherwise use UID/GID 1000. Existing persisted VM - state using 10001 remains compatible. + state retains its recorded or recoverable identity, including legacy 10001; + the driver does not assign 10001 to an overlay without supporting state. Unknown fields and non-table `[openshell.drivers.]` values fail startup. This strict validation prevents misspelled or misplaced security-sensitive @@ -139,8 +145,11 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Guest TLS paths remain gateway settings. Set all three for TLS, or omit all -# three only when TLS is disabled. Driver tables must not repeat these fields. +# Guest TLS paths remain gateway settings. TLS-enabled Docker, Podman, and VM +# gateways require a complete bundle unless package-managed local TLS supplies +# it automatically. Omit all three when TLS is disabled. Kubernetes projects +# sandbox TLS from client_tls_secret_name instead. Driver tables must not repeat +# these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -832,13 +841,18 @@ health_check_interval_secs = 10 # explicit container-reachable TCP endpoint, for provider token exchange. # provider_spiffe_workload_api_socket = "/run/spire/agent.sock" # provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" -# Explicit supervisor-compatible default. RuntimeDefault and Localhost/ -# require Podman to report AppArmor support. -app_armor_profile = "Unconfined" +# Omit app_armor_profile to preserve Podman's runtime-selected profile. +# Set Unconfined only when the supervisor's mount setup requires it. +# Explicit RuntimeDefault and Localhost/ require Podman to report +# AppArmor support. +# app_armor_profile = "Unconfined" ``` Use `ssh_socket_path` for Podman configurations. The legacy -`sandbox_ssh_socket_path` key is rejected. +`sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, +OpenShell sends no override and Podman applies its runtime-selected profile. +Set `Unconfined` explicitly only when the deployment requires the supervisor's +mount setup to bypass that profile. ### MicroVM @@ -873,10 +887,12 @@ vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 # Resolved sandbox UID/GID for new rootfs /etc/passwd entries. -# Defaults to 1000 when unset; matching GID is used if sandbox_gid is empty. -# Existing persisted VM rootfs/overlays with UID 10001 retain that identity. -# Any non-root Linux UID/GID is valid. +# Defaults to the image's sandbox account, or 1000 when the account is absent; +# matching GID is used if sandbox_gid is empty. Persisted overlays recover their +# recorded identity, including 10001, rather than receiving a legacy fallback. +# Values must fall within OpenShell's allowed non-root sandbox identity range. # sandbox_uid = 20001 +# sandbox_gid = 20001 # Corporate TLS egress proxy. These settings are converted to protected # supervisor argv inside the guest; workload environment cannot override them. # https_proxy = "https://proxy.corp.example:8443" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 3f832f0869..da3913d261 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -64,7 +64,7 @@ Common gateway options: |---|---| | `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. For gateway-managed Docker, Podman, and VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` together in `[openshell.gateway]`; driver tables reject those gateway-owned fields. See the [Gateway Configuration File](./gateway-config) reference for the full schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. A TLS-enabled gateway-managed Docker, Podman, or VM driver requires a complete `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` bundle in `[openshell.gateway]`; package-managed local TLS supplies it automatically. Driver tables reject those gateway-owned fields. Kubernetes projects guest TLS through a Secret instead. See the [Gateway Configuration File](./gateway-config) reference for the full schema and migration steps. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -255,6 +255,12 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. +Podman preserves its runtime-selected AppArmor profile when +`app_armor_profile` is omitted. Set `Unconfined` explicitly only when the +supervisor's mount setup requires it. Explicit `RuntimeDefault` and +`Localhost/` selections fail startup when Podman reports that AppArmor +is unavailable. + On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts @@ -585,7 +591,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created so a driver upgrade does not rewrite their ownership contract. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created. An unmarked overlay recovers identity from concrete overlay or prepared-image state, an explicit override, or the current image; the driver never assigns legacy `10001:10001` without persisted evidence. ### Custom Images diff --git a/openshell.spec b/openshell.spec index 3200659d73..a9327f1c3e 100644 --- a/openshell.spec +++ b/openshell.spec @@ -146,6 +146,8 @@ fi # Shipped as a read-only reference in %{_datadir}. The systemd unit seeds a # user-level copy at ~/.config/openshell/gateway.toml on first start. install -Dpm 0644 deploy/rpm/gateway.toml.default %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default +install -Dpm 0644 deploy/rpm/gateway.toml.default.v1 %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default.v1 +install -Dpm 0755 deploy/rpm/migrate-gateway-config.sh %{buildroot}%{_libexecdir}/%{name}-gateway-migrate-config # --- Gateway systemd user unit --- # Installed to the systemd user unit directory so any user can run: @@ -165,11 +167,10 @@ Type=exec # the CLI discovers them automatically. # See /usr/share/doc/openshell-gateway/ for details. -# Seed a default TOML config on first start if the user has not created one. -# The template ships at /usr/share/openshell-gateway/gateway.toml.default. -# Edit ~/.config/openshell/gateway.toml to customize. +# Seed a default TOML config on first start. On upgrade, replace only the exact +# schema-v1 config previously seeded by this package; preserve edited files. # %%E expands to $XDG_CONFIG_HOME (~/.config) in user units. -ExecStartPre=/bin/sh -c 'test -f %%E/openshell/gateway.toml || install -Dm644 /usr/share/openshell-gateway/gateway.toml.default %%E/openshell/gateway.toml' +ExecStartPre=%{_libexecdir}/%{name}-gateway-migrate-config %%E/openshell/gateway.toml /usr/share/openshell-gateway/gateway.toml.default /usr/share/openshell-gateway/gateway.toml.default.v1 # Auto-generate PKI on first start if not present. # The default local TLS dir uses %%h because %%S resolves differently across @@ -253,10 +254,12 @@ PYTHONPATH=%{buildroot}%{python3_sitelib} %{python3} -c "from importlib.metadata # A missing template means first-start seeding silently falls back to the # binary default of 127.0.0.1, which breaks Podman sandbox connectivity. test -f %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default +test -f %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default.v1 +test -x %{buildroot}%{_libexecdir}/%{name}-gateway-migrate-config -# Verify the systemd unit references the template in its ExecStartPre seed step. -# If this grep fails, the first-start seeding logic was removed from the unit. -grep -q 'gateway.toml.default' %{buildroot}%{_userunitdir}/%{name}-gateway.service +# Verify the systemd unit invokes exact-default migration before startup. +grep -q '%{name}-gateway-migrate-config' %{buildroot}%{_userunitdir}/%{name}-gateway.service +grep -q 'gateway.toml.default.v1' %{buildroot}%{_userunitdir}/%{name}-gateway.service %post gateway %systemd_user_post %{name}-gateway.service @@ -284,7 +287,9 @@ grep -q 'gateway.toml.default' %{buildroot}%{_userunitdir}/%{name}-gateway.servi %doc %{_docdir}/%{name}-gateway/TROUBLESHOOTING.md %{_bindir}/%{name}-gateway %{_userunitdir}/%{name}-gateway.service +%{_libexecdir}/%{name}-gateway-migrate-config %{_datadir}/%{name}-gateway/gateway.toml.default +%{_datadir}/%{name}-gateway/gateway.toml.default.v1 %{_mandir}/man8/openshell-gateway.8* %files -n python3-%{name} diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index 49d18f12d2..0d27c57311 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -63,9 +63,28 @@ def test_generate_homebrew_formula_uses_tagged_macos_driver_asset_without_defaul flags=re.DOTALL, ) assert generated_config is not None + assert "version = 2" in generated_config.group("contents") assert "[openshell.gateway]" in generated_config.group("contents") assert "bind_address =" not in generated_config.group("contents") - assert 'bind_address = "[::1]:17670"' in formula + + legacy_empty_config = re.search( + r"legacy_empty_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_empty_config is not None + assert "version = 1" in legacy_empty_config.group("contents") + assert "bind_address =" not in legacy_empty_config.group("contents") + + legacy_ipv6_config = re.search( + r"legacy_ipv6_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_ipv6_config is not None + assert "version = 1" in legacy_ipv6_config.group("contents") + assert 'bind_address = "[::1]:17670"' in legacy_ipv6_config.group("contents") + assert "gateway_config.read == legacy_empty_gateway_config_contents ||" in formula assert "gateway_config.read == legacy_ipv6_gateway_config_contents" in formula assert "gateway_config.write gateway_config_contents" in formula assert '# compute_driver = "vm"' not in formula @@ -130,12 +149,15 @@ def test_snap_wrapper_uses_optional_gateway_config_without_generating_toml() -> assert 'exec "${SNAP}/bin/openshell-gateway" "$@"' in wrapper -def test_rpm_spec_uses_gateway_defaults_without_config_helper() -> None: +def test_rpm_spec_seeds_and_migrates_gateway_defaults() -> None: repo_root = Path(__file__).resolve().parents[2] spec = (repo_root / "openshell.spec").read_text(encoding="utf-8") assert "init-gateway-config.sh" not in spec assert "init-pki.sh" not in spec + assert "migrate-gateway-config.sh" in spec + assert "gateway.toml.default.v1" in spec + assert "%{name}-gateway-migrate-config" in spec assert "Environment=OPENSHELL_LOCAL_TLS_DIR=%%h/.local/state/openshell/tls" in spec assert ( "openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR}" diff --git a/python/openshell/rpm_gateway_config_migration_test.py b/python/openshell/rpm_gateway_config_migration_test.py new file mode 100644 index 0000000000..af846678f3 --- /dev/null +++ b/python/openshell/rpm_gateway_config_migration_test.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +MIGRATOR = REPO_ROOT / "deploy/rpm/migrate-gateway-config.sh" +CURRENT = REPO_ROOT / "deploy/rpm/gateway.toml.default" +LEGACY = REPO_ROOT / "deploy/rpm/gateway.toml.default.v1" + + +def run_migrator( + destination: Path, *, check: bool = True +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["sh", str(MIGRATOR), str(destination), str(CURRENT), str(LEGACY)], + check=check, + text=True, + capture_output=True, + ) + + +def test_migrator_seeds_missing_config_and_is_idempotent(tmp_path: Path) -> None: + destination = tmp_path / "config/openshell/gateway.toml" + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_replaces_only_exact_legacy_default(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + destination.write_bytes(LEGACY.read_bytes()) + + run_migrator(destination) + + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_preserves_edited_legacy_and_current_configs(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + edited = LEGACY.read_text(encoding="utf-8") + "# operator edit\n" + destination.write_text(edited, encoding="utf-8") + + run_migrator(destination) + assert destination.read_text(encoding="utf-8") == edited + + destination.write_bytes(CURRENT.read_bytes()) + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_refuses_symlink_destination(tmp_path: Path) -> None: + target = tmp_path / "target.toml" + target.write_text("operator-owned\n", encoding="utf-8") + destination = tmp_path / "gateway.toml" + destination.symlink_to(target) + + result = run_migrator(destination, check=False) + + assert result.returncode != 0 + assert "non-regular gateway config" in result.stderr + assert target.read_text(encoding="utf-8") == "operator-owned\n" diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 57d20b30d8..77632cb0a4 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -8,16 +8,16 @@ state: implemented ## Summary -Introduce a TOML-based configuration file for the OpenShell gateway that unifies all gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file, while preserving full backwards compatibility with the existing CLI flags and `OPENSHELL_*` environment variables. +Introduce a TOML-based configuration file for the OpenShell gateway that unifies gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file. CLI flags and supported `OPENSHELL_*` environment variables retain higher precedence. Schema version 2 intentionally rejects legacy file fields and locations. ## Motivation -The gateway today is configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This works for simple single-node deployments but breaks down as deployments grow: +Before this RFC, the gateway was configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This worked for simple single-node deployments but broke down as deployments grew: -- **Too many flags** — the gateway has ~40 configurable parameters today (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests are hard to read, diff, and audit. -- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers all live in the same flat CLI namespace, with no structural separation. Most flags only apply to one driver, but there is no way to express that in CLI form. -- **Helm friction** — The chart's `statefulset.yaml` already carries a long `env:` block of `OPENSHELL_*` variables that each map to a `values.yaml` key. A config file can be mounted as a single `ConfigMap` and reduces the chart's templating surface significantly. -- **Secrets management** — Injecting secrets (TLS material paths, database URL, OIDC settings) via environment variables is functional but not idiomatic for Kubernetes. A file-based format opens the door to projected secrets and volume mounts that compose cleanly with the non-secret config. +- **Too many flags** — the gateway exposed roughly 40 configurable parameters (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests were hard to read, diff, and audit. +- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers shared one flat CLI namespace with no structural separation. Most flags applied to only one driver, but CLI syntax did not express that ownership. +- **Helm friction** — The chart's `statefulset.yaml` carried a long `env:` block of `OPENSHELL_*` variables that each mapped to a `values.yaml` key. A mounted configuration file reduces the chart's templating surface. +- **Secrets management** — Environment-only configuration did not compose naturally with Kubernetes `ConfigMap` and projected `Secret` volumes. ## Non-goals @@ -87,10 +87,10 @@ server_sans = ["openshell", "*.dev.openshell.localhost"] enable_loopback_service_http = true # ────────────────────────────────────────────────────────────────────────────── -# TLS / mTLS — when omitted, the gateway listens plaintext (sets --disable-tls) +# TLS / mTLS — package-managed local TLS may supply listener defaults. # ────────────────────────────────────────────────────────────────────────────── -# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. When true, the gateway -# ignores the [openshell.gateway.tls] table below. +# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. Set true explicitly for a +# plaintext listener; guest TLS fields must then be omitted. disable_tls = false # Gateway-owned TLS bundle injected into the selected local driver. @@ -202,9 +202,9 @@ Deserialization uses `#[serde(deny_unknown_fields)]` at every table level. An un The following cross-field validations are applied after merging file + env + CLI: - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. -- When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. +- Gateway listener TLS requires `cert_path` and `key_path`; `client_ca_path` is required only for listener client-certificate verification. TLS-enabled Docker, Podman, and VM drivers also require a complete gateway-owned guest CA, certificate, and key bundle. Kubernetes projects guest TLS through a Secret instead. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list is rejected. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver requires a named table with `socket_path`, unless startup supplies an explicit socket override. The legacy `compute_drivers` list is rejected. ### Schema compatibility @@ -220,7 +220,8 @@ version = 2 bind_address = "0.0.0.0:8080" compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). -# No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). +# The gateway runs plaintext behind Envoy / ingress. +disable_tls = true [openshell.drivers.kubernetes] namespace = "agents" @@ -231,12 +232,7 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ### Helm integration -The Helm chart today renders a long `env:` block in `templates/statefulset.yaml`, with each `OPENSHELL_*` variable mapped to a `values.yaml` key. This RFC's adoption replaces that block with: - -1. A new `gateway.config` value tree (TOML-shaped YAML) in `values.yaml`. -2. A new `ConfigMap` template that renders the values into a TOML document via Helm's `tpl`. -3. A volume mount of the `ConfigMap` at `/etc/openshell/gateway.toml` and a `--config` flag in the gateway container's `args`. -4. Continued use of a `Secret`-backed `env:` entry for `OPENSHELL_DB_URL` (which never lives in the `ConfigMap`), plus optional projections for TLS material paths. The CLI/env precedence above means any `Secret`-backed env var also wins over a value in the `ConfigMap`. +The Helm chart renders schema-v2 gateway TOML into a `ConfigMap`, mounts it at `/etc/openshell/gateway.toml`, and starts the gateway with that file. Secret process inputs such as `OPENSHELL_DB_URL` remain `Secret`-backed environment entries and retain higher precedence. Kubernetes projects sandbox guest TLS through its configured Secret rather than placing host guest-certificate paths in the gateway TOML. ```yaml # values.yaml excerpt @@ -255,23 +251,15 @@ gateway: The chart owners can migrate one section at a time: `OPENSHELL_*` env vars and the `ConfigMap` coexist during the transition, with env continuing to override the file. -## Implementation plan - -No part of this RFC has shipped yet. The work breaks down as: +## Implementation -1. **Add a config-file loader to `openshell-server`** — define a `GatewayConfigFile` struct that mirrors the schema above, parse it with `serde` + `toml`, and merge it into `openshell_core::Config` plus the per-driver structs in `compute/`. -2. **Wire the merge into `cli.rs`** — add `--config` / `OPENSHELL_GATEWAY_CONFIG`, gate each existing flag's "apply from file" path on clap `ValueSource::DefaultValue`, and run cross-field validation after the merge. -3. **Per-driver deserialization** — give each driver crate (`openshell-driver-{kubernetes,docker,podman,vm}`) a `from_toml` (or `serde::Deserialize`) entry point so the gateway can hand each driver its own table. -4. **Test coverage** — file parsing, env-overrides-file, CLI-overrides-env, partial TLS error, port-collision error, unknown-field rejection, missing driver table fallback. -5. **Helm chart migration** — add `gateway.config` value tree, render the `ConfigMap`, mount it, switch the gateway container to `--config`. Keep the `OPENSHELL_*` env names available as opt-in overrides for secrets. -6. **Example file** — ship the per-driver examples on the published docs reference at `docs/reference/gateway-config.mdx`. -7. **Architecture doc update** — reflect the new config sources and precedence in `architecture/gateway.md`. +The implemented gateway loader parses TOML with `serde`, merges file values below environment and CLI sources, and rejects unknown fields. Each compute driver deserializes only its named table. Helm renders schema-v2 TOML into a ConfigMap, while secret process inputs remain environment-backed. Package templates, examples, tests, and the gateway architecture documentation use the same canonical schema. ## Risks -- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Mitigate by treating field renames as breaking, keeping the `version` field reserved for schema migrations, and surfacing rename errors clearly. +- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Treat field renames as versioned schema changes and surface migration errors clearly. - **Secrets in the file** — `database_url` is excluded from the schema entirely (env / CLI only). OIDC settings remain allowed in the file because none of them are credentials in isolation. Operators should still prefer env-var injection for any field that would live in a `Secret` rather than a `ConfigMap` (TLS material paths, restricted-environment OIDC issuers, etc.). Documentation must call this out prominently. -- **Partial TLS configuration** — the hard error on partial TLS config is the right UX, but the error message must clearly identify which source (file vs. CLI/env) is missing which field, since the file's `[openshell.gateway.tls]` table is all-or-nothing while the CLI flags are independent. +- **Partial TLS configuration** — listener and guest TLS are separate complete-bundle contracts. Startup rejects partial bundles and identifies the missing configuration before constructing a driver. - **Driver schema drift** — once each driver owns its own TOML table, driver releases can change field names independently of the gateway. The gateway's `version` field does not protect against driver-side breakage; document driver-config stability separately. ## Alternatives @@ -290,8 +278,7 @@ No part of this RFC has shipped yet. The work breaks down as: ## Open questions -1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. -2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. +1. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. -3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. + Deferred to a follow-on: the single `--config` file is sufficient for the current schema, and the directory loader can be added without changing the file schema. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. +2. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). Schema version 2 allows the listed OIDC fields because they are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index d6c6e1efe6..188c698962 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -24,12 +24,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index d1d86ad4a2..4d990d629f 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -21,12 +21,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -223,6 +225,9 @@ ttl_secs = 3600 default_image = "${SANDBOX_IMAGE}" supervisor_image = "${SUPERVISOR_IMAGE}" image_pull_policy = "${SANDBOX_IMAGE_PULL_POLICY}" +# Local development requires supervisor mount setup that Podman's runtime +# profile may deny. Production configs preserve Podman's default when omitted. +app_armor_profile = "Unconfined" health_check_interval_secs = 10 EOF diff --git a/tasks/scripts/gateway-pull-policy.sh b/tasks/scripts/gateway-pull-policy.sh new file mode 100755 index 0000000000..7c0d1bd741 --- /dev/null +++ b/tasks/scripts/gateway-pull-policy.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Normalize compatibility inputs at the development-script boundary while +# keeping schema-v2 TOML and backend validation strict. +normalize_image_pull_policy() { + local value + value="$(printf '%s' "${1:-}" | LC_ALL=C tr '[:upper:]' '[:lower:]')" + case "${value}" in + always) + printf '%s\n' "always" + ;; + if_not_present|ifnotpresent|missing) + printf '%s\n' "if_not_present" + ;; + never) + printf '%s\n' "never" + ;; + newer) + printf '%s\n' "newer" + ;; + *) + printf 'unsupported image pull policy: %s\n' "${1:-}" >&2 + return 2 + ;; + esac +} diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index da7f91fb68..300b6a8c7e 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -16,6 +16,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" usage() { @@ -206,7 +208,7 @@ GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" diff --git a/tasks/scripts/release.py b/tasks/scripts/release.py index 29a503567a..3b0afe5ecd 100644 --- a/tasks/scripts/release.py +++ b/tasks/scripts/release.py @@ -416,9 +416,17 @@ def post_install [openshell.gateway] TOML + # These are the only v1 configurations emitted by pre-schema-v2 formulas. + # Do not migrate a configuration unless it exactly matches one of them. + legacy_empty_gateway_config_contents = <<~TOML + [openshell] + version = 1 + + [openshell.gateway] + TOML legacy_ipv6_gateway_config_contents = <<~TOML [openshell] - version = 2 + version = 1 [openshell.gateway] bind_address = "[::1]:{LOCAL_GATEWAY_PORT}" @@ -426,9 +434,9 @@ def post_install unless gateway_config.exist? gateway_config.write gateway_config_contents else - # Migrate only the exact config generated by the affected formula. Keep - # any user-edited config untouched. - if gateway_config.read == legacy_ipv6_gateway_config_contents + # Keep any user-edited config untouched. + if gateway_config.read == legacy_empty_gateway_config_contents || + gateway_config.read == legacy_ipv6_gateway_config_contents gateway_config.write gateway_config_contents end end diff --git a/tasks/scripts/test-gateway-pull-policy.sh b/tasks/scripts/test-gateway-pull-policy.sh new file mode 100755 index 0000000000..267c302fd0 --- /dev/null +++ b/tasks/scripts/test-gateway-pull-policy.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" + +assert_policy() { + local input=$1 + local expected=$2 + local actual + actual="$(normalize_image_pull_policy "${input}")" + if [[ "${actual}" != "${expected}" ]]; then + printf 'expected %q -> %q, got %q\n' "${input}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +for input in always Always ALWAYS; do + assert_policy "${input}" always +done +for input in if_not_present IfNotPresent ifnotpresent IFNOTPRESENT missing MISSING; do + assert_policy "${input}" if_not_present +done +for input in never Never NEVER; do + assert_policy "${input}" never +done +for input in newer Newer NEWER; do + assert_policy "${input}" newer +done + +if normalize_image_pull_policy sometimes >/dev/null 2>&1; then + echo "unsupported policy unexpectedly succeeded" >&2 + exit 1 +fi + +for script in gateway.sh gateway-docker.sh gateway-podman.sh; do + if ! grep -q 'normalize_image_pull_policy' "${ROOT}/tasks/scripts/${script}"; then + echo "${script} does not normalize image pull policy" >&2 + exit 1 + fi +done + +echo "gateway pull-policy tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index 2fc3c565e4..6e9f05d2d0 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -12,6 +12,7 @@ depends = [ "test:sbom", "test:install-sh", "test:build-env", + "test:gateway-pull-policy", "test:packaging-assets", "test:codex-security-release-range", "test:docs-website", @@ -40,6 +41,12 @@ run = "tasks/scripts/test-build-env.sh" run_windows = "echo Skipping test:build-env: the Unix build-env.sh helper does not apply on Windows." hide = true +["test:gateway-pull-policy"] +description = "Test development gateway image pull-policy normalization" +run = "tasks/scripts/test-gateway-pull-policy.sh" +run_windows = "echo Skipping test:gateway-pull-policy: Unix gateway scripts do not apply on Windows." +hide = true + ["test:packaging-assets"] description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh"