diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 47588b1aeb..8bcd7c5d9e 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -82,12 +82,26 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash -rg -n 'compute_drivers|socket_path' /etc/openshell/gateway.toml +rg -n '^version|compute_driver|socket_path|guest_tls_' /etc/openshell/gateway.toml stat /run/openshell/.sock journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 ``` +Gateway configuration requires `[openshell] version = 2`, a singular +`compute_driver` selector, and driver-owned settings under +`[openshell.drivers.]`. The gateway rejects legacy `compute_drivers`, +`--drivers`, and `OPENSHELL_DRIVERS` selectors rather than silently migrating +them. Homebrew and RPM package startup migrates only exact package-generated v1 +defaults. If an upgraded package still reports an unsupported version, inspect +the active prefix or `~/.config/openshell/gateway.toml`; an edited v1 file must +follow the published schema-v2 migration steps and must not be overwritten. +Guest TLS CA, certificate, and key paths are the exception to driver ownership: +configure the complete bundle under `[openshell.gateway]`, and the gateway +injects it only into the selected local driver. TLS-enabled Docker, Podman, and +VM drivers fail startup when neither those paths nor the package-managed local +bundle is available; Kubernetes projects its bundle through a Secret. + Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. For configured gateway interceptors, inspect `[[openshell.gateway.interceptors]]`, their Unix or network endpoints, and gateway startup logs: @@ -620,6 +634,11 @@ Use the VM driver logs and host diagnostics available in the user's environment. - The VM driver process is running and reachable by the gateway. - The runtime rootfs exists and matches the expected architecture. +- `mke2fs` or `mkfs.ext4` and `debugfs` from e2fsprogs are installed; explicit + `sandbox_uid`/`sandbox_gid` does not remove this prerequisite. +- A persisted overlay identity error is resolved from its owner marker, overlay + upper layer, prepared rootfs, explicit config, or current image. Do not assign + `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. - The sandbox supervisor can establish its callback connection to the gateway. diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 5e8bbf394c..f70489ea28 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -23,6 +23,10 @@ does not contribute to product usage metrics. `install.sh` defaults to the *latest tagged* release — the canary is therefore checking that the most recent public release still installs, not the just-published `dev` build. The `kubernetes` job is the exception: it pins to `0.0.0-dev` chart + `:dev` images. +The host-package jobs exercise fresh installs, not upgrades from a persisted +schema-v1 gateway config. Validate Homebrew and RPM exact-default migration with +the release-tooling and package lifecycle tests before relying on the canary. + The canary does not install or import `@nvidia/openshell-sdk`. TypeScript SDK validation lives in the `TypeScript SDK` branch check, including a publish dry-run. The tagged release workflow publishes the package to GitHub Packages; @@ -121,7 +125,7 @@ Loopback registration auto-derives the gateway name to `openshell` if `--name` i | Symptom | Likely cause | Where to look | |---|---|---| | `macos`/`ubuntu`/`fedora` job fails on `install.sh` | Latest tagged release missing an asset, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. | -| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_DRIVERS` env in the "Ensure …" step. | +| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_COMPUTE_DRIVER` env in the "Ensure …" step. | | `kubernetes` job fails on `helm install --wait` | Chart did not deploy in 5 min — usually image pull failure or readiness probe failing. | "Diagnostics on failure" step dumps `helm status`, manifest, pod describe, pod logs. | | `kubernetes` job fails on `kubectl wait` | Gateway pod stuck `CrashLoopBackOff` or `ImagePullBackOff`. | Diagnostics dump; check `:dev` image existence at `ghcr.io/nvidia/openshell/gateway`. | | `kubernetes` job fails on `openshell gateway add` or `status` | Port-forward not reachable, or CLI/gateway proto mismatch. | `port-forward.log` and `openshell gateway list` in the diagnostics dump. | diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 937e774db7..71a2a52e93 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Ensure VM driver run: | - launchctl setenv OPENSHELL_DRIVERS vm + launchctl setenv OPENSHELL_COMPUTE_DRIVER vm launchctl setenv OPENSHELL_TELEMETRY_ENABLED "$OPENSHELL_TELEMETRY_ENABLED" - name: Install and check status @@ -48,7 +48,7 @@ jobs: fi sudo systemctl start docker || sudo service docker start mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" docker info @@ -140,7 +140,7 @@ jobs: bash -s <<'EOF' set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" podman info curl -LsSf "${INSTALL_SH_URL}" | sh @@ -276,7 +276,7 @@ jobs: run: | set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\n' > "${HOME}/.config/openshell/gateway.env" + printf 'OPENSHELL_COMPUTE_DRIVER=docker\n' > "${HOME}/.config/openshell/gateway.env" curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh - name: Register kind gateway and check status diff --git a/Cargo.lock b/Cargo.lock index 90240d6b3a..3feba7cde5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4129,6 +4129,7 @@ dependencies = [ "openshell-driver-mxc", "openshell-driver-podman", "openshell-otel", + "openshell-policy", "openshell-server", "rustix 1.1.4", "serde", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..642374ba77 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -127,10 +127,11 @@ defines the available implementation set, while the runtime consumes a generic registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry -to `run_cli_with_compute_drivers`; factories receive merged driver config and -return either an in-process driver or a gateway-managed remote endpoint. The -server constructs the common runtime adapter and snapshots `GetCapabilities` -for either result. A configured UDS endpoint still takes precedence over a +to `run_cli_with_compute_drivers`; factories receive only the selected +`[openshell.drivers.]` table and return either an in-process driver or a +gateway-managed remote endpoint. The server constructs the common runtime +adapter and snapshots `GetCapabilities` for either result. A configured UDS +endpoint still takes precedence over a compiled registration with the same name. The `openshell-gateway` composition crate groups first-party registrations @@ -251,7 +252,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | -| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--compute-driver ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. @@ -288,10 +289,23 @@ pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise they implement and validate the same complete contract. The capability marker is driver-owned supervisor input and is removed from workload environments. -Kubernetes deployments may set an AppArmor profile on sandbox agent containers -through the driver configuration. The Helm chart defaults sandbox agents to -`Unconfined` so runtime/default AppArmor profiles do not block supervisor -network namespace setup on AppArmor-enabled nodes. +Kubernetes, Docker, and Podman share one AppArmor configuration model: +`RuntimeDefault`, `Unconfined`, or `Localhost/`. Each driver translates +that model to its native API and rejects an explicitly requested confined +profile when its backend reports AppArmor unavailable. Docker keeps its +historical explicit `Unconfined` default. Podman sends no override when the +field is omitted, preserving the runtime-selected profile; development paths +that require the supervisor's namespace mount setup opt into `Unconfined` +explicitly. The Helm chart independently uses `Unconfined` for Kubernetes. + +Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, +and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, +and the explicit cleartext-Basic-auth acknowledgement without allowing +workload environment to override them. Local containers project provider SPIFFE +through a dedicated host UNIX-socket parent mount. A VM cannot safely expose +that host socket: it accepts only a separately operated, concrete TCP listener +when `provider_spiffe_allow_guest_tcp = true` explicitly acknowledges guest +access. Host-only sockets are never implicitly forwarded to VM guests. The Kubernetes deployment packaging has two ownership boundaries. The gateway chart owns the gateway workload, configuration, Services, PKI, and @@ -309,10 +323,14 @@ can request a specific number of GPUs or the driver-specific default behaviour. For all in-tree drivers, this is equivalent to selecting a single GPU. VM runtime state paths are derived only from driver-validated sandbox IDs -matching `[A-Za-z0-9._-]{1,128}`. The gateway-owned VM driver socket uses a -private `run/` directory plus Unix peer UID/PID checks. Standalone -unauthenticated TCP mode is disabled unless explicitly enabled for local -development. +matching `[A-Za-z0-9._-]{1,128}`. Each writable overlay records its effective +sandbox UID/GID so later rootfs cache changes cannot rewrite persisted file +ownership. Unmarked pre-migration overlays recover identity from concrete +overlay or prepared-rootfs state, an explicit operator override, or the current +image account. The driver never assumes `10001:10001`; it preserves that legacy +identity only when persisted state reports it. The gateway-owned VM driver +socket uses a private `run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP +mode is disabled unless explicitly enabled for local development. Runtime-specific implementation notes belong in the driver crate README: diff --git a/architecture/gateway.md b/architecture/gateway.md index f86411511b..3665fc9699 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -38,6 +38,20 @@ immediately without a grace period. Finalization is persisted separately from the exit result; the gateway deletes an ephemeral sandbox only after the finalized supervisor session disconnects. +## Configuration Boundary + +The gateway accepts exactly schema version 2. Missing, legacy, and future +versions fail before runtime construction, and driver settings belong only to +`[openshell.drivers.]`. The process does not migrate legacy files. +Package lifecycle code may replace an exact package-generated v1 default, but +it preserves edited configurations for explicit operator migration. + +Gateway listener TLS and sandbox callback TLS are separate inputs. A selected +local Docker, Podman, or VM driver requires a complete guest bundle whenever +the gateway listener uses TLS; package-managed local TLS can supply that bundle. +Kubernetes instead projects guest credentials through its configured Secret. +The gateway validates this requirement before constructing the selected driver. + ## Protocol and Auth The gateway listens on one service port and multiplexes gRPC and HTTP traffic. @@ -246,10 +260,10 @@ controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing deployments. Supervisors renew gateway JWTs in memory before expiry only while the sandbox record still exists. Older tokens are not server-revoked; shared deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. -The config default is -`gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM -gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other -shared deployments should set a positive TTL. +Omitting `gateway_jwt.ttl_secs` selects non-expiring tokens for local +single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0`. +Kubernetes and other shared deployments should set a positive TTL. Explicit +zero is rejected. Gateway JWT signing-key rotation is currently an offline operator action. The runtime loads one active signing key and one matching public verification key @@ -690,9 +704,10 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ``` The TOML file is opt-in via `--config ` / `OPENSHELL_GATEWAY_CONFIG`. -Driver implementation settings live in the TOML driver tables. See -`docs/reference/gateway-config.mdx` for worked per-driver examples and RFC -0003 for the full schema. +Driver implementation settings live exclusively in TOML driver tables. The +selector is the singular `[openshell.gateway] compute_driver`; legacy +`compute_drivers` lists are rejected. See `docs/reference/gateway-config.mdx` +for worked per-driver examples and RFC 0003 for the full schema. Each installation has an operator-assigned gateway name. Configure it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. @@ -706,26 +721,20 @@ aliases, network names, and the sandbox JWT issuer. `database_url` is env-only and rejected when present in the file (`OPENSHELL_DB_URL` / `--db-url`). -### Driver inheritance - -`[openshell.gateway]` carries a small set of values (`sandbox_namespace`, -`default_image`, -`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, -`host_gateway_ip`, `enable_user_namespaces`) that are inherited into each -driver's `[openshell.drivers.]` table when the driver-specific table -does not override them. The allowlist is per-driver so a gateway-wide -default cannot land in a driver that does not understand it (e.g. -`client_tls_secret_name` is K8s-only). +### Driver ownership -`image_pull_policy` is intentionally **not** inheritable: Kubernetes uses -`Always | IfNotPresent | Never` (passed verbatim to the K8s API) while -Podman uses the lowercase enum `always | missing | never | newer`. No -value means the same thing in both, so the key lives only under each -driver's own table. +`[openshell.gateway]` contains gateway process settings only. Each selected +driver reads its own configuration exclusively from +`[openshell.drivers.]`; values are never inherited from gateway scope. +Kubernetes owns `namespace`, `default_image`, `supervisor_image`, +`client_tls_secret_name`, `service_account_name`, `host_gateway_ip`, +`enable_user_namespaces`, and `sa_token_ttl_secs`. Docker uses +`sandbox_label` instead of the legacy `sandbox_namespace` name. Podman and VM +likewise own their image, endpoint, and runtime settings in their tables. -Driver-specific values that are not part of the inheritance allowlist -(e.g. Podman `socket_path`, VM `vcpus`) only come from the driver's own -table. +`image_pull_policy` uses the shared canonical vocabulary +`always | if_not_present | never | newer`. Drivers translate it to their runtime +APIs; `newer` is supported only by Podman and rejected by Docker and Kubernetes. ### OTLP export @@ -786,7 +795,10 @@ system entry instead of pretending to delete package-manager owned state. - Gateway TLS and client certificate distribution are deployment concerns owned by the operator or packaging layer. - Compute runtimes own the mechanics of starting workloads and injecting - callback configuration. + callback configuration. Local Docker, Podman, and VM callback endpoints can + be derived from their fixed host aliases. Kubernetes requires an explicit + endpoint from deployment topology; Helm renders it from the gateway Service + name and namespace rather than inferring it from sandbox placement. - Docker-backed local gateways use Docker's `host-gateway` callback alias on macOS and Docker Desktop-style runtimes. They request IPv4 loopback callback reachability and add a listener only when the primary does not cover it. diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3507011120..c6bb53a260 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -6,7 +6,9 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; +use std::fmt; use std::net::SocketAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -112,6 +114,12 @@ pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; /// Compute drivers may override this through backend configuration. pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; +/// Typed default cgroup PID limit for local container sandboxes. +#[must_use] +pub fn default_sandbox_pids_limit() -> Option { + NonZeroI64::new(DEFAULT_SANDBOX_PIDS_LIMIT) +} + /// Normalize a configured compute driver name. /// /// Built-in driver names and custom remote driver names share the same @@ -133,6 +141,225 @@ pub fn normalize_compute_driver_name(value: &str) -> Result { Ok(value.to_ascii_lowercase()) } +/// Canonical policy controlling when a driver pulls a sandbox image. +/// +/// Backends translate this shared vocabulary to their runtime API. `newer` is +/// supported only by Podman; other backends reject it during configuration. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImagePullPolicy { + /// Always pull, even if a local image is available. + Always, + /// Pull only when a local image is unavailable. + #[default] + IfNotPresent, + /// Never pull; fail when a local image is unavailable. + Never, + /// Pull only when the registry image is newer than the local copy. + Newer, +} + +impl ImagePullPolicy { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Always => "always", + Self::IfNotPresent => "if_not_present", + Self::Never => "never", + Self::Newer => "newer", + } + } +} + +impl fmt::Display for ImagePullPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for ImagePullPolicy { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "always" => Ok(Self::Always), + "if_not_present" => Ok(Self::IfNotPresent), + "never" => Ok(Self::Never), + "newer" => Ok(Self::Newer), + other => Err(format!( + "invalid image pull policy '{other}'; expected one of: always, if_not_present, never, newer" + )), + } + } +} + +/// Canonical `AppArmor` confinement requested for a sandbox container. +/// +/// Drivers translate this model to their runtime API. An omitted value leaves +/// the runtime default unchanged; `Unconfined` is explicit because the +/// supervisor needs mount operations that the default Docker/Podman profile +/// commonly denies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppArmorProfile { + RuntimeDefault, + Unconfined, + Localhost(String), +} + +impl AppArmorProfile { + #[must_use] + pub const fn kubernetes_type(&self) -> &'static str { + match self { + Self::RuntimeDefault => "RuntimeDefault", + Self::Unconfined => "Unconfined", + Self::Localhost(_) => "Localhost", + } + } + + #[must_use] + pub fn localhost_profile(&self) -> Option<&str> { + match self { + Self::Localhost(profile) => Some(profile), + Self::RuntimeDefault | Self::Unconfined => None, + } + } + + /// Translate to the OCI `apparmor=` security option. + /// + /// `RuntimeDefault` deliberately returns `None`: omitting an OCI option + /// asks Docker/Podman to apply their runtime default profile. + #[must_use] + pub fn oci_security_opt(&self) -> Option { + match self { + Self::RuntimeDefault => None, + Self::Unconfined => Some("apparmor=unconfined".to_string()), + Self::Localhost(profile) => Some(format!("apparmor={profile}")), + } + } +} + +impl fmt::Display for AppArmorProfile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RuntimeDefault => f.write_str("RuntimeDefault"), + Self::Unconfined => f.write_str("Unconfined"), + Self::Localhost(profile) => write!(f, "Localhost/{profile}"), + } + } +} + +impl FromStr for AppArmorProfile { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "RuntimeDefault" => Ok(Self::RuntimeDefault), + "Unconfined" => Ok(Self::Unconfined), + other => match other.strip_prefix("Localhost/") { + Some("") => Err( + "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" + .to_string(), + ), + Some(profile) if !profile.contains(char::is_whitespace) => { + Ok(Self::Localhost(profile.to_string())) + } + Some(_) => { + Err("invalid AppArmor localhost profile; whitespace is not allowed".to_string()) + } + None => Err(format!( + "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" + )), + }, + } + } +} + +impl Serialize for AppArmorProfile { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for AppArmorProfile { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +/// Common local-driver corporate forward-proxy settings. +/// +/// This type is `flatten`ed by local compute-driver tables, preserving the +/// established TOML field names while keeping their safety contract shared. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct UpstreamProxyConfig { + pub https_proxy: Option, + pub no_proxy: Option, + pub proxy_auth_file: Option, + pub proxy_auth_allow_insecure: Option, + pub proxy_connect_by_hostname: Option, +} + +impl UpstreamProxyConfig { + /// Validate relationships that are independent of the container backend. + /// Credential contents are intentionally not read here and are never put + /// in an error message; drivers validate and stage them per sandbox. + pub fn validate(&self) -> Result<(), String> { + use crate::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; + + let proxy_secure = if let Some(url) = self.https_proxy.as_deref() { + parse_upstream_proxy_url(url) + .map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials; supply them with proxy_auth_file so they are not stored in configuration or runtime metadata".to_string(), + err => format!("https_proxy {err}"), + })? + .secure + } else { + false + }; + + if self + .no_proxy + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + return Err("no_proxy must not be empty when set; omit it instead".to_string()); + } + if self.no_proxy.is_some() && self.https_proxy.is_none() { + return Err("no_proxy is set but no https_proxy is configured".to_string()); + } + if let Some(path) = self.proxy_auth_file.as_ref() { + if path.as_os_str().is_empty() { + return Err("proxy_auth_file must not be empty when set".to_string()); + } + if self.https_proxy.is_none() { + return Err("proxy_auth_file is set but no https_proxy is configured".to_string()); + } + if !proxy_secure && self.proxy_auth_allow_insecure != Some(true) { + return Err("proxy_auth_file sends a cleartext Basic credential to an http:// proxy; set proxy_auth_allow_insecure = true to acknowledge that exposure".to_string()); + } + } else if self.proxy_auth_allow_insecure.is_some() { + return Err( + "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), + ); + } + if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { + return Err( + "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), + ); + } + Ok(()) + } +} + /// Server configuration. /// /// Built programmatically in [`crate::Config::new`] and the gateway CLI from @@ -194,12 +421,9 @@ pub struct Config { /// Database URL for persistence. pub database_url: String, - /// Compute drivers configured for the gateway. - /// - /// The config shape allows multiple drivers so the gateway can evolve - /// toward multi-backend routing. Current releases require exactly one - /// configured driver. - pub compute_drivers: Vec, + /// Explicit compute driver configured for the gateway. + /// `None` enables runtime auto-detection. + pub compute_driver: Option, /// Operator-provided endpoints for named remote compute drivers. /// @@ -522,18 +746,21 @@ pub struct GatewayJwtConfig { /// `openshell`. #[serde(default = "default_gateway_id")] pub gateway_id: String, - /// Token lifetime in seconds. A value of 0 disables expiration and is - /// intended only for local single-player deployments. - #[serde(default = "default_sandbox_token_ttl_secs")] - pub ttl_secs: u64, + /// Token lifetime in seconds. Omit the field for a non-expiring token. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl_secs: Option, } -fn default_gateway_id() -> String { - "openshell".to_string() +impl GatewayJwtConfig { + /// Effective token lifetime. `None` represents a non-expiring token. + pub fn sandbox_token_ttl(&self) -> Option { + self.ttl_secs.map(|ttl| Duration::from_secs(ttl.get())) + } } -const fn default_sandbox_token_ttl_secs() -> u64 { - 0 +fn default_gateway_id() -> String { + "openshell".to_string() } fn default_roles_claim() -> String { @@ -569,7 +796,7 @@ impl Config { mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, database_url: String::new(), - compute_drivers: vec![], + compute_driver: None, compute_driver_endpoints: BTreeMap::new(), credential_drivers: Vec::new(), default_credential_driver: None, @@ -620,17 +847,10 @@ impl Config { self } - /// Create a new configuration with the configured compute drivers. + /// Create a new configuration with an explicit compute driver. #[must_use] - pub fn with_compute_drivers(mut self, drivers: I) -> Self - where - I: IntoIterator, - D: ToString, - { - self.compute_drivers = drivers - .into_iter() - .map(|driver| driver.to_string()) - .collect(); + pub fn with_compute_driver(mut self, driver: impl ToString) -> Self { + self.compute_driver = Some(driver.to_string()); self } @@ -825,9 +1045,9 @@ const fn default_ssh_session_ttl_secs() -> u64 { #[cfg(test)] mod tests { use super::{ - Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, + AppArmorProfile, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, + GatewayProviderProfileSourceConfig, ImagePullPolicy, PolicyValidationFailureMode, normalize_compute_driver_name, }; use std::net::SocketAddr; @@ -922,7 +1142,61 @@ mod tests { })) .expect("gateway JWT config should deserialize with default ttl"); - assert_eq!(cfg.ttl_secs, 0); + assert_eq!(cfg.ttl_secs, None); + assert_eq!(cfg.sandbox_token_ttl(), None); + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert!(serialized.get("ttl_secs").is_none()); + } + #[test] + fn gateway_jwt_positive_ttl_serializes_and_has_effective_duration() { + let cfg: GatewayJwtConfig = serde_json::from_value(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 3600 + })) + .expect("gateway JWT config should deserialize with positive ttl"); + + assert_eq!(cfg.sandbox_token_ttl(), Some(Duration::from_secs(3600))); + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert_eq!(serialized["ttl_secs"], 3600); + } + + #[test] + fn gateway_jwt_ttl_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 0 + })) + .expect_err("zero TTL must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); + } + + #[test] + fn image_pull_policy_uses_canonical_vocabulary() { + for (value, expected) in [ + ("always", ImagePullPolicy::Always), + ("if_not_present", ImagePullPolicy::IfNotPresent), + ("never", ImagePullPolicy::Never), + ("newer", ImagePullPolicy::Newer), + ] { + assert_eq!(value.parse::(), Ok(expected)); + assert_eq!(expected.to_string(), value); + assert_eq!(serde_json::to_value(expected).unwrap(), value); + } + assert!("missing".parse::().is_err()); + assert!("IfNotPresent".parse::().is_err()); + } + + #[test] + fn app_armor_profile_rejects_whitespace_in_localhost_name() { + assert!( + "Localhost/openshell profile" + .parse::() + .is_err() + ); } #[test] diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index c63e4bcdd8..97c9a0220d 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -63,6 +63,7 @@ pub const VM_GUEST_TLS_CA_PATH: &str = "/opt/openshell/tls/ca.crt"; pub const VM_GUEST_TLS_CERT_PATH: &str = "/opt/openshell/tls/tls.crt"; pub const VM_GUEST_TLS_KEY_PATH: &str = "/opt/openshell/tls/tls.key"; pub const VM_GUEST_SANDBOX_TOKEN_PATH: &str = "/opt/openshell/auth/sandbox.jwt"; +pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy"; pub const VM_GUEST_INIT_DROPIN_DIR: &str = "/opt/openshell/init.d"; pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest"; pub const VM_UMOCI_PATH: &str = "/opt/openshell/bin/umoci"; @@ -101,6 +102,7 @@ mod tests { VM_GUEST_TLS_CERT_PATH, VM_GUEST_TLS_KEY_PATH, VM_GUEST_SANDBOX_TOKEN_PATH, + VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, VM_GUEST_INIT_DROPIN_DIR, VM_GUEST_INIT_DROPIN_MANIFEST, VM_UMOCI_PATH, diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index c8be114ebd..061330b9cf 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -7,6 +7,58 @@ use std::path::{Path, PathBuf}; use crate::proto::compute::v1::DriverSandbox; +/// Built-in sandbox network topologies used to derive a callback endpoint +/// when an operator does not configure a per-driver `grpc_endpoint` override. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GatewayCallbackTopology { + /// A Docker container reaches the host through Docker's gateway alias. + Docker, + /// A Podman container reaches the host through Podman's gateway alias. + Podman, + /// A libkrun guest reaches the host through gvproxy's gateway alias. + Vm, +} + +/// Build the endpoint a sandbox uses to call its gateway for a known topology. +/// +/// The result is deliberately derived by the gateway rather than baked into +/// individual driver defaults. A configured `grpc_endpoint` remains an +/// operator override for remote or non-standard deployments. +#[must_use] +pub fn gateway_callback_endpoint( + topology: GatewayCallbackTopology, + gateway_port: u16, + gateway_tls_enabled: bool, +) -> String { + let scheme = if gateway_tls_enabled { "https" } else { "http" }; + let host = match topology { + GatewayCallbackTopology::Docker | GatewayCallbackTopology::Vm => "host.openshell.internal", + GatewayCallbackTopology::Podman => "host.containers.internal", + }; + format!("{scheme}://{host}:{gateway_port}") +} + +#[cfg(test)] +mod callback_endpoint_tests { + use super::{GatewayCallbackTopology, gateway_callback_endpoint}; + + #[test] + fn derives_endpoint_for_each_builtin_topology() { + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Docker, 17670, false), + "http://host.openshell.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Podman, 17670, true), + "https://host.containers.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Vm, 17670, true), + "https://host.openshell.internal:17670" + ); + } +} + // --------------------------------------------------------------------------- // Sandbox container/pod label keys (openshell.ai/ namespace) // --------------------------------------------------------------------------- @@ -433,6 +485,68 @@ pub fn read_upstream_proxy_credential_file(path: &str) -> Result /// Container-side directory where the provider SPIFFE Workload API socket is mounted. pub const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = "/spiffe-workload-api"; +/// Validate a host UNIX socket selected for provider SPIFFE projection. +/// +/// Local container drivers bind-mount the socket's dedicated parent directory, +/// not a broad host root. TCP endpoints are deliberately rejected here: a +/// container projection must be a filesystem socket, while VM guest TCP +/// exposure has its own explicit acknowledgement contract. +pub fn validate_provider_spiffe_unix_socket(path: &Path) -> Result<(), String> { + let raw = path + .to_str() + .ok_or_else(|| "provider_spiffe_workload_api_socket must be valid UTF-8".to_string())?; + if raw.trim() != raw || raw.is_empty() { + return Err("provider_spiffe_workload_api_socket must not be empty or contain surrounding whitespace".to_string()); + } + if raw.starts_with("tcp:") || raw.starts_with("unix:") { + return Err("provider_spiffe_workload_api_socket must be an absolute host UNIX socket path, not a URI".to_string()); + } + if !path.is_absolute() || path.parent().is_none_or(|parent| parent == Path::new("/")) { + return Err("provider_spiffe_workload_api_socket must be an absolute UNIX socket path below a dedicated parent directory".to_string()); + } + Ok(()) +} + +/// Return the guest/container path for a projected provider SPIFFE socket. +pub fn projected_provider_spiffe_socket_path(path: &Path) -> Result { + validate_provider_spiffe_unix_socket(path)?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .ok_or_else(|| "provider_spiffe_workload_api_socket must name a socket file".to_string())?; + Ok(format!( + "{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}/{file_name}" + )) +} + +/// Validate an explicitly operator-acknowledged guest-reachable SPIFFE TCP endpoint. +/// +/// The `tcp:` spelling is the SPIFFE Workload API endpoint grammar accepted by +/// the client. The address must be concrete; wildcard and host-only UNIX +/// sockets are never silently exposed to VM guests. +pub fn validate_guest_spiffe_tcp_endpoint( + endpoint: &str, + acknowledged: bool, +) -> Result<(), String> { + if endpoint.trim() != endpoint || endpoint.is_empty() { + return Err("provider_spiffe_workload_api_tcp_endpoint must not be empty or contain surrounding whitespace".to_string()); + } + if !acknowledged { + return Err("provider_spiffe_workload_api_tcp_endpoint exposes a Workload API to VM guests; set provider_spiffe_allow_guest_tcp = true only after explicitly acknowledging that exposure".to_string()); + } + let address = endpoint.strip_prefix("tcp:").ok_or_else(|| { + "provider_spiffe_workload_api_tcp_endpoint must use tcp:host:port (for example tcp:192.0.2.10:8081)".to_string() + })?; + let address: std::net::SocketAddr = address.parse().map_err(|_| { + "provider_spiffe_workload_api_tcp_endpoint must use a concrete IP address and non-zero port".to_string() + })?; + if address.ip().is_unspecified() || address.port() == 0 { + return Err("provider_spiffe_workload_api_tcp_endpoint must not use an unspecified address or port 0".to_string()); + } + Ok(()) +} + /// Return the XDG state path for a driver's sandbox JWT token file. /// /// The resulting path is `$XDG_STATE_HOME/openshell/[/]//sandbox.jwt`. @@ -871,6 +985,28 @@ mod tests { } #[cfg(unix)] + #[test] + fn projected_spiffe_socket_requires_dedicated_absolute_unix_path() { + assert_eq!( + projected_provider_spiffe_socket_path(Path::new("/run/spire/agent.sock")).unwrap(), + "/spiffe-workload-api/agent.sock" + ); + for path in ["relative.sock", "/agent.sock", "tcp:127.0.0.1:8081"] { + assert!( + validate_provider_spiffe_unix_socket(Path::new(path)).is_err(), + "{path}" + ); + } + } + + #[test] + fn guest_spiffe_tcp_requires_acknowledgement_and_concrete_endpoint() { + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", true).is_ok()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", false).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:0.0.0.0:8081", true).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("unix:/run/spire/agent.sock", true).is_err()); + } + #[test] fn credential_file_rejects_fifo_without_hanging() { // A FIFO with no writer would block a blocking open() forever. The diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 9aecfcb1f1..d4fb7c61ef 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -51,10 +51,11 @@ pub mod time; pub mod transport_errors; pub use config::{ - Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, - GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, - GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, - PolicyValidationFailureMode, TlsConfig, + AppArmorProfile, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, + GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, + GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, + ImagePullPolicy, MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, + UpstreamProxyConfig, }; pub use dynamic_string_allowlist::DynamicStringAllowlist; pub use error::{ComputeDriverError, Error, Result}; diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..10e085ecf0 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -104,7 +104,7 @@ contract: | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | +| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. `[openshell.drivers.docker].sandbox_pids_limit` defaults to `2048`; explicit `0` is invalid. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | | `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | @@ -187,6 +187,25 @@ mounted into the container and exposed with: HTTP endpoints reject TLS material because the supervisor would not use it. +## Corporate proxy, SPIFFE, and AppArmor + +`https_proxy`, `no_proxy`, and `proxy_auth_file` in +`[openshell.drivers.docker]` are operator-owned supervisor settings. Docker +passes the proxy URL and bypass list on the supervisor command line and mounts +an optional `user:pass` auth file read-only at a root-only path. Credentials +never appear in container environment or Docker labels. An auth file used with +an `http://` proxy requires `proxy_auth_allow_insecure = true`; an `https://` +proxy protects the Basic-auth header in its TLS session. + +Set `provider_spiffe_workload_api_socket` to an absolute host UNIX socket to +project its dedicated parent directory into the supervisor and set +`OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to the guest path. TCP URIs are +rejected for this projection. `app_armor_profile` uses the shared +`RuntimeDefault`, `Unconfined`, or `Localhost/` vocabulary. Docker +uses explicit `Unconfined` by default because the supervisor's namespace mount +setup is incompatible with `docker-default`; requested confined profiles fail +at startup if Docker does not report AppArmor support. + ## Environment Ownership The driver merges template environment and sandbox spec environment first, then diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d599cac169..12b4d09522 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -21,13 +21,14 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackTopology, LABEL_MANAGED_BY, + LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -54,7 +55,9 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{Error, Result as CoreResult}; +use openshell_core::{ + AppArmorProfile, Error, ImagePullPolicy, Result as CoreResult, UpstreamProxyConfig, +}; use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; use std::future::Future; @@ -81,6 +84,10 @@ const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = + openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH; +const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = + openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; @@ -121,10 +128,10 @@ pub struct DockerComputeConfig { pub default_image: String, /// Image pull policy for sandbox images. - pub image_pull_policy: String, + pub image_pull_policy: ImagePullPolicy, - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, + /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. + pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, @@ -157,13 +164,33 @@ pub struct DockerComputeConfig { /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] pub enable_bind_mounts: bool, + + /// Corporate forward-proxy settings supplied to the supervisor on argv. + /// The flattened fields retain the common `https_proxy`, `no_proxy`, and + /// `proxy_auth_*` gateway TOML contract. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Host UNIX socket to project into sandbox supervisors for provider + /// SPIFFE token exchange. + pub provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` confinement requested for sandbox containers. The explicit + /// default preserves the prior supervisor-compatible Docker behavior. + #[serde(skip_serializing_if = "Option::is_none")] + pub app_armor_profile: Option, } impl Default for DockerComputeConfig { @@ -171,8 +198,8 @@ impl Default for DockerComputeConfig { Self { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + image_pull_policy: ImagePullPolicy::default(), + sandbox_label: "default".to_string(), grpc_endpoint: String::new(), supervisor_bin: None, supervisor_image: None, @@ -182,8 +209,11 @@ impl Default for DockerComputeConfig { network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } } @@ -198,8 +228,8 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { default_image: String, - image_pull_policy: String, - sandbox_namespace: String, + image_pull_policy: ImagePullPolicy, + sandbox_label: String, grpc_endpoint: String, network_name: String, gateway_route: DockerGatewayRoute, @@ -212,8 +242,11 @@ struct DockerDriverRuntimeConfig { daemon_version: String, supports_gpu: bool, allow_all_default_gpu: bool, - sandbox_pids_limit: i64, + sandbox_pids_limit: Option, enable_bind_mounts: bool, + upstream_proxy: UpstreamProxyConfig, + provider_spiffe_workload_api_socket: Option, + app_armor_profile: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -590,6 +623,17 @@ impl DockerComputeDriver { let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + validate_image_pull_policy(docker_config.image_pull_policy)?; + docker_config + .upstream_proxy + .validate() + .map_err(Error::config)?; + validate_docker_proxy_auth_file(&docker_config.upstream_proxy)?; + if let Some(socket) = docker_config.provider_spiffe_workload_api_socket.as_deref() { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(Error::config)?; + } + validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = gateway_bind_address.port(); if gateway_port == 0 { return Err(Error::config( @@ -605,13 +649,11 @@ impl DockerComputeDriver { docker_gateway_callback_bind_address(&gateway_route, gateway_bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); + docker_config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Docker, + gateway_port, + docker_guest_tls_configured(&docker_config), + ); } let grpc_endpoint = docker_container_openshell_endpoint( &docker_config.grpc_endpoint, @@ -626,8 +668,8 @@ impl DockerComputeDriver { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), + image_pull_policy: docker_config.image_pull_policy, + sandbox_label: docker_config.sandbox_label.clone(), grpc_endpoint, network_name, gateway_route, @@ -642,6 +684,11 @@ impl DockerComputeDriver { allow_all_default_gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + upstream_proxy: docker_config.upstream_proxy.clone(), + provider_spiffe_workload_api_socket: docker_config + .provider_spiffe_workload_api_socket + .clone(), + app_armor_profile: docker_config.app_armor_profile.clone(), }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -933,7 +980,7 @@ impl DockerComputeDriver { ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, )); @@ -1334,7 +1381,7 @@ impl DockerComputeDriver { PendingSandboxRecord { sandbox: pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, ), @@ -1389,7 +1436,7 @@ impl DockerComputeDriver { cleanup_sandbox_token_file(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, error_condition(failure.reason, &failure.message), false, ); @@ -1595,7 +1642,7 @@ impl DockerComputeDriver { } async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); + let filters = managed_container_label_filters(&self.config.sandbox_label, []); self.docker .list_containers(Some( ListContainersOptionsBuilder::default() @@ -1620,7 +1667,7 @@ impl DockerComputeDriver { } let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); + managed_container_label_filters(&self.config.sandbox_label, label_filter_values); let containers = self .docker .list_containers(Some( @@ -1638,7 +1685,7 @@ impl DockerComputeDriver { }; let namespace_matches = labels .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); + .is_some_and(|value| value == &self.config.sandbox_label); let id_matches = sandbox_id.is_empty() || labels .get(LABEL_SANDBOX_ID) @@ -1656,9 +1703,8 @@ impl DockerComputeDriver { sandbox_id: &str, image: &str, ) -> Result { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - let inspect = match policy.as_str() { - "" | "ifnotpresent" => { + let inspect = match self.config.image_pull_policy { + ImagePullPolicy::IfNotPresent => { if let Ok(inspect) = self.docker.inspect_image(image).await { self.publish_docker_progress( sandbox_id, @@ -1675,14 +1721,14 @@ impl DockerComputeDriver { .map_err(|err| internal_status("inspect Docker image after pull", err))? } } - "always" => { + ImagePullPolicy::Always => { self.pull_image(sandbox_id, image).await?; self.docker .inspect_image(image) .await .map_err(|err| internal_status("inspect Docker image after pull", err))? } - "never" => match self.docker.inspect_image(image).await { + ImagePullPolicy::Never => match self.docker.inspect_image(image).await { Ok(inspect) => { self.publish_docker_progress( sandbox_id, @@ -1694,15 +1740,15 @@ impl DockerComputeDriver { } Err(err) if is_not_found_error(&err) => { return Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" + "docker image '{image}' is not present locally and image_pull_policy = \"never\"" ))); } Err(err) => return Err(internal_status("inspect Docker image", err)), }, - other => { - return Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))); + ImagePullPolicy::Newer => { + return Err(Status::failed_precondition( + "image_pull_policy = \"newer\" is supported only by the Podman compute driver", + )); } }; @@ -2688,6 +2734,49 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } +/// Verify the configured credential without exposing its contents. Docker +/// bind-mounts the root-owned file directly, unlike Podman which uses a native +/// secret object; this preflight makes a bad file fail before any sandbox is +/// created. +fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<()> { + let Some(path) = config.proxy_auth_file.as_ref() else { + return Ok(()); + }; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file( + path.to_str() + .ok_or_else(|| Error::config("proxy_auth_file must be valid UTF-8"))?, + ) + .map_err(Error::config)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map_err(|error| Error::config(format!("proxy_auth_file is invalid: {error}")))?; + Ok(()) +} + +/// Build immutable operator-owned proxy arguments. Credentials never appear on +/// argv: only the fixed in-container root-only file path is supplied. +fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { + let mut args = Vec::new(); + if let Some(url) = config.https_proxy.as_ref() { + args.extend(["--upstream-proxy".to_string(), url.clone()]); + } + if let Some(no_proxy) = config.no_proxy.as_ref() { + args.extend(["--upstream-no-proxy".to_string(), no_proxy.clone()]); + } + if config.proxy_auth_file.is_some() { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if config.proxy_auth_allow_insecure == Some(true) { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if config.proxy_connect_by_hostname == Some(true) { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args +} + fn build_binds( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -2717,6 +2806,23 @@ fn build_binds( SANDBOX_TOKEN_MOUNT_PATH )); } + if let Some(path) = config.upstream_proxy.proxy_auth_file.as_ref() { + binds.push(format!( + "{}:{}:ro,z", + path.display(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH + )); + } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { + let parent = socket.parent().ok_or_else(|| { + Status::failed_precondition("provider SPIFFE socket has no parent directory") + })?; + binds.push(format!( + "{}:{}:ro", + parent.display(), + PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + )); + } Ok(binds) } @@ -2733,7 +2839,7 @@ fn sandbox_token_host_path_by_id( ) -> Result { openshell_core::driver_utils::sandbox_token_path( "docker-sandbox-tokens", - Some(&config.sandbox_namespace), + Some(&config.sandbox_label), sandbox_id, ) .map_err(|err| { @@ -2899,6 +3005,15 @@ fn build_environment_for_oci_user( TLS_KEY_MOUNT_PATH.to_string(), ); } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() + && let Ok(path) = + openshell_core::driver_utils::projected_provider_spiffe_socket_path(socket) + { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + path, + ); + } environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); @@ -3090,13 +3205,13 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_WORKSPACE.to_string(), sandbox.workspace.clone(), ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use + // The list/get/find paths filter by `config.sandbox_label`, so use // the same value here. `DriverSandbox.namespace` is unset on the request // path (the gateway elides it), and using it would produce containers // that the driver itself cannot find afterwards. labels.insert( LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), + config.sandbox_label.clone(), ); Ok(ContainerCreateBody { @@ -3109,7 +3224,11 @@ fn build_container_create_body_for_image( entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Replace the image CMD with the supervisor's resolved workspace // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + cmd: { + let mut args = vec!["--workdir".to_string(), workspace_root]; + args.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); + Some(args) + }, labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3131,17 +3250,13 @@ fn build_container_create_body_for_image( "SYS_PTRACE".to_string(), "SYSLOG".to_string(), ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), + // The default is explicitly Unconfined because the supervisor + // needs mount operations commonly denied by docker-default. + security_opt: config + .app_armor_profile + .as_ref() + .and_then(AppArmorProfile::oci_security_opt) + .map(|option| vec![option]), network_mode: Some(config.network_name.clone()), extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), ..Default::default() @@ -3426,26 +3541,55 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { +fn validate_sandbox_pids_limit(value: Option) -> CoreResult<()> { + if value.is_some_and(|limit| limit.get() < 0) { return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", + "docker sandbox_pids_limit must be positive when set", )); } Ok(()) } -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", +fn validate_image_pull_policy(policy: ImagePullPolicy) -> CoreResult<()> { + if policy == ImagePullPolicy::Newer { + return Err(Error::config( + "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", )); } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) + Ok(()) +} + +fn validate_docker_app_armor_profile( + profile: Option<&AppArmorProfile>, + info: &SystemInfo, +) -> CoreResult<()> { + let requires_apparmor = matches!( + profile, + Some(AppArmorProfile::RuntimeDefault | AppArmorProfile::Localhost(_)) + ); + if !requires_apparmor { + return Ok(()); + } + let available = info.security_options.as_ref().is_some_and(|options| { + options + .iter() + .any(|option| option.to_ascii_lowercase().contains("apparmor")) + }); + if !available { + return Err(Error::config( + "app_armor_profile requires AppArmor, but Docker reports it is unavailable; enable AppArmor on the daemon host or set app_armor_profile = \"Unconfined\" explicitly", + )); } + Ok(()) +} + +fn docker_pids_limit(value: Option) -> Result, Status> { + if value.is_some_and(|limit| limit.get() < 0) { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be positive when set", + )); + } + Ok(value.map(std::num::NonZeroI64::get)) } #[allow(clippy::cast_possible_truncation)] @@ -3698,12 +3842,12 @@ fn label_filters(values: impl IntoIterator) -> HashMap, ) -> HashMap> { let mut values = vec![ format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_label}"), ]; values.extend(extra_values); label_filters(values) @@ -4067,8 +4211,8 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult bool { docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() + || docker_config.guest_tls_cert.is_some() + || docker_config.guest_tls_key.is_some() } pub(crate) fn docker_guest_tls_paths( diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 07fb2b0ffb..101cae80d6 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -95,8 +95,8 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + image_pull_policy: ImagePullPolicy::IfNotPresent, + sandbox_label: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { @@ -122,11 +122,110 @@ fn runtime_config() -> DockerDriverRuntimeConfig { daemon_version: "28.0.0".to_string(), supports_gpu: false, allow_all_default_gpu: false, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } +#[test] +fn docker_config_uses_canonical_sandbox_label_name() { + let config: DockerComputeConfig = + serde_json::from_value(serde_json::json!({ "sandbox_label": "tenant-a" })).unwrap(); + assert_eq!(config.sandbox_label, "tenant-a"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["sandbox_label"], "tenant-a"); + assert!(serialized.get("sandbox_namespace").is_none()); +} + +#[test] +fn docker_config_rejects_legacy_sandbox_namespace() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_namespace": "tenant-a" + })) + .expect_err("legacy sandbox_namespace must be rejected"); + assert!(error.to_string().contains("sandbox_namespace")); +} + +#[test] +fn docker_config_keeps_explicit_unconfined_apparmor_default() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!(config.app_armor_profile, Some(AppArmorProfile::Unconfined)); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], "Unconfined"); +} + +#[test] +fn docker_config_defaults_to_driver_owned_pids_limit() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!( + config.sandbox_pids_limit.map(std::num::NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); +} + +#[test] +fn docker_config_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 + })) + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: DockerComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = validate_sandbox_pids_limit(negative.sandbox_pids_limit).unwrap_err(); + assert!(error.to_string().contains("must be positive")); +} + +#[test] +fn docker_rejects_newer_image_pull_policy() { + let error = validate_image_pull_policy(ImagePullPolicy::Newer).unwrap_err(); + assert!(error.to_string().contains("supported only by the Podman")); +} + +#[test] +fn docker_config_uses_shared_proxy_contract_and_explicit_apparmor_default() { + let config: DockerComputeConfig = toml::from_str( + r#" +https_proxy = "http://proxy.example:8080" +no_proxy = ".svc" +proxy_auth_file = "/run/secrets/proxy-auth" +proxy_auth_allow_insecure = true +app_armor_profile = "Localhost/openshell-supervisor" +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +"#, + ) + .unwrap(); + assert_eq!( + config.upstream_proxy.https_proxy.as_deref(), + Some("http://proxy.example:8080") + ); + assert_eq!( + config.app_armor_profile, + Some(AppArmorProfile::Localhost( + "openshell-supervisor".to_string() + )) + ); + assert!(config.upstream_proxy.validate().is_ok()); + assert!( + openshell_core::driver_utils::validate_provider_spiffe_unix_socket( + config + .provider_spiffe_workload_api_socket + .as_deref() + .unwrap() + ) + .is_ok() + ); +} + fn json_struct(value: serde_json::Value) -> prost_types::Struct { let serde_json::Value::Object(object) = value else { panic!("expected JSON object"); @@ -389,7 +488,7 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { .build(); let subscriber = tracing_subscriber::registry().with(otel_tracing::layer(&provider)); let mut config = runtime_config(); - config.image_pull_policy = "unsupported".to_string(); + config.image_pull_policy = ImagePullPolicy::Newer; let driver = test_driver_with_config(config); driver @@ -1048,13 +1147,13 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { } #[test] -fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { +fn docker_pids_limit_uses_runtime_default_when_omitted() { assert_eq!( - docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), - Some(DEFAULT_SANDBOX_PIDS_LIMIT) + docker_pids_limit(std::num::NonZeroI64::new(2048)).unwrap(), + Some(2048) ); - assert_eq!(docker_pids_limit(0).unwrap(), None); - assert!(docker_pids_limit(-1).is_err()); + assert_eq!(docker_pids_limit(None).unwrap(), None); + assert!(docker_pids_limit(std::num::NonZeroI64::new(-1)).is_err()); } #[test] @@ -1064,10 +1163,10 @@ fn docker_compute_config_disables_bind_mounts_by_default() { } #[test] -fn container_create_body_sets_driver_owned_pids_limit() { +fn container_create_body_omits_pids_limit_by_default() { let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); + assert_eq!(host_config.pids_limit, None); } #[test] @@ -2001,6 +2100,45 @@ fn build_environment_uses_token_file_without_raw_token_env() { ))); } +#[test] +fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { + let mut config = runtime_config(); + config.upstream_proxy = UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }; + config.provider_spiffe_workload_api_socket = Some(PathBuf::from("/run/spire/agent.sock")); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + let command = body.cmd.unwrap(); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy", "https://proxy.example:8443"]) + ); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy-auth-file", UPSTREAM_PROXY_AUTH_MOUNT_PATH]) + ); + let binds = body.host_config.unwrap().binds.unwrap(); + assert!( + binds + .iter() + .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) + ); + assert!(binds.contains(&format!( + "/run/spire:{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}:ro" + ))); + assert!(binds.iter().all(|bind| !bind.contains("rbind"))); + let env = body.env.unwrap(); + assert!(env.iter().any(|entry| entry + == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); + assert!(!env.iter().any(|entry| entry.contains("proxy-auth"))); +} + #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = @@ -2532,10 +2670,10 @@ fn build_container_create_body_uses_runtime_namespace_label() { // runtime config, not from `DriverSandbox.namespace`. The gateway // does not populate `DriverSandbox.namespace`, so a container created // with that empty value would not match subsequent list/get/find - // queries (which filter on `config.sandbox_namespace`), leaking + // queries (which filter on `config.sandbox_label`), leaking // sandboxes that the driver itself cannot observe. let mut config = runtime_config(); - config.sandbox_namespace = "tenant-a".to_string(); + config.sandbox_label = "tenant-a".to_string(); let mut sandbox = test_sandbox(); sandbox.namespace = "ignored-by-driver".to_string(); diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 02dcfe5e87..b64bf0c6e4 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -96,8 +96,11 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC. ## Credentials, TLS, and Relay The driver injects gateway callback configuration, sandbox identity, TLS client -material, and the supervisor SSH socket path into the workload. Driver-owned -values must override image-provided environment variables. +material, and the supervisor SSH socket path into the workload. The callback +endpoint is required because the sandbox namespace does not identify the +Gateway Service; Helm renders it from the release topology, while standalone +and raw TOML configurations must set it explicitly. Driver-owned values must +override image-provided environment variables. Sandbox pods run as `service_account_name` and keep `automountServiceAccountToken: false`. The only Kubernetes token exposed to the @@ -152,14 +155,13 @@ abstract socket whose peer PID must match that authenticated supervisor. Both supervisors exit if the control connection closes, coupling their container restart lifecycle before a new authoritative client can be established. -The driver can request a Kubernetes AppArmor profile through -`app_armor_profile`. - +The driver uses the shared AppArmor model through `app_armor_profile`. Supported values are `Unconfined`, `RuntimeDefault`, and -`Localhost/`. An empty or unset value omits -`securityContext.appArmorProfile`. Helm deployments default sandbox agent -containers to `Unconfined` because runtime/default AppArmor profiles can block -the supervisor's network namespace mount setup on AppArmor-enabled nodes. +`Localhost/`; an empty or unset value omits +`securityContext.appArmorProfile`. Docker and Podman translate the same values +to OCI security options. Helm deployments default sandbox agent containers to +`Unconfined` because runtime/default AppArmor profiles can block the +supervisor's network namespace mount setup on AppArmor-enabled nodes. ## GPU Support diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 805c0314b0..bfdceff9d4 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,8 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub use openshell_core::AppArmorProfile; pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; -use openshell_core::config; +use openshell_core::{ImagePullPolicy, config}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; #[cfg(test)] @@ -177,83 +178,6 @@ impl KubernetesSidecarConfig { } } -/// Kubernetes `AppArmor` profile requested for the sandbox agent container. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AppArmorProfile { - RuntimeDefault, - Unconfined, - Localhost(String), -} - -impl AppArmorProfile { - #[must_use] - pub fn to_k8s_type(&self) -> &'static str { - match self { - Self::RuntimeDefault => "RuntimeDefault", - Self::Unconfined => "Unconfined", - Self::Localhost(_) => "Localhost", - } - } - - #[must_use] - pub fn localhost_profile(&self) -> Option<&str> { - match self { - Self::Localhost(profile) => Some(profile), - Self::RuntimeDefault | Self::Unconfined => None, - } - } -} - -impl std::fmt::Display for AppArmorProfile { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::RuntimeDefault => f.write_str("RuntimeDefault"), - Self::Unconfined => f.write_str("Unconfined"), - Self::Localhost(profile) => write!(f, "Localhost/{profile}"), - } - } -} - -impl FromStr for AppArmorProfile { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "RuntimeDefault" => Ok(Self::RuntimeDefault), - "Unconfined" => Ok(Self::Unconfined), - other => match other.strip_prefix("Localhost/") { - Some("") => Err( - "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" - .to_string(), - ), - Some(profile) => Ok(Self::Localhost(profile.to_string())), - None => Err(format!( - "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" - )), - }, - } - } -} - -impl Serialize for AppArmorProfile { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> Deserialize<'de> for AppArmorProfile { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::from_str(&value).map_err(serde::de::Error::custom) - } -} - fn deserialize_optional_app_armor_profile<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -263,7 +187,8 @@ where let value = Option::::deserialize(deserializer)?; match value.as_deref() { None | Some("") => Ok(None), - Some(value) => AppArmorProfile::from_str(value) + Some(value) => value + .parse::() .map(Some) .map_err(serde::de::Error::custom), } @@ -306,7 +231,9 @@ pub struct KubernetesComputeConfig { /// the driver's `TokenReview` bootstrap authenticator. pub service_account_name: String, pub default_image: String, - pub image_pull_policy: String, + /// Pull policy for sandbox images. Omit to use Kubernetes's image default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_pull_policy: Option, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, /// Managed-mode SSH ingress isolation. When enabled, the driver creates a @@ -317,9 +244,10 @@ pub struct KubernetesComputeConfig { /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. pub supervisor_image: String, - /// Kubernetes `imagePullPolicy` for the supervisor image. - /// Empty string delegates to the Kubernetes default. - pub supervisor_image_pull_policy: String, + /// Pull policy for the supervisor image. Omit to use Kubernetes's image + /// default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supervisor_image_pull_policy: Option, /// How the supervisor binary is delivered into sandbox pods. pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. @@ -439,15 +367,13 @@ impl Default for KubernetesComputeConfig { operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), - // Default empty so the gateway omits `imagePullPolicy` from pod - // specs and Kubernetes applies its own default (Always for `latest`, - // IfNotPresent otherwise). `DEFAULT_IMAGE_PULL_POLICY` ("missing") - // is Podman vocabulary and is not a valid Kubernetes value. - image_pull_policy: String::new(), + // Omit the field so Kubernetes applies its own default (Always for + // `latest`, IfNotPresent otherwise). + image_pull_policy: None, image_pull_secrets: Vec::new(), managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), - supervisor_image_pull_policy: String::new(), + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), @@ -506,17 +432,52 @@ impl KubernetesComputeConfig { self.sidecar.validate_proxy_uid() } + /// Reject pull policies Kubernetes cannot express before creating pods. + pub fn validate_image_pull_policies(&self) -> Result<(), String> { + for (field, policy) in [ + ("image_pull_policy", self.image_pull_policy), + ( + "supervisor_image_pull_policy", + self.supervisor_image_pull_policy, + ), + ] { + if policy == Some(ImagePullPolicy::Newer) { + return Err(format!( + "{field} = \"newer\" is supported only by the Podman compute driver" + )); + } + } + Ok(()) + } + + /// Translate a validated shared policy to Kubernetes's API vocabulary. + #[must_use] + pub fn image_pull_policy_value(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "Always", + ImagePullPolicy::IfNotPresent => "IfNotPresent", + ImagePullPolicy::Never => "Never", + ImagePullPolicy::Newer => unreachable!("newer must be rejected during validation"), + } + } + /// Validate the operator-owned corporate upstream proxy configuration. pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - if let Some(url) = &self.https_proxy { - parse_upstream_proxy_url(url).map_err(|err| match err { - UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), - UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), - err => format!("https_proxy {err}"), - })?; - } + let proxy_addr = self + .https_proxy + .as_deref() + .map(|url| { + parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => { + "https_proxy must not be empty when set".to_string() + } + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), + err => format!("https_proxy {err}"), + }) + }) + .transpose()?; if let Some(list) = self.no_proxy.as_deref() { if list.trim().is_empty() { @@ -575,7 +536,9 @@ impl KubernetesComputeConfig { .to_string(), ); } - if self.proxy_auth_allow_insecure != Some(true) { + if proxy_addr.as_ref().is_some_and(|proxy| !proxy.secure) + && self.proxy_auth_allow_insecure != Some(true) + { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } if self.topology == SupervisorTopology::Combined { @@ -928,6 +891,34 @@ mod tests { assert!(cfg.sidecar.process_binary_aware_network_policy); } + #[test] + fn image_pull_policy_uses_shared_canonical_values() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "image_pull_policy": "if_not_present", + "supervisor_image_pull_policy": "never" + })) + .unwrap(); + assert_eq!(cfg.image_pull_policy, Some(ImagePullPolicy::IfNotPresent)); + assert_eq!( + cfg.supervisor_image_pull_policy, + Some(ImagePullPolicy::Never) + ); + assert_eq!( + KubernetesComputeConfig::image_pull_policy_value(ImagePullPolicy::IfNotPresent), + "IfNotPresent" + ); + } + + #[test] + fn image_pull_policy_rejects_newer() { + let cfg = KubernetesComputeConfig { + image_pull_policy: Some(ImagePullPolicy::Newer), + ..KubernetesComputeConfig::default() + }; + let error = cfg.validate_image_pull_policies().unwrap_err(); + assert!(error.contains("supported only by the Podman")); + } + #[test] fn serde_override_topology_sidecar() { let json = serde_json::json!({ @@ -1387,6 +1378,18 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } + #[test] + fn upstream_proxy_config_accepts_tls_protected_secret_credentials() { + let cfg = KubernetesComputeConfig { + topology: SupervisorTopology::Sidecar, + https_proxy: Some("https://proxy.corp.example:8443".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + #[test] fn toml_deserializes_sidecar_upstream_proxy_settings() { let cfg: KubernetesComputeConfig = toml::from_str( diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index afe3f579e0..96e5ac0759 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -508,6 +508,9 @@ impl KubernetesComputeDriver { config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_image_pull_policies() + .map_err(KubernetesDriverError::Precondition)?; config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; @@ -1505,10 +1508,16 @@ impl KubernetesComputeDriver { let params = SandboxPodParams { default_image: &self.config.default_image, - image_pull_policy: &self.config.image_pull_policy, + image_pull_policy: self + .config + .image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), image_pull_secrets: &self.config.image_pull_secrets, supervisor_image: &self.config.supervisor_image, - supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, + supervisor_image_pull_policy: self + .config + .supervisor_image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, proxy_uid: self.config.sidecar.proxy_uid, @@ -2753,13 +2762,13 @@ fn supervisor_volume_mount() -> serde_json::Value { /// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. fn supervisor_image_volume( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let mut image_spec = serde_json::json!({ "reference": supervisor_image, }); - if !supervisor_image_pull_policy.is_empty() { - image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + image_spec["pullPolicy"] = serde_json::json!(policy); } serde_json::json!({ "name": SUPERVISOR_VOLUME_NAME, @@ -2777,7 +2786,7 @@ fn supervisor_image_volume( /// emissary executor. fn supervisor_init_container( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); let mut spec = serde_json::json!({ @@ -2795,8 +2804,8 @@ fn supervisor_init_container( "readOnly": false }] }); - if !supervisor_image_pull_policy.is_empty() { - spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + spec["imagePullPolicy"] = serde_json::json!(policy); } spec } @@ -2804,7 +2813,7 @@ fn supervisor_init_container( fn apply_supervisor_binary_source( spec: &mut serde_json::Map, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, ) { let volumes = spec @@ -2938,7 +2947,7 @@ fn apply_supervisor_sideload_with_params( fn apply_supervisor_sideload( pod_template: &mut serde_json::Value, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, sandbox_uid: u32, sandbox_gid: u32, @@ -3148,8 +3157,8 @@ fn supervisor_sidecar_container( .into_iter() .map(serde_json::Value::String), ); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if params.provider_spiffe_enabled { container["volumeMounts"] @@ -3211,8 +3220,8 @@ fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_jso sidecar_tls_volume_mount(), ] }); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if !params.client_tls_secret_name.is_empty() { container["volumeMounts"] @@ -3412,7 +3421,7 @@ fn apply_supervisor_sidecar_topology( fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, - image_pull_policy: &str, + image_pull_policy: Option<&str>, sandbox_gid: u32, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { @@ -3502,8 +3511,8 @@ fn apply_workspace_persistence( "mountPath": WORKSPACE_INIT_MOUNT_PATH }] }); - if !image_pull_policy.is_empty() { - init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + if let Some(policy) = image_pull_policy { + init_spec["imagePullPolicy"] = serde_json::json!(policy); } init_containers.push(init_spec); } @@ -3550,10 +3559,10 @@ fn default_workspace_volume_claim_templates( #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { default_image: &'a str, - image_pull_policy: &'a str, + image_pull_policy: Option<&'a str>, image_pull_secrets: &'a [String], supervisor_image: &'a str, - supervisor_image_pull_policy: &'a str, + supervisor_image_pull_policy: Option<&'a str>, supervisor_sideload_method: SupervisorSideloadMethod, topology: SupervisorTopology, proxy_uid: u32, @@ -3591,10 +3600,10 @@ impl Default for SandboxPodParams<'_> { fn default() -> Self { Self { default_image: "", - image_pull_policy: "", + image_pull_policy: None, image_pull_secrets: &[], supervisor_image: "", - supervisor_image_pull_policy: "", + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, @@ -3919,11 +3928,8 @@ fn sandbox_template_to_k8s_with_validated_config( }; if !image.is_empty() { container.insert("image".to_string(), serde_json::json!(image)); - if !params.image_pull_policy.is_empty() { - container.insert( - "imagePullPolicy".to_string(), - serde_json::json!(params.image_pull_policy), - ); + if let Some(policy) = params.image_pull_policy { + container.insert("imagePullPolicy".to_string(), serde_json::json!(policy)); } } @@ -4225,7 +4231,7 @@ fn image_pull_secret_refs(secrets: &[String]) -> Vec { fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { let mut value = serde_json::json!({ - "type": profile.to_k8s_type() + "type": profile.kubernetes_type() }); if let Some(localhost_profile) = profile.localhost_profile() { value["localhostProfile"] = serde_json::json!(localhost_profile); @@ -6200,7 +6206,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "custom-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, // sandbox_uid 1500, // sandbox_gid @@ -6237,7 +6243,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, 1600, @@ -6284,7 +6290,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6311,7 +6317,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6398,7 +6404,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6441,7 +6447,7 @@ mod tests { } #[test] - fn supervisor_image_volume_omits_pull_policy_when_empty() { + fn supervisor_image_volume_omits_pull_policy_when_unspecified() { let mut pod_template = serde_json::json!({ "spec": { "containers": [{ @@ -6454,7 +6460,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "", + None, SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6464,7 +6470,7 @@ mod tests { assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); assert!( volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when empty" + "pullPolicy should be omitted when unspecified" ); } @@ -6474,7 +6480,7 @@ mod tests { topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: "IfNotPresent", + supervisor_image_pull_policy: Some("IfNotPresent"), grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", client_tls_secret_name: "openshell-client-tls", proxy_uid: 2200, @@ -7352,7 +7358,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "openshell/sandbox:latest", - "IfNotPresent", + Some("IfNotPresent"), 1000, // sandbox_gid ); @@ -7411,7 +7417,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "my-custom-image:v2", - "IfNotPresent", + Some("IfNotPresent"), 1000, ); @@ -7435,7 +7441,7 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence(&mut pod_template, "img:latest", Some("Always"), 1000); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 7690ccee85..cedc6a333f 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,8 +10,8 @@ use tracing::info; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; -use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::otel_tracing::compute_driver_rpc_layer; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, @@ -75,7 +75,7 @@ struct Args { sandbox_image: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, + sandbox_image_pull_policy: Option, #[arg( long, @@ -97,8 +97,10 @@ struct Args { )] managed_ssh_gateway_pod_selector: Vec, + /// Gateway callback endpoint reachable from sandbox pods. Kubernetes + /// service topology cannot be inferred from the sandbox namespace. #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, + grpc_endpoint: String, #[arg( long, @@ -117,7 +119,7 @@ struct Args { supervisor_image: Option, #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, + supervisor_image_pull_policy: Option, #[arg( long, @@ -260,7 +262,7 @@ async fn main() -> Result<()> { operator_namespace_file: args.operator_namespace_file, service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, image_pull_secrets: args.sandbox_image_pull_secrets, managed_ssh_ingress: ManagedSshIngressConfig { enabled: args.managed_ssh_ingress_enabled, @@ -270,7 +272,7 @@ async fn main() -> Result<()> { supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_image_pull_policy: args.supervisor_image_pull_policy, supervisor_sideload_method: args.supervisor_sideload_method, topology: args.topology, sidecar: KubernetesSidecarConfig { @@ -284,7 +286,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + grpc_endpoint: args.grpc_endpoint, ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), @@ -353,6 +355,13 @@ async fn main() -> Result<()> { mod tests { use super::*; + #[test] + fn requires_explicit_gateway_callback_endpoint() { + let error = Args::try_parse_from(["openshell-driver-kubernetes"]) + .expect_err("Kubernetes service topology must be explicit"); + assert!(error.to_string().contains("--grpc-endpoint")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ @@ -361,6 +370,8 @@ mod tests { "http://collector.example:4317", "--gateway-name", "kubernetes-dev", + "--grpc-endpoint", + "http://openshell.example:8080", ]) .expect("OTLP endpoint should parse"); diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 9f8baa3446..837913ed1f 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -240,7 +240,7 @@ Step "Prepare DemoDir $DemoDir" New-Item -ItemType Directory -Force $DemoDir | Out-Null Ok "DemoDir ready" -$env:OPENSHELL_DRIVERS = "mxc" +$env:OPENSHELL_COMPUTE_DRIVER = "mxc" $env:OPENSHELL_MXC_SHARE_DIR = $DemoDir # ── Start gateway ───────────────────────────────────────────────────────────── diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index e53ddc9f3f..5781594158 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -275,9 +275,9 @@ Podman follows the same end-to-end contract as the Kubernetes and VM drivers for the in-container SSH relay: gateway config to `PodmanComputeConfig` to sandbox environment to supervisor session registration on that path. -1. `openshell-core` `Config::sandbox_ssh_socket_path` is copied into - `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the - in-process driver. +1. `[openshell.drivers.podman].ssh_socket_path` is deserialized into + `PodmanComputeConfig::ssh_socket_path` when the gateway builds the in-process + driver. The field defaults to `/run/openshell/ssh.sock` when omitted. 2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that value, alongside required vars such as `OPENSHELL_ENDPOINT` and `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template @@ -356,8 +356,12 @@ signals succeeds: - `test -S` on the configured supervisor Unix socket path. - The prior TCP check for a listener on the in-container SSH port. -The Unix socket check allows relay-only readiness when the supervisor exposes -the socket without the old marker or published-port signal. +The Unix socket check allows relay-only backend readiness when the supervisor +exposes the socket without the old marker or published-port signal. Omitting +`health_check_interval_secs` disables these Podman/conmon probes, but it does +not bypass public readiness gating: the gateway keeps a backend-ready sandbox +in `Provisioning` with `SupervisorNotConnected` until its supervisor control +session is connected. ### Deletion Flow @@ -380,14 +384,14 @@ Podman resources after out-of-band container removal or label drift. |---|---|---|---| | `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | | `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, or `newer`. | +| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `if_not_present` | Pull policy: `always`, `if_not_present`, `never`, or `newer`. | | `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | | `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | | `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | +| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Omission uses OpenShell's `2048` default; explicit `0` is invalid. | | `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | | `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | | `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | @@ -405,6 +409,17 @@ Through the gateway, the same settings are the `https_proxy`, `no_proxy`, and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see `docs/reference/gateway-config.mdx`. +`provider_spiffe_workload_api_socket` accepts either an absolute host UNIX +Workload API socket, projected through a dedicated read-only mount, or an +explicit container-reachable `tcp:IP:port` endpoint. The driver sets the +supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. +`app_armor_profile` shares the canonical +`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and +Kubernetes. When omitted, the driver sends no override and preserves Podman's +runtime-selected profile. Set `Unconfined` explicitly only when the deployment +requires the supervisor's mount setup to bypass that profile. Explicit confined +choices fail early when Podman reports AppArmor unavailable. + This is an operator-owned egress boundary: the driver passes the settings on the supervisor's command line, so sandbox and template environment — and any `ENV` baked into the sandbox image — cannot override them, and the diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 508b604ce7..5e0dadfb7f 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -272,9 +272,13 @@ pub struct HostInfo { /// Podman returns `host.security.rootless: true` when the daemon is /// running without root privileges (rootless mode). #[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "camelCase")] pub struct SecurityInfo { #[serde(default)] pub rootless: bool, + /// Whether the Podman host has `AppArmor` support enabled. + #[serde(default)] + pub apparmor_enabled: bool, } // ── Client ─────────────────────────────────────────────────────────────── @@ -959,13 +963,17 @@ mod tests { "cgroupVersion": "v2", "networkBackend": "netavark", "rootlessNetworkCmd": "pasta", - "security": {"rootless": true} + "security": { + "rootless": true, + "apparmorEnabled": true + } } }"#, ) .unwrap(); assert!(info.host.security.rootless); + assert!(info.host.security.apparmor_enabled); assert_eq!(info.host.rootless_network_cmd, "pasta"); } diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 42571f00f1..855b7f0e24 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::net::IpAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; -use std::str::FromStr; + +use openshell_core::{AppArmorProfile, ImagePullPolicy}; /// Default Podman bridge network name. pub const DEFAULT_NETWORK_NAME: &str = "openshell"; @@ -11,59 +13,14 @@ pub const MACOS_PODMAN_MACHINE_HOST_GATEWAY_IP: &str = "192.168.127.254"; /// Default Podman stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_PODMAN_STOP_TIMEOUT_SECS: u32 = 45; -// Re-export the shared default so existing imports inside this crate keep working. -pub use openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT; - -/// Image pull policy for sandbox and supervisor images. -/// -/// Controls when the Podman driver fetches a newer copy of an OCI image -/// from the registry. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ImagePullPolicy { - /// Always pull, even if a local copy exists. - Always, - /// Pull only when no local copy exists (default). - #[default] - Missing, - /// Never pull; fail if not available locally. - Never, - /// Pull only if the remote image is newer. - Newer, -} - -impl ImagePullPolicy { - /// Return the policy string expected by the Podman libpod API. - #[must_use] - pub fn as_str(&self) -> &'static str { - match self { - Self::Always => "always", - Self::Missing => "missing", - Self::Never => "never", - Self::Newer => "newer", - } - } -} - -impl std::fmt::Display for ImagePullPolicy { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ImagePullPolicy { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "always" => Ok(Self::Always), - "missing" => Ok(Self::Missing), - "never" => Ok(Self::Never), - "newer" => Ok(Self::Newer), - other => Err(format!( - "invalid pull policy '{other}'; expected one of: always, missing, never, newer" - )), - } +/// Translate the shared pull-policy vocabulary to the Podman libpod API. +#[must_use] +pub const fn podman_image_pull_policy(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "always", + ImagePullPolicy::IfNotPresent => "missing", + ImagePullPolicy::Never => "never", + ImagePullPolicy::Newer => "newer", } } @@ -90,7 +47,7 @@ pub struct PodmanComputeConfig { /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. - pub sandbox_ssh_socket_path: String, + pub ssh_socket_path: String, /// Name of the Podman bridge network. /// Created automatically if it does not exist. pub network_name: String, @@ -120,8 +77,13 @@ pub struct PodmanComputeConfig { pub guest_tls_key: Option, /// Container cgroup PID limit for Podman-managed sandboxes. /// - /// Set to `0` to leave Podman's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] @@ -129,14 +91,19 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, + /// `AppArmor` confinement requested for sandbox containers. Omission sends + /// no override and preserves Podman's runtime-selected profile. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_armor_profile: Option, /// Health check interval in seconds for sandbox containers. /// /// Podman runs the health check command at this interval to determine /// container readiness. Lower values detect readiness faster but /// increase process churn (each check spawns a conmon subprocess). - /// Set to `0` to disable health checks entirely. - /// Defaults to [`DEFAULT_HEALTH_CHECK_INTERVAL_SECS`] (10 seconds). - pub health_check_interval_secs: u64, + /// Omit the field to disable health checks entirely. Explicit zero is + /// invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub health_check_interval_secs: Option, /// Corporate forward proxy URL passed to the in-container supervisor /// (e.g. `http://proxy.corp.com:8080` or `https://proxy.corp.com:3130`). /// @@ -217,8 +184,6 @@ pub struct PodmanComputeConfig { pub gidmap: Vec, } -pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; - /// Parse a single `"container_id:host_id:size"` mapping entry. /// /// Returns `(container_id, host_id, size)` on success. @@ -298,9 +263,9 @@ impl PodmanComputeConfig { /// Validate runtime resource-limit configuration. pub fn validate_runtime_limits(&self) -> Result<(), crate::client::PodmanApiError> { - if self.sandbox_pids_limit < 0 { + if self.sandbox_pids_limit.is_some_and(|limit| limit.get() < 0) { return Err(crate::client::PodmanApiError::InvalidInput( - "sandbox_pids_limit must be zero or greater".to_string(), + "sandbox_pids_limit must be positive when set".to_string(), )); } Ok(()) @@ -318,91 +283,19 @@ impl PodmanComputeConfig { /// the URL is rejected because it would otherwise be stored in /// `gateway.toml` and exposed in container metadata. pub fn validate_proxy_config(&self) -> Result<(), crate::client::PodmanApiError> { - use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - let proxy_secure = if let Some(url) = &self.https_proxy { - let addr = parse_upstream_proxy_url(url).map_err(|err| { - crate::client::PodmanApiError::InvalidInput(match err { - UpstreamProxyUrlError::Empty => { - "https_proxy must not be empty when set".to_string() - } - UpstreamProxyUrlError::InlineCredentials => { - "https_proxy must not embed credentials in the URL; supply them via \ - proxy_auth_file so they are not stored in config or container metadata" - .to_string() - } - err => format!("https_proxy {err}"), - }) - })?; - addr.secure - } else { - false - }; - - // The supervisor treats a present-but-empty driver-supplied argument - // as a fatal misconfiguration, so never accept (and later pass) one. - if let Some(list) = self.no_proxy.as_deref() { - if list.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy must not be empty when set; omit it instead".to_string(), - )); - } - // A bypass list only makes sense relative to a proxy boundary. An - // operator who set one believed proxying was in effect, so accepting - // it while all egress dials directly would hide a fail-open state. - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy is set but no https_proxy is configured".to_string(), - )); - } - } - - if let Some(path) = self.proxy_auth_file.as_deref() { - if path.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file must not be empty when set".to_string(), - )); - } - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file is set but no https_proxy is configured".to_string(), - )); - } - // Basic auth over the plain-TCP proxy connection is readable by - // anyone on the network path; sending it requires an explicit - // operator acknowledgement rather than being an implicit side - // effect of configuring credentials. For an https:// proxy the - // credential is inside the verified TLS session, so the - // acknowledgement is unnecessary (but tolerated). - if self.proxy_auth_allow_insecure != Some(true) && !proxy_secure { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file sends the credential as cleartext Basic auth over the \ - plain-TCP connection to the http:// proxy; set proxy_auth_allow_insecure \ - = true to accept that exposure, or remove proxy_auth_file" - .to_string(), - )); - } - } else if self.proxy_auth_allow_insecure.is_some() { - // The acknowledgement without credentials means the operator - // believed an auth file was configured; surface the mismatch. - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), - )); - } - - // The CONNECT-target mode only means something relative to a proxy - // boundary the operator believed was in effect. - if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), - )); + // Keep the Podman-only CA-bundle behaviour below, but delegate the + // shared URL, bypass-list, credential-file, and acknowledgement + // contract to openshell-core so Docker and VM cannot drift. + openshell_core::UpstreamProxyConfig { + https_proxy: self.https_proxy.clone(), + no_proxy: self.no_proxy.clone(), + proxy_auth_file: self.proxy_auth_file.as_ref().map(PathBuf::from), + proxy_auth_allow_insecure: self.proxy_auth_allow_insecure, + proxy_connect_by_hostname: self.proxy_connect_by_hostname, } + .validate() + .map_err(crate::client::PodmanApiError::InvalidInput)?; - // A CA bundle only makes sense relative to a proxy boundary (an - // https:// proxy handshake, or a TLS-intercepting proxy's re-sign CA). - // Mirror the proxy_auth_file pairing so a stray setting cannot hide a - // fail-open state. The file's readability and certificate content are - // checked at sandbox-create time (see the driver) and fail closed in - // the supervisor. if let Some(path) = self.proxy_ca_bundle.as_deref() { if path.trim().is_empty() { return Err(crate::client::PodmanApiError::InvalidInput( @@ -501,6 +394,18 @@ impl PodmanComputeConfig { } /// Validate optional host gateway override. + /// Validate `AppArmor` syntax before contacting the runtime. Drivers check + /// runtime availability after querying their backend. + pub fn validate_app_armor_profile(&self) -> Result<(), crate::client::PodmanApiError> { + if matches!(self.app_armor_profile, Some(AppArmorProfile::Localhost(ref name)) if name.is_empty()) + { + return Err(crate::client::PodmanApiError::InvalidInput( + "app_armor_profile Localhost profile must not be empty".to_string(), + )); + } + Ok(()) + } + pub fn validate_host_gateway_ip(&self) -> Result<(), crate::client::PodmanApiError> { let trimmed = self.host_gateway_ip.trim(); if trimmed.is_empty() { @@ -536,7 +441,7 @@ impl Default for PodmanComputeConfig { image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, - sandbox_ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), + ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS, @@ -544,10 +449,11 @@ impl Default for PodmanComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, - health_check_interval_secs: DEFAULT_HEALTH_CHECK_INTERVAL_SECS, + app_armor_profile: None, + health_check_interval_secs: None, https_proxy: None, no_proxy: None, proxy_auth_file: None, @@ -566,10 +472,10 @@ impl std::fmt::Debug for PodmanComputeConfig { f.debug_struct("PodmanComputeConfig") .field("socket_path", &self.socket_path) .field("default_image", &self.default_image) - .field("image_pull_policy", &self.image_pull_policy.as_str()) + .field("image_pull_policy", &self.image_pull_policy) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) - .field("sandbox_ssh_socket_path", &self.sandbox_ssh_socket_path) + .field("ssh_socket_path", &self.ssh_socket_path) .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) .field("stop_timeout_secs", &self.stop_timeout_secs) @@ -583,6 +489,7 @@ impl std::fmt::Debug for PodmanComputeConfig { "provider_spiffe_workload_api_socket", &self.provider_spiffe_workload_api_socket, ) + .field("app_armor_profile", &self.app_armor_profile) .field( "health_check_interval_secs", &self.health_check_interval_secs, @@ -606,14 +513,60 @@ mod tests { use super::*; #[test] - fn default_config_sets_health_check_interval() { - let cfg = PodmanComputeConfig::default(); + fn config_uses_canonical_ssh_socket_path_name() { + let config: PodmanComputeConfig = + serde_json::from_value(serde_json::json!({ "ssh_socket_path": "/run/test.sock" })) + .unwrap(); + assert_eq!(config.ssh_socket_path, "/run/test.sock"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["ssh_socket_path"], "/run/test.sock"); + assert!(serialized.get("sandbox_ssh_socket_path").is_none()); + } + + #[test] + fn config_rejects_legacy_sandbox_ssh_socket_path() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_ssh_socket_path": "/run/test.sock" + })) + .expect_err("legacy sandbox_ssh_socket_path must be rejected"); + assert!(error.to_string().contains("sandbox_ssh_socket_path")); + } + + #[test] + fn default_config_disables_health_checks() { assert_eq!( - cfg.health_check_interval_secs, - DEFAULT_HEALTH_CHECK_INTERVAL_SECS + PodmanComputeConfig::default().health_check_interval_secs, + None ); } + #[test] + fn omitted_apparmor_profile_preserves_runtime_default() { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("omitted AppArmor profile should deserialize"); + assert_eq!(config.app_armor_profile, None); + + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert!(serialized.get("app_armor_profile").is_none()); + } + + #[test] + fn explicit_apparmor_profiles_round_trip() { + for value in [ + "RuntimeDefault", + "Unconfined", + "Localhost/openshell-supervisor", + ] { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "app_armor_profile": value, + })) + .expect("explicit AppArmor profile should deserialize"); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], value); + } + } + #[test] fn default_config_sets_podman_stop_timeout() { let cfg = PodmanComputeConfig::default(); @@ -623,9 +576,21 @@ mod tests { #[test] fn default_config_sets_driver_owned_pids_limit() { let cfg = PodmanComputeConfig::default(); - assert_eq!(cfg.sandbox_pids_limit, DEFAULT_SANDBOX_PIDS_LIMIT); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); assert!(!cfg.enable_bind_mounts); - assert!(cfg.validate_runtime_limits().is_ok()); + } + + #[test] + fn omitted_pids_limit_uses_driver_owned_default() { + let cfg: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Podman config should deserialize"); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); } #[test] @@ -655,13 +620,28 @@ mod tests { } #[test] - fn runtime_limit_validation_rejects_negative_pids_limit() { - let cfg = PodmanComputeConfig { - sandbox_pids_limit: -1, - ..PodmanComputeConfig::default() - }; - let err = cfg.validate_runtime_limits().unwrap_err(); - assert!(err.to_string().contains("sandbox_pids_limit")); + fn runtime_limit_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 + })) + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = negative.validate_runtime_limits().unwrap_err(); + assert!(error.to_string().contains("must be positive")); + } + + #[test] + fn health_check_interval_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "health_check_interval_secs": 0 + })) + .expect_err("zero health-check interval must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); } // ── Proxy config validation ─────────────────────────────────────── diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a81ee13e1d..4918c9210a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -216,8 +216,14 @@ struct ContainerSpec { cap_add: Vec, no_new_privileges: bool, seccomp_profile_path: String, + /// Podman's container create API accepts `AppArmor` through the dedicated + /// `apparmor_profile` `SpecGenerator` field. This is not Docker's + /// `security_opt` representation. + #[serde(skip_serializing_if = "Option::is_none")] + apparmor_profile: Option, image_pull_policy: String, - healthconfig: HealthConfig, + #[serde(skip_serializing_if = "Option::is_none")] + healthconfig: Option, resource_limits: ResourceLimits, /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. /// Podman's libpod `SpecGenerator` uses `secret_env` (a flat map) for @@ -526,7 +532,7 @@ fn build_env( ); env.insert( openshell_core::sandbox_env::SSH_SOCKET_PATH.into(), - config.sandbox_ssh_socket_path.clone(), + config.ssh_socket_path.clone(), ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) @@ -652,14 +658,10 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids_limit: podman_pids_limit(config.sandbox_pids_limit), + pids_limit: config.sandbox_pids_limit.map(std::num::NonZeroI64::get), } } -fn podman_pids_limit(value: i64) -> Option { - if value > 0 { Some(value) } else { None } -} - pub fn podman_driver_volume_mount_sources( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -940,6 +942,14 @@ fn validate_tmpfs_options(options: &[String]) -> Result, String> { .collect() } +fn podman_apparmor_profile(profile: Option<&openshell_core::AppArmorProfile>) -> Option { + match profile { + None | Some(openshell_core::AppArmorProfile::RuntimeDefault) => None, + Some(openshell_core::AppArmorProfile::Unconfined) => Some("unconfined".to_string()), + Some(openshell_core::AppArmorProfile::Localhost(profile)) => Some(profile.clone()), + } +} + /// Build the Podman container creation JSON spec. #[cfg(test)] #[must_use] @@ -1175,21 +1185,22 @@ pub fn build_container_spec_for_image( // locks itself down. no_new_privileges: true, seccomp_profile_path: "unconfined".into(), + apparmor_profile: podman_apparmor_profile(config.app_armor_profile.as_ref()), image_pull_policy: "never".to_string(), - healthconfig: HealthConfig { + healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { test: vec![ "CMD-SHELL".into(), format!( "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", - config.sandbox_ssh_socket_path, + config.ssh_socket_path, openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: config.health_check_interval_secs * 1_000_000_000, + interval: interval_secs.get() * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, - }, + }), resource_limits, secret_env: BTreeMap::new(), secrets: { @@ -1561,7 +1572,8 @@ mod tests { }), ..Default::default() }); - let config = test_config(); + let mut config = test_config(); + config.sandbox_pids_limit = std::num::NonZeroI64::new(2048); let spec = build_container_spec(&sandbox, &config); assert_eq!( @@ -1572,22 +1584,53 @@ mod tests { spec["resource_limits"]["memory"]["limit"].as_u64(), Some(2 * 1024 * 1024 * 1024) ); - assert_eq!( - spec["resource_limits"]["PidsLimit"].as_i64(), - Some(crate::config::DEFAULT_SANDBOX_PIDS_LIMIT) - ); + assert_eq!(spec["resource_limits"]["PidsLimit"].as_i64(), Some(2048)); } #[test] fn container_spec_can_inherit_runtime_pids_limit() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.sandbox_pids_limit = 0; + config.sandbox_pids_limit = None; let spec = build_container_spec(&sandbox, &config); assert!(spec["resource_limits"].get("PidsLimit").is_none()); } + #[test] + fn container_spec_uses_podman_apparmor_profile_field() { + let sandbox = test_sandbox("test-id", "test-name"); + + for (profile, expected) in [ + (openshell_core::AppArmorProfile::Unconfined, "unconfined"), + ( + openshell_core::AppArmorProfile::Localhost("openshell-supervisor".to_string()), + "openshell-supervisor", + ), + ] { + let mut config = test_config(); + config.app_armor_profile = Some(profile); + let spec = build_container_spec(&sandbox, &config); + + assert_eq!(spec["apparmor_profile"].as_str(), Some(expected)); + assert!(spec.get("security_opt").is_none()); + } + } + + #[test] + fn container_spec_omits_podman_apparmor_profile_for_runtime_default() { + let sandbox = test_sandbox("test-id", "test-name"); + + for profile in [None, Some(openshell_core::AppArmorProfile::RuntimeDefault)] { + let mut config = test_config(); + config.app_armor_profile = profile; + let spec = build_container_spec(&sandbox, &config); + + assert!(spec.get("apparmor_profile").is_none()); + assert!(spec.get("security_opt").is_none()); + } + } + #[test] fn container_name_is_workspace_qualified() { assert_eq!( @@ -1961,7 +2004,8 @@ mod tests { #[test] fn container_spec_healthcheck_accepts_supervisor_socket() { let sandbox = test_sandbox("test-id", "test-name"); - let config = test_config(); + let mut config = test_config(); + config.health_check_interval_secs = std::num::NonZeroU64::new(10); let spec = build_container_spec(&sandbox, &config); let healthcheck = spec["healthconfig"]["test"] @@ -1977,11 +2021,18 @@ mod tests { ); } + #[test] + fn container_spec_omits_healthcheck_when_disabled() { + let sandbox = test_sandbox("test-id", "test-name"); + let spec = build_container_spec(&sandbox, &test_config()); + assert!(spec.get("healthconfig").is_none()); + } + #[test] fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.health_check_interval_secs = 30; + config.health_check_interval_secs = std::num::NonZeroU64::new(30); let spec = build_container_spec(&sandbox, &config); let interval = spec["healthconfig"]["Interval"] @@ -2407,7 +2458,7 @@ mod tests { default_image: "test-image:latest".to_string(), grpc_endpoint: "http://localhost:50051".to_string(), host_gateway_ip: String::new(), - sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), ..PodmanComputeConfig::default() } } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index aa41017663..04322236f5 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -4,7 +4,7 @@ //! Podman compute driver. use crate::client::{ContainerListEntry, PodmanApiError, PodmanClient, VolumeInspect}; -use crate::config::PodmanComputeConfig; +use crate::config::{PodmanComputeConfig, podman_image_pull_policy}; use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, PodmanSandboxDriverConfig}; use crate::watcher::{ self, LifecycleEventFences, WatchStream, driver_sandbox_from_inspect, @@ -13,8 +13,9 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + GatewayCallbackTopology, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -373,6 +374,22 @@ impl PodmanComputeDriver { config.validate_runtime_limits()?; config.validate_host_gateway_ip()?; config.validate_proxy_config()?; + config.validate_app_armor_profile()?; + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_deref() { + let raw = socket.to_str().ok_or_else(|| { + PodmanApiError::InvalidInput( + "provider_spiffe_workload_api_socket must be valid UTF-8".to_string(), + ) + })?; + // Preserve Podman's established pass-through support for an + // explicitly configured container-reachable Workload API TCP + // endpoint. The Workload API client validates its endpoint grammar + // when it connects. + if !raw.starts_with("tcp:") { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(PodmanApiError::InvalidInput)?; + } + } config.canonicalize_userns()?; config.validate_userns_mappings()?; @@ -411,11 +428,16 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } + validate_apparmor_support( + config.app_armor_profile.as_ref(), + info.host.security.apparmor_enabled, + )?; info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, rootless = info.host.security.rootless, rootless_network_cmd = %info.host.rootless_network_cmd, + apparmor_enabled = info.host.security.apparmor_enabled, "Connected to Podman" ); (info.host.security.rootless, info.host.rootless_network_cmd) @@ -437,14 +459,10 @@ impl PodmanComputeDriver { // Auto-detect the gRPC callback endpoint before deciding whether this // topology needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = format!( - "{scheme}://host.containers.internal:{}", - config.gateway_port + config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Podman, + config.gateway_port, + config.tls_enabled(), ); info!( grpc_endpoint = %config.grpc_endpoint, @@ -791,7 +809,7 @@ impl PodmanComputeDriver { .to_string(), )); } - let pull_policy = self.config.image_pull_policy.as_str(); + let pull_policy = podman_image_pull_policy(self.config.image_pull_policy); info!(image = %image, policy = %pull_policy, "Ensuring sandbox image"); self.client .pull_image(image, pull_policy) @@ -1387,6 +1405,26 @@ impl PodmanComputeDriver { } } +fn validate_apparmor_support( + profile: Option<&openshell_core::AppArmorProfile>, + apparmor_enabled: bool, +) -> Result<(), PodmanApiError> { + let requires_apparmor = matches!( + profile, + Some( + openshell_core::AppArmorProfile::RuntimeDefault + | openshell_core::AppArmorProfile::Localhost(_) + ) + ); + if requires_apparmor && !apparmor_enabled { + return Err(PodmanApiError::InvalidInput( + "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" + .to_string(), + )); + } + Ok(()) +} + fn supervisor_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" @@ -2198,6 +2236,26 @@ mod tests { ); } + #[test] + fn confined_apparmor_profiles_follow_podman_capability() { + use openshell_core::AppArmorProfile; + + for profile in [ + AppArmorProfile::RuntimeDefault, + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + ] { + validate_apparmor_support(Some(&profile), true) + .expect("confined profile should be accepted when Podman reports AppArmor"); + let error = validate_apparmor_support(Some(&profile), false) + .expect_err("confined profile must fail when AppArmor is unavailable"); + assert!(error.to_string().contains("AppArmor is unavailable")); + } + validate_apparmor_support(Some(&AppArmorProfile::Unconfined), false) + .expect("Unconfined does not require AppArmor support"); + validate_apparmor_support(None, false) + .expect("an omitted profile preserves Podman's runtime behavior"); + } + #[test] #[cfg(target_os = "linux")] fn rootless_pasta_requests_default_route_interface() { diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index a0fa85d018..b3be4d273a 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -5,17 +5,15 @@ use clap::Parser; use miette::{IntoDiagnostic, Result}; use std::future::Future; use std::net::SocketAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use tracing::info; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; -use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; +use openshell_core::{AppArmorProfile, ImagePullPolicy, VERSION}; +use openshell_driver_podman::config::{DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS}; use openshell_driver_podman::otel_tracing::compute_driver_rpc_layer; use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; @@ -53,7 +51,7 @@ struct Args { #[arg( long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing + default_value_t = ImagePullPolicy::IfNotPresent )] sandbox_image_pull_policy: ImagePullPolicy, @@ -92,14 +90,19 @@ struct Args { #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] stop_timeout: u32, - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit - /// Podman's runtime/default PID limit. + /// Container cgroup PID limit for sandbox containers. Omit to use + /// `OpenShell`'s 2048-process default. + #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT", default_value = "2048")] + sandbox_pids_limit: Option, + + /// Health check interval in seconds. Omit it in gateway TOML to disable + /// health checks; the standalone driver keeps its prior 10-second default. #[arg( long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + env = "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS", + default_value = "10" )] - sandbox_pids_limit: i64, + health_check_interval_secs: Option, /// OCI image containing the openshell-sandbox supervisor binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] @@ -117,6 +120,14 @@ struct Args { #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] podman_tls_key: Option, + /// Host UNIX socket projected into supervisors for provider SPIFFE token exchange. + #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] + provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` model: `RuntimeDefault`, `Unconfined`, or `Localhost/`. + #[arg(long, env = "OPENSHELL_APP_ARMOR_PROFILE")] + app_armor_profile: Option, + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, /// in explicit `http://host:port` form (scheme and port required). /// Credentials must not be embedded in the URL; use @@ -205,7 +216,7 @@ async fn main() -> Result<()> { host_gateway_ip: args .host_gateway_ip .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, supervisor_image: args @@ -214,7 +225,10 @@ async fn main() -> Result<()> { guest_tls_ca: args.podman_tls_ca, guest_tls_cert: args.podman_tls_cert, guest_tls_key: args.podman_tls_key, + provider_spiffe_workload_api_socket: args.provider_spiffe_workload_api_socket, + app_armor_profile: args.app_armor_profile, sandbox_pids_limit: args.sandbox_pids_limit, + health_check_interval_secs: args.health_check_interval_secs, https_proxy: args.sandbox_https_proxy, no_proxy: args.sandbox_no_proxy, proxy_auth_file: args.sandbox_proxy_auth_file, @@ -225,7 +239,6 @@ async fn main() -> Result<()> { uidmap: args.uidmap, gidmap: args.gidmap, enable_bind_mounts: args.enable_bind_mounts, - ..PodmanComputeConfig::default() }) .await .into_diagnostic()?; @@ -327,4 +340,25 @@ mod tests { ); assert_eq!(args.gateway_name.as_deref(), Some("production-us-west")); } + + #[test] + fn standalone_defaults_preserve_health_checks_and_reject_zero_limits() { + let defaults = Args::try_parse_from(["openshell-driver-podman"]) + .expect("standalone driver defaults should parse"); + assert_eq!( + defaults.health_check_interval_secs.map(NonZeroU64::get), + Some(10) + ); + assert_eq!( + defaults.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); + + for flag in ["--sandbox-pids-limit", "--health-check-interval-secs"] { + let result = Args::try_parse_from(["openshell-driver-podman", flag, "0"]); + assert!(result.is_err(), "zero must be rejected for {flag}"); + let error = result.err().expect("error was asserted above"); + assert!(error.to_string().contains("invalid value"), "flag: {flag}"); + } + } } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index c3903a0067..257d649a76 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -439,7 +439,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { Some(HealthState { status }) if status == "starting" => { ("False", "HealthCheckStarting", String::new()) } - _ => ("False", CONDITION_STARTING, String::new()), + None => ( + "True", + CONDITION_RUNNING, + "Container is running".to_string(), + ), + Some(_) => ("False", CONDITION_STARTING, String::new()), }, "created" => ("False", "ContainerCreated", String::new()), "exited" | "stopped" => { @@ -578,6 +583,25 @@ mod tests { assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); } + #[test] + fn condition_running_without_healthcheck_is_ready() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: None, + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let cond = condition_from_state(&state); + assert_eq!(cond.r#type, "Ready"); + assert_eq!(cond.status, "True"); + assert_eq!(cond.reason, CONDITION_RUNNING); + assert_eq!(cond.message, "Container is running"); + assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); + } + #[test] fn condition_oom_killed() { let state = ContainerState { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 4fc9ace415..6e7013e215 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -113,22 +113,23 @@ codesign \ mkdir -p /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm cat > .cache/gateway-vm/gateway.toml <` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | +| `grpc_endpoint` | topology-derived | Optional override for the URL the sandbox guest dials to reach the gateway. The gateway derives `http(s)://host.openshell.internal:` when absent. Use `host.containers.internal`, `host.docker.internal`, or another routable host only for a non-standard topology. Loopback URLs are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | | `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | @@ -151,9 +152,15 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | -| `guest_tls_cert` | unset | Guest client certificate. | -| `guest_tls_key` | unset | Guest client private key. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers identity from its upper layer or prepared rootfs, an explicit override, or the current image. Legacy `10001:10001` is retained only when persisted state reports it. | +| `https_proxy`, `no_proxy`, `proxy_auth_file` | unset | Operator-owned corporate TLS proxy settings. The driver injects only URL/list/path controls into protected guest startup; it copies a validated `user:pass` auth file into the private overlay, never into logs or process arguments. An `http://` proxy with credentials requires `proxy_auth_allow_insecure = true`. | +| `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | + +For gateway-managed VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and +`guest_tls_key` together under `[openshell.gateway]`; the gateway validates and +injects that bundle into only the selected local driver. The standalone +`openshell-driver-vm` CLI retains its `--guest-tls-*` inputs for independent +operation. See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. @@ -249,7 +256,7 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain -- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection +- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation, identity inspection, and QEMU environment injection. Explicit `sandbox_uid`/`sandbox_gid` values do not remove this runtime prerequisite. - Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. @@ -274,8 +281,8 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical On Debian-family Linux amd64 and arm64 systems, `install.sh` installs the Debian package from the selected `OPENSHELL_VERSION` release tag. That package includes `openshell-gateway` and `openshell-driver-vm`, but leaves -`OPENSHELL_DRIVERS` unset so the gateway uses its normal runtime -auto-detection. Set `OPENSHELL_DRIVERS=vm` to force the VM driver. +`OPENSHELL_COMPUTE_DRIVER` unset so the gateway uses its normal runtime +auto-detection. Set `OPENSHELL_COMPUTE_DRIVER=vm` to force the VM driver. On RPM-family Linux x86_64 and aarch64 systems, `install.sh` installs the `openshell` and `openshell-gateway` RPM packages from the selected release tag. @@ -286,7 +293,7 @@ formula from the selected release in the `nvidia/openshell` Homebrew tap. Homebrew installs `openshell`, `openshell-gateway`, and `openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement in `post_install`, and owns the `brew services` gateway lifecycle. The service -also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless +also leaves `OPENSHELL_COMPUTE_DRIVER` unset so driver choice remains automatic unless the user explicitly overrides it. ## TODOs diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 14dbc0466b..ada2fb0a54 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -29,7 +29,6 @@ BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) GVPROXY_GATEWAY_IP="192.168.127.1" GVPROXY_HOST_LOOPBACK_IP="192.168.127.254" GATEWAY_IP="$GVPROXY_GATEWAY_IP" -SANDBOX_OWNER_NORMALIZED_MARKER="/opt/openshell/.sandbox-owner-normalized" GPU_ENABLED="${GPU_ENABLED:-false}" VM_NET_IP="${VM_NET_IP:-}" @@ -90,7 +89,9 @@ sandbox_owner_from_passwd() { done < "$passwd_path" fi - printf '10001:10001\n' + # New images use the conventional sandbox UID. Existing images retain the + # sandbox account read above, including the legacy 10001:10001 identity. + printf '1000:1000\n' } source_overlay_env_if_present() { @@ -103,6 +104,21 @@ source_overlay_env_if_present() { ensure_target_runtime() { local image_root="$1" + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local replace_account=0 + + # An omitted identity means the image owns its sandbox account contract. + # Fall back to 1000 only when the image has no sandbox account at all. + if [ -n "$sandbox_uid" ] || [ -n "$sandbox_gid" ]; then + sandbox_uid="${sandbox_uid:-1000}" + sandbox_gid="${sandbox_gid:-$sandbox_uid}" + replace_account=1 + elif ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sandbox_uid=1000 + sandbox_gid=1000 + replace_account=1 + fi mkdir -p \ "$image_root/srv" \ @@ -119,31 +135,31 @@ ensure_target_runtime() { fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" - if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then - printf 'sandbox:x:10001:\n' >> "$image_root/etc/group" - fi - if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then - printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" - fi - if ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then - printf 'sandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n' >> "$image_root/etc/passwd" - fi - if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then - printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + if [ "$replace_account" -eq 1 ]; then + if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" + fi + if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then + printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" + fi + if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" + fi + if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then + printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + fi fi local owner - local owner_normalized=0 owner="$(sandbox_owner_for_root "$image_root")" - if chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 - elif chown -R 10001:10001 "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 + if ! chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then + ts "FATAL: failed to apply sandbox image ownership (${owner})" + exit 1 fi chmod 0755 "$image_root/sandbox" - if [ "$owner_normalized" -eq 1 ]; then - mkdir -p "$image_root/opt/openshell" - printf '1\n' > "$image_root${SANDBOX_OWNER_NORMALIZED_MARKER}" - fi } prepare_guest_image_rootfs() { @@ -214,14 +230,14 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" "$@" fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$supervisor" "$@" fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox --workdir /sandbox + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox "$@" fi done @@ -546,6 +562,50 @@ setup_gpu() { fi } +reconcile_sandbox_account() { + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local etc + + [ -n "$sandbox_uid" ] && [ -n "$sandbox_gid" ] || return 0 + [[ "$sandbox_uid" =~ ^[0-9]+$ ]] && [[ "$sandbox_gid" =~ ^[0-9]+$ ]] || { + ts "FATAL: invalid requested sandbox identity" + exit 1 + } + etc="$(root_path /etc)" + mkdir -p "$etc" + touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow" + if grep -q '^sandbox:' "$etc/group"; then + if ! awk -F: -v OFS=: -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = gid } { print }' \ + "$etc/group" >"$etc/group.openshell"; then + rm -f "$etc/group.openshell" + ts "FATAL: failed to reconcile sandbox group" + exit 1 + fi + mv "$etc/group.openshell" "$etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group" + fi + if grep -q '^sandbox:' "$etc/passwd"; then + # Preserve image-owned account metadata (home, shell, and description) + # while restoring the UID/GID contract recorded for this overlay. + if ! awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = uid; $4 = gid } { print }' \ + "$etc/passwd" >"$etc/passwd.openshell"; then + rm -f "$etc/passwd.openshell" + ts "FATAL: failed to reconcile sandbox account" + exit 1 + fi + mv "$etc/passwd.openshell" "$etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd" + fi + grep -q '^sandbox:' "$etc/gshadow" || printf 'sandbox:!::\n' >> "$etc/gshadow" + grep -q '^sandbox:' "$etc/shadow" || printf 'sandbox:!:20123:0:99999:7:::\n' >> "$etc/shadow" + ts "reconciled sandbox account (${sandbox_uid}:${sandbox_gid})" +} + setup_sandbox_workdir() { local sandbox_dir local owner @@ -554,10 +614,13 @@ setup_sandbox_workdir() { owner="$(sandbox_owner)" mkdir -p "$sandbox_dir" current_owner="$(stat -c '%u:%g' "$sandbox_dir" 2>/dev/null || true)" - if [ "$current_owner" != "$owner" ] \ - || [ ! -f "$(root_path "$SANDBOX_OWNER_NORMALIZED_MARKER")" ]; then + if [ "$owner" = "10001:10001" ]; then + ts "preserving legacy sandbox ownership (10001:10001)" + fi + if [ "$current_owner" != "$owner" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then - chown -R 10001:10001 "$sandbox_dir" + ts "FATAL: failed to apply sandbox ownership (${owner})" + exit 1 fi fi chmod 0755 "$sandbox_dir" @@ -684,6 +747,7 @@ run_post_overlay_setup() { echo 1 > /proc/sys/net/netfilter/nf_log_all_netns 2>/dev/null || true fi + reconcile_sandbox_account setup_sandbox_workdir configure_hostname @@ -833,11 +897,32 @@ if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}" fi +# Proxy settings are driver-owned environment values. Turn them into +# supervisor argv only after all user/template values have been overwritten; +# the credential itself stays in the root-only overlay file and is never put +# in argv or logged. +set -- --workdir /sandbox +if [ -n "${OPENSHELL_VM_UPSTREAM_PROXY:-}" ]; then + set -- "$@" --upstream-proxy "$OPENSHELL_VM_UPSTREAM_PROXY" +fi +if [ -n "${OPENSHELL_VM_UPSTREAM_NO_PROXY:-}" ]; then + set -- "$@" --upstream-no-proxy "$OPENSHELL_VM_UPSTREAM_NO_PROXY" +fi +if [ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE:-}" ]; then + set -- "$@" --upstream-proxy-auth-file "$OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE" +fi +if [ "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE:-}" = "true" ]; then + set -- "$@" --upstream-proxy-auth-allow-insecure +fi +if [ "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME:-}" = "true" ]; then + set -- "$@" --upstream-proxy-connect-by-hostname +fi + ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then - exec_supervisor_in_newroot + exec_supervisor_in_newroot "$@" fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox +exec /opt/openshell/bin/openshell-sandbox "$@" } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 8adcc79f92..2356b107df 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11,6 +11,7 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, + sandbox_guest_user_ids_from_image, sandbox_guest_user_ids_from_overlay_image, set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; @@ -29,6 +30,7 @@ use oci_client::manifest::{ }; use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; +use openshell_core::UpstreamProxyConfig; use openshell_core::gpu::{ driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; @@ -152,6 +154,8 @@ const GUEST_TLS_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CA const GUEST_TLS_CERT_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CERT_PATH; const GUEST_TLS_KEY_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_KEY_PATH; const GUEST_SANDBOX_TOKEN_PATH: &str = openshell_core::container_paths::VM_GUEST_SANDBOX_TOKEN_PATH; +const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = + openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH; const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_DIR; /// Guest path of the driver-authored manifest enumerating which /// `init.d` drop-ins the guest init script is allowed to execute. @@ -167,6 +171,9 @@ const IMAGE_CACHE_ROOTFS_IMAGE: &str = "rootfs.ext4"; const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; const OVERLAY_TEMPLATE_CACHE_LAYOUT_VERSION: &str = "sandbox-overlay-ext4-v1"; const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; +const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; +const SANDBOX_OWNER_STATE_V1: &str = "sandbox-owner-v1"; +const SANDBOX_OWNER_STATE_V2: &str = "sandbox-owner-v2"; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -182,6 +189,7 @@ const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; const IMAGE_PREP_INIT_MODE: &str = "image-prep"; static IMAGE_CACHE_BUILD_COUNTER: AtomicU64 = AtomicU64::new(0); +static OWNER_STATE_WRITE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone)] struct VmDriverTlsPaths { @@ -218,8 +226,9 @@ enum GuestImagePayloadSource { } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct VmDriverConfig { - pub openshell_endpoint: String, + pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, pub default_image: String, @@ -232,26 +241,36 @@ pub struct VmDriverConfig { pub guest_tls_ca: Option, pub guest_tls_cert: Option, pub guest_tls_key: Option, + /// Corporate forward proxy settings delivered to the guest init script. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + /// Guest-reachable SPIFFE Workload API TCP endpoint. A VM cannot safely + /// project a host UNIX socket; this must be a deliberately exposed TCP + /// listener and requires `provider_spiffe_allow_guest_tcp`. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - /// Resolved sandbox UID for rootfs `/etc/passwd` entry. - /// When empty, defaults to 10001 (the legacy hardcoded value). + /// Optional UID override for the sandbox account in newly prepared rootfs images. + /// When both identity fields are empty, an image-provided sandbox account is preserved. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, - /// Resolved sandbox GID for rootfs `/etc/passwd` and `/etc/group` entries. - /// When empty, defaults to the resolved UID. + /// Optional GID override for rootfs `/etc/passwd` and `/etc/group` entries. + /// When one override is supplied, its missing counterpart defaults to the UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, } -/// Default sandbox UID used by the VM driver when no config value is set. -pub const DEFAULT_SANDBOX_UID: u32 = 10001; +/// Fallback sandbox UID for images without a `sandbox` account and partial +/// operator identity overrides. +pub const DEFAULT_SANDBOX_UID: u32 = 1000; impl Default for VmDriverConfig { fn default() -> Self { Self { - openshell_endpoint: String::new(), + grpc_endpoint: String::new(), state_dir: PathBuf::from("target/openshell-vm-driver"), launcher_bin: None, default_image: String::new(), @@ -264,6 +283,9 @@ impl Default for VmDriverConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, gpu_enabled: false, gpu_mem_mib: 8192, gpu_vcpus: 4, @@ -274,16 +296,29 @@ impl Default for VmDriverConfig { } impl VmDriverConfig { - /// Resolve the sandbox UID, falling back to `DEFAULT_SANDBOX_UID`. + /// Resolve a fallback sandbox UID for an image that has no sandbox account. pub fn resolve_sandbox_uid(&self) -> u32 { self.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID) } - /// Resolve the sandbox GID, falling back to the resolved UID. + /// Resolve a fallback sandbox GID from the selected UID. pub fn resolve_sandbox_gid(&self, resolved_uid: u32) -> u32 { self.sandbox_gid.unwrap_or(resolved_uid) } + pub fn validate_runtime_security_config(&self) -> Result<(), String> { + self.upstream_proxy.validate()?; + if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + self.provider_spiffe_allow_guest_tcp, + )?; + } else if self.provider_spiffe_allow_guest_tcp { + return Err("provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured".to_string()); + } + Ok(()) + } + pub fn validate_sandbox_identity(&self) -> Result<(), String> { let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; if let Some(uid) = self.sandbox_uid @@ -308,7 +343,7 @@ impl VmDriverConfig { } fn requires_tls_materials(&self) -> bool { - self.openshell_endpoint.starts_with("https://") + self.grpc_endpoint.starts_with("https://") } fn tls_paths(&self) -> Result, String> { @@ -401,6 +436,25 @@ enum OverlayPreparation { PreserveExisting, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SandboxOwnerIdentity { + uid: u32, + gid: u32, +} + +impl SandboxOwnerIdentity { + fn guest_environment(self) -> [String; 2] { + [ + format!("OPENSHELL_VM_SANDBOX_UID={}", self.uid), + format!("OPENSHELL_VM_SANDBOX_GID={}", self.gid), + ] + } + + fn marker_contents(self) -> String { + format!("{SANDBOX_OWNER_STATE_V2}:{}:{}\n", self.uid, self.gid) + } +} + fn provisioning_span( parent: &opentelemetry::Context, sandbox_id: &str, @@ -447,10 +501,11 @@ impl VmDriver { .validate() .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; - if config.openshell_endpoint.trim().is_empty() { + config.validate_runtime_security_config()?; + if config.grpc_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } - validate_openshell_endpoint(&config.openshell_endpoint)?; + validate_openshell_endpoint(&config.grpc_endpoint)?; let _ = config.tls_paths()?; #[cfg(target_os = "linux")] @@ -746,6 +801,7 @@ impl VmDriver { let disk_paths = sandbox_runtime_disk_paths(&state_dir); let root_disk = image_plan.root_disk; let image_disk = image_plan.image_disk; + let owner_source_disk = image_disk.as_ref().unwrap_or(&root_disk).clone(); let overlay_disk = disk_paths.overlay_disk; self.publish_platform_event( @@ -757,9 +813,11 @@ impl VmDriver { "Preparing writable VM overlay disk".to_string(), ), ); - if let Err(err) = self + let sandbox_owner_state = self .prepare_runtime_overlay( + &state_dir, &overlay_disk, + &owner_source_disk, tls_paths.as_ref(), sandbox .spec @@ -769,11 +827,7 @@ impl VmDriver { overlay_preparation, ) .await - { - return Err(Status::internal(format!( - "prepare guest overlay disk failed: {err}" - ))); - } + .map_err(|err| Status::internal(format!("prepare guest overlay disk failed: {err}")))?; self.ensure_provisioning_active(&sandbox.id).await?; if let Err(err) = @@ -910,7 +964,7 @@ impl VmDriver { let endpoint_override = if plan.backend == VmBackend::Qemu { plan.host_ip.as_deref().map(|host_ip| { - guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) + guest_visible_openshell_endpoint_for_tap(&self.config.grpc_endpoint, host_ip) }) } else { None @@ -974,6 +1028,9 @@ impl VmDriver { for env in &plan.env { command.arg("--vm-env").arg(env); } + for env in sandbox_owner_state.guest_environment() { + command.arg("--vm-env").arg(env); + } info!( sandbox_id = %sandbox.id, @@ -1713,7 +1770,7 @@ impl VmDriver { "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] )); - plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + plan.gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); Ok(()) } @@ -1845,7 +1902,7 @@ impl VmDriver { mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] ); let tap = tap_device_name(sandbox_id); - let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + let gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); let (vcpus, mem_mib) = if is_gpu { (self.config.gpu_vcpus, self.config.gpu_mem_mib) @@ -2042,17 +2099,20 @@ impl VmDriver { )] async fn prepare_runtime_overlay( &self, + state_dir: &Path, overlay_disk: &Path, + owner_source_disk: &Path, tls_paths: Option<&VmDriverTlsPaths>, sandbox_token: Option<&str>, preparation: OverlayPreparation, - ) -> Result<(), String> { + ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); let tls_materials = match tls_paths { Some(paths) => Some(read_guest_tls_materials(paths).await?), None => None, }; let sandbox_token = sandbox_token.map(str::to_string); + let proxy_auth = self.read_proxy_auth_credential().await?; let overlay_disk = overlay_disk.to_path_buf(); let overlay_size_bytes = self .config @@ -2064,6 +2124,22 @@ impl VmDriver { self.config.overlay_disk_mib ) })?; + let (owner_state, write_owner_state) = sandbox_owner_state_for_launch( + state_dir, + &overlay_disk, + owner_source_disk, + &self.config, + preparation, + ) + .await?; + let owner_state_written_before_prepare = + write_owner_state && preparation == OverlayPreparation::Fresh; + if owner_state_written_before_prepare { + // Persist the selected identity before creating the overlay. A + // crash during preparation can then retry without misclassifying + // the partial overlay as legacy state. + write_sandbox_owner_state(state_dir, owner_state).await?; + } let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes); if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? { @@ -2082,13 +2158,38 @@ impl VmDriver { &overlay_disk, tls_materials.as_ref(), sandbox_token.as_deref(), + proxy_auth.as_deref(), preparation, overlay_size_bytes, ) }) .await .map_err(|err| format!("overlay image preparation panicked: {err}"))?; - span_status.finish(result) + result?; + if write_owner_state && !owner_state_written_before_prepare { + write_sandbox_owner_state(state_dir, owner_state).await?; + } + span_status.finish(Ok(owner_state)) + } + + async fn read_proxy_auth_credential(&self) -> Result, String> { + let Some(path) = self.config.upstream_proxy.proxy_auth_file.as_ref() else { + return Ok(None); + }; + let path = path.clone(); + Ok(Some( + tokio::task::spawn_blocking(move || { + let path = path + .to_str() + .ok_or_else(|| "proxy_auth_file must be valid UTF-8".to_string())?; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file(path)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map(str::to_owned) + .map_err(|error| format!("proxy_auth_file is invalid: {error}")) + }) + .await + .map_err(|error| format!("proxy_auth_file read task failed: {error}"))??, + )) } fn resolved_sandbox_image(&self, sandbox: &Sandbox) -> Option { @@ -2463,7 +2564,7 @@ impl VmDriver { image_identity: &str, bootstrap_root_disk: &Path, ) -> Result { - let cache_identity = prepared_image_cache_identity(image_identity); + let cache_identity = prepared_image_cache_identity(image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2572,7 +2673,7 @@ impl VmDriver { "failed to resolve vm sandbox image '{image_ref}': {err}" )) })?; - let cache_identity = prepared_image_cache_identity(&source_image_identity); + let cache_identity = prepared_image_cache_identity(&source_image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2774,6 +2875,7 @@ impl VmDriver { Ok(()) } + #[allow(clippy::similar_names)] async fn run_image_prep_vm( &self, bootstrap_root_disk: &Path, @@ -2802,6 +2904,14 @@ impl VmDriver { command .arg("--vm-env") .arg(format!("OPENSHELL_VM_INIT_MODE={IMAGE_PREP_INIT_MODE}")); + if let Some((uid, gid)) = configured_sandbox_identity(&self.config) { + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_UID={uid}")); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_GID={gid}")); + } let mut child = command .spawn() @@ -2919,19 +3029,20 @@ impl VmDriver { let image_identity_owned = image_identity.to_string(); let exported_rootfs_for_build = exported_rootfs.clone(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!( - "Preparing VM rootfs for local image \"{image_ref}\" (sandbox uid={sandbox_uid})" - ), + format!("Preparing VM rootfs for local image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "local_docker".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -3060,17 +3171,20 @@ impl VmDriver { let image_ref_owned = image_ref.to_string(); let image_identity_owned = image_identity.to_string(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!("Preparing VM rootfs for image \"{image_ref}\" (sandbox uid={sandbox_uid})"), + format!("Preparing VM rootfs for image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "registry".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -4402,7 +4516,7 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { /// Rewrites loopback host references in a gateway URL to a hostname the guest /// can reach via gvproxy. /// -/// The driver receives the gateway endpoint from `--openshell-endpoint`, which +/// The driver receives the gateway endpoint from `--grpc-endpoint`, which /// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is /// useless inside the guest because the guest's loopback interface is its own, /// not the host's. Inside the guest we need a name that gvproxy will translate @@ -4466,7 +4580,7 @@ fn build_guest_environment( endpoint_override: Option<&str>, ) -> Vec { let openshell_endpoint = endpoint_override.map_or_else( - || guest_visible_openshell_endpoint(&config.openshell_endpoint), + || guest_visible_openshell_endpoint(&config.grpc_endpoint), String::from, ); // 1. User-supplied environment (lowest priority). @@ -4535,6 +4649,39 @@ fn build_guest_environment( GUEST_TLS_KEY_PATH.to_string(), ); } + if let Some(url) = config.upstream_proxy.https_proxy.as_ref() { + environment.insert("OPENSHELL_VM_UPSTREAM_PROXY".to_string(), url.clone()); + } + if let Some(no_proxy) = config.upstream_proxy.no_proxy.as_ref() { + environment.insert( + "OPENSHELL_VM_UPSTREAM_NO_PROXY".to_string(), + no_proxy.clone(), + ); + } + if config.upstream_proxy.proxy_auth_file.is_some() { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE".to_string(), + GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string(), + ); + } + if config.upstream_proxy.proxy_auth_allow_insecure == Some(true) { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE".to_string(), + "true".to_string(), + ); + } + if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME".to_string(), + "true".to_string(), + ); + } + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + endpoint.clone(), + ); + } environment.insert( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), @@ -4618,6 +4765,198 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { } } +/// Select the exact identity the guest must use for this overlay and whether a +/// successful preparation must create or upgrade its state marker. +/// +/// Persisted overlays are resolved only from concrete state. In particular, +/// absence of a marker is not evidence that an overlay used the historical +/// 10001 identity: it can also mean fresh provisioning was interrupted. +async fn sandbox_owner_state_for_launch( + state_dir: &Path, + overlay_disk: &Path, + owner_source_disk: &Path, + config: &VmDriverConfig, + preparation: OverlayPreparation, +) -> Result<(SandboxOwnerIdentity, bool), String> { + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + match tokio::fs::read_to_string(&marker_path).await { + Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_V1 => { + // The v1 marker recorded no identity. Resolve it through the same + // evidence-based migration path as an unmarked overlay. + } + Ok(contents) => { + let identity = parse_sandbox_owner_state(&contents).map_err(|error| { + format!( + "invalid sandbox owner state {}: {error}", + marker_path.display() + ) + })?; + return Ok((identity, false)); + } + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!( + "read sandbox owner state {}: {error}", + marker_path.display() + )); + } + Err(_) => {} + } + + let overlay_exists = match tokio::fs::metadata(overlay_disk).await { + Ok(metadata) => metadata.is_file(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "stat overlay disk {}: {error}", + overlay_disk.display() + )); + } + }; + + if preparation == OverlayPreparation::PreserveExisting && overlay_exists { + match sandbox_owner_identity_from_overlay(overlay_disk).await { + Ok(Some(identity)) => return Ok((identity, true)), + Ok(None) => {} + Err(error) => warn!( + overlay_path = %overlay_disk.display(), + error = %error, + "could not read sandbox identity from VM overlay upper layer" + ), + } + if let Some(identity) = persisted_sandbox_owner_identity(state_dir, config).await? { + return Ok((identity, true)); + } + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + return sandbox_owner_identity_from_image(owner_source_disk) + .await + .map(|identity| (identity, true)); + } + + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + sandbox_owner_identity_from_image(owner_source_disk) + .await + .map(|identity| (identity, true)) +} + +async fn persisted_sandbox_owner_identity( + state_dir: &Path, + config: &VmDriverConfig, +) -> Result, String> { + let prior_identity = match tokio::fs::read_to_string(state_dir.join(IMAGE_IDENTITY_FILE)).await + { + Ok(identity) if !identity.trim().is_empty() => identity, + Ok(_) => String::new(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(format!("read persisted VM image identity: {error}")), + }; + + if !prior_identity.is_empty() { + let prior_disk = image_cache_rootfs_image(&config.state_dir, prior_identity.trim()); + if tokio::fs::metadata(&prior_disk).await.is_ok() { + match sandbox_owner_identity_from_image(&prior_disk).await { + Ok(identity) => return Ok(Some(identity)), + Err(error) => warn!( + image_path = %prior_disk.display(), + error = %error, + "could not read sandbox identity from persisted VM rootfs; using compatibility fallback" + ), + } + } + } + + Ok(None) +} + +async fn sandbox_owner_identity_from_image( + image_path: &Path, +) -> Result { + let image_path = image_path.to_path_buf(); + let identity = + tokio::task::spawn_blocking(move || sandbox_guest_user_ids_from_image(&image_path)) + .await + .map_err(|error| format!("read sandbox identity task failed: {error}"))??; + let (uid, gid) = identity.unwrap_or((DEFAULT_SANDBOX_UID, DEFAULT_SANDBOX_UID)); + Ok(SandboxOwnerIdentity { uid, gid }) +} + +async fn sandbox_owner_identity_from_overlay( + overlay_path: &Path, +) -> Result, String> { + let overlay_path = overlay_path.to_path_buf(); + let identity = tokio::task::spawn_blocking(move || { + sandbox_guest_user_ids_from_overlay_image(&overlay_path) + }) + .await + .map_err(|error| format!("read sandbox overlay identity task failed: {error}"))??; + Ok(identity.map(|(uid, gid)| SandboxOwnerIdentity { uid, gid })) +} + +fn parse_sandbox_owner_state(contents: &str) -> Result { + let mut fields = contents.trim().split(':'); + if fields.next() != Some(SANDBOX_OWNER_STATE_V2) { + return Err("unsupported version".to_string()); + } + let uid = fields + .next() + .ok_or_else(|| "missing uid".to_string())? + .parse::() + .map_err(|error| format!("invalid uid: {error}"))?; + let gid = fields + .next() + .ok_or_else(|| "missing gid".to_string())? + .parse::() + .map_err(|error| format!("invalid gid: {error}"))?; + if fields.next().is_some() { + return Err("unexpected fields".to_string()); + } + validate_sandbox_owner_identity(uid, gid)?; + Ok(SandboxOwnerIdentity { uid, gid }) +} + +async fn write_sandbox_owner_state( + state_dir: &Path, + identity: SandboxOwnerIdentity, +) -> Result<(), String> { + validate_sandbox_owner_identity(identity.uid, identity.gid)?; + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + let sequence = OWNER_STATE_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary_path = state_dir.join(format!( + ".{SANDBOX_OWNER_STATE_FILE}.{}.{sequence}.tmp", + std::process::id() + )); + write_private_file(&temporary_path, identity.marker_contents().into_bytes()) + .await + .map_err(|err| format!("write temporary sandbox owner state: {err}"))?; + if let Err(error) = tokio::fs::rename(&temporary_path, &marker_path).await { + let _ = tokio::fs::remove_file(&temporary_path).await; + return Err(format!("install sandbox owner state: {error}")); + } + Ok(()) +} + +fn validate_sandbox_owner_identity(uid: u32, gid: u32) -> Result<(), String> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + if !range.contains(&uid) { + return Err(format!( + "uid {uid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + if !range.contains(&gid) { + return Err(format!( + "gid {gid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + Ok(()) +} + #[allow(clippy::result_large_err)] fn validate_sandbox_state_dir(root: &Path, state_dir: &Path) -> Result<(), Status> { let sandboxes_root = sandboxes_root_dir(root); @@ -4775,9 +5114,20 @@ fn bootstrap_image_cache_identity(image_identity: &str) -> String { ) } -fn prepared_image_cache_identity(image_identity: &str) -> String { +fn configured_sandbox_identity(config: &VmDriverConfig) -> Option<(u32, u32)> { + (config.sandbox_uid.is_some() || config.sandbox_gid.is_some()).then(|| { + let uid = config.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + (uid, config.sandbox_gid.unwrap_or(uid)) + }) +} + +fn prepared_image_cache_identity(image_identity: &str, config: &VmDriverConfig) -> String { + let identity = configured_sandbox_identity(config).map_or_else( + || "image-account".to_string(), + |(uid, gid)| format!("configured-{uid}-{gid}"), + ); format!( - "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}", + "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{identity}:{image_identity}", openshell_core::VERSION ) } @@ -4996,6 +5346,7 @@ fn create_sandbox_overlay_image_from_template( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, ) -> Result<(), String> { clone_or_copy_sparse_file(template_path, overlay_disk)?; if let Some(tls) = tls_materials { @@ -5004,6 +5355,9 @@ fn create_sandbox_overlay_image_from_template( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } Ok(()) } @@ -5012,6 +5366,7 @@ fn prepare_sandbox_overlay_image( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, preparation: OverlayPreparation, expected_size_bytes: u64, ) -> Result<(), String> { @@ -5024,6 +5379,9 @@ fn prepare_sandbox_overlay_image( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } return Ok(()); } Ok(metadata) if metadata.is_file() => { @@ -5055,6 +5413,7 @@ fn prepare_sandbox_overlay_image( overlay_disk, tls_materials, sandbox_token, + proxy_auth, ) } @@ -5083,6 +5442,12 @@ fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), St set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600) } +fn inject_guest_proxy_auth(overlay_disk: &Path, credential: &str) -> Result<(), String> { + let path = overlay_upper_path(GUEST_UPSTREAM_PROXY_AUTH_PATH); + write_rootfs_image_file(overlay_disk, &path, format!("{credential}\n").as_bytes())?; + set_rootfs_image_file_mode(overlay_disk, &path, 0o600) +} + #[allow(clippy::result_large_err)] #[tracing::instrument( name = "vm.prepare_guest", @@ -5607,6 +5972,38 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + + #[test] + fn vm_config_uses_canonical_grpc_endpoint_name() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let serialized = serde_json::to_value(&config).unwrap(); + assert_eq!(serialized["grpc_endpoint"], "http://127.0.0.1:8080"); + assert!(serialized.get("openshell_endpoint").is_none()); + + let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); + } + + #[test] + fn vm_config_rejects_legacy_openshell_endpoint() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let mut serialized = serde_json::to_value(config).unwrap(); + serialized.as_object_mut().unwrap().insert( + "openshell_endpoint".to_string(), + serde_json::json!("http://127.0.0.1:8080"), + ); + + let error = serde_json::from_value::(serialized) + .expect_err("legacy openshell_endpoint must be rejected as unknown"); + assert!(error.to_string().contains("openshell_endpoint")); + } + struct TestTracing { exporter: opentelemetry_sdk::trace::InMemorySpanExporter, _provider: opentelemetry_sdk::trace::SdkTracerProvider, @@ -6064,7 +6461,14 @@ mod tests { let parent = tracing::info_span!("vm.provision"); let result = driver - .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh) + .prepare_runtime_overlay( + Path::new("/unused"), + Path::new("/unused"), + Path::new("/unused"), + None, + None, + OverlayPreparation::Fresh, + ) .instrument(parent) .await; assert!(result.is_err(), "overflow should stop before disk I/O"); @@ -6650,6 +7054,310 @@ mod tests { } } + #[tokio::test] + async fn unmarked_overlay_uses_current_image_instead_of_blind_legacy_identity() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let source = dir.join("current-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let current_rootfs = dir.join("current-rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, ¤t_rootfs, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + ¤t_rootfs, + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_without_identity_evidence_fails_safely() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let error = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .expect_err("ambiguous overlay must not receive a guessed identity"); + + assert!(error.contains("missing-current-rootfs")); + assert!(!error.contains("10001")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_overlay_uses_explicit_identity_without_image_inspection() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::Fresh, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000 + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_image_without_sandbox_account_uses_default_identity() { + let dir = unique_temp_dir(); + let source = dir.join("rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write(source.join("etc/passwd"), "root:x:0:0:root:/root:/bin/sh\n").unwrap(); + let rootfs = dir.join("rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, &rootfs, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, _) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + &rootfs, + &config, + OverlayPreparation::Fresh, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: DEFAULT_SANDBOX_UID, + gid: DEFAULT_SANDBOX_UID, + } + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_legacy_overlay_uses_explicit_config_when_old_rootfs_is_missing() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_owner_marker_rejects_malformed_or_unknown_state() { + for marker in [ + "sandbox-owner-v3:1000:1000", + "sandbox-owner-v2", + "sandbox-owner-v2:nope:1000", + "sandbox-owner-v2:0:1000", + "sandbox-owner-v2:1000:0", + "sandbox-owner-v2:1000:1000:extra", + ] { + assert!( + parse_sandbox_owner_state(marker).is_err(), + "marker should be rejected: {marker}" + ); + } + } + + #[tokio::test] + async fn persisted_owner_marker_preserves_exact_identity() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"current overlay").unwrap(); + let expected = SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + }; + write_sandbox_owner_state(&dir, expected).await.unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!(identity, expected); + assert!(!write_marker); + assert_eq!( + std::fs::read_to_string(dir.join(SANDBOX_OWNER_STATE_FILE)).unwrap(), + "sandbox-owner-v2:4242:4343\n" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_upper_passwd() { + let dir = unique_temp_dir(); + let overlay_source = dir.join("overlay-source"); + std::fs::create_dir_all(overlay_source.join("upper/etc")).unwrap(); + std::fs::write( + overlay_source.join("upper/etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:10001:10001:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + create_ext4_image_from_dir_with_size(&overlay_source, &overlay, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 10001, + gid: 10001, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_persisted_rootfs() { + let root = unique_temp_dir(); + let state_dir = root.join("sandboxes/sandbox-1"); + std::fs::create_dir_all(&state_dir).unwrap(); + let overlay = state_dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let prior_identity = "legacy-cache:sha256:abc"; + std::fs::write( + state_dir.join(IMAGE_IDENTITY_FILE), + format!("{prior_identity}\n"), + ) + .unwrap(); + let source = root.join("legacy-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let prior_disk = image_cache_rootfs_image(&root, prior_identity); + create_ext4_image_from_dir_with_size(&source, &prior_disk, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: root.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &state_dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn sandbox_state_dir_rejects_path_unsafe_ids() { let err = sandbox_state_dir(Path::new("/tmp/openshell-vm"), "../escape") @@ -6798,6 +7506,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "saved-overlay".len() as u64, ) @@ -6821,6 +7530,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "fresh-overlay".len() as u64, ) @@ -7051,7 +7761,7 @@ mod tests { #[test] fn build_guest_environment_sets_supervisor_defaults() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7073,6 +7783,16 @@ mod tests { ))); } + #[test] + fn new_vm_sandbox_identity_defaults_to_1000() { + let config = VmDriverConfig::default(); + assert_eq!(config.resolve_sandbox_uid(), 1000); + assert_eq!( + config.resolve_sandbox_gid(config.resolve_sandbox_uid()), + 1000 + ); + } + #[test] fn validate_sandbox_identity_accepts_non_root_system_ids() { let config = VmDriverConfig { @@ -7086,7 +7806,7 @@ mod tests { #[test] fn persisted_legacy_sandbox_without_command_uses_scratch_main() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; // Requests persisted before the canonical-main contract have a @@ -7120,7 +7840,7 @@ mod tests { #[test] fn build_guest_environment_preserves_main_command_spaces() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8080".to_string(), + grpc_endpoint: "https://127.0.0.1:8080".to_string(), ..Default::default() }; let command = vec![ @@ -7157,7 +7877,7 @@ mod tests { #[test] fn build_guest_environment_uses_token_file_without_raw_token_env() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7189,7 +7909,7 @@ mod tests { #[test] fn build_guest_environment_strips_gateway_tls_server_name() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7226,7 +7946,7 @@ mod tests { )], || { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7265,7 +7985,7 @@ mod tests { #[test] fn build_guest_environment_clears_unsupported_network_capabilities() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7295,7 +8015,7 @@ mod tests { #[test] fn build_guest_environment_uses_endpoint_override_for_tap() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7494,10 +8214,67 @@ mod tests { ); } + #[test] + fn vm_proxy_and_spiffe_config_require_explicit_safe_acknowledgements() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: None, + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: false, + ..Default::default() + }; + let error = config.validate_runtime_security_config().unwrap_err(); + assert!(error.contains("provider_spiffe_allow_guest_tcp")); + + let config = VmDriverConfig { + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + assert!(config.validate_runtime_security_config().is_ok()); + } + + #[test] + fn build_guest_environment_projects_operator_proxy_and_spiffe_endpoint() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + let sandbox = Sandbox { + id: "vm-spiffe".to_string(), + name: "vm-spiffe".to_string(), + ..Default::default() + }; + let env = build_guest_environment(&sandbox, &config, None); + assert!( + env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY=https://proxy.example:8443".to_string()) + ); + assert!(env.contains(&format!( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE={GUEST_UPSTREAM_PROXY_AUTH_PATH}" + ))); + assert!(env.contains( + &"OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=tcp:192.0.2.10:8081".to_string() + )); + assert!(!env.iter().any(|value| value.contains("user:pass"))); + } + #[test] fn build_guest_environment_includes_tls_paths_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), guest_tls_cert: Some(PathBuf::from("/host/tls.crt")), guest_tls_key: Some(PathBuf::from("/host/tls.key")), @@ -7519,7 +8296,7 @@ mod tests { #[test] fn vm_driver_config_requires_tls_materials_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), ..Default::default() }; let err = config @@ -7776,9 +8553,9 @@ mod tests { #[test] fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() { assert_eq!( - prepared_image_cache_identity("sha256:local-image"), + prepared_image_cache_identity("sha256:local-image", &VmDriverConfig::default()), format!( - "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:sha256:local-image", + "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:sha256:local-image", openshell_core::VERSION ) ); @@ -7814,7 +8591,10 @@ mod tests { &staging_dir, &GuestImagePayload { image_ref: "ghcr.io/example/app:latest".to_string(), - image_identity: prepared_image_cache_identity("sha256:abc"), + image_identity: prepared_image_cache_identity( + "sha256:abc", + &VmDriverConfig::default(), + ), source: GuestImagePayloadSource::RegistryOciLayout { layout_dir }, }, ) @@ -8054,7 +8834,7 @@ mod tests { let (events, _) = broadcast::channel(WATCH_BUFFER); VmDriver { config: VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), vcpus: 2, mem_mib: 2048, gpu_vcpus: 8, diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 95ebf0f8b2..b9468afd2b 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -94,8 +94,8 @@ struct Args { #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, + #[arg(long = "grpc-endpoint", env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] default_image: String, @@ -119,6 +119,47 @@ struct Args { #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] guest_tls_key: Option, + /// Corporate forward proxy for supervisor TLS egress. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY")] + upstream_proxy: Option, + + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_NO_PROXY")] + upstream_no_proxy: Option, + + /// Root-owned gateway-host file containing `user:pass` proxy credentials. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE")] + upstream_proxy_auth_file: Option, + + /// Explicitly acknowledge cleartext Basic authentication to an http proxy. + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE", + default_value_t = false + )] + upstream_proxy_auth_allow_insecure: bool, + + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", + default_value_t = false + )] + upstream_proxy_connect_by_hostname: bool, + + /// Guest-reachable SPIFFE Workload API endpoint (`tcp:IP:port`). + #[arg( + long = "provider-spiffe-workload-api-tcp-endpoint", + env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT" + )] + provider_spiffe_workload_api_tcp_endpoint: Option, + + /// Explicit acknowledgement that the configured Workload API listener is exposed to VM guests. + #[arg( + long, + env = "OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP", + default_value_t = false + )] + provider_spiffe_allow_guest_tcp: bool, + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] krun_log_level: u32, @@ -223,8 +264,8 @@ async fn main() -> Result<()> { } let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint + grpc_endpoint: args + .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, state_dir: args.state_dir.clone(), launcher_bin: None, @@ -238,6 +279,17 @@ async fn main() -> Result<()> { guest_tls_ca: args.guest_tls_ca.clone(), guest_tls_cert: args.guest_tls_cert.clone(), guest_tls_key: args.guest_tls_key.clone(), + upstream_proxy: openshell_core::UpstreamProxyConfig { + https_proxy: args.upstream_proxy.clone(), + no_proxy: args.upstream_no_proxy.clone(), + proxy_auth_file: args.upstream_proxy_auth_file.clone(), + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname.then_some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: args + .provider_spiffe_workload_api_tcp_endpoint + .clone(), + provider_spiffe_allow_guest_tcp: args.provider_spiffe_allow_guest_tcp, gpu_enabled: args.gpu, gpu_mem_mib: args.gpu_mem_mib, gpu_vcpus: args.gpu_vcpus, @@ -695,6 +747,28 @@ mod tests { assert!(err.contains("--bind-socket is required")); } + #[test] + fn accepts_canonical_grpc_endpoint_flag() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--grpc-endpoint", + "http://127.0.0.1:8080", + ]) + .unwrap(); + assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + } + + #[test] + fn rejects_legacy_openshell_endpoint_flag() { + let error = Args::try_parse_from([ + "openshell-driver-vm", + "--openshell-endpoint", + "http://127.0.0.1:8080", + ]) + .expect_err("legacy --openshell-endpoint must be rejected"); + assert!(error.to_string().contains("--openshell-endpoint")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..4821844a7d 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -16,8 +16,7 @@ const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; -const SANDBOX_OWNER_NORMALIZED_MARKER: &str = - openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; +const DEFAULT_SANDBOX_UID: u32 = 1000; const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024; const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024; const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024; @@ -31,8 +30,8 @@ pub const fn sandbox_guest_init_path() -> &'static str { pub fn prepare_sandbox_rootfs_from_image_root( rootfs: &Path, image_identity: &str, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { prepare_sandbox_rootfs(rootfs, sandbox_uid, sandbox_gid)?; validate_sandbox_rootfs(rootfs)?; @@ -353,7 +352,11 @@ fn append_symlink_to_archive( } #[allow(clippy::similar_names)] -fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> Result<(), String> { +fn prepare_sandbox_rootfs( + rootfs: &Path, + sandbox_uid: Option, + sandbox_gid: Option, +) -> Result<(), String> { for relative in ["opt/openshell/.initialized", "opt/openshell/.rootfs-type"] { remove_rootfs_path(rootfs, relative)?; } @@ -567,8 +570,7 @@ fn normalize_sandbox_owner_in_rootfs_image(source: &Path, image_path: &Path) -> return Ok(()); } - run_debugfs_batch(image_path, &commands)?; - write_rootfs_image_file(image_path, SANDBOX_OWNER_NORMALIZED_MARKER, b"1\n") + run_debugfs_batch(image_path, &commands) } fn collect_sandbox_owner_commands( @@ -655,6 +657,82 @@ fn debugfs_quote_argument(argument: &str) -> Option { Some(quoted) } +/// Read the sandbox account identity directly from an ext4 rootfs image. +/// +/// Persisted VM overlays may outlive the prepared-image cache layout that +/// created them. Reading the matching old lower disk lets the driver preserve +/// that overlay's real ownership contract during an upgrade. +pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result, String> { + sandbox_guest_user_ids_from_image_path(image_path, "/etc/passwd") +} + +/// Read a sandbox account copied into an overlay upper layer. +/// +/// An upper-layer passwd file is the most direct evidence of the identity an +/// existing overlay observed, so migration consults it before any lower image. +pub fn sandbox_guest_user_ids_from_overlay_image( + image_path: &Path, +) -> Result, String> { + sandbox_guest_user_ids_from_image_path(image_path, "/upper/etc/passwd") +} + +fn sandbox_guest_user_ids_from_image_path( + image_path: &Path, + guest_path: &str, +) -> Result, String> { + let metadata = fs::metadata(image_path) + .map_err(|error| format!("stat rootfs image {}: {error}", image_path.display()))?; + if !metadata.is_file() { + return Err(format!( + "rootfs image {} is not a regular file", + image_path.display() + )); + } + + let quoted_path = debugfs_quote_absolute_path(guest_path) + .expect("the static passwd path is a valid debugfs path"); + let command = format!("cat {quoted_path}"); + let mut last_error = None; + + for candidate in e2fs_tool_candidates("debugfs") { + let label = candidate.display().to_string(); + match Command::new(&candidate) + .arg("-R") + .arg(&command) + .arg(image_path) + .output() + { + Ok(output) if output.status.success() => { + let passwd = String::from_utf8(output.stdout).map_err(|error| { + format!( + "read {guest_path} from {} as UTF-8: {error}", + image_path.display() + ) + })?; + return parse_sandbox_guest_user_ids(&passwd, &image_path.display().to_string()); + } + Ok(output) => { + last_error = Some(format!( + "{label} failed with status {}\nstdout: {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + last_error = Some(format!("{label} not found")); + } + Err(error) => last_error = Some(format!("run {label}: {error}")), + } + } + + Err(format!( + "debugfs command '{command}' failed for {}: {}. Install e2fsprogs (debugfs) and retry", + image_path.display(), + last_error.unwrap_or_else(|| "debugfs not found".to_string()) + )) +} + fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd_path = rootfs.join("etc/passwd"); if !passwd_path.exists() { @@ -663,6 +741,10 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd = fs::read_to_string(&passwd_path) .map_err(|e| format!("read {}: {e}", passwd_path.display()))?; + parse_sandbox_guest_user_ids(&passwd, &passwd_path.display().to_string()) +} + +fn parse_sandbox_guest_user_ids(passwd: &str, source: &str) -> Result, String> { for line in passwd.lines() { let mut parts = line.split(':'); if parts.next() != Some("sandbox") { @@ -671,14 +753,14 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let _password = parts.next(); let uid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing uid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing uid"))? .parse::() - .map_err(|e| format!("sandbox uid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox uid in {source} is invalid: {e}"))?; let gid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing gid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing gid"))? .parse::() - .map_err(|e| format!("sandbox gid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox gid in {source} is invalid: {e}"))?; return Ok(Some((uid, gid))); } @@ -789,9 +871,18 @@ fn temporary_injection_path(image_path: &Path) -> PathBuf { #[allow(clippy::similar_names)] fn ensure_sandbox_guest_user( rootfs: &Path, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { + // An image's sandbox account is part of its filesystem contract. Leave it + // intact unless an operator explicitly configured either side of the + // identity. A missing account still gets the OpenShell default. + if sandbox_uid.is_none() && sandbox_gid.is_none() && sandbox_guest_user_ids(rootfs)?.is_some() { + return Ok(()); + } + + let sandbox_uid = sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + let sandbox_gid = sandbox_gid.unwrap_or(sandbox_uid); let etc_dir = rootfs.join("etc"); fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?; @@ -822,23 +913,32 @@ fn ensure_line_in_file( line: &str, exists: impl Fn(&str) -> bool, ) -> Result<(), String> { - let mut contents = if path.exists() { + let contents = if path.exists() { fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))? } else { String::new() }; - if contents.lines().any(exists) { - return Ok(()); + let mut replaced = false; + let mut updated = String::new(); + for existing in contents.lines() { + if exists(existing) { + if !replaced { + updated.push_str(line); + updated.push('\n'); + replaced = true; + } + } else { + updated.push_str(existing); + updated.push('\n'); + } } - - if !contents.is_empty() && !contents.ends_with('\n') { - contents.push('\n'); + if !replaced { + updated.push_str(line); + updated.push('\n'); } - contents.push_str(line); - contents.push('\n'); - fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display())) + fs::write(path, updated).map_err(|e| format!("write {}: {e}", path.display())) } fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { @@ -959,10 +1059,10 @@ mod tests { write_fake_runtime_binaries(&rootfs); fs::write( rootfs.join("etc/passwd"), - "root:x:0:0:root:/root:/bin/bash\n", + "root:x:0:0:root:/root:/bin/bash\nsandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n", ) .expect("write passwd"); - fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group"); + fs::write(rootfs.join("etc/group"), "root:x:0:\nsandbox:x:10001:\n").expect("write group"); fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts"); fs::create_dir_all(rootfs.join("bin")).expect("create bin"); fs::create_dir_all(rootfs.join("sbin")).expect("create sbin"); @@ -974,7 +1074,7 @@ mod tests { // Use a non-standard UID so the test doesn't collide with the default. let uid = 20001; - prepare_sandbox_rootfs(&rootfs, uid, uid).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(uid), Some(uid)).expect("prepare sandbox rootfs"); validate_sandbox_rootfs(&rootfs).expect("validate sandbox rootfs"); assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); @@ -1002,6 +1102,12 @@ mod tests { .expect("read group") .contains(&format!("sandbox:x:{uid}:")) ); + assert!( + !fs::read_to_string(rootfs.join("etc/passwd")) + .expect("read passwd") + .contains("sandbox:x:10001:"), + "newly prepared rootfs must replace the legacy sandbox account" + ); assert_eq!( fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"), "127.0.0.1 localhost\n" @@ -1020,7 +1126,7 @@ mod tests { fs::create_dir_all(rootfs.join("sandbox")).expect("create sandbox workdir"); fs::write(rootfs.join("sandbox/app.py"), "print('hello')\n").expect("write app"); - prepare_sandbox_rootfs(&rootfs, 10001, 10001).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(10001), Some(10001)).expect("prepare sandbox rootfs"); assert!(rootfs.join("sandbox").is_dir()); assert_eq!( @@ -1100,6 +1206,61 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn sandbox_user_preserves_image_account_when_identity_is_omitted() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((4242, 4343))); + assert!( + fs::read_to_string(rootfs.join("etc/passwd")) + .unwrap() + .contains("Image:/image-home:/bin/false") + ); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_defaults_to_1000_when_image_has_no_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((1000, 1000))); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_explicit_identity_overrides_image_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, Some(2000), Some(3000)).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((2000, 3000))); + assert!( + fs::read_to_string(rootfs.join("etc/group")) + .unwrap() + .contains("sandbox:x:3000:") + ); + let _ = fs::remove_dir_all(dir); + } + #[test] fn sandbox_guest_user_ids_reads_existing_sandbox_user() { let dir = unique_temp_dir(); diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index f9d02027e0..e9685adad4 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -26,6 +26,7 @@ tokio = { workspace = true } openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +openshell-policy = { path = "../openshell-policy", optional = true } hyper-util = { workspace = true, optional = true } nix = { workspace = true, optional = true } serde = { workspace = true, optional = true } @@ -43,6 +44,7 @@ in-tree-compute-drivers = [ "dep:openshell-driver-docker", "dep:openshell-driver-kubernetes", "dep:openshell-driver-podman", + "dep:openshell-policy", "dep:openshell-otel", "dep:hyper-util", "dep:nix", diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index c5a7ebe0e7..a63a58068b 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -117,16 +117,6 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("kubernetes")) .without_mtls_user_auth() .with_tracing_setup(kubernetes_tracing_setup) - .with_inherited_config_keys(&[ - "namespace", - "default_image", - "supervisor_image", - "client_tls_secret_name", - "service_account_name", - "host_gateway_ip", - "enable_user_namespaces", - "sa_token_ttl_secs", - ]) }), ComputeDriverRegistration::new( "podman", @@ -139,14 +129,6 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("podman")) .with_local_singleplayer() .with_tracing_setup(podman_tracing_setup) - .with_inherited_config_keys(&[ - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ComputeDriverRegistration::new( "docker", @@ -159,26 +141,11 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("docker")) .with_local_singleplayer() .with_tracing_setup(docker_tracing_setup) - .with_inherited_config_keys(&[ - "sandbox_namespace", - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ComputeDriverRegistration::new("vm", u16::MAX, None, VmFactory).map(|registration| { registration .with_telemetry_category(TelemetryComputeDriver::anonymous_category("vm")) .with_local_singleplayer() - .with_inherited_config_keys(&[ - "default_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ] { registry @@ -278,6 +245,11 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { ) -> openshell_core::Result { let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = context.driver_config()?; + if config.grpc_endpoint.trim().is_empty() { + return Err(openshell_core::Error::config( + "kubernetes compute driver requires grpc_endpoint in [openshell.drivers.kubernetes]; the gateway service location cannot be inferred from the sandbox namespace", + )); + } if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { config.workspace_default_storage_size = size; } @@ -309,6 +281,7 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "docker")?; apply_guest_tls( &mut config.guest_tls_ca, &mut config.guest_tls_cert, @@ -341,6 +314,7 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "podman")?; config.gateway_port = context.gateway_port(); if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { config.socket_path = Some(path.into()); @@ -379,18 +353,16 @@ impl openshell_server::ComputeDriverFactory for VmFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: vm::VmComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "vm")?; if config.state_dir.as_os_str().is_empty() { config.state_dir = vm::VmComputeConfig::default_state_dir(); } - if config.grpc_endpoint.trim().is_empty() - && (!context.gateway_tls_enabled() || context.guest_tls_paths().is_some()) - { - let scheme = if context.gateway_tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port()); + if config.grpc_endpoint.trim().is_empty() { + config.grpc_endpoint = openshell_core::driver_utils::gateway_callback_endpoint( + openshell_core::driver_utils::GatewayCallbackTopology::Vm, + context.gateway_port(), + context.gateway_tls_enabled(), + ); } apply_guest_tls( &mut config.guest_tls_ca, @@ -411,6 +383,32 @@ impl openshell_server::ComputeDriverFactory for VmFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn require_guest_tls_for_local_driver( + context: &openshell_server::ComputeDriverBuildContext<'_>, + driver_name: &str, +) -> openshell_core::Result<()> { + validate_local_driver_guest_tls( + context.gateway_tls_enabled(), + context.guest_tls_paths().is_some(), + driver_name, + ) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn validate_local_driver_guest_tls( + gateway_tls_enabled: bool, + has_guest_tls: bool, + driver_name: &str, +) -> openshell_core::Result<()> { + if gateway_tls_enabled && !has_guest_tls { + return Err(openshell_core::Error::config(format!( + "gateway TLS requires guest_tls_ca, guest_tls_cert, and guest_tls_key in [openshell.gateway] when using the {driver_name} compute driver" + ))); + } + Ok(()) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn apply_guest_tls( ca: &mut Option, @@ -429,6 +427,47 @@ fn apply_guest_tls( } } +#[cfg(all(test, not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +mod local_driver_tests { + use super::{apply_guest_tls, validate_local_driver_guest_tls}; + use std::path::{Path, PathBuf}; + + #[test] + fn tls_enabled_local_drivers_require_a_guest_bundle() { + for driver_name in ["docker", "podman", "vm"] { + let error = validate_local_driver_guest_tls(true, false, driver_name) + .expect_err("TLS-enabled local driver must require guest TLS"); + let message = error.to_string(); + assert!(message.contains(driver_name)); + assert!(message.contains("guest_tls_ca")); + } + validate_local_driver_guest_tls(true, true, "docker") + .expect("a complete guest bundle satisfies the requirement"); + validate_local_driver_guest_tls(false, false, "docker") + .expect("plaintext gateways do not require guest TLS"); + } + + #[test] + fn package_managed_guest_bundle_is_injected_when_driver_paths_are_absent() { + let mut ca = None; + let mut cert = None; + let mut key = None; + apply_guest_tls( + &mut ca, + &mut cert, + &mut key, + Some(( + Path::new("/managed/ca.pem"), + Path::new("/managed/client.pem"), + Path::new("/managed/client-key.pem"), + )), + ); + assert_eq!(ca, Some(PathBuf::from("/managed/ca.pem"))); + assert_eq!(cert, Some(PathBuf::from("/managed/client.pem"))); + assert_eq!(key, Some(PathBuf::from("/managed/client-key.pem"))); + } +} + #[cfg(all(test, target_os = "windows", feature = "in-tree-compute-drivers"))] mod windows_tests { use super::*; diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index e86de28c12..8bdec15f07 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -31,7 +31,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{Error, Result}; +use openshell_core::{Error, Result, UpstreamProxyConfig}; #[cfg(unix)] use openshell_otel::TraceContextInterceptor; use openshell_server::AcquiredRemoteDriverEndpoint; @@ -91,6 +91,12 @@ pub struct VmComputeConfig { /// Writable overlay disk size for each VM sandbox, in MiB. pub overlay_disk_mib: u64, + /// Optional UID override for the VM guest sandbox account. + pub sandbox_uid: Option, + + /// Optional GID override for the VM guest sandbox account. + pub sandbox_gid: Option, + /// Host-side CA certificate for the guest's mTLS client bundle. pub guest_tls_ca: Option, @@ -99,6 +105,17 @@ pub struct VmComputeConfig { /// Host-side private key for the guest's mTLS client bundle. pub guest_tls_key: Option, + + /// Corporate forward-proxy settings passed to the VM driver. Flattening + /// preserves the shared local-driver TOML field names. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Explicit guest-reachable SPIFFE Workload API TCP listener. VM guests + /// cannot receive a host UNIX socket, so this requires acknowledgement. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, } impl VmComputeConfig { @@ -160,9 +177,14 @@ impl Default for VmComputeConfig { vcpus: Self::default_vcpus(), mem_mib: Self::default_mem_mib(), overlay_disk_mib: Self::default_overlay_disk_mib(), + sandbox_uid: None, + sandbox_gid: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, } } } @@ -460,6 +482,22 @@ pub async fn spawn( )); } + validate_vm_sandbox_identity(vm_config)?; + vm_config.upstream_proxy.validate().map_err(Error::config)?; + if let Some(endpoint) = vm_config + .provider_spiffe_workload_api_tcp_endpoint + .as_deref() + { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + vm_config.provider_spiffe_allow_guest_tcp, + ) + .map_err(Error::config)?; + } else if vm_config.provider_spiffe_allow_guest_tcp { + return Err(Error::config( + "provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured", + )); + } let driver_bin = resolve_compute_driver_bin(vm_config)?; let socket_path = compute_driver_socket_path(vm_config); let guest_tls_paths = compute_driver_guest_tls_paths(vm_config)?; @@ -476,9 +514,7 @@ pub async fn spawn( .arg(std::process::id().to_string()); command.arg("--log-level").arg(gateway_log_level); append_otlp_args(&mut command, otlp_config, gateway_name); - command - .arg("--openshell-endpoint") - .arg(&vm_config.grpc_endpoint); + command.arg("--grpc-endpoint").arg(&vm_config.grpc_endpoint); command.arg("--state-dir").arg(&vm_config.state_dir); if !vm_config.default_image.trim().is_empty() { command.arg("--default-image").arg(&vm_config.default_image); @@ -496,11 +532,13 @@ pub async fn spawn( command .arg("--overlay-disk-mib") .arg(vm_config.overlay_disk_mib.to_string()); + append_vm_identity_args(&mut command, vm_config); if let Some(tls) = guest_tls_paths { command.arg("--guest-tls-ca").arg(tls.ca); command.arg("--guest-tls-cert").arg(tls.cert); command.arg("--guest-tls-key").arg(tls.key); } + append_vm_proxy_and_spiffe_args(&mut command, vm_config); let mut child = command.spawn().map_err(|e| { Error::execution(format!( @@ -515,7 +553,61 @@ pub async fn spawn( )) } +fn validate_vm_sandbox_identity(config: &VmComputeConfig) -> Result<()> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + for (field, value) in [ + ("sandbox_uid", config.sandbox_uid), + ("sandbox_gid", config.sandbox_gid), + ] { + if let Some(value) = value + && !range.contains(&value) + { + return Err(Error::config(format!( + "{field} {value} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + ))); + } + } + Ok(()) +} + +#[cfg(unix)] +fn append_vm_identity_args(command: &mut Command, config: &VmComputeConfig) { + if let Some(uid) = config.sandbox_uid { + command.arg("--sandbox-uid").arg(uid.to_string()); + } + if let Some(gid) = config.sandbox_gid { + command.arg("--sandbox-gid").arg(gid.to_string()); + } +} + #[cfg(unix)] +fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConfig) { + let proxy = &config.upstream_proxy; + if let Some(url) = proxy.https_proxy.as_ref() { + command.arg("--upstream-proxy").arg(url); + } + if let Some(no_proxy) = proxy.no_proxy.as_ref() { + command.arg("--upstream-no-proxy").arg(no_proxy); + } + if let Some(auth_file) = proxy.proxy_auth_file.as_ref() { + command.arg("--upstream-proxy-auth-file").arg(auth_file); + } + if proxy.proxy_auth_allow_insecure == Some(true) { + command.arg("--upstream-proxy-auth-allow-insecure"); + } + if proxy.proxy_connect_by_hostname == Some(true) { + command.arg("--upstream-proxy-connect-by-hostname"); + } + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + command + .arg("--provider-spiffe-workload-api-tcp-endpoint") + .arg(endpoint); + command.arg("--provider-spiffe-allow-guest-tcp"); + } +} + fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { if let Some(config) = otlp_config { command.arg("--otlp-endpoint").arg(&config.endpoint); @@ -606,9 +698,10 @@ async fn connect_compute_driver(socket_path: &Path) -> Result { #[cfg(all(test, unix))] mod tests { use super::{ - VmComputeConfig, append_otlp_args, compute_driver_guest_tls_paths, + VmComputeConfig, append_otlp_args, append_vm_identity_args, compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, + validate_vm_sandbox_identity, }; use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; @@ -644,6 +737,36 @@ mod tests { ); } + #[test] + fn vm_driver_command_includes_configured_sandbox_identity() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + let config = VmComputeConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + validate_vm_sandbox_identity(&config).expect("valid identity"); + append_vm_identity_args(&mut command, &config); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(args, ["--sandbox-uid", "2000", "--sandbox-gid", "3000"]); + } + + #[test] + fn vm_gateway_config_rejects_root_identity() { + let config = VmComputeConfig { + sandbox_uid: Some(0), + ..Default::default() + }; + let error = validate_vm_sandbox_identity(&config).expect_err("root UID must fail"); + assert!(error.to_string().contains("sandbox_uid 0")); + } + #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 9dc10b8401..0a16000999 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -84,7 +84,7 @@ pub struct SandboxJwtIssuer { kid: String, issuer: String, audience: String, - ttl: Duration, + ttl: Option, } impl std::fmt::Debug for SandboxJwtIssuer { @@ -110,10 +110,14 @@ impl SandboxJwtIssuer { signing_key_pem: &[u8], kid: String, gateway_id: &str, - ttl: Duration, + ttl: Option, ) -> Result { crate::install_jsonwebtoken_crypto_provider(); + if ttl.is_some_and(|ttl| ttl.is_zero()) { + return Err("sandbox token TTL must be positive when configured".to_string()); + } + let encoding_key = EncodingKey::from_ed_pem(signing_key_pem) .map_err(|e| format!("failed to parse Ed25519 signing key PEM: {e}"))?; let identity = format!("openshell-gateway:{gateway_id}"); @@ -132,11 +136,9 @@ impl SandboxJwtIssuer { crate::install_jsonwebtoken_crypto_provider(); let now = now_secs(); - let exp = if self.ttl.is_zero() { - 0 - } else { - now.saturating_add(i64::try_from(self.ttl.as_secs()).unwrap_or(3_600)) - }; + let exp = self.ttl.map_or(0, |ttl| { + now.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(3_600)) + }); let claims = SandboxJwtClaims { sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"), iss: self.issuer.clone(), @@ -226,7 +228,7 @@ impl SandboxJwtIssuer { }) } - pub fn ttl(&self) -> Duration { + pub fn sandbox_token_ttl(&self) -> Option { self.ttl } } @@ -417,10 +419,10 @@ mod tests { } fn pair() -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { - pair_with_ttl(Duration::from_secs(3600)) + pair_with_ttl(Some(Duration::from_secs(3600))) } - fn pair_with_ttl(ttl: Duration) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { + fn pair_with_ttl(ttl: Option) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { let mat = generate_jwt_key().expect("jwt key"); let issuer = SandboxJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), @@ -446,6 +448,7 @@ mod tests { async fn mint_and_validate_round_trip() { let (issuer, auth) = pair(); let minted = issuer.mint("sandbox-a").unwrap(); + assert!(minted.expires_at_ms > 0); let principal = auth .authenticate(&header_map_with_bearer(&minted.token), "/anything") .await @@ -472,7 +475,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); let auth = SandboxJwtAuthenticator::from_pem( @@ -514,8 +517,8 @@ mod tests { } #[tokio::test] - async fn ttl_zero_mints_non_expiring_token() { - let (issuer, auth) = pair_with_ttl(Duration::ZERO); + async fn ttl_none_mints_non_expiring_token() { + let (issuer, auth) = pair_with_ttl(None); let minted = issuer.mint("sandbox-never").unwrap(); assert_eq!(minted.expires_at_ms, 0); @@ -537,6 +540,19 @@ mod tests { assert_eq!(decoded.claims.exp, 0); } + #[test] + fn ttl_some_zero_is_rejected() { + let mat = generate_jwt_key().expect("jwt key"); + let error = SandboxJwtIssuer::from_pem( + mat.signing_key_pem.as_bytes(), + mat.kid, + "test-gateway", + Some(Duration::ZERO), + ) + .expect_err("Some(Duration::ZERO) must not reintroduce a sentinel"); + assert!(error.contains("must be positive")); + } + #[tokio::test] async fn token_signed_by_other_key_is_rejected() { let (_, auth_a) = pair(); @@ -582,7 +598,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "g", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); let auth = @@ -613,7 +629,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::ZERO, + None, ) .expect("issuer"); let decoding_key = DecodingKey::from_ed_pem(mat.public_key_pem.as_bytes()).unwrap(); @@ -662,7 +678,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); @@ -692,13 +708,9 @@ mod tests { #[test] fn extension_token_rejects_wrong_audience() { let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid, - "gateway-a", - Duration::ZERO, - ) - .expect("issuer"); + let issuer = + SandboxJwtIssuer::from_pem(mat.signing_key_pem.as_bytes(), mat.kid, "gateway-a", None) + .expect("issuer"); let minted = issuer .mint_extension_token( &extension_audience("service-a"), @@ -734,7 +746,7 @@ mod tests { #[test] fn extension_token_enforces_positive_bounded_ttl_and_caller_shape() { - let (issuer, _) = pair_with_ttl(Duration::ZERO); + let (issuer, _) = pair_with_ttl(None); for ttl in [ Duration::ZERO, MAX_EXTENSION_TOKEN_TTL + Duration::from_secs(1), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index f18dbe07aa..b746fb247b 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -101,29 +101,25 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_DB_URL")] db_url: Option, - /// Compute drivers configured for this gateway. + /// Compute driver configured for this gateway. /// - /// Accepts a comma-delimited list of registered driver names. The - /// configuration format is future-proofed for multiple drivers, but the - /// gateway currently requires exactly one. When unset, the gateway runs - /// detection probes supplied by the drivers compiled into the binary. + /// When unset, the gateway runs detection probes supplied by the drivers + /// compiled into the binary. #[arg( - long, - alias = "driver", - env = "OPENSHELL_DRIVERS", - value_delimiter = ',', + long = "compute-driver", + env = "OPENSHELL_COMPUTE_DRIVER", value_parser = parse_compute_driver )] - drivers: Vec, + compute_driver: Option, /// Path to a Unix domain socket served by a remote compute driver /// implementing `compute_driver.proto`. /// - /// When set, the socket is associated with the single driver name supplied - /// by `--drivers` or `OPENSHELL_DRIVERS` and replaces normal construction - /// for that selected name, including a compiled registration with the same - /// name. The gateway connects to this operator-provided endpoint; it does - /// not provision the remote driver. + /// When set, the socket is associated with the driver name supplied by + /// `--compute-driver` or `OPENSHELL_COMPUTE_DRIVER` and replaces normal + /// construction for that selected name, including a compiled registration + /// with the same name. The gateway connects to this operator-provided + /// endpoint; it does not provision the remote driver. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] compute_driver_socket: Option, @@ -248,11 +244,22 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result Result<()> { + if std::env::var_os("OPENSHELL_DRIVERS").is_some() { + return Err(miette::miette!( + "OPENSHELL_DRIVERS is no longer supported; use OPENSHELL_COMPUTE_DRIVER with exactly one driver name" + )); + } + Ok(()) +} + fn prepare_server_config_with_drivers( args: &mut RunArgs, matches: &ArgMatches, compute_drivers: &ComputeDriverRegistry, ) -> Result { + reject_legacy_driver_selector_env()?; + // Load TOML when explicitly requested, or from the default XDG location // when that file exists. Missing default config is not an error: runtime // defaults and OPENSHELL_* env vars are enough for package-managed starts. @@ -267,12 +274,17 @@ fn prepare_server_config_with_drivers( } normalize_compute_driver_socket_args(args, matches)?; let compute_driver = compute_drivers - .select(&args.drivers) + .select(args.compute_driver.as_deref()) .map_err(|error| miette::miette!("{error}"))?; let selected_registration = compute_drivers.get(compute_driver.name()); let local_tls = apply_runtime_defaults(args)?; - let guest_tls = local_tls.as_ref().map(GuestTlsPaths::from); + let guest_tls = GuestTlsPaths::resolve( + file.as_ref().map(|file| &file.openshell.gateway), + local_tls.as_ref(), + args.disable_tls, + ) + .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; let local_jwt = defaults::complete_local_jwt_config()?; let bind = SocketAddr::new(args.bind_address, args.port); @@ -411,7 +423,6 @@ fn prepare_server_config_with_drivers( config = config .with_database_url(db_url) - .with_compute_drivers(args.drivers.clone()) .with_grpc_rate_limit( args.grpc_rate_limit_requests, args.grpc_rate_limit_window_seconds, @@ -423,6 +434,9 @@ fn prepare_server_config_with_drivers( ) .with_server_sans(args.server_sans.clone()) .with_loopback_service_http(args.enable_loopback_service_http); + if let Some(driver) = &args.compute_driver { + config = config.with_compute_driver(driver); + } if let Some(sources) = file .as_ref() .and_then(|file| file.openshell.gateway.provider_profile_sources.clone()) @@ -444,8 +458,8 @@ fn prepare_server_config_with_drivers( )?; if let Some(socket) = args.compute_driver_socket.clone() { let driver = args - .drivers - .first() + .compute_driver + .as_ref() .expect("normalize_compute_driver_socket_args sets a driver for socket endpoints"); config = config.with_compute_driver_endpoint(driver.clone(), socket); } @@ -699,10 +713,10 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.log_level.clone_from(level); } - if let Some(drivers) = &file.compute_drivers - && arg_defaulted(matches, "drivers") + if let Some(driver) = &file.compute_driver + && arg_defaulted(matches, "compute_driver") { - args.drivers.clone_from(drivers); + args.compute_driver = Some(driver.clone()); } if let Some(sans) = &file.server_sans && args.server_sans.is_empty() @@ -803,24 +817,21 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches "--compute-driver-socket must not be an empty path" )); } - if arg_defaulted(matches, "drivers") { + if arg_defaulted(matches, "compute_driver") { return Err(miette::miette!( - "--compute-driver-socket requires --drivers or OPENSHELL_DRIVERS= to select a compute driver name" + "--compute-driver-socket requires --compute-driver or OPENSHELL_COMPUTE_DRIVER= to select a compute driver name" )); } - match args.drivers.as_slice() { - [driver] => { - let driver = openshell_core::config::normalize_compute_driver_name(driver) - .map_err(|err| miette::miette!("{err}"))?; - args.drivers[0] = driver; - Ok(()) - } - drivers => Err(miette::miette!( - "--compute-driver-socket requires exactly one compute driver name, got: {}", - drivers.join(",") - )), - } + let driver = args + .compute_driver + .as_deref() + .expect("explicit compute driver is required for socket endpoints"); + args.compute_driver = Some( + openshell_core::config::normalize_compute_driver_name(driver) + .map_err(|err| miette::miette!("{err}"))?, + ); + Ok(()) } fn is_singleplayer_driver(registration: Option<&crate::ComputeDriverRegistration>) -> bool { @@ -1359,7 +1370,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "local", "--tls-cert", "/tmp/server.crt", @@ -1387,7 +1398,7 @@ mod tests { let _state = EnvVarGuard::set("XDG_STATE_HOME", state.path().to_str().unwrap()); let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config.path().to_str().unwrap()); let _mtls = EnvVarGuard::remove("OPENSHELL_ENABLE_MTLS_AUTH"); - let _drivers = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _drivers = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); let (mut args, matches) = parse_with_args(&[ @@ -1407,7 +1418,7 @@ mod tests { super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); assert_eq!(prepared.compute_driver.name(), "local"); - assert!(prepared.config.compute_drivers.is_empty()); + assert!(prepared.config.compute_driver.is_none()); assert!(prepared.config.mtls_auth.enabled); assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 1); } @@ -1423,7 +1434,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "shared", "--tls-cert", "/tmp/server.crt", @@ -1452,7 +1463,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "local", "--tls-cert", "/tmp/server.crt", @@ -1726,13 +1737,13 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "Kyma", "--compute-driver-socket", "/run/openshell/kyma.sock", @@ -1742,7 +1753,7 @@ ssh_session_ttl_secs = 1234 args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1751,7 +1762,7 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", @@ -1763,7 +1774,7 @@ ssh_session_ttl_secs = 1234 let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); assert!( - err.to_string().contains("requires --drivers "), + err.to_string().contains("requires --compute-driver "), "unexpected error: {err}" ); } @@ -1774,19 +1785,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "docker", "--compute-driver-socket", "/run/openshell/extension.sock", ]); super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["docker"]); + assert_eq!(args.compute_driver.as_deref(), Some("docker")); } #[test] @@ -1795,19 +1806,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "vm", "--compute-driver-socket", "/run/openshell/vm.sock", ]); super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["vm"]); + assert_eq!(args.compute_driver.as_deref(), Some("vm")); } #[test] @@ -1819,7 +1830,7 @@ ssh_session_ttl_secs = 1234 "OPENSHELL_COMPUTE_DRIVER_SOCKET", "/var/run/openshell/kyma.sock", ); - let _g2 = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + let _g2 = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "kyma"); let (mut args, matches) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); @@ -1828,7 +1839,7 @@ ssh_session_ttl_secs = 1234 args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/var/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1899,6 +1910,9 @@ enable_loopback_service_http = false std::fs::write( &config_path, r#" +[openshell] +version = 2 + [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" @@ -1917,7 +1931,7 @@ mem_mib = "not-a-number" config_path.to_str().unwrap(), "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "podman", "--disable-tls", ]); @@ -1925,7 +1939,7 @@ mem_mib = "not-a-number" let prepared = super::prepare_server_config(&mut args, &matches).expect("server config is prepared"); - assert_eq!(prepared.config.compute_drivers, vec!["podman".to_string()]); + assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman")); assert_eq!( prepared.config.policy_validation_failure_mode, openshell_core::PolicyValidationFailureMode::RetainLastValid diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index d06e6fbc8f..d7488270a7 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -3,9 +3,10 @@ //! Selected compute-driver config construction. //! -//! This module owns loading the selected driver config from TOML and applying -//! gateway startup defaults and endpoint overrides. It does not acquire, -//! connect to, or start compute drivers. +//! This module owns loading the selected driver config from TOML, applying +//! gateway startup defaults and endpoint overrides, and enforcing that +//! driver-specific configuration remains in the selected driver table. It does +//! not acquire, connect to, or start compute drivers. use crate::config_file; use crate::defaults::LocalTlsPaths; @@ -27,13 +28,68 @@ impl GuestTlsPaths { } } -impl From<&LocalTlsPaths> for GuestTlsPaths { - fn from(paths: &LocalTlsPaths) -> Self { - Self { +impl GuestTlsPaths { + /// Resolve gateway-owned guest TLS inputs. Explicit TOML values take + /// precedence over the package-managed local bundle; partial bundles are + /// rejected before any driver is deserialized or constructed. + pub(crate) fn resolve( + gateway: Option<&config_file::GatewayFileSection>, + local: Option<&LocalTlsPaths>, + tls_disabled: bool, + ) -> std::result::Result, String> { + let configured = gateway.map(|gateway| { + ( + gateway.guest_tls_ca.as_ref(), + gateway.guest_tls_cert.as_ref(), + gateway.guest_tls_key.as_ref(), + ) + }); + let provided = configured + .is_some_and(|(ca, cert, key)| ca.is_some() || cert.is_some() || key.is_some()); + + if tls_disabled { + if provided { + return Err( + "guest_tls_ca, guest_tls_cert, and guest_tls_key require gateway TLS; remove them or omit --disable-tls" + .to_string(), + ); + } + return Ok(None); + } + + if let Some((ca, cert, key)) = configured + && (ca.is_some() || cert.is_some() || key.is_some()) + { + let (Some(ca), Some(cert), Some(key)) = (ca, cert, key) else { + return Err( + "guest TLS requires one complete bundle: guest_tls_ca, guest_tls_cert, and guest_tls_key" + .to_string(), + ); + }; + for (field, path) in [ + ("guest_tls_ca", ca), + ("guest_tls_cert", cert), + ("guest_tls_key", key), + ] { + if !path.is_file() { + return Err(format!( + "{field} '{}' does not exist or is not a file", + path.display() + )); + } + } + return Ok(Some(Self { + ca: ca.clone(), + cert: cert.clone(), + key: key.clone(), + })); + } + + Ok(local.map(|paths| Self { ca: paths.ca.clone(), cert: paths.client_cert.clone(), key: paths.client_key.clone(), - } + })) } } @@ -52,11 +108,8 @@ pub fn remote_driver_config_from_context( ) -> Result { let mut cfg = RemoteDriverConfig::default(); if let Some(file) = context.file { - let merged = config_file::driver_table( - name, - &file.openshell.gateway, - file.openshell.drivers.get(name), - ); + let merged = config_file::driver_table(file.openshell.drivers.get(name)); + reject_driver_owned_guest_tls_fields(&merged)?; if let Some(socket_path) = merged.get("socket_path").and_then(toml::Value::as_str) { cfg.socket_path = PathBuf::from(socket_path); } @@ -75,31 +128,25 @@ pub struct RemoteDriverConfig { pub fn driver_config_from_context( context: DriverStartupContext<'_>, driver_name: &str, - inherited_config_keys: &[&str], ) -> Result where T: Default + serde::de::DeserializeOwned, { - driver_config_from_file(context.file, driver_name, inherited_config_keys) + driver_config_from_file(context.file, driver_name) } fn driver_config_from_file( file: Option<&config_file::ConfigFile>, driver_name: &str, - inherited_config_keys: &[&str], ) -> Result where T: Default + serde::de::DeserializeOwned, { - let Some(file) = file else { - return Ok(T::default()); - }; - let merged = config_file::driver_table_with_inherited_keys( - driver_name, - &file.openshell.gateway, - file.openshell.drivers.get(driver_name), - inherited_config_keys, + let merged = file.map_or_else( + || config_file::driver_table(None), + |file| config_file::driver_table(file.openshell.drivers.get(driver_name)), ); + reject_driver_owned_guest_tls_fields(&merged)?; merged.try_into().map_err(|e| { Error::config(format!( "invalid [openshell.drivers.{driver_name}] table: {e}" @@ -107,6 +154,23 @@ where }) } +/// Reject TLS paths in gateway driver tables. These credentials are gateway +/// inputs and are injected only into the selected local driver after the +/// gateway has validated the complete bundle. +fn reject_driver_owned_guest_tls_fields(table: &toml::Value) -> Result<()> { + let Some(table) = table.as_table() else { + return Ok(()); + }; + for field in ["guest_tls_ca", "guest_tls_cert", "guest_tls_key"] { + if table.contains_key(field) { + return Err(Error::config(format!( + "{field} belongs in [openshell.gateway], not a [openshell.drivers.*] table" + ))); + } + } + Ok(()) +} + fn apply_remote_driver_overrides( cfg: &mut RemoteDriverConfig, context: DriverStartupContext<'_>, @@ -130,6 +194,7 @@ fn validate_remote_driver_config(cfg: &RemoteDriverConfig, name: &str) -> Result mod tests { use super::*; use std::collections::BTreeMap; + use std::path::Path; fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = @@ -150,6 +215,111 @@ mod tests { } } + #[test] + fn gateway_guest_tls_resolves_explicit_complete_bundle() { + let dir = tempfile::tempdir().expect("temp dir"); + let ca = dir.path().join("ca.pem"); + let cert = dir.path().join("cert.pem"); + let key = dir.path().join("key.pem"); + for path in [&ca, &cert, &key] { + std::fs::write(path, b"test").expect("write TLS fixture"); + } + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(ca.clone()), + guest_tls_cert: Some(cert.clone()), + guest_tls_key: Some(key.clone()), + ..Default::default() + }; + + let resolved = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect("complete guest TLS should resolve") + .expect("guest TLS bundle"); + + assert_eq!( + resolved.as_paths(), + (ca.as_path(), cert.as_path(), key.as_path()) + ); + } + + #[test] + fn gateway_guest_tls_rejects_every_partial_bundle() { + let path = PathBuf::from("/tmp/guest-tls.pem"); + for (ca, cert, key) in [ + (Some(path.clone()), None, None), + (None, Some(path.clone()), None), + (None, None, Some(path.clone())), + (Some(path.clone()), Some(path.clone()), None), + (Some(path.clone()), None, Some(path.clone())), + (None, Some(path.clone()), Some(path)), + ] { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: ca, + guest_tls_cert: cert, + guest_tls_key: key, + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("partial guest TLS must fail"); + assert!(error.contains("one complete bundle")); + } + } + + #[test] + fn gateway_guest_tls_rejects_missing_explicit_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(dir.path().join("missing-ca.pem")), + guest_tls_cert: Some(dir.path().join("missing-cert.pem")), + guest_tls_key: Some(dir.path().join("missing-key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("missing explicit file must fail"); + assert!(error.contains("guest_tls_ca")); + assert!(error.contains("does not exist")); + } + + #[test] + fn gateway_guest_tls_uses_package_managed_bundle() { + let local = LocalTlsPaths { + ca: PathBuf::from("/managed/ca.pem"), + server_cert: PathBuf::from("/managed/server-cert.pem"), + server_key: PathBuf::from("/managed/server-key.pem"), + client_cert: PathBuf::from("/managed/client-cert.pem"), + client_key: PathBuf::from("/managed/client-key.pem"), + }; + let resolved = GuestTlsPaths::resolve(None, Some(&local), false) + .expect("managed bundle should resolve") + .expect("guest TLS bundle"); + assert_eq!( + resolved.as_paths(), + ( + Path::new("/managed/ca.pem"), + Path::new("/managed/client-cert.pem"), + Path::new("/managed/client-key.pem"), + ) + ); + } + + #[test] + fn gateway_guest_tls_can_be_absent() { + assert!(GuestTlsPaths::resolve(None, None, false).unwrap().is_none()); + assert!(GuestTlsPaths::resolve(None, None, true).unwrap().is_none()); + } + + #[test] + fn gateway_guest_tls_rejects_plaintext_gateway() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: Some(PathBuf::from("/tmp/cert.pem")), + guest_tls_key: Some(PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, true) + .expect_err("guest TLS and plaintext gateway conflict"); + assert!(error.contains("require gateway TLS")); + } + #[test] fn remote_driver_config_reads_socket_path_from_named_table() { let file: config_file::ConfigFile = toml::from_str( @@ -170,8 +340,8 @@ socket_path = "/run/openshell/kyma.sock" fn remote_driver_config_ignores_in_process_driver_fields() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "sandboxes" +[openshell] +version = 2 [openshell.drivers.kubernetes] socket_path = "/run/openshell/kubernetes.sock" diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 65709748b1..5270247b3e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -6,9 +6,8 @@ //! See `rfc/0003-gateway-configuration/README.md` for the file format. This //! module parses the file into [`ConfigFile`], rejects fields that must be //! supplied via env/CLI (database URL), and provides -//! [`driver_table`] which overlays shared `[openshell.gateway]` defaults onto -//! a `[openshell.drivers.]` table so each driver crate's -//! `Deserialize` impl sees a fully-populated table. +//! [`driver_table`] which returns a driver-owned +//! `[openshell.drivers.]` table without gateway-level inheritance. //! //! The merge precedence for gateway process settings is: //! ```text @@ -33,8 +32,8 @@ use openshell_core::{ }; use serde::{Deserialize, Serialize}; -/// Latest schema version this build understands. -pub const SCHEMA_VERSION: u32 = 1; +/// Gateway configuration schema version supported by this build. +pub const SCHEMA_VERSION: u32 = 2; /// Root of the gateway TOML config file. /// @@ -52,8 +51,8 @@ pub struct ConfigFile { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct OpenShellRoot { - /// Reserved for future schema migrations. Versions greater than - /// [`SCHEMA_VERSION`] are rejected at load time. + /// Gateway configuration schema version. Loaded files must set this to + /// [`SCHEMA_VERSION`]. #[serde(default)] pub version: Option, @@ -64,8 +63,8 @@ pub struct OpenShellRoot { pub supervisor: SupervisorFileSection, /// `[openshell.drivers.]` tables — passed verbatim to each driver - /// crate's `Deserialize` impl after the gateway-side inheritance merge. - /// Stored as raw [`toml::Value`] so each driver can evolve its schema + /// crate's `Deserialize` impl. Stored as raw [`toml::Value`] so each + /// driver can evolve its schema /// independently of this crate. #[serde(default)] pub drivers: BTreeMap, @@ -80,9 +79,8 @@ pub struct OpenShellRoot { /// /// All fields are `Option` so the loader can tell whether a key was set /// in the file (`Some`) or not (`None` — value is taken from CLI/env/default). -/// -/// The fields under "Shared driver defaults" are inherited into -/// `[openshell.drivers.]` tables per [`inheritable_keys`]. +/// Driver-specific settings belong exclusively in +/// `[openshell.drivers.]` tables. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GatewayFileSection { @@ -104,8 +102,9 @@ pub struct GatewayFileSection { pub log_level: Option, // ── Drivers ────────────────────────────────────────────────────────── - #[serde(default)] - pub compute_drivers: Option>, + /// Explicit compute driver selection. `None` enables auto-detection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compute_driver: Option, #[serde(default)] pub credential_drivers: Option>, #[serde(default)] @@ -115,8 +114,6 @@ pub struct GatewayFileSection { // ── Sandbox / SSH ──────────────────────────────────────────────────── #[serde(default)] - pub sandbox_namespace: Option, - #[serde(default)] pub ssh_session_ttl_secs: Option, #[serde(default)] pub grpc_rate_limit_requests: Option, @@ -135,24 +132,7 @@ pub struct GatewayFileSection { #[serde(default)] pub enable_loopback_service_http: Option, - // ── Shared driver defaults (inherited into [openshell.drivers.]) ─ - #[serde(default)] - pub default_image: Option, - #[serde(default)] - pub supervisor_image: Option, - #[serde(default)] - pub client_tls_secret_name: Option, - #[serde(default)] - pub service_account_name: Option, - #[serde(default)] - pub host_gateway_ip: Option, - #[serde(default)] - pub enable_user_namespaces: Option, - /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet - /// writes for the `IssueSandboxToken` bootstrap exchange. Driver - /// clamps to `[600, 86400]`. - #[serde(default)] - pub sa_token_ttl_secs: Option, + // ── Sandbox client TLS ─────────────────────────────────────────────── #[serde(default)] pub guest_tls_ca: Option, #[serde(default)] @@ -335,7 +315,11 @@ pub enum ConfigFileError { source: toml::de::Error, }, #[error( - "unsupported gateway config version {version}; this build only supports version {SCHEMA_VERSION}" + "gateway config schema version is required; add `[openshell]` and `version = {SCHEMA_VERSION}`" + )] + MissingVersion, + #[error( + "unsupported gateway config version {version}; this build requires version {SCHEMA_VERSION}; migrate legacy fields to the version {SCHEMA_VERSION} schema" )] UnsupportedVersion { version: u32 }, #[error( @@ -351,6 +335,8 @@ pub enum ConfigFileError { field: &'static str, message: &'static str, }, + #[error("invalid gateway config field `openshell.drivers.{name}`: expected a TOML table")] + InvalidDriverTable { name: String }, #[error( "failed to read TLS CA certificate for supervisor middleware '{name}' from '{}': {source}", path.display() @@ -374,8 +360,8 @@ pub enum ConfigFileError { /// Load and validate a TOML config file. /// -/// Returns `Ok(ConfigFile::default())` for an empty file (the gateway then -/// falls back entirely to CLI/env/built-in defaults). +/// Configuration files must declare exactly [`SCHEMA_VERSION`]. Running +/// without a config file still uses CLI, environment, and built-in defaults. #[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] pub fn load(path: &Path) -> Result { let contents = std::fs::read_to_string(path).map_err(|source| ConfigFileError::Io { @@ -383,17 +369,17 @@ pub fn load(path: &Path) -> Result { source, })?; if contents.trim().is_empty() { - return Ok(ConfigFile::default()); + return Err(ConfigFileError::MissingVersion); } let file: ConfigFile = toml::from_str(&contents).map_err(|source| ConfigFileError::Parse { path: path.to_path_buf(), source, })?; - if let Some(version) = file.openshell.version - && version > SCHEMA_VERSION - { - return Err(ConfigFileError::UnsupportedVersion { version }); + match file.openshell.version { + Some(SCHEMA_VERSION) => {} + Some(version) => return Err(ConfigFileError::UnsupportedVersion { version }), + None => return Err(ConfigFileError::MissingVersion), } if file.openshell.gateway.database_url.is_some() { @@ -415,80 +401,34 @@ pub fn load(path: &Path) -> Result { message: "omit the field to use default encrypted gateway credential storage, or specify exactly one external credential driver", }); } - - Ok(file) -} - -/// Build the merged TOML table for `driver` by overlaying inheritable -/// `[openshell.gateway]` defaults onto `[openshell.drivers.]`. -/// -/// The returned [`toml::Value`] is a Table ready to feed into the driver's -/// `Deserialize` impl — keys present in `raw` win over the gateway defaults. -/// Keys outside [`inheritable_keys`] for this driver are never copied from -/// the gateway section, which keeps each driver's `deny_unknown_fields` -/// invariant intact. -pub fn driver_table( - driver_name: &str, - gateway: &GatewayFileSection, - raw: Option<&toml::Value>, -) -> toml::Value { - driver_table_with_inherited_keys(driver_name, gateway, raw, &[]) -} - -pub(crate) fn driver_table_with_inherited_keys( - _driver_name: &str, - gateway: &GatewayFileSection, - raw: Option<&toml::Value>, - inheritable_keys: &[&str], -) -> toml::Value { - let mut merged = match raw { - Some(toml::Value::Table(table)) => table.clone(), - _ => toml::Table::new(), - }; - - for key in inheritable_keys { - if merged.contains_key(*key) { - continue; - } - if let Some(value) = gateway_inherited_value(gateway, key) { - merged.insert((*key).to_string(), value); - } + if let Some((name, _)) = file + .openshell + .drivers + .iter() + .find(|(_, value)| !value.is_table()) + { + return Err(ConfigFileError::InvalidDriverTable { name: name.clone() }); } - toml::Value::Table(merged) + Ok(file) } -fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { - match key { - "namespace" | "sandbox_namespace" => g.sandbox_namespace.as_deref().map(string_value), - "default_image" => g.default_image.as_deref().map(string_value), - "supervisor_image" => g.supervisor_image.as_deref().map(string_value), - "client_tls_secret_name" => g.client_tls_secret_name.as_deref().map(string_value), - "service_account_name" => g.service_account_name.as_deref().map(string_value), - "host_gateway_ip" => g.host_gateway_ip.as_deref().map(string_value), - "enable_user_namespaces" => g.enable_user_namespaces.map(toml::Value::Boolean), - "sa_token_ttl_secs" => g.sa_token_ttl_secs.map(toml::Value::Integer), - "guest_tls_ca" => g.guest_tls_ca.as_deref().map(path_value), - "guest_tls_cert" => g.guest_tls_cert.as_deref().map(path_value), - "guest_tls_key" => g.guest_tls_key.as_deref().map(path_value), - _ => None, +/// Return a driver's table without gateway-level inheritance. +/// Driver-specific configuration belongs exclusively to +/// `[openshell.drivers.]` in schema version 2. +pub fn driver_table(raw: Option<&toml::Value>) -> toml::Value { + match raw { + Some(toml::Value::Table(table)) => toml::Value::Table(table.clone()), + _ => toml::Value::Table(toml::Table::new()), } } -fn string_value(s: &str) -> toml::Value { - toml::Value::String(s.to_owned()) -} - -fn path_value(p: &Path) -> toml::Value { - toml::Value::String(p.display().to_string()) -} - #[cfg(test)] mod tests { use super::*; use std::io::Write; - fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + fn write_raw_tmp(contents: &str) -> tempfile::NamedTempFile { let mut tmp = tempfile::Builder::new() .suffix(".toml") .tempfile() @@ -497,35 +437,103 @@ mod tests { tmp } + fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + if contents.contains("[openshell]") { + write_raw_tmp(contents) + } else { + write_raw_tmp(&format!("[openshell]\nversion = 2\n\n{contents}")) + } + } + #[test] - fn empty_file_yields_default_config() { - let tmp = write_tmp(""); - let file = load(tmp.path()).expect("empty file parses"); - assert!(file.openshell.version.is_none()); - assert!(file.openshell.gateway.bind_address.is_none()); - assert!(file.openshell.drivers.is_empty()); + fn empty_file_requires_schema_version() { + let tmp = write_raw_tmp(""); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); + } + + #[test] + fn compute_driver_entries_must_be_tables() { + for value in ["\"not-a-table\"", "[\"also\", \"not-a-table\"]", "42"] { + let tmp = write_raw_tmp(&format!( + "[openshell]\nversion = 2\n\n[openshell.drivers]\ndocker = {value}\n" + )); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::InvalidDriverTable { ref name }) if name == "docker" + )); + } + } + + #[test] + fn canonical_compute_driver_is_singular() { + let file: ConfigFile = toml::from_str( + r#" +[openshell.gateway] +compute_driver = "docker" +"#, + ) + .expect("canonical compute driver parses"); + + assert_eq!( + file.openshell.gateway.compute_driver.as_deref(), + Some("docker") + ); + } + + #[test] + fn legacy_compute_drivers_list_is_rejected() { + let error = + toml::from_str::("[openshell.gateway]\ncompute_drivers = [\"docker\"]\n") + .expect_err("legacy compute_drivers must be rejected"); + assert!(error.to_string().contains("compute_drivers")); + } + + #[test] + fn compute_driver_rejects_non_string_values() { + let error = toml::from_str::( + r" +[openshell.gateway] +compute_driver = 42 +", + ) + .expect_err("compute driver must be a string"); + assert!(error.to_string().contains("invalid type")); + } + + #[test] + fn compute_driver_serialization_uses_scalar_name() { + let file = ConfigFile { + openshell: OpenShellRoot { + gateway: GatewayFileSection { + compute_driver: Some("docker".to_string()), + ..Default::default() + }, + ..Default::default() + }, + }; + + let serialized = toml::to_string(&file).expect("config serializes"); + assert!(serialized.contains("compute_driver = \"docker\"")); } #[test] fn parses_full_example() { let toml = r#" [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "agents" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 policy_validation_failure_mode = "retain_last_valid" -default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -client_tls_secret_name = "openshell-sandbox-tls" -service_account_name = "openshell-sandbox" [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" @@ -538,6 +546,10 @@ audience = "openshell-cli" [openshell.drivers.kubernetes] namespace = "agents" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +client_tls_secret_name = "openshell-sandbox-tls" +service_account_name = "openshell-sandbox" grpc_endpoint = "https://openshell-gateway.agents.svc:8080" [openshell.credential_drivers.kubernetes-secrets] @@ -547,10 +559,6 @@ namespace = "agents" let file = load(tmp.path()).expect("valid file parses"); let gw = &file.openshell.gateway; assert_eq!(gw.log_level.as_deref(), Some("info")); - assert_eq!( - gw.default_image.as_deref(), - Some("ghcr.io/nvidia/openshell/sandbox:latest") - ); assert_eq!(gw.grpc_rate_limit_requests, Some(120)); assert_eq!(gw.grpc_rate_limit_window_seconds, Some(60)); assert_eq!( @@ -895,6 +903,27 @@ nonsense = true assert!(matches!(err, ConfigFileError::Parse { .. })); } + #[test] + fn rejects_removed_driver_fields_at_gateway_scope() { + for field in [ + "sandbox_namespace = \"agents\"", + "default_image = \"sandbox:latest\"", + "supervisor_image = \"supervisor:latest\"", + "client_tls_secret_name = \"sandbox-tls\"", + "service_account_name = \"sandbox-sa\"", + "host_gateway_ip = \"10.0.0.1\"", + "enable_user_namespaces = true", + "sa_token_ttl_secs = 3600", + ] { + let tmp = write_tmp(&format!("[openshell.gateway]\n{field}\n")); + let err = load(tmp.path()).expect_err("gateway-scoped driver field must be rejected"); + assert!( + matches!(err, ConfigFileError::Parse { .. }), + "field: {field}" + ); + } + } + #[test] fn rejects_unknown_field_in_nested_gateway_jwt_table() { // Regression guard for the class of silent-misconfig bug fixed in @@ -927,164 +956,61 @@ ssh_gateway_port = 8080 } #[test] - fn rejects_unsupported_version() { - let toml = r" -[openshell] -version = 2 -"; - let tmp = write_tmp(toml); - let err = load(tmp.path()).expect_err("version > 1 must be rejected"); + fn rejects_legacy_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 1\n"); + let err = load(tmp.path()).expect_err("version 1 must be rejected"); assert!(matches!( err, - ConfigFileError::UnsupportedVersion { version: 2 } + ConfigFileError::UnsupportedVersion { version: 1 } )); } #[test] - fn driver_table_inherits_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), - supervisor_image: Some("ghcr.io/nvidia/openshell/supervisor:0.9".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - namespace = "agents" - }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - Some(&toml::Value::Table(raw)), - &["default_image", "supervisor_image"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("namespace").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/sandbox:0.9") - ); - assert_eq!( - table.get("supervisor_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/supervisor:0.9") - ); + fn rejects_missing_version_in_nonempty_file() { + let tmp = write_raw_tmp("[openshell]\n\n[openshell.gateway]\nname = \"test\"\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); } #[test] - fn registered_driver_table_inherits_selected_gateway_defaults() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("agents".to_string()), - default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - None, - &["sandbox_namespace", "default_image", "host_gateway_ip"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("sandbox_namespace").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/sandbox:0.9") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("10.0.0.1") - ); + fn rejects_future_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 3\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::UnsupportedVersion { version: 3 }) + )); } #[test] - fn registered_driver_table_can_select_network_defaults() { - let gateway = GatewayFileSection { - default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), - host_gateway_ip: Some("192.168.127.254".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys( - "beta", - &gateway, - None, - &["default_image", "host_gateway_ip"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/sandbox:0.9") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("192.168.127.254") - ); + fn accepts_current_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 2\n"); + load(tmp.path()).expect("schema version 2 must be accepted"); } #[test] - fn driver_table_specific_value_overrides_gateway_default() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default".to_string()), - ..Default::default() - }; + fn driver_table_uses_only_driver_owned_values() { let raw = toml::toml! { default_image = "driver-specific" + socket_path = "/run/openshell/driver.sock" }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - Some(&toml::Value::Table(raw)), - &["default_image"], - ); + let table = driver_table(Some(&toml::Value::Table(raw))); + let table = table.as_table().expect("driver table"); assert_eq!( - merged - .as_table() - .unwrap() - .get("default_image") - .and_then(|v| v.as_str()), + table.get("default_image").and_then(toml::Value::as_str), Some("driver-specific") ); - } - - #[test] - fn driver_table_does_not_leak_keys_outside_allowlist() { - // Fields not selected by the registration must remain gateway-only. - let gateway = GatewayFileSection { - client_tls_secret_name: Some("openshell-sandbox-tls".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys("alpha", &gateway, None, &["default_image"]); - assert!( - !merged - .as_table() - .unwrap() - .contains_key("client_tls_secret_name") + assert_eq!( + table.get("socket_path").and_then(toml::Value::as_str), + Some("/run/openshell/driver.sock") ); } #[test] - fn remote_driver_table_does_not_inherit_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default:1.0".to_string()), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - socket_path = "/run/openshell/kyma.sock" - }; - - let merged = driver_table("kyma", &gateway, Some(&toml::Value::Table(raw))); - let table = merged.as_table().expect("table"); - - assert_eq!( - table.get("socket_path").and_then(|v| v.as_str()), - Some("/run/openshell/kyma.sock") - ); - assert!(!table.contains_key("default_image")); - assert!(!table.contains_key("host_gateway_ip")); + fn driver_table_does_not_inject_gateway_values() { + let table = driver_table(None); + assert!(table.as_table().expect("driver table").is_empty()); } #[test] @@ -1102,7 +1028,7 @@ version = 2 /// - template corruption or unknown fields (`deny_unknown_fields`) /// - schema drift (version bump or field renames) /// - accidental addition of a wildcard bind-address override - /// - accidental changes to the compute driver list + /// - accidental changes to the configured compute driver #[test] fn rpm_default_config_parses_and_has_podman_defaults() { let path = @@ -1118,15 +1044,20 @@ version = 2 ); } - let drivers = gw - .compute_drivers - .as_ref() - .expect("compute_drivers must be explicitly set in the RPM default config"); assert_eq!( - drivers, - &["podman".to_string()], - "RPM default must pin compute_drivers to [podman] to prevent unexpected \ + gw.compute_driver.as_deref(), + Some("podman"), + "RPM default must pin compute_driver to podman to prevent unexpected \ driver selection when Docker is also installed" ); + + let podman = driver_table(config.openshell.drivers.get("podman")); + assert_eq!( + podman + .get("health_check_interval_secs") + .and_then(toml::Value::as_integer), + Some(10), + "RPM defaults must retain Podman's readiness health check" + ); } } diff --git a/crates/openshell-server/src/defaults.rs b/crates/openshell-server/src/defaults.rs index b5a5a5e924..21e66b02bd 100644 --- a/crates/openshell-server/src/defaults.rs +++ b/crates/openshell-server/src/defaults.rs @@ -104,7 +104,7 @@ pub fn complete_local_jwt_config() -> Result> { public_key_path: paths.public_key, kid_path: paths.kid, gateway_id: "openshell".to_string(), - ttl_secs: 0, + ttl_secs: None, })), _ => Err(miette::miette!( "partial local sandbox JWT state in {}: expected jwt/signing.pem, jwt/public.pem, and jwt/kid", @@ -237,6 +237,6 @@ mod tests { assert_eq!(config.public_key_path, tmp.path().join("jwt/public.pem")); assert_eq!(config.kid_path, tmp.path().join("jwt/kid")); assert_eq!(config.gateway_id, "openshell"); - assert_eq!(config.ttl_secs, 0); + assert_eq!(config.ttl_secs, None); } } diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs index 104d639584..35325f3bbf 100644 --- a/crates/openshell-server/src/grpc/auth_rpc.rs +++ b/crates/openshell-server/src/grpc/auth_rpc.rs @@ -226,11 +226,11 @@ fn mint_extension_credentials( .iter() .map(|service| (service.name.as_str(), service)) .collect(); - let ttl = if issuer.ttl().is_zero() { - DEFAULT_EXTENSION_TOKEN_TTL - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - }; + let ttl = issuer + .sandbox_token_ttl() + .map_or(DEFAULT_EXTENSION_TOKEN_TTL, |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }); requested_names .iter() @@ -323,7 +323,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid, "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); state.sandbox_jwt_issuer = Some(Arc::new(issuer)); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 2acd34fc44..4b64cf40a0 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -97,11 +97,11 @@ struct GatewayExtensionCredential { } fn extension_token_ttl(issuer: &auth::sandbox_jwt::SandboxJwtIssuer) -> Duration { - if issuer.ttl().is_zero() { - Duration::from_secs(15 * 60) - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - } + issuer + .sandbox_token_ttl() + .map_or(Duration::from_secs(15 * 60), |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }) } /// Mint the gateway-caller credential for one extension registration. @@ -496,7 +496,7 @@ pub(crate) async fn run_server( &signing_pem, kid.clone(), &jwt.gateway_id, - Duration::from_secs(jwt.ttl_secs), + jwt.sandbox_token_ttl(), ) .map_err(Error::config)?, ); @@ -506,7 +506,7 @@ pub(crate) async fn run_server( ); info!( gateway_id = %jwt.gateway_id, - ttl_secs = jwt.ttl_secs, + ttl_secs = jwt.ttl_secs.map(std::num::NonZeroU64::get), "gateway-minted sandbox JWT enabled" ); (Some(issuer), Some(authenticator)) @@ -1106,7 +1106,6 @@ pub struct ComputeDriverRegistration { detect: Option bool>, factory: Arc, telemetry_category: TelemetryComputeDriver, - inherited_config_keys: &'static [&'static str], local_singleplayer: bool, supports_mtls_user_auth: bool, tracing_setup: Option, @@ -1139,20 +1138,12 @@ impl ComputeDriverRegistration { detect, factory: Arc::new(factory), telemetry_category: TelemetryComputeDriver::custom(), - inherited_config_keys: &[], local_singleplayer: false, supports_mtls_user_auth: true, tracing_setup: None, }) } - /// Select gateway-wide defaults understood by this driver's config type. - #[must_use] - pub fn with_inherited_config_keys(mut self, keys: &'static [&'static str]) -> Self { - self.inherited_config_keys = keys; - self - } - /// Assign a bounded telemetry category chosen by the binary composition /// boundary. Runtime driver names are never used as telemetry values. #[must_use] @@ -1296,27 +1287,23 @@ impl ComputeDriverRegistry { ComputeDriverDetection { available } } - pub(crate) fn select(&self, configured_drivers: &[String]) -> Result { - match configured_drivers { - [] => { + pub(crate) fn select(&self, configured_driver: Option<&str>) -> Result { + match configured_driver { + None => { let detection = self.detect(); if detection.selected().is_none() { return Err(Error::config( "no compute driver configured and auto-detection found no suitable installed \ - driver; set --drivers or OPENSHELL_DRIVERS=", + driver; set --compute-driver or OPENSHELL_COMPUTE_DRIVER=", )); } Ok(ComputeDriverSelection::AutoDetected(detection)) } - [driver] => { + Some(driver) => { let name = openshell_core::config::normalize_compute_driver_name(driver) .map_err(Error::config)?; Ok(ComputeDriverSelection::Configured { name }) } - drivers => Err(Error::config(format!( - "multiple compute drivers are not supported yet; configured drivers: {}", - drivers.join(",") - ))), } } } @@ -1328,7 +1315,6 @@ pub struct ComputeDriverBuildContext<'a> { gateway_log_level: &'a str, driver_startup: compute::driver_config::DriverStartupContext<'a>, shutdown_rx: watch::Receiver, - inherited_config_keys: &'static [&'static str], } impl ComputeDriverBuildContext<'_> { @@ -1370,16 +1356,12 @@ impl ComputeDriverBuildContext<'_> { .map(compute::driver_config::GuestTlsPaths::as_paths) } - /// Deserialize the selected driver's merged TOML table. + /// Deserialize the selected driver's TOML table. pub fn driver_config(&self) -> Result where T: Default + serde::de::DeserializeOwned, { - compute::driver_config::driver_config_from_context( - self.driver_startup, - &self.driver_name, - self.inherited_config_keys, - ) + compute::driver_config::driver_config_from_context(self.driver_startup, &self.driver_name) } #[must_use] @@ -1414,7 +1396,7 @@ async fn build_compute_runtime( if config .gateway_jwt .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) + .is_some_and(|jwt| jwt.sandbox_token_ttl().is_none()) && !driver.is_local_singleplayer(registry) { warn!( @@ -1431,7 +1413,6 @@ async fn build_compute_runtime( gateway_log_level: &config.log_level, driver_startup, shutdown_rx, - inherited_config_keys: registration.inherited_config_keys, }; let instance = registration.factory.build(build_context).await?; match instance { @@ -1534,7 +1515,7 @@ fn configured_compute_driver( config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { - let selection = registry.select(&config.compute_drivers)?; + let selection = registry.select(config.compute_driver.as_deref())?; resolve_configured_compute_driver(registry, selection.name(), driver_startup) } @@ -1624,7 +1605,7 @@ mod tests { BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, + configured_compute_driver, extension_token_ttl, is_benign_tls_handshake_failure, mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ @@ -1670,18 +1651,30 @@ mod tests { } fn extension_test_issuer() -> Arc { + extension_test_issuer_with_ttl(Some(Duration::from_secs(900))) + } + + fn extension_test_issuer_with_ttl( + ttl: Option, + ) -> Arc { let material = openshell_bootstrap::jwt::generate_jwt_key().expect("jwt key"); Arc::new( crate::auth::sandbox_jwt::SandboxJwtIssuer::from_pem( material.signing_key_pem.as_bytes(), material.kid, "gateway-a", - Duration::from_secs(900), + ttl, ) .expect("issuer"), ) } + #[test] + fn non_expiring_sandbox_tokens_use_finite_extension_ttl() { + let issuer = extension_test_issuer_with_ttl(None); + assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(15 * 60)); + } + #[test] fn plaintext_extension_endpoint_is_rejected_unless_explicitly_opted_out() { let issuer = extension_test_issuer(); @@ -2131,7 +2124,7 @@ mod tests { .unwrap(), ) .unwrap(); - let config = Config::new(None).with_compute_drivers(std::iter::empty::()); + let config = Config::new(None); let result = configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) .unwrap(); @@ -2200,25 +2193,9 @@ mod tests { ); } - #[test] - fn configured_compute_driver_rejects_multiple_entries() { - let config = Config::new(None).with_compute_drivers(["alpha", "beta"]); - let err = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap_err(); - assert!( - err.to_string() - .contains("multiple compute drivers are not supported yet") - ); - assert!(err.to_string().contains("alpha,beta")); - } - #[test] fn configured_compute_driver_accepts_registered_name() { - let config = Config::new(None).with_compute_drivers(["beta"]); + let config = Config::new(None).with_compute_driver("beta"); let registry = test_compute_drivers(); let driver = configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) @@ -2235,7 +2212,7 @@ mod tests { #[test] fn configured_compute_driver_resolves_named_remote() { - let config = Config::new(None).with_compute_drivers(["kyma"]); + let config = Config::new(None).with_compute_driver("kyma"); let registry = test_compute_drivers(); let driver = @@ -2262,7 +2239,7 @@ mod tests { #[test] fn configured_compute_driver_uses_endpoint_override() { let config = Config::new(None) - .with_compute_drivers(["alpha"]) + .with_compute_driver("alpha") .with_compute_driver_endpoint("alpha", "/run/openshell/alpha.sock"); let registry = test_compute_drivers(); @@ -2282,7 +2259,7 @@ mod tests { #[test] fn configured_compute_driver_uses_builtin_endpoint_override() { let config = Config::new(None) - .with_compute_drivers(["beta"]) + .with_compute_driver("beta") .with_compute_driver_endpoint("beta", "/run/openshell/beta.sock"); let driver = configured_compute_driver( diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index 4fe84d633a..9fbd574035 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -22,7 +22,7 @@ # - "host.openshell.internal:host-gateway" [openshell] -version = 1 +version = 2 [openshell.gateway] # Bind to loopback only. The Docker driver adds an extra listener on the @@ -30,7 +30,7 @@ version = 1 bind_address = "127.0.0.1:8080" health_bind_address = "127.0.0.1:8081" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.drivers.docker] @@ -40,12 +40,15 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # first start. The binary is cached to XDG_DATA_HOME and reused on restart. supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # Only pull images that are not already cached locally. -image_pull_policy = "IfNotPresent" -# Prefix applied to sandbox container names. -sandbox_namespace = "openshell" +image_pull_policy = "if_not_present" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "openshell" # Address sandbox containers use to call back to the gateway. # The Docker driver replaces the host with host.openshell.internal and the # port with the gateway's own bind port (8080). Only the scheme survives. # The gateway must be published on port 8080 on the Docker host so that # host.openshell.internal:8080 resolves to the gateway container. grpc_endpoint = "http://host.openshell.internal:8080" +# Explicit supervisor-compatible Docker default. Set RuntimeDefault or +# Localhost/ only when the daemon host has AppArmor available. +app_armor_profile = "Unconfined" diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 7dbd7964d0..b9bd807aac 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -276,7 +276,7 @@ discovery endpoint or its TLS CA. | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `""` | Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev clusters so new images are picked up without manual eviction. | +| server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -297,7 +297,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | +| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 15ff87554e..706df3eca5 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -3,7 +3,7 @@ # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). server: - sandboxImagePullPolicy: IfNotPresent + sandboxImagePullPolicy: if_not_present otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). @@ -13,4 +13,4 @@ server: supervisor: image: - pullPolicy: IfNotPresent + pullPolicy: if_not_present diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3d9f2f3e0b..98243627dd 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -252,6 +252,26 @@ database requires persistent per-pod storage. {{- default "statefulset" (get $workload "kind") | lower -}} {{- end }} +{{/* +Translate chart image pull policy values to the canonical gateway vocabulary. +The Kubernetes spellings remain accepted so existing values files continue to +work across the schema-v2 chart upgrade. +*/}} +{{- define "openshell.canonicalImagePullPolicy" -}} +{{- $policy := printf "%v" . -}} +{{- if eq $policy "Always" -}} +always +{{- else if eq $policy "IfNotPresent" -}} +if_not_present +{{- else if eq $policy "Never" -}} +never +{{- else if has $policy (list "always" "if_not_present" "never") -}} +{{- $policy -}} +{{- else -}} +{{- fail (printf "image pull policy %q must be one of: always, if_not_present, never, Always, IfNotPresent, Never" $policy) -}} +{{- end -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 083748aee3..2c2b8216ee 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -32,7 +32,7 @@ metadata: data: gateway.toml: | [openshell] - version = 1 + version = 2 [openshell.gateway] name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} @@ -44,29 +44,17 @@ data: metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" {{- end }} log_level = {{ .Values.server.logLevel | quote }} + compute_driver = "kubernetes" {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} - sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} {{- end }} policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} - default_image = {{ .Values.server.sandboxImage | quote }} - {{- if include "openshell.supervisorImageOverrideEnabled" . }} - supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- end }} - {{- if .Values.server.hostGatewayIP }} - host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} - {{- end }} - {{- if .Values.server.enableUserNamespaces }} - enable_user_namespaces = true - {{- end }} {{- if .Values.server.disableTls }} disable_tls = true - {{- else }} - client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} {{- end }} enable_loopback_service_http = {{ .Values.server.enableLoopbackServiceHttp }} {{- $sans := list -}} @@ -148,10 +136,24 @@ data: {{- end }} [openshell.drivers.kubernetes] + namespace = {{ include "openshell.sandboxNamespace" . | quote }} + default_image = {{ .Values.server.sandboxImage | quote }} + {{- if include "openshell.supervisorImageOverrideEnabled" . }} + supervisor_image = {{ include "openshell.supervisorImage" . | quote }} + {{- end }} + {{- if .Values.server.hostGatewayIP }} + host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} + {{- end }} + {{- if not .Values.server.disableTls }} + client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} + {{- end }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.enableUserNamespaces }} + enable_user_namespaces = true + {{- end }} {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} {{- end }} @@ -183,7 +185,7 @@ data: provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} {{- if .Values.server.sandboxImagePullPolicy }} - image_pull_policy = {{ .Values.server.sandboxImagePullPolicy | quote }} + image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.server.sandboxImagePullPolicy | quote }} {{- end }} {{- $sandboxImagePullSecretNames := list -}} {{- range .Values.server.sandboxImagePullSecrets }} @@ -207,7 +209,7 @@ data: app_armor_profile = {{ .Values.server.appArmorProfile | quote }} {{- end }} {{- if .Values.supervisor.image.pullPolicy }} - supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} + supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} {{- end }} [openshell.drivers.kubernetes.managed_ssh_ingress] diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 1380cb18c5..7130871f22 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -27,6 +27,16 @@ tests: path: data["gateway.toml"] pattern: '(?m)^name\s*=\s*"production-us-west"$' + - it: renders schema version 2 and the Kubernetes compute driver selector + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\]\s*version\s*=\s*2' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?compute_driver\s*=\s*"kubernetes"' + # Regression for Drew's P2: a ConfigMap-only mutation in `helm upgrade` # must roll the StatefulSet, otherwise pods keep running with stale config. - it: annotates the StatefulSet pod template with a ConfigMap checksum @@ -138,12 +148,77 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?grpc_endpoint' - - it: renders the sandbox service account name under [openshell.drivers.kubernetes] + - it: omits pull policies by default so Kubernetes applies its own defaults + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^\s*(image_pull_policy|supervisor_image_pull_policy)\s*=' + + - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: if_not_present + supervisor.image.pullPolicy: never + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' + + - it: translates legacy Kubernetes pull policy values to canonical gateway values + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Always + supervisor.image.pullPolicy: IfNotPresent asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"always".*?supervisor_image_pull_policy\s*=\s*"if_not_present"' + + - it: rejects unsupported sandbox image pull policies + template: templates/statefulset.yaml + set: + server.sandboxImagePullPolicy: Sometimes + asserts: + - failedTemplate: + errorMessage: 'image pull policy "Sometimes" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + + - it: rejects unsupported supervisor image pull policies + template: templates/statefulset.yaml + set: + supervisor.image.pullPolicy: newer + asserts: + - failedTemplate: + errorMessage: 'image pull policy "newer" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + + - it: renders driver-owned Kubernetes settings only in its driver table + template: templates/gateway-config.yaml + set: + server.hostGatewayIP: 10.0.0.1 + server.enableUserNamespaces: true + supervisor.image.tag: test + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace".*?default_image\s*=.*?supervisor_image\s*=.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?client_tls_secret_name\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?enable_user_namespaces\s*=\s*true.*?sa_token_ttl_secs\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'guest_tls_(ca|cert|key)\s*=' + + - it: renders user namespace enablement under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + server.enableUserNamespaces: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?enable_user_namespaces\s*=\s*true' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces' - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/tests/grpc_endpoint_test.yaml b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml new file mode 100644 index 0000000000..f84442f605 --- /dev/null +++ b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: Kubernetes gateway callback endpoint + +templates: + - templates/gateway-config.yaml + +release: + name: team-a + namespace: gateway-system + +tests: + - it: derives callback from the gateway Service when sandbox namespace differs + set: + server.sandboxNamespace: agent-sandboxes + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^namespace\s*=\s*"agent-sandboxes"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=\s*"https://team-a-openshell\.gateway-system\.svc\.cluster\.local:8080"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=.*agent-sandboxes' diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 864e3a8512..337c991961 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -13,21 +13,27 @@ release: namespace: my-namespace tests: - - it: defaults sandbox_namespace to release namespace in the TOML config + - it: defaults the Kubernetes driver namespace to release namespace template: templates/gateway-config.yaml asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"my-namespace"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - - it: uses explicit sandboxNamespace when set + - it: uses explicit sandboxNamespace for the Kubernetes driver template: templates/gateway-config.yaml set: server.sandboxNamespace: other-ns asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"other-ns"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"other-ns"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - it: defaults NetworkPolicy namespace to release namespace template: templates/networkpolicy.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 8f5b6fa51a..a4f5b4a533 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -33,8 +33,10 @@ supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor - # -- Supervisor image pull policy. Defaults to the gateway image pull policy when empty. - pullPolicy: "" + # -- Sandbox supervisor pull policy. Leave unset to use the Kubernetes + # image default. Prefer always, if_not_present, or never; the chart also + # accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. + pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" # -- How the supervisor binary is delivered into sandbox pods. @@ -224,10 +226,11 @@ server: externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default - # (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev - # clusters so new images are picked up without manual eviction. - sandboxImagePullPolicy: "" + # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image + # default (Always for :latest, IfNotPresent otherwise). Prefer always, + # if_not_present, or never; the chart also accepts legacy Kubernetes spellings + # Always, IfNotPresent, and Never. + sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. sandboxImagePullSecrets: [] diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 2d584c4ba1..68439ea596 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -58,12 +58,11 @@ TLS. stores SQLite state under *~/.local/state/openshell/gateway/*. Environment: **OPENSHELL_DB_URL**. -**--drivers** *DRIVER*\[,*DRIVER*\] -: Compute driver. Accepts a comma-delimited list. The gateway - currently requires exactly one driver. Options: **podman**, +**--compute-driver** *DRIVER* +: Compute driver. Selects exactly one driver. Options: **podman**, **docker**, **kubernetes**, **vm**. When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. VM is opt-in. - Environment: **OPENSHELL_DRIVERS**. + Environment: **OPENSHELL_COMPUTE_DRIVER**. **--tls-cert** *PATH* : Path to server TLS certificate file. Defaults to the local generated diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 4fc18e6215..f6889e4fb9 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -17,10 +17,10 @@ The defaults are tuned for rootless Podman use: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" ``` The RPM does not override `bind_address`. The primary listener uses the @@ -28,15 +28,17 @@ built-in `127.0.0.1:17670` default. The Podman driver reports the callback interface it needs, and the gateway adds a separate listener scoped to that interface. This keeps the general API off unrelated host interfaces. -`compute_drivers = ["podman"]` pins the compute driver to Podman. Without +`compute_driver = "podman"` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver selection if Docker is also installed on the host. ### Customizing the configuration -Edit `~/.config/openshell/gateway.toml` directly. The template at -`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime -and is not overwritten by RPM upgrades. +Edit `~/.config/openshell/gateway.toml` directly. The package-owned template at +`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime and +may change during an RPM upgrade. The active user copy is preserved. During a +schema-v2 upgrade, the service replaces only an exact package-generated v1 +copy; it never rewrites an edited configuration. To apply environment variable overrides that persist across upgrades without editing the TOML file, add them to `~/.config/openshell/gateway.env`: @@ -215,10 +217,10 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | -| `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | -| `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | -| `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | +| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | +| `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | +| `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | @@ -232,14 +234,15 @@ settings: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -compute_drivers = ["podman"] -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +compute_driver = "podman" [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell" stop_timeout_secs = 10 ``` @@ -247,7 +250,7 @@ stop_timeout_secs = 10 ### Image management The gateway pulls container images automatically on first sandbox -creation. The default pull policy is `missing`, which means images are +creation. The default pull policy is `if_not_present`, which means images are pulled once and then cached by Podman. To update cached images: @@ -260,9 +263,10 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest Or set `image_pull_policy = "always"` in `[openshell.drivers.podman]` to pull on every sandbox creation. -To pin specific image versions instead of `:latest`: +To pin specific image versions instead of `:latest`, set these values in +`[openshell.drivers.podman]`: -```shell +```toml supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" ``` diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 103ce3bf9d..58de9e7856 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -182,7 +182,7 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ### Images not updating -The default image pull policy is `missing` -- images are pulled once +The default image pull policy is `if_not_present` -- images are pulled once and cached. To update: ```shell @@ -227,9 +227,14 @@ non-functional until restarted, causing the gateway to fail with a connection error on `/run/user//podman/podman.sock`. The gateway retries briefly on startup, but a stale socket will not recover on its own. -Package upgrades do not overwrite `~/.config/openshell/gateway.toml` when you -create one. New gateway process options can be added manually by referencing -CONFIGURATION.md or running `openshell-gateway --help`. +Package upgrades preserve edited `~/.config/openshell/gateway.toml` files. On +the schema-v2 upgrade, the user service replaces only an exact copy of the v1 +file previously seeded by the RPM. If you edited that file, migrate it manually +before restarting the service; direct `dnf` or `rpm` upgrades do not use the +breaking-upgrade guard in `install.sh`. See the +[Gateway Configuration File](https://docs.nvidia.com/openshell/latest/reference/gateway-config#migrate-to-schema-version-2) +for the field-by-field migration steps. New gateway process options are listed +in CONFIGURATION.md and `openshell-gateway --help`. To pick up new container images after an upgrade: @@ -238,6 +243,17 @@ podman pull ghcr.io/nvidia/openshell/supervisor:latest podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ``` +### Migrating a TLS-enabled local driver to schema version 2 + +Docker, Podman, and VM sandboxes connect back to the gateway with a guest TLS +bundle. Package-managed installs use the complete bundle generated under +`~/.local/state/openshell/tls`, so the RPM default requires no additional TOML. +If you override the listener with custom `--tls-cert` and `--tls-key` inputs and +do not use that managed bundle, configure all three `guest_tls_ca`, +`guest_tls_cert`, and `guest_tls_key` paths under `[openshell.gateway]`. The +gateway now fails at startup instead of allowing sandboxes to fail later. Omit +all three fields when TLS is disabled. + ### Migrating from gateway.env Previous releases generated `~/.config/openshell/gateway.env` on first @@ -255,7 +271,7 @@ and map the relevant variables: | Environment variable | TOML equivalent | |---|---| | `OPENSHELL_BIND_ADDRESS=A` + `OPENSHELL_SERVER_PORT=P` | `bind_address = "A:P"` under `[openshell.gateway]` | -| `OPENSHELL_DRIVERS=podman` | `compute_drivers = ["podman"]` under `[openshell.gateway]` | +| `OPENSHELL_COMPUTE_DRIVER=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | | `OPENSHELL_DISABLE_TLS=true` | `disable_tls = true` under `[openshell.gateway]` | | `OPENSHELL_TLS_CERT=PATH` | `cert_path = "PATH"` under `[openshell.gateway.tls]` | | `OPENSHELL_TLS_KEY=PATH` | `key_path = "PATH"` under `[openshell.gateway.tls]` | diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index cd7e0d99c3..a0a6e296f0 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -15,7 +15,7 @@ # systemctl --user edit openshell-gateway [openshell] -version = 1 +version = 2 [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The @@ -25,4 +25,9 @@ version = 1 # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. -compute_drivers = ["podman"] +compute_driver = "podman" + +[openshell.drivers.podman] +# Keep the packaged local-gateway readiness behavior after health checks became +# opt-in in the Podman driver. Omit this setting only to disable health checks. +health_check_interval_secs = 10 diff --git a/deploy/rpm/gateway.toml.default.v1 b/deploy/rpm/gateway.toml.default.v1 new file mode 100644 index 0000000000..ba76f873b2 --- /dev/null +++ b/deploy/rpm/gateway.toml.default.v1 @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Default gateway configuration for RPM installs. +# +# This file is seeded to ~/.config/openshell/gateway.toml on first start +# of the openshell-gateway.service systemd user unit. Edit that copy to +# customize. This file is not read directly at runtime. +# +# Configuration precedence (highest to lowest): +# CLI flag > OPENSHELL_* env var > TOML file > built-in default +# +# To override settings without editing this file, set OPENSHELL_* variables +# in ~/.config/openshell/gateway.env or run: +# systemctl --user edit openshell-gateway + +[openshell] +version = 1 + +[openshell.gateway] +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. + +# Pin to the Podman compute driver. Without this, the gateway auto-detects +# in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver +# selection if Docker is also installed on the host. +compute_driver = "podman" diff --git a/deploy/rpm/migrate-gateway-config.sh b/deploy/rpm/migrate-gateway-config.sh new file mode 100755 index 0000000000..5c75af1f9f --- /dev/null +++ b/deploy/rpm/migrate-gateway-config.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +if [ "$#" -ne 3 ]; then + echo "usage: $0 DESTINATION CURRENT_DEFAULT LEGACY_DEFAULT" >&2 + exit 2 +fi + +destination=$1 +current_default=$2 +legacy_default=$3 + +for source in "$current_default" "$legacy_default"; do + if [ ! -f "$source" ]; then + echo "gateway config migration source is not a regular file: $source" >&2 + exit 1 + fi +done + +if [ -L "$destination" ] || { [ -e "$destination" ] && [ ! -f "$destination" ]; }; then + echo "refusing to replace non-regular gateway config: $destination" >&2 + exit 1 +fi + +if [ ! -e "$destination" ]; then + install -Dm 0644 "$current_default" "$destination" + exit 0 +fi + +# Replace only the exact config seeded by the schema-v1 RPM. Any edit, +# including whitespace or comments, makes the operator-owned file authoritative. +if ! cmp -s "$legacy_default" "$destination"; then + exit 0 +fi + +destination_dir=$(dirname "$destination") +temporary=$(mktemp "$destination_dir/.gateway.toml.XXXXXX") +trap 'rm -f "$temporary"' EXIT HUP INT TERM +install -m 0644 "$current_default" "$temporary" +mv -f "$temporary" "$destination" +trap - EXIT HUP INT TERM diff --git a/docs/about/container-gateway.mdx b/docs/about/container-gateway.mdx index 42fe2c7858..57be7d0142 100644 --- a/docs/about/container-gateway.mdx +++ b/docs/about/container-gateway.mdx @@ -59,7 +59,7 @@ docker run -d \ -v openshell-state:/var/openshell \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=http://host.openshell.internal:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ @@ -128,7 +128,7 @@ docker run -d \ -v "$HOME/.local/state/openshell:/home/openshell/.local/state/openshell" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=https://127.0.0.1:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/home/openshell/.local/state/openshell/openshell.db \ @@ -188,7 +188,7 @@ podman run -d \ -p 127.0.0.1:8080:8080 \ -v openshell-state:/var/openshell \ -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/podman.sock" \ - -e OPENSHELL_DRIVERS=podman \ + -e OPENSHELL_COMPUTE_DRIVER=podman \ -e OPENSHELL_PODMAN_SOCKET=/var/run/podman.sock \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ -e OPENSHELL_DISABLE_TLS=true \ diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index a733a1b881..7064fe7a56 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -30,7 +30,7 @@ Use `openshell status` to confirm the CLI can reach the gateway. ## Supported Compute Drivers -OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_drivers` in the gateway TOML when you need to pin a specific driver. +OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_driver` in the gateway TOML when you need to pin a specific driver. | Compute Driver | How It Is Configured | System Requirements | |---|---|---| @@ -44,7 +44,7 @@ For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sand On macOS, the install script uses Homebrew. The Homebrew package installs the `openshell` CLI, the gateway binary, and a Homebrew-managed gateway service. -The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew preserves user-edited prefix and user configs during upgrades; it removes the IPv6 bind only from an unchanged config generated by the affected formula. +The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew upgrades migrate exact package-generated schema-v1 prefix configs, including the affected IPv6 variant. They preserve edited prefix configs and all user configs. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) for an edited v1 file. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. @@ -63,7 +63,7 @@ On Debian and Ubuntu, the install script uses a Debian package. The Debian packa Linux packages require glibc 2.28 or newer. The installer checks libc before downloading packages and exits with an error on older glibc versions, Alpine, musl-based distributions, or unknown libc environments. -The Linux user service listens on `https://127.0.0.1:17670`, starts from built-in defaults, and generates a local mTLS bundle before the gateway starts. Create `~/.config/openshell/gateway.toml` only when you need to override those defaults. +The Linux user service listens on `https://127.0.0.1:17670` and generates a local mTLS bundle before the gateway starts. Debian uses built-in gateway defaults unless you create a config. RPM seeds `~/.config/openshell/gateway.toml` from its packaged Podman template on first start. RPM upgrades migrate only an unchanged package-generated schema-v1 file; they preserve edited files. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) when upgrading an edited v1 configuration. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index c04c0040d0..7a0a4e4156 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -22,26 +22,28 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ## Package-Managed Locations -Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. +Package-managed gateways use either built-in defaults or a package-seeded TOML file. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. -| Package | Optional Gateway TOML location | +| Package | Gateway TOML location | |---|---| | Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | -| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | +| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. Upgrades preserve user-edited configs and migrate only an unchanged prefix config generated with the affected IPv6-loopback default. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. + +Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. ## Layout -The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Shared compute-driver keys set at gateway scope are inherited into compute driver tables when not overridden. +The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Driver-specific values are never inherited from gateway scope. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] # ... gateway-wide settings ... @@ -59,6 +61,47 @@ version = 1 # ... credential-driver-specific settings ... ``` +The gateway selector is `compute_driver = ""`. It accepts one scalar driver name. Omit it to retain auto-detection. + +## Migrate to schema version 2 + +Schema version 2 is an intentional breaking cutover. The gateway rejects files +that omit `[openshell] version`, declare version 1, or declare an unsupported +future version. To migrate an existing file: + +1. Set `[openshell] version = 2`. +2. Replace `compute_drivers = [""]` with the scalar + `compute_driver = ""`. Replace `--drivers` and `OPENSHELL_DRIVERS` + with `--compute-driver` and `OPENSHELL_COMPUTE_DRIVER`. +3. Move every compute-driver option into `[openshell.drivers.]`. Schema + version 2 does not inherit driver defaults from `[openshell.gateway]`. + Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway + scope. A TLS-enabled Docker, Podman, or VM gateway requires one complete + guest bundle. Set all three paths unless the package-managed local TLS + bundle supplies them. When TLS is disabled, omit all three. Kubernetes + projects sandbox TLS through `client_tls_secret_name` instead. +4. Rename Docker `sandbox_namespace` to `sandbox_label`, Podman + `sandbox_ssh_socket_path` to `ssh_socket_path`, and VM + `openshell_endpoint` to `grpc_endpoint`. +5. Use canonical image pull policies: `always`, `if_not_present`, `never`, or + Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` + spelling are rejected. +6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, + omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit + of 2048, and omit Podman `health_check_interval_secs` to disable health + checks. Explicit zero values are invalid. +7. Remove local Docker, Podman, or VM `grpc_endpoint` when the topology-derived + callback is correct, or retain it as an explicit override. Kubernetes raw + TOML requires an explicit endpoint; Helm derives one from the release's + gateway Service. New VM root filesystems use an image-provided `sandbox` + account when present and otherwise use UID/GID 1000. Existing persisted VM + state retains its recorded or recoverable identity, including legacy 10001; + the driver does not assign 10001 to an overlay without supporting state. + +Unknown fields and non-table `[openshell.drivers.]` values fail startup. +This strict validation prevents misspelled or misplaced security-sensitive +settings from being silently ignored. + ## Full Example A complete gateway configuration covering every section. Trim to the fields you need. @@ -68,7 +111,7 @@ A complete gateway configuration covering every section. Trim to the fields you # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] name = "production-us-west" @@ -78,15 +121,14 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" -# When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. +# When omitted, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" # Optional external provider credential storage backend. Omit this key to use # the gateway's default encrypted database credential storage. credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 # Reject invalid policy generations securely by default. Set @@ -103,16 +145,11 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Shared driver defaults. These inherit into [openshell.drivers.] tables -# when the driver-specific table does not override them. -default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -# Defaults to the gateway version; override to pin a specific build. -# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -client_tls_secret_name = "openshell-client-tls" -service_account_name = "openshell-sandbox" -host_gateway_ip = "10.0.0.1" -enable_user_namespaces = false -sa_token_ttl_secs = 3600 +# Guest TLS paths remain gateway settings. TLS-enabled Docker, Podman, and VM +# gateways require a complete bundle unless package-managed local TLS supplies +# it automatically. Omit all three when TLS is disabled. Kubernetes projects +# sandbox TLS from client_tls_secret_name instead. Driver tables must not repeat +# these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -157,7 +194,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit or set to 0 only for local single-player Docker, Podman, or VM gateways. +# Omit only for local single-player Docker, Podman, or VM gateways. ttl_secs = 3600 [openshell.gateway.auth] @@ -200,6 +237,19 @@ failure_policy = "fail_closed" rpc = "openshell.v1.OpenShell/UpdateConfig" phases = ["validate"] +[openshell.drivers.kubernetes] +namespace = "openshell" +# Required in raw TOML; Helm derives this from the gateway Service. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +client_tls_secret_name = "openshell-client-tls" +service_account_name = "openshell-sandbox" +host_gateway_ip = "10.0.0.1" +enable_user_namespaces = false +sa_token_ttl_secs = 3600 + [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" allow_reference_namespace = false @@ -211,7 +261,7 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. +`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token `exp` claim and `expires_at_ms` response field become `0`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -355,7 +405,7 @@ The gateway validates snapshot structure and provider-profile semantics. It trea `failure_policy` accepts `fail_closed` or `fail_open`. `timeout` accepts `ms` and `s` suffixes. In `dynamic` mode, binding overrides may select a manifest binding by `id`, `rpc`, or `service` plus `method`; they can disable a binding, narrow its phases, or override its failure policy. -`image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. +`image_pull_policy` is a shared driver setting with the canonical values `always`, `if_not_present`, `never`, and `newer`. Set it inside the relevant driver table. Drivers translate these values to their runtime APIs; `newer` is supported only by Podman and is rejected at Docker and Kubernetes startup. ## Credential Drivers @@ -444,7 +494,9 @@ args = [ ## Driver References -Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. + +Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. ### Kubernetes @@ -452,14 +504,14 @@ The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS ma ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" @@ -482,11 +534,11 @@ workspace_mode = "shared" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. @@ -521,7 +573,10 @@ topology = "combined" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true -grpc_endpoint = "https://openshell-gateway.agents.svc:8080" +# Required in raw gateway TOML because `namespace` identifies sandbox +# placement, not the gateway Service. Helm renders this from the release's +# gateway Service name and namespace. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" @@ -591,33 +646,36 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required). Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -# Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. -image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" -# Empty auto-detects https://host.openshell.internal: when guest TLS is set. +# Canonical values: always | if_not_present | never. `newer` is Podman-only. +image_pull_policy = "if_not_present" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "docker-dev" +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for this topology. grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" ssh_socket_path = "/run/openshell/ssh.sock" @@ -625,22 +683,42 @@ ssh_socket_path = "/run/openshell/ssh.sock" # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Docker's runtime default unchanged. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 +# Explicit supervisor-compatible default. RuntimeDefault requires Docker to +# report AppArmor support; Localhost/ requires an operator-loaded profile. +app_armor_profile = "Unconfined" +# Corporate TLS egress proxy. These are supervisor argv settings, not workload +# environment variables. Do not embed credentials in the URL. +https_proxy = "https://proxy.corp.example:8443" +no_proxy = ".svc.cluster.local,10.0.0.0/8" +# Optional root-owned host file containing user:pass. An http:// proxy also +# requires proxy_auth_allow_insecure = true as an explicit acknowledgement. +proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# Project a host Unix Workload API socket into the supervisor for provider +# token exchange. The socket parent must be a dedicated absolute directory. +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" ``` +Use `sandbox_label` for Docker configurations. The legacy +`sandbox_namespace` key is rejected. + ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -649,7 +727,8 @@ compute_drivers = ["podman"] # one. Set this to pin a specific Podman machine instead. socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "missing" # always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer +# Optional override. When omitted, the gateway derives this endpoint. grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 @@ -657,22 +736,19 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" -sandbox_ssh_socket_path = "/run/openshell/ssh.sock" +ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Podman's runtime default unchanged. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Health check interval in seconds. Lower values detect readiness faster -# but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to disable health checks entirely. Default: 10. +# Health check interval in seconds. Omit to disable health checks; explicit 0 +# is invalid. Lower values detect readiness faster but increase process churn +# (each check spawns a conmon subprocess). health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -761,41 +837,74 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# Project a host Workload API Unix socket into the supervisor, or use an +# explicit container-reachable TCP endpoint, for provider token exchange. +# provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +# provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" +# Omit app_armor_profile to preserve Podman's runtime-selected profile. +# Set Unconfined only when the supervisor's mount setup requires it. +# Explicit RuntimeDefault and Localhost/ require Podman to report +# AppArmor support. +# app_armor_profile = "Unconfined" ``` +Use `ssh_socket_path` for Podman configurations. The legacy +`sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, +OpenShell sends no override and Podman applies its runtime-selected profile. +Set `Unconfined` explicitly only when the deployment requires the supervisor's +mount setup to bypass that profile. + ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. -compute_drivers = ["vm"] +compute_driver = "vm" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -grpc_endpoint = "https://host.containers.internal:17670" +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for the VM topology. +grpc_endpoint = "https://host.openshell.internal:17670" # Empty falls back to default_image. bootstrap_image = "ghcr.io/nvidia/openshell/sandbox:latest" krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" -# Resolved sandbox UID/GID for the rootfs /etc/passwd entry. -# Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. -# Any non-root Linux UID/GID is valid. +# Resolved sandbox UID/GID for new rootfs /etc/passwd entries. +# Defaults to the image's sandbox account, or 1000 when the account is absent; +# matching GID is used if sandbox_gid is empty. Persisted overlays recover their +# recorded identity, including 10001, rather than receiving a legacy fallback. +# Values must fall within OpenShell's allowed non-root sandbox identity range. # sandbox_uid = 20001 +# sandbox_gid = 20001 +# Corporate TLS egress proxy. These settings are converted to protected +# supervisor argv inside the guest; workload environment cannot override them. +# https_proxy = "https://proxy.corp.example:8443" +# no_proxy = ".svc.cluster.local,10.0.0.0/8" +# proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# An http:// proxy with proxy_auth_file requires this explicit acknowledgement: +# proxy_auth_allow_insecure = true +# VM guests cannot mount a host Workload API Unix socket. Configure only a +# separately operated guest-reachable TCP listener and explicitly acknowledge +# the exposure; host-only sockets are never exposed automatically. +# provider_spiffe_workload_api_tcp_endpoint = "tcp:192.0.2.10:8081" +# provider_spiffe_allow_guest_tcp = true ``` ### Extension Driver @@ -808,12 +917,12 @@ key used for driver-owned sandbox config such as `template.driver_config.` ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index f00c49afd6..da3913d261 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -42,11 +42,11 @@ with the exact exit code. Driver and supervisor failures remain `Error`. ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set the singular `compute_driver` key in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["docker"] +compute_driver = "docker" ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. @@ -54,15 +54,17 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. + +`compute_driver` accepts exactly one scalar driver name. The legacy `compute_drivers` list is rejected by schema version 2. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. A TLS-enabled gateway-managed Docker, Podman, or VM driver requires a complete `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` bundle in `[openshell.gateway]`; package-managed local TLS supplies it automatically. Driver tables reject those gateway-owned fields. Kubernetes projects guest TLS through a Secret instead. See the [Gateway Configuration File](./gateway-config) reference for the full schema and migration steps. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -70,7 +72,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -81,8 +83,8 @@ socket path. The endpoint replaces normal driver construction for that name, including canonical built-in names: ```shell -openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock -openshell-gateway --drivers docker --compute-driver-socket /run/openshell/docker.sock +openshell-gateway --compute-driver kyma --compute-driver-socket /run/openshell/kyma.sock +openshell-gateway --compute-driver docker --compute-driver-socket /run/openshell/docker.sock ``` The gateway connects to the operator-provided endpoint; it does not provision @@ -161,7 +163,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -239,7 +241,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -253,6 +255,12 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. +Podman preserves its runtime-selected AppArmor profile when +`app_armor_profile` is omitted. Set `Unconfined` explicitly only when the +supervisor's mount setup requires it. Explicit `RuntimeDefault` and +`Localhost/` selections fail startup when Podman reports that AppArmor +is unavailable. + On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts @@ -333,16 +341,16 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: +Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" ``` -For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, and `krun_log_level` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. @@ -375,23 +383,24 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`; schema version 2 rejects their historical `[openshell.gateway]` locations. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | +| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | -| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | +| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | +| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. Raw TOML and the standalone Kubernetes driver require an explicit endpoint because the sandbox namespace does not identify the gateway Service. Helm derives it from the release's gateway Service when the value is empty. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | +| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | @@ -582,7 +591,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/etc/group`, and `/etc/gshadow` during rootfs preparation. Default UID is `10001`; configure `sandbox_uid` in `[openshell.drivers.vm]` to use a different value. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created. An unmarked overlay recovers identity from concrete overlay or prepared-image state, an explicit override, or the current image; the driver never assigns legacy `10001:10001` without persisted evidence. ### Custom Images diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 1c541f8f8e..53a68364e5 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -71,7 +71,7 @@ This provides defense-in-depth: even if a container escape vulnerability exists, | Aspect | Detail | |---|---| -| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in the gateway config to enable cluster-wide. | +| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in `[openshell.drivers.kubernetes]` to enable cluster-wide. | | What you can change | Enable cluster-wide through Helm or gateway config. Override per-sandbox through the `user_namespaces` field on `SandboxTemplate` in the API. | | Prerequisites | Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), a container runtime that supports user namespaces (containerd 2.0+, CRI-O 1.25+), and Linux 5.12+ for ID-mapped mounts. | | Risk if enabled with GPU | NVIDIA device plugin compatibility with user namespaces is unverified. OpenShell logs a warning when both GPU and user namespaces are active on the same sandbox. | diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 59baed1d7d..63e587ef62 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.gateway.auth] @@ -18,10 +18,10 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" -sandbox_namespace = "openshell-e2e" +image_pull_policy = "if_not_present" +sandbox_label = "openshell-e2e" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index c1549cd933..6eff3b5615 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] @@ -18,11 +18,13 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.podman] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" +ssh_socket_path = "/run/openshell/ssh.sock" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 5d650bae89..4bda320440 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -11,16 +11,16 @@ ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes -ENV OPENSHELL_DRIVERS=kubernetes \ +ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/var/run/openshell-compute/driver/driver.sock \ OPENSHELL_GATEWAY_ID=openshell \ OPENSHELL_SANDBOX_NAMESPACE=openshell \ OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT=openshell-sandbox \ OPENSHELL_SANDBOX_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ - OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ OPENSHELL_K8S_TOPOLOGY=combined diff --git a/e2e/run.sh b/e2e/run.sh index 875186394c..8d9cee8dac 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -142,7 +142,11 @@ if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then fi gateway_driver="$(python3 -c ' import sys, tomllib -print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) +gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] +driver = gateway.get("compute_driver") +if not driver: + raise SystemExit("gateway config must explicitly select a compute driver") +print(driver) ' "${gateway_config}")" if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 9acc633d65..1cfe0d27a7 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -256,11 +256,14 @@ e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" cat >"${GATEWAY_CONFIG}" <&2 exit 2 @@ -501,8 +501,11 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" { - printf '[openshell]\nversion = 1\n\n' - printf '[openshell.gateway]\nlog_level = "info"\n\n' + printf '[openshell]\nversion = 2\n\n' + printf '[openshell.gateway]\nlog_level = "info"\n' + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-docker-${HOST_PORT}" if [ "${OIDC_MODE}" != "1" ]; then e2e_write_gateway_mtls_auth_config @@ -514,14 +517,11 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then @@ -532,7 +532,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then { - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" @@ -560,7 +560,7 @@ GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" - --drivers docker + --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fc3419e182..efc829cb62 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -452,22 +452,29 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # Start from the RPM default template so this e2e test exercises the same TOML # config path that RPM users get on first start. The template leaves -# bind_address unset and sets compute_drivers = ["podman"]. On Podman Machine, +# bind_address unset and sets compute_driver = "podman". On Podman Machine, # the driver reserves IPv4 loopback for its callback-only listener, so the # primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default. # # We append the driver-specific table and override the port via CLI flag # (CLI > TOML in the merge precedence) so the test can use an ephemeral port. cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" -{ - e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" - if [ "${OIDC_MODE}" != "1" ]; then - e2e_write_gateway_mtls_auth_config - if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then - e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" - fi +# The TLS listener credentials are supplied by CLI below. Schema v2 keeps the +# supervisor client bundle gateway-owned, so add it to [openshell.gateway] +# before the RPM template opens the Podman driver table. +GATEWAY_CONFIG_WITH_TLS="${GATEWAY_CONFIG}.tls" +while IFS= read -r line; do + if [ "${line}" = "[openshell.drivers.podman]" ]; then + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" fi - printf '\n[openshell.drivers.podman]\n' + printf '%s\n' "${line}" +done <"${GATEWAY_CONFIG}" >"${GATEWAY_CONFIG_WITH_TLS}" +mv "${GATEWAY_CONFIG_WITH_TLS}" "${GATEWAY_CONFIG}" +{ + # The RPM template ends in [openshell.drivers.podman]. Append driver-owned + # overrides before opening any nested gateway tables below. if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else @@ -475,14 +482,12 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" printf 'gateway_port = %s\n' "${HOST_PORT}" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = "missing"\n' + printf 'image_pull_policy = "if_not_present"\n' + # The RPM template already opts into the 10-second Podman health check. # Keep CI teardown bounded while the production Podman driver default stays # conservative for real user workloads. printf 'stop_timeout_secs = %s\n' "${PODMAN_STOP_TIMEOUT_SECS}" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' if [ -n "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" ]; then printf 'provider_spiffe_workload_api_socket = %s\n' "$(toml_string "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" @@ -496,13 +501,22 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" fi fi + + e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" + if [ "${OIDC_MODE}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then + e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" + fi + fi } >> "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="missing" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="if_not_present" \ + OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ @@ -519,7 +533,7 @@ fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # compute_drivers comes from the RPM template. Override the loopback address + # compute_driver comes from the RPM template. Override the loopback address # and port so Podman Machine can keep its IPv4 callback listener distinct. --bind-address "${PRIMARY_BIND_IP}" --port "${HOST_PORT}" @@ -549,6 +563,7 @@ e2e_export_gateway_restart_metadata \ "${GATEWAY_LOG}" \ "${GATEWAY_PID_FILE}" +OPENSHELL_LOCAL_TLS_DIR="${PKI_DIR}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}" >"${GATEWAY_LOG}" 2>&1 & diff --git a/examples/aws-s3-sts.md b/examples/aws-s3-sts.md index ed0c204d6b..94b73c73cc 100644 --- a/examples/aws-s3-sts.md +++ b/examples/aws-s3-sts.md @@ -87,13 +87,11 @@ if your gateway cache directory differs): ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -compute_drivers = ["podman"] -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +compute_driver = "podman" disable_tls = true -supervisor_image = "localhost/openshell/supervisor:dev" [openshell.gateway.auth] allow_unauthenticated_users = true @@ -106,7 +104,10 @@ gateway_id = "podman-dev" ttl_secs = 3600 [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +supervisor_image = "localhost/openshell/supervisor:dev" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 ``` If the JWT key files do not exist yet, run `mise run gateway` once to generate @@ -118,7 +119,7 @@ Start the gateway: eval "$(aws configure export-credentials --format env)" ./target/debug/openshell-gateway \ --config .cache/gateway-podman/gateway.toml \ - --port 18080 --log-level info --drivers podman --disable-tls \ + --port 18080 --log-level info --compute-driver podman --disable-tls \ --db-url "sqlite:.cache/gateway-podman/gateway.db?mode=rwc" ``` diff --git a/examples/governance-interceptor/smoke.sh b/examples/governance-interceptor/smoke.sh index e25ee0d3cb..8e777f921f 100755 --- a/examples/governance-interceptor/smoke.sh +++ b/examples/governance-interceptor/smoke.sh @@ -325,7 +325,7 @@ generate_gateway_jwt_bundle() { write_gateway_config() { cat >"$GATEWAY_CONFIG" <"$config_path" <"$config_path" <"$GATEWAY_CONFIG" <.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_empty_config is not None + assert "version = 1" in legacy_empty_config.group("contents") + assert "bind_address =" not in legacy_empty_config.group("contents") + + legacy_ipv6_config = re.search( + r"legacy_ipv6_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_ipv6_config is not None + assert "version = 1" in legacy_ipv6_config.group("contents") + assert 'bind_address = "[::1]:17670"' in legacy_ipv6_config.group("contents") + assert "gateway_config.read == legacy_empty_gateway_config_contents ||" in formula assert "gateway_config.read == legacy_ipv6_gateway_config_contents" in formula assert "gateway_config.write gateway_config_contents" in formula - assert '# compute_drivers = ["vm"]' not in formula + assert '# compute_driver = "vm"' not in formula assert ( "openshell gateway add https://localhost:17670 --local --name openshell" in formula @@ -130,12 +149,15 @@ def test_snap_wrapper_uses_optional_gateway_config_without_generating_toml() -> assert 'exec "${SNAP}/bin/openshell-gateway" "$@"' in wrapper -def test_rpm_spec_uses_gateway_defaults_without_config_helper() -> None: +def test_rpm_spec_seeds_and_migrates_gateway_defaults() -> None: repo_root = Path(__file__).resolve().parents[2] spec = (repo_root / "openshell.spec").read_text(encoding="utf-8") assert "init-gateway-config.sh" not in spec assert "init-pki.sh" not in spec + assert "migrate-gateway-config.sh" in spec + assert "gateway.toml.default.v1" in spec + assert "%{name}-gateway-migrate-config" in spec assert "Environment=OPENSHELL_LOCAL_TLS_DIR=%%h/.local/state/openshell/tls" in spec assert ( "openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR}" @@ -143,7 +165,7 @@ def test_rpm_spec_uses_gateway_defaults_without_config_helper() -> None: ) assert "EnvironmentFile=-%%E/openshell/gateway.env" in spec assert "%%S/openshell/tls" not in spec - assert "Environment=OPENSHELL_DRIVERS" not in spec + assert "Environment=OPENSHELL_COMPUTE_DRIVER" not in spec assert "Environment=OPENSHELL_BIND_ADDRESS" not in spec assert "Environment=OPENSHELL_PODMAN_TLS_CA" not in spec assert "ExecStart=/usr/bin/openshell-gateway" in spec diff --git a/python/openshell/rpm_gateway_config_migration_test.py b/python/openshell/rpm_gateway_config_migration_test.py new file mode 100644 index 0000000000..af846678f3 --- /dev/null +++ b/python/openshell/rpm_gateway_config_migration_test.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +MIGRATOR = REPO_ROOT / "deploy/rpm/migrate-gateway-config.sh" +CURRENT = REPO_ROOT / "deploy/rpm/gateway.toml.default" +LEGACY = REPO_ROOT / "deploy/rpm/gateway.toml.default.v1" + + +def run_migrator( + destination: Path, *, check: bool = True +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["sh", str(MIGRATOR), str(destination), str(CURRENT), str(LEGACY)], + check=check, + text=True, + capture_output=True, + ) + + +def test_migrator_seeds_missing_config_and_is_idempotent(tmp_path: Path) -> None: + destination = tmp_path / "config/openshell/gateway.toml" + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_replaces_only_exact_legacy_default(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + destination.write_bytes(LEGACY.read_bytes()) + + run_migrator(destination) + + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_preserves_edited_legacy_and_current_configs(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + edited = LEGACY.read_text(encoding="utf-8") + "# operator edit\n" + destination.write_text(edited, encoding="utf-8") + + run_migrator(destination) + assert destination.read_text(encoding="utf-8") == edited + + destination.write_bytes(CURRENT.read_bytes()) + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_refuses_symlink_destination(tmp_path: Path) -> None: + target = tmp_path / "target.toml" + target.write_text("operator-owned\n", encoding="utf-8") + destination = tmp_path / "gateway.toml" + destination.symlink_to(target) + + result = run_migrator(destination, check=False) + + assert result.returncode != 0 + assert "non-regular gateway config" in result.stderr + assert target.read_text(encoding="utf-8") == "operator-owned\n" diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 2b7c095065..77632cb0a4 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -8,16 +8,16 @@ state: implemented ## Summary -Introduce a TOML-based configuration file for the OpenShell gateway that unifies all gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file, while preserving full backwards compatibility with the existing CLI flags and `OPENSHELL_*` environment variables. +Introduce a TOML-based configuration file for the OpenShell gateway that unifies gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file. CLI flags and supported `OPENSHELL_*` environment variables retain higher precedence. Schema version 2 intentionally rejects legacy file fields and locations. ## Motivation -The gateway today is configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This works for simple single-node deployments but breaks down as deployments grow: +Before this RFC, the gateway was configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This worked for simple single-node deployments but broke down as deployments grew: -- **Too many flags** — the gateway has ~40 configurable parameters today (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests are hard to read, diff, and audit. -- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers all live in the same flat CLI namespace, with no structural separation. Most flags only apply to one driver, but there is no way to express that in CLI form. -- **Helm friction** — The chart's `statefulset.yaml` already carries a long `env:` block of `OPENSHELL_*` variables that each map to a `values.yaml` key. A config file can be mounted as a single `ConfigMap` and reduces the chart's templating surface significantly. -- **Secrets management** — Injecting secrets (TLS material paths, database URL, OIDC settings) via environment variables is functional but not idiomatic for Kubernetes. A file-based format opens the door to projected secrets and volume mounts that compose cleanly with the non-secret config. +- **Too many flags** — the gateway exposed roughly 40 configurable parameters (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests were hard to read, diff, and audit. +- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers shared one flat CLI namespace with no structural separation. Most flags applied to only one driver, but CLI syntax did not express that ownership. +- **Helm friction** — The chart's `statefulset.yaml` carried a long `env:` block of `OPENSHELL_*` variables that each mapped to a `values.yaml` key. A mounted configuration file reduces the chart's templating surface. +- **Secrets management** — Environment-only configuration did not compose naturally with Kubernetes `ConfigMap` and projected `Secret` volumes. ## Non-goals @@ -48,7 +48,7 @@ The file path is provided via: OPENSHELL_GATEWAY_CONFIG=/path/to/gateway.toml ``` -The file must have a `.toml` extension. A missing path is a hard error; an empty existing file is treated as "no configuration" — the gateway falls back to defaults and to whatever the CLI/env supply. +The file must have a `.toml` extension. A missing path is a hard error. A configured file must declare the exact supported schema version; an empty existing file is rejected. ### TOML schema @@ -58,7 +58,7 @@ The file is rooted at an `[openshell]` table. This namespacing reserves room for ```toml [openshell] -version = 1 # optional; reserved for future schema migrations +version = 2 # required schema version # ────────────────────────────────────────────────────────────────────────────── # Gateway-wide settings @@ -72,10 +72,9 @@ metrics_bind_address = "0.0.0.0:9090" # optional; omit to disable # Logging log_level = "info" -# Compute drivers — list of driver names whose [openshell.drivers.] -# tables should be activated. When empty, the gateway auto-detects a driver -# (kubernetes → podman → docker). VM is never auto-detected. -compute_drivers = ["kubernetes"] +# Compute driver — exactly one driver may be active. When omitted, the gateway +# auto-detects a driver (kubernetes → podman → docker). VM is never auto-detected. +compute_driver = "kubernetes" # Note: database_url is a secret and must be supplied via OPENSHELL_DB_URL # (or --db-url) — it is NOT permitted in the file. @@ -88,12 +87,17 @@ server_sans = ["openshell", "*.dev.openshell.localhost"] enable_loopback_service_http = true # ────────────────────────────────────────────────────────────────────────────── -# TLS / mTLS — when omitted, the gateway listens plaintext (sets --disable-tls) +# TLS / mTLS — package-managed local TLS may supply listener defaults. # ────────────────────────────────────────────────────────────────────────────── -# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. When true, the gateway -# ignores the [openshell.gateway.tls] table below. +# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. Set true explicitly for a +# plaintext listener; guest TLS fields must then be omitted. disable_tls = false +# Gateway-owned TLS bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" + [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" @@ -113,15 +117,15 @@ scopes_claim = "" # empty disables scope enforcement # ────────────────────────────────────────────────────────────────────────────── # Compute drivers — each table is owned and parsed by its driver crate. -# Only tables for drivers listed in compute_drivers are activated. +# Only the selected or auto-detected driver's table is activated. # ────────────────────────────────────────────────────────────────────────────── [openshell.drivers.kubernetes] namespace = "openshell" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" grpc_endpoint = "https://host.openshell.internal:8080" client_tls_secret_name = "openshell-sandbox-tls" host_gateway_ip = "10.0.0.1" @@ -129,26 +133,20 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +image_pull_policy = "if_not_present" +sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] socket_path = "/run/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -image_pull_policy = "missing" # Podman vocabulary: always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" network_name = "openshell" stop_timeout_secs = 10 -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -157,9 +155,6 @@ grpc_endpoint = "https://host.containers.internal:8080" vcpus = 2 mem_mib = 2048 krun_log_level = 1 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` ### Driver configuration @@ -167,12 +162,12 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" Each `[openshell.drivers.]` table is extracted from the parsed file and handed to the driver's initialization function as a raw TOML value. The driver is then responsible for: 1. **Parsing** — deserializing the table into its own typed config struct (e.g. `KubernetesComputeConfig`, `DockerComputeConfig`, `PodmanComputeConfig`, `VmComputeConfig`). -2. **Validation** — applying cross-field checks specific to that driver (e.g. requiring TLS triplets when sandbox-side mTLS is enabled). +2. **Validation** — applying cross-field checks specific to that driver. Gateway-owned guest TLS paths are validated as one bundle and injected only into the selected local driver before this step. 3. **Consumption** — using the resulting struct to initialize internal state. Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. -`[openshell.drivers.]` tables for drivers not listed in `compute_drivers` (and not the auto-detected driver) are parsed for syntax but not activated. +`[openshell.drivers.]` tables for drivers other than the selected or auto-detected driver are parsed for syntax but not activated. ### Merge semantics @@ -207,25 +202,26 @@ Deserialization uses `#[serde(deny_unknown_fields)]` at every table level. An un The following cross-field validations are applied after merging file + env + CLI: - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. -- When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. +- Gateway listener TLS requires `cert_path` and `key_path`; `client_ca_path` is required only for listener client-certificate verification. TLS-enabled Docker, Podman, and VM drivers also require a complete gateway-owned guest CA, certificate, and key bundle. Kubernetes projects guest TLS through a Secret instead. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_drivers` may be empty; in that case the gateway falls back to auto-detection. If the list contains a driver name with no matching `[openshell.drivers.]` table, the driver runs with its built-in defaults. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver requires a named table with `socket_path`, unless startup supplies an explicit socket override. The legacy `compute_drivers` list is rejected. -### Backwards compatibility +### Schema compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). +Schema version 2 requires `version = 2`, a singular `compute_driver` when a driver is selected, and driver-owned fields under `[openshell.drivers.]`. Legacy schema versions and `compute_drivers` lists are rejected. `OPENSHELL_DB_URL` remains a required process input and is not accepted from the file. ### Example: minimal Kubernetes deployment ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -bind_address = "0.0.0.0:8080" -compute_drivers = ["kubernetes"] +bind_address = "0.0.0.0:8080" +compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). -# No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). +# The gateway runs plaintext behind Envoy / ingress. +disable_tls = true [openshell.drivers.kubernetes] namespace = "agents" @@ -236,12 +232,7 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ### Helm integration -The Helm chart today renders a long `env:` block in `templates/statefulset.yaml`, with each `OPENSHELL_*` variable mapped to a `values.yaml` key. This RFC's adoption replaces that block with: - -1. A new `gateway.config` value tree (TOML-shaped YAML) in `values.yaml`. -2. A new `ConfigMap` template that renders the values into a TOML document via Helm's `tpl`. -3. A volume mount of the `ConfigMap` at `/etc/openshell/gateway.toml` and a `--config` flag in the gateway container's `args`. -4. Continued use of a `Secret`-backed `env:` entry for `OPENSHELL_DB_URL` (which never lives in the `ConfigMap`), plus optional projections for TLS material paths. The CLI/env precedence above means any `Secret`-backed env var also wins over a value in the `ConfigMap`. +The Helm chart renders schema-v2 gateway TOML into a `ConfigMap`, mounts it at `/etc/openshell/gateway.toml`, and starts the gateway with that file. Secret process inputs such as `OPENSHELL_DB_URL` remain `Secret`-backed environment entries and retain higher precedence. Kubernetes projects sandbox guest TLS through its configured Secret rather than placing host guest-certificate paths in the gateway TOML. ```yaml # values.yaml excerpt @@ -250,7 +241,7 @@ gateway: bind_address: "0.0.0.0:8080" health_bind_address: "0.0.0.0:8081" metrics_bind_address: "0.0.0.0:9090" - compute_drivers: ["kubernetes"] + compute_driver: "kubernetes" drivers: kubernetes: namespace: agents @@ -260,23 +251,15 @@ gateway: The chart owners can migrate one section at a time: `OPENSHELL_*` env vars and the `ConfigMap` coexist during the transition, with env continuing to override the file. -## Implementation plan - -No part of this RFC has shipped yet. The work breaks down as: +## Implementation -1. **Add a config-file loader to `openshell-server`** — define a `GatewayConfigFile` struct that mirrors the schema above, parse it with `serde` + `toml`, and merge it into `openshell_core::Config` plus the per-driver structs in `compute/`. -2. **Wire the merge into `cli.rs`** — add `--config` / `OPENSHELL_GATEWAY_CONFIG`, gate each existing flag's "apply from file" path on clap `ValueSource::DefaultValue`, and run cross-field validation after the merge. -3. **Per-driver deserialization** — give each driver crate (`openshell-driver-{kubernetes,docker,podman,vm}`) a `from_toml` (or `serde::Deserialize`) entry point so the gateway can hand each driver its own table. -4. **Test coverage** — file parsing, env-overrides-file, CLI-overrides-env, partial TLS error, port-collision error, unknown-field rejection, missing driver table fallback. -5. **Helm chart migration** — add `gateway.config` value tree, render the `ConfigMap`, mount it, switch the gateway container to `--config`. Keep the `OPENSHELL_*` env names available as opt-in overrides for secrets. -6. **Example file** — ship the per-driver examples on the published docs reference at `docs/reference/gateway-config.mdx`. -7. **Architecture doc update** — reflect the new config sources and precedence in `architecture/gateway.md`. +The implemented gateway loader parses TOML with `serde`, merges file values below environment and CLI sources, and rejects unknown fields. Each compute driver deserializes only its named table. Helm renders schema-v2 TOML into a ConfigMap, while secret process inputs remain environment-backed. Package templates, examples, tests, and the gateway architecture documentation use the same canonical schema. ## Risks -- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Mitigate by treating field renames as breaking, keeping the `version` field reserved for schema migrations, and surfacing rename errors clearly. +- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Treat field renames as versioned schema changes and surface migration errors clearly. - **Secrets in the file** — `database_url` is excluded from the schema entirely (env / CLI only). OIDC settings remain allowed in the file because none of them are credentials in isolation. Operators should still prefer env-var injection for any field that would live in a `Secret` rather than a `ConfigMap` (TLS material paths, restricted-environment OIDC issuers, etc.). Documentation must call this out prominently. -- **Partial TLS configuration** — the hard error on partial TLS config is the right UX, but the error message must clearly identify which source (file vs. CLI/env) is missing which field, since the file's `[openshell.gateway.tls]` table is all-or-nothing while the CLI flags are independent. +- **Partial TLS configuration** — listener and guest TLS are separate complete-bundle contracts. Startup rejects partial bundles and identifies the missing configuration before constructing a driver. - **Driver schema drift** — once each driver owns its own TOML table, driver releases can change field names independently of the gateway. The gateway's `version` field does not protect against driver-side breakage; document driver-config stability separately. ## Alternatives @@ -295,8 +278,7 @@ No part of this RFC has shipped yet. The work breaks down as: ## Open questions -1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. -2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. +1. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (e.g. `compute_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. -3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. + Deferred to a follow-on: the single `--config` file is sufficient for the current schema, and the directory loader can be added without changing the file schema. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. +2. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). Schema version 2 allows the listed OIDC fields because they are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. diff --git a/rfc/0011-multi-player-design/README.md b/rfc/0011-multi-player-design/README.md index 32a2f584e1..5fcb67fe51 100644 --- a/rfc/0011-multi-player-design/README.md +++ b/rfc/0011-multi-player-design/README.md @@ -755,13 +755,14 @@ use the workspace to select the target Kubernetes namespace instead of encoding it in the resource name. The label-based lookup and annotation patterns established here carry over unchanged. -**Docker and Podman drivers.** The Docker driver's `sandbox_namespace` label -provides a foundation for workspace mapping, but the driver currently uses a -single configured namespace rather than per-sandbox values. The driver contract -must be updated so that workspace flows through `DriverSandbox` and the driver -applies it as the container label filter. The same applies to Podman and other -local drivers — workspace isolation is enforced at the gateway level and does -not require Kubernetes. +**Docker and Podman drivers.** The Docker driver's `sandbox_label` +configuration value is stored in the `openshell.sandbox_namespace` container +label and provides a foundation for workspace mapping, but the driver currently +uses a single configured value rather than per-sandbox values. The driver +contract must be updated so that workspace flows through `DriverSandbox` and +the driver applies it as the container label filter. The same applies to Podman +and other local drivers — workspace isolation is enforced at the gateway level +and does not require Kubernetes. ### Compute Driver Trust Model diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 6826829fd8..188c698962 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -24,12 +24,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" @@ -52,6 +54,18 @@ linux_target_triple() { esac } +# Escape a value for a TOML basic string before copying an operator-provided +# proxy path or URL into the generated local configuration. +toml_escape() { + local s=$1 + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/\\n} + s=${s//$'\r'/\\r} + s=${s//$'\t'/\\t} + printf '%s' "${s}" +} + port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then @@ -211,11 +225,11 @@ mkdir -p "${STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" < only on a Docker host with AppArmor enabled. +app_armor_profile = "Unconfined" EOF +# Keep the local task's proxy inputs aligned with [openshell.drivers.docker]. +# Credentials stay in the referenced root-owned file; do not echo their value. +if [[ -n "${OPENSHELL_SANDBOX_HTTPS_PROXY+x}" ]]; then + printf 'https_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_HTTPS_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then + printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -256,6 +294,6 @@ exec "${GATEWAY_BIN}" \ --config "${CONFIG_PATH}" \ --port "${PORT}" \ --log-level "${LOG_LEVEL}" \ - --drivers docker \ + --compute-driver docker \ --disable-tls \ --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index ab166865ef..4d990d629f 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -21,12 +21,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -90,19 +92,6 @@ ensure_podman_supervisor_image() { fi } -podman_pull_policy() { - case "$1" in - Always|always) echo "always" ;; - IfNotPresent|ifnotpresent|missing|"") echo "missing" ;; - Never|never) echo "never" ;; - Newer|newer) echo "newer" ;; - *) - echo "ERROR: unsupported Podman image pull policy '$1'" >&2 - exit 2 - ;; - esac -} - # Escape a value for embedding in a double-quoted TOML basic string, so # quotes, backslashes, or control characters in an environment value cannot # corrupt gateway.toml or inject extra configuration keys. @@ -215,12 +204,11 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" <&2 + return 2 + ;; + esac +} diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 3818dca364..6853ad7e5d 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -39,7 +39,7 @@ STATE_DIR="${OPENSHELL_VM_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-vm}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-vm-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" VM_BOOTSTRAP_IMAGE="${OPENSHELL_VM_BOOTSTRAP_IMAGE:-}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" DRIVER_DIR_DEFAULT="${ROOT}/target/debug" @@ -70,6 +70,17 @@ normalize_bool() { esac } +# Escape values that are copied from the local environment to gateway TOML. +toml_escape() { + local s=$1 + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/\\n} + s=${s//$'\r'/\\r} + s=${s//$'\t'/\\t} + printf '%s' "${s}" +} + port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then @@ -336,11 +347,11 @@ chmod 700 "${VM_DRIVER_STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT+x}" ]]; then + printf 'provider_spiffe_workload_api_tcp_endpoint = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP+x}" ]]; then + printf 'provider_spiffe_allow_guest_tcp = %s\n' "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP}" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -386,7 +422,7 @@ GATEWAY_ARGS=( --config "${CONFIG_PATH}" --port "${PORT}" --log-level "${LOG_LEVEL}" - --drivers vm + --compute-driver vm --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" ) diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 019d1b1b63..300b6a8c7e 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -10,12 +10,14 @@ # # VM/MicroVM is intentionally explicit-only because it requires runtime setup. # Use either: -# OPENSHELL_DRIVERS=vm mise run gateway +# OPENSHELL_COMPUTE_DRIVER=vm mise run gateway # mise run gateway:vm set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" usage() { @@ -33,7 +35,7 @@ Options: -h, --help Show this help. Environment: - OPENSHELL_DRIVERS Driver override used by openshell-gateway. + OPENSHELL_COMPUTE_DRIVER Driver override used by openshell-gateway. OPENSHELL_GATEWAY_NAME Gateway name for delegated or Kubernetes runs. OPENSHELL_BIND_ADDRESS Gateway listener address. Defaults to 127.0.0.1, or ::1 for Podman Machine on macOS. @@ -104,7 +106,7 @@ detect_driver() { fi echo "ERROR: no compute driver detected." >&2 - echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_DRIVERS." >&2 + echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_COMPUTE_DRIVER." >&2 exit 2 } @@ -171,17 +173,17 @@ while [[ "$#" -gt 0 ]]; do esac done -if [[ -n "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - echo "ERROR: use either --driver or OPENSHELL_DRIVERS, not both" >&2 +if [[ -n "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + echo "ERROR: use either --driver or OPENSHELL_COMPUTE_DRIVER, not both" >&2 exit 2 fi -if [[ -z "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - if [[ "${OPENSHELL_DRIVERS}" == *,* ]]; then - echo "ERROR: mise run gateway supports one driver; got OPENSHELL_DRIVERS=${OPENSHELL_DRIVERS}" >&2 +if [[ -z "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + if [[ "${OPENSHELL_COMPUTE_DRIVER}" == *,* ]]; then + echo "ERROR: mise run gateway supports one driver; got OPENSHELL_COMPUTE_DRIVER=${OPENSHELL_COMPUTE_DRIVER}" >&2 exit 2 fi - explicit_driver="$(normalize_driver "${OPENSHELL_DRIVERS}")" + explicit_driver="$(normalize_driver "${OPENSHELL_COMPUTE_DRIVER}")" fi DRIVER="${explicit_driver:-$(detect_driver)}" @@ -206,7 +208,7 @@ GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -244,12 +246,11 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" < %q, got %q\n' "${input}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +for input in always Always ALWAYS; do + assert_policy "${input}" always +done +for input in if_not_present IfNotPresent ifnotpresent IFNOTPRESENT missing MISSING; do + assert_policy "${input}" if_not_present +done +for input in never Never NEVER; do + assert_policy "${input}" never +done +for input in newer Newer NEWER; do + assert_policy "${input}" newer +done + +if normalize_image_pull_policy sometimes >/dev/null 2>&1; then + echo "unsupported policy unexpectedly succeeded" >&2 + exit 1 +fi + +for script in gateway.sh gateway-docker.sh gateway-podman.sh; do + if ! grep -q 'normalize_image_pull_policy' "${ROOT}/tasks/scripts/${script}"; then + echo "${script} does not normalize image pull policy" >&2 + exit 1 + fi +done + +echo "gateway pull-policy tests passed" diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh index 7d0b05334d..6d66fbb5d4 100755 --- a/tasks/scripts/vm/smoke-orphan-cleanup.sh +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -54,10 +54,10 @@ start_gateway() { mkdir -p "$STATE_DIR" cat >"$config" <