From 0fd8b67bfd46fc4dec647359b2d0434ab8f34fcb Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 14:46:00 +0930 Subject: [PATCH 01/16] docs: add Pyodide Python runtime QRSPI artifacts (#118) Adds the full question/research/design/structure/plan artifact set for the Pyodide Python runtime work: python_runtime enum migration, OS-isolated Deno subprocess execution, committed IPC with frame-integrity invariants, file diff-and-drain, concurrency/residency caps, and the adversarial escape suite. --- thoughts/issue-118-pyodide-runtime/design.md | 247 +++++++ thoughts/issue-118-pyodide-runtime/plan.md | 635 ++++++++++++++++++ .../issue-118-pyodide-runtime/questions.md | 34 + .../issue-118-pyodide-runtime/research.md | 127 ++++ .../issue-118-pyodide-runtime/structure.md | 280 ++++++++ thoughts/issue-118-pyodide-runtime/task.md | 5 + 6 files changed, 1328 insertions(+) create mode 100644 thoughts/issue-118-pyodide-runtime/design.md create mode 100644 thoughts/issue-118-pyodide-runtime/plan.md create mode 100644 thoughts/issue-118-pyodide-runtime/questions.md create mode 100644 thoughts/issue-118-pyodide-runtime/research.md create mode 100644 thoughts/issue-118-pyodide-runtime/structure.md create mode 100644 thoughts/issue-118-pyodide-runtime/task.md diff --git a/thoughts/issue-118-pyodide-runtime/design.md b/thoughts/issue-118-pyodide-runtime/design.md new file mode 100644 index 0000000..c845edd --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/design.md @@ -0,0 +1,247 @@ +# Design Discussion — Pyodide Python runtime (issue #118) + +> **Revision (review pass).** This version resolves an 8-point design review. Decisions: the Deno boundary +> is an **explicit single-layer** boundary (Deno/V8 escape is out of the threat model); the **IPC transport +> is committed** (stdin/stdout + realm-lockdown, fixed frame-integrity invariants); **OOM isolation is not +> guaranteed** (accepted availability risk); the DB migration is **rolling-deploy-safe**. Corrections folded +> in: Deno module-loading flags, timeout-must-throw, first-class manager ownership, atomic LRU admission. + +## Current State + +A sandbox's Python capability is a single boolean toggling just-bash's built-in **CPython-emscripten** +`python3` (plain CPython in WASM — *no* JS↔Python bridge; only host contact is the `ctx.fs` SharedArrayBuffer +bridge + stdout/stderr, so it is genuinely air-gapped). The flag travels a fixed chain: + +- **Validate** — inline Zod `python: z.boolean().optional()`, defaulted `?? false` (`routes/sandboxes.ts:15-23,50-52`). +- **Types** — `SandboxMeta`/`SandboxListEntry` carry `readonly python/javascript/network: boolean` + (`sql-fs/types.ts:61-82`); `RuntimeOptions` + `DEFAULT_RUNTIME_OPTIONS` (`session-manager.ts:110-122`). +- **Persist** — `updateSandboxMeta` `UPDATE sandboxes SET … python,javascript,network` (`dialects/postgres.ts:345-362`); + read via `getSandboxMeta` (`:316-343`). +- **Rehydrate** — cold start reads meta, threads `resolvedRuntime` into `new Bash({ python: … })` + (`session-manager.ts:930-959,487-497`), stored on `Session.runtimeOptions` (`:522`). +- **Respond** — create 201 echoes all three (`sandboxes.ts:134`); list maps all three (`:142-159`); + **single GET omits them** (`:179-196`) — a known asymmetry. + +`new Bash({ python: true })` registers the built-in `python3`, which spawns a `worker_threads` worker; output +files persist to SqlFs *only because* its SharedArrayBuffer bridge calls `ctx.fs` per op (`research.md` Q6). +A prior audit (**C1**) rejected a host-process `py-exec` for sandbox-escape risk; Python is WASM-only +(`session-manager.ts:475-477`). The existing `nodeCommand` (`commands/node-command.ts:48`) is the template: +`defineCommand`, registered only when `javascript` is on, delegating via `ctx.exec`; custom commands **shadow +built-ins** (`Bash.d.ts:111`); `python` is a *separate* registry entry from `python3` (no alias mechanism). + +Concurrency: `pythonSem`/`jsSem` (`session-manager.ts:313-314,357-374`), limits default 5, routed by regex +gated on the session flag (`:143-146,1235-1237`). Timeout/cancel is enforced **in routes** via +`AbortController`→`bash.exec` (`exec.ts:154-202,219-229`); a command sees only `ctx.signal`. The readOnly exec +path holds the per-session RWLock in **shared** mode → **concurrent readers** (`session-manager.ts:640-647`). +Sessions evict on idle (`SESSION_IDLE_MS`, default 600 000 ms); cleanup today only disconnects `session.fs` +(`session-manager.ts:1038,1088`). Migrations: **only `migrations/postgres/` exists**, **no tracking table** — +*every* `.sql` re-runs on *every* boot under `pg_advisory_lock`, so **every migration MUST be idempotent** +(`migrations.ts:40-65`). The `python` field is duplicated across both SDKs, MCP, OpenAPI, docs (`research.md` Q5). + +## Desired End State + +`python_runtime: "stdlib" | "pyodide" | null` replaces boolean `python`. + +- `"stdlib"` → `new Bash({ python: true })` (built-in CPython-emscripten, unchanged, air-gapped). +- `"pyodide"` → register **two** custom commands (`python3` + `python`, one shared handler) running + numpy/pandas/scipy/openpyxl **fully offline inside an OS-isolated Deno subprocess** (Decision 1). + `python: undefined` so the custom commands are the only Python. +- `null` → no Python. + +**Verify:** create a `pyodide` sandbox; upload a CSV; `python3 analyze.py` runs `import pandas`, writes +`out.xlsx`, and the file is **retrievable via the files API** (the issue's core requirement). `python_runtime` +round-trips create/list/**get**, both SDKs, MCP, OpenAPI; `pnpm typecheck && lint && test:unit` green; the +migration integration test asserts the column. + +**Threat model (explicit — finding 1).** The boundary is **single-layer**: a zero-permission Deno subprocess +spawned with a **scrubbed env**. A successful escape *out of Deno itself* (a Deno/V8 0-day) is **out of the +threat model** — it would expose the host under the same uid. Compensating controls keep that escape low-value: +**no secrets in the child** (`env:{}` — `AUTH_SECRET`/`DATABASE_URL` are simply absent) and `--allow-read` +scoped to read-only Pyodide assets only. We **recommend** (do not require) operators add a gVisor/seccomp layer. +**Security acceptance is first-class:** an adversarial suite proves `import js; js.process.env`, `js.fetch(...)`, +`pyodide.code.run_js(...)`, `ctypes.CDLL(None)`, `import('node:child_process')` **fail closed** (no secret read, +no network, no host-FS reach) **and** that escaped JS **cannot forge, interleave, or replay an IPC control +frame** — proving capability denial, not merely a thrown error. + +## Patterns to Follow + +- **Custom command shape** — `defineCommand` + `ExecResult {stdout,stderr,exitCode}`; parse `-c CODE` / + script-path / stdin / `--version`; resolve args with `ctx.fs.resolvePath(ctx.cwd, arg)`; match the built-in + `python3` arg surface (`research.md` Q2). Register `python3` + `python` as two commands, one handler. +- **Bash construction** — extend the existing `new Bash({…})` block (`session-manager.ts:487-497`); no parallel + path. Store resolved runtime on `Session` (`:522`). +- **First-class manager ownership (required — finding 7)** — `CommandContext` exposes no `Session` handle, but + the manager must **not** live only inside a command closure (it has to participate in the global residency + LRU and complete teardown). Add a first-class field `Session.pyodideSandbox?: PyodideSandbox` (or a generic + session-resource disposer); the command handler reaches it via the session captured at `getOrCreate` time. + Teardown MUST run on **every** path: session destroy, **reaper eviction**, **shutdown**, partial + `getOrCreate` failure, runtime timeout, IPC corruption, LRU eviction, unexpected child exit — extend the + `session.fs`-only cleanup at `session-manager.ts:1038,1088`. +- **Per-subprocess serialization (required)** — readOnly execs run concurrently (`session-manager.ts:640`), so + the manager serializes execs to its single subprocess via a mutex/queue; a queued waiter is removed on + `ctx.signal` abort. +- **Cancellation = throw, never return (required — finding 6)** — `bash.exec` cancel is cooperative and the + route's `AbortController` is invisible to the command (only `ctx.signal` is). The handler observes + `ctx.signal` + its own timer and **kills the subprocess** (`child.kill('SIGKILL')`), then: + - **route abort** → **reject with `AbortError`**, aborting the script transaction (route returns **408**); + - **internal runtime timeout** → **throw a typed timeout error**; + - **never drain files** from a timed-out / protocol-invalid run; **never reuse that subprocess generation**. + Returning a normal `{exitCode}` is forbidden — it would let `bash.exec` resolve, the route return 200, and the + script transaction **commit** a partial/garbage drain. Throwing triggers SqlFs `abortScriptScope` rollback + (`research.md` Q6). +- **Migration** — numbered, **fully idempotent** SQL in `migrations/postgres/` (re-runs every boot), reflected + in `types.ts`/`postgres.ts`; integration test asserts the column. + +**Do NOT follow / avoid:** +- No in-process / `worker_threads` Pyodide. Pyodide-on-Node is **not** a security boundary: `import js`→ + `globalThis` exposes `process.env`/`fetch`; `pyodide.code.run_js`, `ctypes`, `_pyodide._base.eval_code`, + `__subclasses__` survive `jsglobals`/`unregisterJsModule` hardening (live RCEs: n8n CVE-2025-68668 9.9, Grist + CVE-2026-24002 9.0). worker_threads can scrub `env:{}` but cannot block network or `import('node:fs')`. +- Do **not** reintroduce host-*capability* Python (audit C1) — the Deno child has no net/run/write/env/ffi/sys + permissions and read-only access only to vendored assets. +- Do **not** rely on NODEFS to bridge SqlFs; do **not** use `micropip`/PyPI at runtime — pre-stage offline. +- Do **not** trust subprocess output — untrusted code controls it. Enforce the frame invariants (Decision 1) and + **validate every drain path stays under cwd** (reject `..`, absolute, null-byte) before applying to SqlFs. +- Do **not** repeat the single-GET omission — fix GET to echo `python_runtime`. + +## Design Decisions + +1. **Isolation = explicit single-layer Deno subprocess (headline; findings 1, 2, 3).** The per-session + `PyodideSandbox` spawns a **Deno subprocess** with a **scrubbed env** (`env:{}` plus `DENO_NO_UPDATE_CHECK=1`). + Flags (spike-validated against the pinned Deno version, S1): `--no-prompt` + the deny belt `--deny-net + --deny-run --deny-write --deny-env --deny-ffi --deny-sys --deny-import`, **plus the module-loading air-gap + flags `--no-remote --no-npm --cached-only --no-config`** (finding 2 — `--deny-net` alone does not gate the + module graph; remote registries load by default), with `--allow-read` scoped to the vendored asset dir only. + Note `--deny-import` blocks *remote* imports only — **local** dynamic imports under `--allow-read` remain + possible, contained because the read scope is read-only Pyodide assets. Deno gates the *JS layer itself*, so a + full Python→JS escape lands capability-less. + - **IPC is committed, not an alternative (finding 3).** Transport = **Node↔Deno over the child's stdin/stdout**, + with **realm-lockdown**: at startup the harness captures the writer, then deletes `Deno`/`console`/other + write primitives from `globalThis` **before** any untrusted Python runs. Every frame (both directions) is + **length-prefixed JSON** carrying **mandatory integrity fields**: random **requestId**, monotonic + **sequence number**, exact **message type**, **child-generation id**; with **max per-frame and aggregate + size caps** and **exactly one response per request**. The Node side **kills the child immediately** on any + malformed, oversized, duplicate, out-of-sequence, wrong-generation, or unexpected frame. Defense-in-depth: + Node independently re-enforces cwd path-validation, RLS, and size caps, so even a forged frame cannot + escalate beyond what the sandboxed code could already write to its own cwd. Spike S2 **confirms feasibility** + of this committed design (it does not choose between transports). +2. **Capability field — full external break, rolling-safe expand/contract migration (findings 5).** Add + `python_runtime TEXT` (`CHECK IN ('stdlib','pyodide')`, nullable). **Migration N (this release):** + `ADD COLUMN IF NOT EXISTS`; idempotent backfill **only `WHERE python_runtime IS NULL`** (so repeated boot + re-runs never clobber `pyodide`), guarded by a `DO` block checking `pg_attribute` for `python` + (`python=true → 'stdlib'`). **Mixed-version safety:** new **reads** use + `COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END)` so an old replica's `python=true`/NULL row is + read correctly; new **writes dual-write** `python` (`stdlib → true`, `pyodide → false`, `null → false`) so old + replicas keep working. **Migration N+1 (later release):** `DROP COLUMN IF EXISTS python` and remove the + COALESCE/dual-write. API/SDKs/MCP/OpenAPI/docs accept/return only `python_runtime`; legacy `python: bool` + rejected. `javascript`/`network` unchanged. Fix single-GET to echo capabilities (`sandboxes.ts:179-196`). +3. **Execution model — per-session warm Deno subprocess, serialized (Q1 + findings 2,3).** One subprocess per + sandbox session, spawned lazily on first `pyodide` exec, reused across that session's execs (amortizes the + multi-second cold start for the iterative LibreChat loop), serialized by the per-subprocess mutex. Between + execs: fresh Python `globals` + wipe staged MEMFS paths (bounds variable scope + staged files only; + `sys.modules`/package globals persist within a session — same trust boundary). Cross-session isolation comes + from per-session subprocesses. Timeout/abort kills the subprocess; next exec re-inits (new generation). +4. **Residency LRU — explicit state machine + atomic admission (findings 7, 8).** A **global registry** caps + resident subprocesses at `MAX_RESIDENT_PYODIDE` (small default, e.g. 2), independent of `SESSION_IDLE_MS` + (else warm subprocesses accumulate per active session); a shorter `PYODIDE_IDLE_MS` idle-kills them. Each + worker has an explicit state: `cold → starting → idle → busy → terminating → dead`. **`starting` and `busy` + are never evictable.** A **registry mutex** makes admission atomic — it covers *reserve a slot → select an + eviction victim → spawn → roll back on failed init* as one critical section, so concurrent cold starts cannot + both observe a free slot and exceed the cap. **Capacity is reserved before** expensive Pyodide init. +5. **Concurrency + memory — dedicated semaphore; OOM isolation NOT guaranteed (Q4 + finding 4).** New + `MAX_CONCURRENT_PYODIDE` (low default, **2**) + queue/wait-timeout env vars mirroring the python set; routed by + `python_runtime==="pyodide"` so `stdlib` keeps `MAX_CONCURRENT_PYTHON=5`. The semaphore caps *in-flight execs*, + not RAM; Decision 4 caps residency. **Invariant `MAX_RESIDENT_PYODIDE >= MAX_CONCURRENT_PYODIDE`** (a busy + proc cannot be evicted). **OOM is an accepted availability risk, not an isolation guarantee:** a container + memory limit covers Node + *all* Deno children together, so a runaway child may OOM-kill Node or the whole + container. `prlimit --as` is **unsuitable** (V8 reserves huge virtual address space); on `node:22-slim` as + non-root `app`, per-process cgroup v2 `memory.max` is **best-effort** (likely no cgroup-write access). The + **operator-set container memory limit is the real guard** — operators size `MAX_RESIDENT × per-proc ceiling` + (`MAX_RESIDENT=1` on small hosts). The Pyodide ~2 GB WASM cap is only a per-instance heap ceiling. The manager + reports an error + respawns (new generation) on child exit. +6. **File staging — cwd subtree + script over IPC, diff-and-drain with explicit semantics (Q2 + finding 5/6).** + Before run: ship the cwd subtree **plus the resolved script path** (even if outside cwd, for `python3 FILE` + parity) to Deno, written into MEMFS. After a **successful** run only: Deno reports the created/modified/deleted + set; Node applies it to `ctx.fs` **inside the existing script transaction** (atomic rollback on failure, + `research.md` Q6). **Never drain from a timed-out / aborted / protocol-invalid run** (finding 6). **Semantics:** + reject **symlinks** (SqlFs default-deny); files written 0644 default (`sql-fs.ts:738`), exec-bit via `chmod` + only if needed; dirs-before-files, delete depth-first; hardlinks drained as independent copies; **cwd-scoped + only** (keep script + inputs under cwd). Per-file + total byte caps on both directions. +7. **Offline packaging — Deno + full distribution + lockfile tooling (finding 6 prior).** Pin pyodide to stable + **0.29.x** (not the `314.0.0-alpha` `next` tag). The build/Docker step bakes the **Deno binary**, the full + `pyodide-0.29.x` distribution (numpy/pandas/scipy wheels + `pyodide-lock.json`), and a **custom lock** + incorporating openpyxl + `et_xmlfile` + transitive deps via tooling (`micropip.freeze`) — **not** hand-edited. + `loadPyodide({ indexURL, lockFileURL, packageBaseUrl })` from local paths. Preload {numpy,pandas,scipy,openpyxl} + at subprocess init. + +## Required Tests (acceptance contract) + +Each test names the decision it protects; `/4-structure` distributes these into per-phase checkpoints. + +- **Migration (D2):** an old-replica-style `python=true` write is read back as `stdlib` (COALESCE); re-running + the migration **after** the N+1 `python`-drop is idempotent (no error, no clobber). +- **Residency / LRU (D4):** concurrent admissions never exceed `MAX_RESIDENT_PYODIDE` (atomic-admission mutex); + LRU **never evicts a `busy` or `starting` worker**. +- **Cancellation (Patterns + D6, finding 6):** abort while **waiting on the subprocess mutex**; abort **during + init / package preload**; abort **after the child response but before drain completion** — each kills the + child, **retires that generation**, and **drains nothing**. +- **Teardown (Patterns, finding 7):** session **reaper, destroy, shutdown, and failed `getOrCreate`** each kill + the child. +- **Drain / FS (D6):** a read-only exec's MEMFS mutation is rejected (`EREADONLY_VIOLATION`), never silently + dropped; a script **resolved outside cwd cannot drain outside its cwd** (`..` / absolute / null-byte rejected). +- **IPC integrity (D1):** **malformed, oversized, duplicate, stale-generation, and forged** frames each kill the + child; the size cap is measured on the **base64-encoded wire size** (accounts for ~33% expansion), not raw bytes. +- **Deny-belt (D1 + escape suite):** **remote import, npm import, update check, filesystem write, env read, + subprocess spawn, FFI (`Deno.dlopen`), and network** are each denied (fail closed). + +## What We're NOT Doing + +- No in-process / `worker_threads` Pyodide (Decision 1). **No required gVisor/microVM** — Deno permissions + env + scrub are the boundary; a Deno/V8 runtime escape is **out of the threat model**; gVisor/seccomp is a + **recommended** operator add-on, not a hard requirement (finding 1). +- **No guaranteed per-child OOM isolation** — whole-service termination on a runaway child is an accepted + documented risk; cgroup `memory.max` is best-effort only (finding 4). +- No mysql/azure-sql migrations (none exist; postgres-only). No `micropip`/PyPI or arbitrary packages at runtime. + No NODEFS-backed Python; no network egress from Pyodide. +- No `python_runtime` mutation after create (capabilities immutable, `endpoints.md:148`). No broader + `schema.ts`/Drizzle fix. No change to `javascript`/`network` or `node`. + +## Required Spikes (pre-implementation merge gates) + +Each must pass a POC before its dependent phase is built; fail any → revisit the architecture with the user. + +1. **Pyodide-on-Deno feasibility (gates P3).** Under the **exact** committed flags (Decision 1) + scoped + `--allow-read`, a Deno subprocess `loadPyodide` from the **local offline** `indexURL`, `loadPackage` + numpy/pandas/scipy from disk, install the **frozen openpyxl+et_xmlfile** lock, run a pandas→openpyxl + round-trip, and capture stdout/stderr — all with **zero network**. +2. **IPC integrity (gates P3/P4) — confirm the *committed* design, do not re-choose transport.** The + stdin/stdout + realm-lockdown framing works under the deny flags with binary payloads, and after lockdown an + adversarial `Deno.stdout.write`/`console.log`/raw-fd attempt **cannot forge, interleave, or replay** a control + frame; verify generation-id rejection of a killed-generation message. +3. **Per-child memory behavior (gates P6).** Confirm `node:22-slim` non-root cannot reliably set cgroup + `memory.max` (and that `prlimit --as` is unusable with V8); validate the container-limit guard + accepted + availability risk (Decision 5) is the operating model. + +## Open Risks + +- **New runtime dependency: Deno** must be vendored and available on self-hosted installs — adds image size + a + portability constraint. Pin the Deno version + the full flag set cleanly. +- **C1 reversal needs security sign-off** — reintroduces a Python-running *subprocess*, justified by Deno's + zero-permission boundary + env scrub. The adversarial escape/forgery suite is the explicit gate; get + maintainer/security review before merge, and have them confirm the **single-layer threat model** (Deno/V8 + escape out of scope) is acceptable for the deployment. +- **IPC integrity** — untrusted code shares the harness realm, so it can send hostile payloads *and* (via escaped + JS on stdout) attempt to forge frames. The committed invariants (generation id, sequence, single-response, + size caps, kill-on-anomaly) + realm-lockdown are the defense; both are spike- and adversarially-tested. +- **Memory under load** — worst case ≈ `MAX_RESIDENT_PYODIDE × ~2 GB` (WASM heap ceiling) + Node baseline + DB + pools. **A runaway child can OOM the whole container** (accepted risk, Decision 5); operators must size the + container against `MAX_RESIDENT × per-proc ceiling` (`MAX_RESIDENT=1` on small hosts). Multi-replica + restart + mitigates the availability hit. Validate defaults + `PYODIDE_IDLE_MS` on the target host. +- **Cold-start UX** — first `pyodide` exec (or any after eviction) pays Deno spawn + Pyodide init + package load + (several seconds); exec timeout defaults must account for it. +- **Intra-session state persistence** — warm subprocess preserves `sys.modules`/package globals across a session's + execs (Decision 3); add a fresh-globals isolation test; fall back to kill-and-respawn per exec if per-exec + purity is ever required. +- **Coordinated breaking release** — every `research.md` Q5 surface ships together; reconcile the second-class + `network` field (absent from MCP/OpenAPI/SDK records) at the same time. The DB layer rolls safely (Decision 2), + but the API/SDK break still requires clients to upgrade. diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md new file mode 100644 index 0000000..f443bd9 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -0,0 +1,635 @@ +# Implementation Plan — Pyodide Python runtime (issue #118) + +## Overview + +Replace the boolean `python` capability with `python_runtime: "stdlib" | "pyodide" | null` end-to-end (rolling-safe expand/contract migration), then implement the `pyodide` runtime as an **OS-isolated, zero-permission Deno subprocess** that loads Pyodide fully offline, runs untrusted Python (numpy/pandas/scipy/openpyxl), speaks a committed length-prefixed JSON IPC with realm lockdown + frame-integrity invariants, and drains cwd-scoped file changes into SqlFs inside the script transaction. The issue's core requirement: a `pyodide` sandbox can run `python3 analyze.py` (import pandas, write `out.xlsx`) and the file is retrievable via the files API. + +**Resolved decisions carried from `design.md`/`structure.md`:** +- **Vendored assets are git-ignored and fetched by a build script** (`scripts/fetch-pyodide-assets.mjs`), invoked in the Dockerfile and runnable locally. (Resolves the "gitignore/LFS decided in plan" open item — no LFS; keep the repo lean, reproduce assets from a pinned manifest.) +- **Shared protocol contract** lives in `src/pyodide-runner/protocol.ts`, written runtime-agnostically (`Uint8Array`/`DataView`, no `Buffer`/`Deno` globals) so tsc compiles it for the Node side and Deno imports the raw `.ts`. The Deno entry `runner.ts` is **excluded from the tsc build** (uses Deno globals) and shipped to `dist/` as raw `.ts`. + +> **Coordinated-release note.** All seven phases are implementation slices on **one branch → one breaking release**. The DB layer is rolling-deploy-safe (Phase 1); the API/SDK enum break still requires clients to upgrade. Between Phase 2 and Phase 5, a `pyodide` sandbox creates fine but `python3` inside it is "not registered" — an intermediate state that never reaches a cut release. + +--- + +## Phase 0: Spikes (merge gates — throwaway POCs, not shippable) + +Three POCs from `design.md` §"Required Spikes". Each must pass before its dependent phase; **fail any → stop and revisit the architecture with the user** before writing the corresponding product phase. + +### Changes + +#### 1. Spike scratch area +**Files**: `thoughts/issue-118-pyodide-runtime/spikes/` (create) — one scratch script + one short `SN-findings.md` note per spike. +**Action**: create + +- **S1 — Pyodide-on-Deno offline** (gates Phase 3). Script `spikes/s1-pyodide-deno.sh` + `spikes/s1_runner.ts`. Download Deno (pinned version) + the `pyodide-0.29.x` full distribution locally into `spikes/assets/`. Run (note: `DENO_NO_UPDATE_CHECK` is set in the **parent shell/spawn env** — it is read by the Deno runtime, not via `Deno.env`, which `--deny-env` blocks; the asset dir is passed as **argv**): + ``` + DENO_NO_UPDATE_CHECK=1 deno run --no-prompt --deny-net --deny-run --deny-write \ + --deny-env --deny-ffi --deny-sys --deny-import --no-remote --no-npm \ + --cached-only --no-config --allow-read= s1_runner.ts + ``` + `s1_runner.ts` reads `` from `Deno.args` (**not** `Deno.env`), then `loadPyodide({ indexURL, lockFileURL, packageBaseUrl })` from local paths, `loadPackage(["numpy","pandas","scipy"])`, install the frozen openpyxl+et_xmlfile lock, run a pandas→openpyxl round-trip (DataFrame → `out.xlsx` bytes → read back), and print the round-trip byte length. **Assert zero network** (the deny flags + a packet-capture or a deliberate offline run prove it). +- **S2 — IPC integrity** (gates Phase 3/4). Script `spikes/s2-ipc.ts`. Implement length-prefixed JSON framing over stdin/stdout with a binary payload; after realm lockdown (capture stdout writer, delete `Deno`/`console`/write primitives from `globalThis`), run an adversarial snippet that tries `Deno.stdout.write`/`console.log`/raw-fd to **forge, interleave, or replay** a control frame, and a stale-`generation` message. Assert all are rejected. +- **S3 — Per-child memory** (gates Phase 6). Script `spikes/s3-memory.sh`. In a `node:22-slim` non-root context, attempt to set cgroup v2 `memory.max` for a child and attempt `prlimit --as`; confirm both are unavailable/unusable (V8 vaddr reservation). Document that the **container memory limit + accepted availability risk (design D5)** is the operating model. + +### Phase 0: Success Criteria + +#### Phase 0: Programmatic Verification +- [ ] `spikes/s1-pyodide-deno.sh` exits 0 and prints the pandas→openpyxl round-trip byte length with zero network access +- [ ] `spikes/s2-ipc.ts` exits 0 and prints PASS for each of: forged-frame rejected, interleave rejected, replay rejected, stale-generation rejected +- [ ] `spikes/s3-memory.sh` exits 0 and prints the memory-limit probe result (cgroup write denied / prlimit unusable) + +#### Phase 0: Agent Verification +- [ ] Agent reviews `S1-findings.md`, `S2-findings.md`, `S3-findings.md` and confirms each gate's pass/fail is **explicit** before Phase 3/Phase 6 begin +- [ ] Agent confirms the exact Deno version and pyodide version validated in S1 match what Phase 3 pins + +### Phase 0: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 0._ + +--- + +## Phase 1: `python_runtime` field — server-side break, rolling-safe (stdlib + null end-to-end) + +Replace boolean `python` with the nullable enum across DB, types, persistence, validation, runtime resolution, and HTTP responses, safely under a mixed-version rolling deploy. `stdlib`→`new Bash({ python: true })`; `null`→no Python; `pyodide` is a valid stored value but Python stays unregistered until Phase 5. + +### Changes + +#### 1. Migration `0006_python_runtime.sql` +**File**: `src/sql-fs/migrations/postgres/0006_python_runtime.sql` +**Action**: create + +Idempotent (re-runs every boot under the advisory lock — see `migrations.ts`). Adds the nullable enum column, backfills only NULL rows from legacy `python`, and survives the later `python`-drop release via a `pg_attribute` guard. + +```sql +-- Migration 0006: replace boolean `python` with nullable `python_runtime` enum. +-- Expand/contract step N (this release). Rolling-deploy-safe: reads COALESCE the +-- legacy column, writes dual-write it (see postgres.ts). Step N+1 (later release) +-- drops `python` and removes the COALESCE/dual-write. + +ALTER TABLE sandboxes + ADD COLUMN IF NOT EXISTS python_runtime TEXT; + +-- CHECK constraint (idempotent: add only if absent). +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'sandboxes_python_runtime_check' + ) THEN + ALTER TABLE sandboxes + ADD CONSTRAINT sandboxes_python_runtime_check + CHECK (python_runtime IN ('stdlib','pyodide')); + END IF; +END $$; + +-- Backfill ONLY rows not yet migrated, and ONLY while the legacy `python` column +-- still exists (so this is a no-op after the N+1 drop release — never errors). +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = 'sandboxes'::regclass + AND attname = 'python' AND NOT attisdropped + ) THEN + UPDATE sandboxes + SET python_runtime = CASE WHEN python THEN 'stdlib' END + WHERE python_runtime IS NULL; + END IF; +END $$; +``` + +#### 2. Shared types +**File**: `src/sql-fs/types.ts` +**Action**: modify (`SandboxMeta` ~:61-70, `SandboxListEntry` ~:73-82) + +```ts +/** Python runtime selection. null = no Python. */ +export type PythonRuntime = "stdlib" | "pyodide" | null; +``` + +- `SandboxMeta`: replace `readonly python: boolean;` with `readonly python_runtime: PythonRuntime;` (keep `javascript`/`network`). +- `SandboxListEntry`: same replacement. + +#### 3. Postgres dialect — rolling-safe read + dual-write +**File**: `src/sql-fs/dialects/postgres.ts` +**Action**: modify `getSandboxMeta` (:316-343), `updateSandboxMeta` (:345-362), `listSandboxes` (:364-402) + +- **`getSandboxMeta`** — select the COALESCE expression so an old replica's `python=true`/`python_runtime IS NULL` row reads as `stdlib`: + ```sql + SELECT owner, name, + COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END) AS python_runtime, + javascript, network, created_at + FROM sandboxes WHERE id = ${sandboxId} + ``` + Map row → `python_runtime: r.python_runtime as PythonRuntime` (NULL → `null`). +- **`listSandboxes`** — same COALESCE in both the owner-filtered and unfiltered queries; map `python_runtime`. +- **`updateSandboxMeta`** — dual-write both columns so old replicas keep working: + ```sql + UPDATE sandboxes + SET owner = ${meta.owner}, name = ${meta.name}, + python_runtime = ${meta.python_runtime}, + python = ${meta.python_runtime === "stdlib"}, + javascript = ${meta.javascript}, network = ${meta.network ?? false} + WHERE id = ${sandboxId} RETURNING id + ``` + (`stdlib → python=true`; `pyodide`/`null → python=false`.) + +#### 4. RuntimeOptions + Bash construction +**File**: `src/api/session-manager.ts` +**Action**: modify `RuntimeOptions` (:110-120), `DEFAULT_RUNTIME_OPTIONS` (:122), `getOrCreate` Bash block (:487-497), `rehydrateAndExec` (:945-947), `rehydrateAndExecRead` (:898-900) + +- `RuntimeOptions`: replace `readonly python: boolean;` with `readonly pythonRuntime: PythonRuntime;` (import `PythonRuntime` from `../sql-fs/types.js`). +- `DEFAULT_RUNTIME_OPTIONS`: `{ pythonRuntime: null, javascript: false, network: false }`. +- Bash block: `python: resolvedRuntime.pythonRuntime === "stdlib" || undefined,` — `pyodide` leaves `python: undefined` (Phase 5 adds the custom commands). +- Both rehydrate paths: build `resolvedRuntime` as `{ pythonRuntime: meta.python_runtime, javascript: meta.javascript, network: meta.network }`. +- `execWithRuntimeThrottle` (:1236): `const usesPython = session.runtimeOptions.pythonRuntime === "stdlib" && PYTHON_INVOCATION_REGEX.test(script);` (only `stdlib` routes through `pythonSem`; `pyodide` routing comes in Phase 6). + +#### 5. Route validation + responses +**File**: `src/api/routes/sandboxes.ts` +**Action**: modify `createBodySchema` (:15-23), create handler (:33-134), list (:142-159), **GET (:179-196 — fix the omission)** + +- `createBodySchema`: drop `python: z.boolean()`; add `python_runtime: z.enum(["stdlib", "pyodide"]).nullable().optional()`. Keep `javascript`/`network`. (Legacy `python: bool` is no longer in the schema → Zod `.strict()` is not currently used, so an extra `python` key is ignored; to **reject** it explicitly, add `.strict()` or a refine — see note below.) +- Replace the local `let python = false;` with `let pythonRuntime: PythonRuntime = null;`; parse `pythonRuntime = result.data.python_runtime ?? null;`. +- `persistSandboxMeta` call: pass `python_runtime: pythonRuntime` (drop `python`). +- `withSession` runtimeOptions arg: `{ pythonRuntime, javascript, network }`. +- Create 201 response: echo `python_runtime` (drop `python`); keep `javascript`, `network`. +- List response map: `python_runtime: s.python_runtime` (drop `python`). +- **GET fix** — both the cold-DB branch (:179-185) and warm-session branch (:190-196) must echo capabilities. Cold branch already has `meta`; add `python_runtime: meta.python_runtime, javascript: meta.javascript, network: meta.network`. Warm branch has no capabilities on `Session` directly — read them from `session.runtimeOptions`: `python_runtime: session.runtimeOptions.pythonRuntime, javascript: session.runtimeOptions.javascript, network: session.runtimeOptions.network`. + +> **Reject-legacy note:** the design says "reject legacy `python: bool`". Implement by adding `.strict()` to `createBodySchema` so an unknown `python` key returns 400 `INVALID_INPUT`. Confirm no existing test sends `python:` (Phase 2 updates those that do). + +#### 6. Update existing tests that assert `python` +**Files**: `src/api/tests/unit/session-manager.rehydrate.test.ts` (asserts `runtimeOptions.python:true` ~:63-71, 114-150), `src/sql-fs/dialects/tests/unit/postgres.advisory-lock.test.ts` (:131-170), `src/api/tests/unit/sandboxes.test.ts` (:64-101) +**Action**: modify — change `python: true` assertions to `pythonRuntime: "stdlib"` (RuntimeOptions) / `python_runtime: "stdlib"` (meta rows), and any mock `SandboxMeta` literals to the new shape. + +#### 7. Migration integration test — assert the column + rolling-safe reads +**File**: `src/api/tests/integration/migrations.integration.test.ts` +**Action**: modify + +**Test A — extend the existing `it(...)`** (column + CHECK + COALESCE read). Add, after the existing table/proc assertions: +```ts +// 0006: python_runtime column + CHECK exist. +const col = await sql<{ n: string }[]>` + SELECT count(*)::text AS n FROM information_schema.columns + WHERE table_name = 'sandboxes' AND column_name = 'python_runtime'`; +expect(col[0]?.n).toBe("1"); + +// Old-replica-style row (python=true, python_runtime NULL) reads back as stdlib via COALESCE. +await sql`INSERT INTO sandboxes (id, python, python_runtime) VALUES ('mig-legacy', true, NULL)`; +const legacy = await sql<{ pr: string | null }[]>` + SELECT COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END) AS pr + FROM sandboxes WHERE id = 'mig-legacy'`; +expect(legacy[0]?.pr).toBe("stdlib"); +``` + +**Test B — the simulated `python`-drop idempotency, in its OWN isolated ephemeral database** (do NOT do this in Test A's DB). The `python`-drop is destructive: Phase 1 dialect code (`getSandboxMeta`/`updateSandboxMeta`/`listSandboxes`) still references `python` for COALESCE reads + dual-writes until release N+1, so dropping it on a shared DB would make any later dialect call fail for the wrong reason. Keep this check **migration-SQL-only** in a throwaway DB created/dropped within the test, with no dialect calls after the drop: +```ts +it("re-runs idempotently after a simulated python-column drop (N+1)", async () => { + // own ephemeral DB, mirroring beforeAll's create/teardown pattern + const dropDb = `vfs_mig_drop_${randomBytes(8).toString("hex")}`; + await admin!.unsafe(`CREATE DATABASE ${dropDb}`); + const dropUrl = withDatabase(base, dropDb); + const dropCfg = loadTenantConfig({ TENANT_DATABASES: JSON.stringify({ default: dropUrl }) }); + const s = postgres(dropUrl, { prepare: false, max: 1 }); + try { + await runMigrations(dropCfg); + await s`ALTER TABLE sandboxes DROP COLUMN IF EXISTS python`; // simulate the N+1 drop + await expect(runMigrations(dropCfg)).resolves.toBeUndefined(); // pg_attribute guard → no error + } finally { + await s.end({ timeout: 5 }); + await admin!`SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = ${dropDb} AND pid <> pg_backend_pid()`; + await admin!.unsafe(`DROP DATABASE IF EXISTS ${dropDb}`); + } +}); +``` +(The existing second-run idempotency assertion in Test A stays.) + +### Phase 1: Success Criteria + +#### Phase 1: Programmatic Verification +- [ ] `pnpm typecheck` passes +- [ ] `pnpm lint:fix` passes (no remaining `python:` boolean in server code) +- [ ] `pnpm test:unit` passes (updated rehydrate/advisory-lock/sandboxes assertions green) +- [ ] `pnpm test:integration` migration test passes: `python_runtime` column + CHECK exist; legacy `python=true` row reads back `stdlib`; migration re-runs cleanly after a simulated `python` drop +- [ ] Server starts (`pnpm dev`) without migration errors + +#### Phase 1: Agent Verification +_(Dev-server protocol: see Success Criteria Guidelines — if no server is running for this worktree, ask the user to start `pnpm dev` or authorise the agent to manage it.)_ +- [ ] Against the running dev server, create a `stdlib` sandbox and a `null` (no `python_runtime`) sandbox; confirm **create-201, list, AND GET** all echo `python_runtime` +- [ ] In the `stdlib` sandbox, `bash_exec` `python3 -c "print(1)"` returns `1`, exit 0 +- [ ] In the `null` sandbox, `python3 -c "print(1)"` reports command-not-found (Python not registered) +- [ ] Agent reviews `postgres.ts` `getSandboxMeta`/`updateSandboxMeta` to confirm COALESCE read + dual-write are both present + +### Phase 1: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 1._ + +--- + +## Phase 2: `python_runtime` — client & contract surfaces (SDKs, MCP, OpenAPI, docs) + +Propagate the enum to every external representation (research Q5) and reconcile the second-class `network` field in the same coordinated break. + +### Changes + +#### 1. TypeScript SDK +**File**: `clients/typescript/src/models.ts` (:4-11, :81-90), `clients/typescript/src/client.ts` (:10-17, :72-80) +**Action**: modify + +- `models.ts`: add `export type PythonRuntime = "stdlib" | "pyodide" | null;`. In `SandboxRecord`, replace `python: boolean` with `python_runtime: PythonRuntime`; **add `network: boolean`** (reconcile asymmetry). In `sandboxRecordFromApi`, parse `python_runtime: (payload.python_runtime ?? null) as PythonRuntime`, `network: Boolean(payload.network)`. +- `client.ts`: in `CreateSandboxOptions`, replace `python?: boolean` with `python_runtime?: PythonRuntime`. In `create()`, replace the `options.python` block with `if (options.python_runtime !== undefined) body.python_runtime = options.python_runtime;`. + +#### 2. Python SDK +**File**: `clients/python/src/sqlfs/models.py` (:14-34), `clients/python/src/sqlfs/client.py` (:103-146) +**Action**: modify + +- `models.py`: add `PythonRuntime = Literal["stdlib", "pyodide"]` type alias (Optional in field). In `SandboxRecord`, replace `python: bool` with `python_runtime: Optional[PythonRuntime]`; **add `network: bool`**. In `from_api`, `python_runtime=payload.get("python_runtime"), network=bool(payload.get("network", False))`. +- `client.py`: in `create()`, replace `python: bool = False` param with `python_runtime: Optional[Literal["stdlib","pyodide"]] = None`; body: `if python_runtime is not None: body["python_runtime"] = python_runtime`. Update the docstring (drop `python:`, document `python_runtime`). + +#### 3. MCP tools +**File**: `src/api/mcp/tools.ts` (:34-78, :81-116, bash_exec prose :153-180) +**Action**: modify + +- `sandbox_create` input schema: replace `python: z.boolean().optional()` with `python_runtime: z.enum(["stdlib","pyodide"]).nullable().optional()`. Build `runtimeOptions = { pythonRuntime: args.python_runtime ?? null, javascript: args.javascript ?? false, network: false }`. `persistSandboxMeta` with `python_runtime: runtimeOptions.pythonRuntime`. Echo `python_runtime` in the response JSON. +- `sandbox_list` map: echo `python_runtime: s.python_runtime` (drop `python`). +- Update the `sandbox_create` tool **description** and `bash_exec` prose: explain `python_runtime: "stdlib"` (CPython WASM, air-gapped) vs `"pyodide"` (numpy/pandas/scipy/openpyxl, OS-isolated Deno subprocess). + +#### 4. OpenAPI spec +**File**: `src/api/openapi-spec.ts` (sandboxSchema :16-27, create body :321-322) +**Action**: modify + +- `sandboxSchema`: replace `python: { type: "boolean" }` with `python_runtime: { type: "string", enum: ["stdlib","pyodide"], nullable: true }`; **add `network: { type: "boolean" }`**. Update `required` → `["id","name","owner","createdAt","python_runtime","javascript","network"]`. +- Create-body schema: replace `python` with `python_runtime: { type: "string", enum: ["stdlib","pyodide"], nullable: true, description: "Python runtime: stdlib (CPython WASM) or pyodide (numpy/pandas/scipy/openpyxl, OS-isolated)" }`; **add `network`** to the create body too. + +#### 5. Changeset (version bump — do NOT hand-edit version/CHANGELOG) +**File**: `.changeset/.md` +**Action**: create via `pnpm changeset` — describe the breaking change (`python` boolean → `python_runtime` enum; add Pyodide runtime). Pick **major** (breaking API/SDK contract). + +#### 6. Plugin docs +**File**: `plugins/sql-fs/skills/api/ref/endpoints.md` (:119-168), `api/ref/bash.md` (:48-78), `py-sdk/ref/models.md` (:25-37), `py-sdk/ref/client.md` (:113-142), `py-sdk/SKILL.md` (:204-208), `api/SETUP.md` (:61) +**Action**: modify — replace `python` boolean rows/examples with `python_runtime` (values `stdlib`/`pyodide`/null), keep the immutability note (`endpoints.md:148`), and ensure `network` appears in the response shapes. + +#### 7. SDK + MCP tests +**Files**: `clients/python/tests/test_client.py` (:82-160), `clients/typescript/tests/sandboxes.test.ts` (:15-38), `src/api/tests/unit/mcp-tools.test.ts` / `mcp.test.ts`, examples (`clients/*/examples/*`) +**Action**: modify — assert `python_runtime` round-trips (request + record), `network` now present in `SandboxRecord`; add an MCP `sandbox_create`/`sandbox_list` assertion echoing `python_runtime`. + +### Phase 2: Success Criteria + +#### Phase 2: Programmatic Verification +- [ ] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass +- [ ] Python SDK suite passes (`cd clients/python && `); TS SDK suite passes (`cd clients/typescript && `) +- [ ] `mcp-tools.test.ts` passes with the new `python_runtime` assertions +- [ ] OpenAPI spec still serializes (server boots; `GET /openapi.json` valid JSON) and `.changeset/*.md` exists + +#### Phase 2: Agent Verification +- [ ] Agent diffs every research-Q5 surface against a checklist (Python SDK model+client, TS SDK model+client, MCP create+list, OpenAPI record+create, all docs) and confirms **no remaining boolean `python`** +- [ ] Agent confirms `network` now appears in both SDK `SandboxRecord` types and the OpenAPI record + create schemas + +### Phase 2: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 2._ + +--- + +## Phase 3: Offline assets + Deno harness (the untrusted side) + +Vendor the runtime assets and write the Deno-side harness that loads Pyodide offline, runs untrusted Python, locks down its realm, and speaks the committed IPC. Hardens S1/S2 into product. Standalone-testable without Node. **Gated by spike S1 + S2.** + +### Changes + +#### 1. Asset fetch + lock-build tooling +**File**: `scripts/fetch-pyodide-assets.mjs` (new), `scripts/build-pyodide-lock.mjs` (new) +**Action**: create + +- `fetch-pyodide-assets.mjs`: downloads the pinned **Deno** binary into `vendor/deno/` and the full **`pyodide-0.29.x`** distribution (wasm + `python_stdlib.zip` + numpy/pandas/scipy wheels + base `pyodide-lock.json`) into `vendor/pyodide/`. Pins versions in a constant at the top of the file. Idempotent (skips if present + checksum matches). +- `build-pyodide-lock.mjs`: runs Pyodide once (Node or Deno) to `micropip.freeze` the **openpyxl + et_xmlfile + transitive** set against the local distribution, producing a **custom `vendor/pyodide/pyodide-lock.custom.json`**. **Never hand-edit the lock.** If the freeze tooling is unavailable offline, the fallback is documented in the file header: re-run with network once on a build host, commit the resulting lock to the asset manifest (not the wheels). +- `.gitignore`: add `vendor/deno/` and `vendor/pyodide/` (assets are reproduced by the fetch script, not committed). + +#### 2. Shared protocol contract +**File**: `src/pyodide-runner/protocol.ts` (new) +**Action**: create — runtime-agnostic (`Uint8Array`/`DataView`, no `Buffer`/`Deno`). + +```ts +export const PROTOCOL_VERSION = 1; +export type FrameType = "run" | "result" | "error" | "ready"; + +export interface RunRequest { + readonly type: "run"; + readonly requestId: string; // random, set by Node + readonly seq: number; // monotonic per child + readonly generation: number; // child generation id + readonly code: string; // resolved script or -c body + readonly argv: readonly string[]; + readonly stdin: string; // base64 + readonly files: ReadonlyArray<{ path: string; mode: number; data: string /*base64*/ }>; + readonly cwd: string; +} + +export interface RunResponse { + readonly type: "result" | "error"; + readonly requestId: string; + readonly seq: number; + readonly generation: number; + readonly stdout: string; // base64 + readonly stderr: string; // base64 + readonly exitCode: number; + readonly created: ReadonlyArray<{ path: string; mode: number; data: string /*base64*/ }>; + readonly modified: ReadonlyArray<{ path: string; mode: number; data: string /*base64*/ }>; + readonly deleted: readonly string[]; +} + +// `ready` is a ONE-TIME pre-run handshake (no requestId/seq), validated separately +// from per-request frames — see ipc.ts integrity rules. It carries `generation` only. +export interface ReadyFrame { readonly type: "ready"; readonly generation: number; } +export type Frame = RunRequest | RunResponse | ReadyFrame; + +// Length-prefixed framing: 4-byte big-endian uint32 length + UTF-8 JSON body. +export function encodeFrame(obj: Frame): Uint8Array { /* DataView header + TextEncoder body */ } +export function decodeFrames(buf: Uint8Array): { frames: Frame[]; rest: Uint8Array } { /* parse complete frames */ } +``` + +#### 3. Deno harness (untrusted side) +**File**: `src/pyodide-runner/runner.ts` (new — Deno entry, **excluded from tsc build**) +**Action**: create + +- Resolve asset paths (`indexURL`, `lockFileURL` (the custom lock), `packageBaseUrl` — all local, derived from the absolute asset dir) from `Deno.args` (**argv only**, passed by Node). **Never `Deno.env`** — it is blocked by `--deny-env`. (`DENO_NO_UPDATE_CHECK` is set in Node's spawn env and read by the Deno runtime, not by the program.) +- `const pyodide = await loadPyodide({ indexURL, lockFileURL, packageBaseUrl, stdout, stderr })`; preload `await pyodide.loadPackage(["numpy","pandas","scipy","openpyxl"])`. +- **Realm lockdown BEFORE any untrusted Python:** capture the raw stdout writer (`Deno.stdout.write` bound), then delete `Deno`, `console`, and other write primitives from `globalThis`. All control-frame writes go through the captured writer only. +- IPC loop: read length-prefixed frames from stdin (`Deno.stdin`), for each `run` frame: stage `files` into MEMFS (`FS.mkdirTree` + `FS.writeFile`), reset Python `globals`, run `pyodide.runPythonAsync(code, ...)`, capture stdout/stderr via Pyodide stream callbacks into base64 fields, compute the `{created,modified,deleted}` diff against the staged input set, and emit exactly one `result`/`error` frame carrying the matching `requestId`/`seq`/`generation`. +- Between execs: fresh `globals` + wipe staged MEMFS paths (bounds variable scope + staged files; `sys.modules`/package globals persist within a session — same trust boundary, per design D3). + +#### 4. Build wiring +**File**: `package.json` (`build` script), `scripts/copy-postgres-migrations.mjs` sibling, `Dockerfile` +**Action**: modify + +- Add a `copy-pyodide-runner.mjs` (or extend the build) to copy `src/pyodide-runner/*.ts` → `dist/pyodide-runner/` raw (Deno runs the `.ts`). tsc compiles `protocol.ts` → `dist/pyodide-runner/protocol.js` for the Node side; **exclude `src/pyodide-runner/runner.ts`** from `tsconfig.json` `include`/add to `exclude` (Deno globals). +- `Dockerfile`: in the builder stage run `node scripts/fetch-pyodide-assets.mjs && node scripts/build-pyodide-lock.mjs`; `COPY --from=builder --chown=app:app /app/vendor ./vendor` into the runtime stage, and copy `dist/pyodide-runner`. Set `ENV PYODIDE_ASSET_DIR=/app/vendor/pyodide DENO_BIN_PATH=/app/vendor/deno/deno`. + +### Phase 3: Success Criteria + +#### Phase 3: Programmatic Verification +- [ ] `node scripts/fetch-pyodide-assets.mjs && node scripts/build-pyodide-lock.mjs` produce `vendor/pyodide/` + the custom lock +- [ ] `pnpm typecheck` passes (protocol.ts compiles; runner.ts excluded) +- [ ] Running the built `runner.ts` under the committed flags with a fixture `run` frame on stdin returns a valid `result` frame whose pandas→openpyxl output bytes decode correctly — **zero network**: + `deno run --allow-read=$PYODIDE_ASSET_DIR dist/pyodide-runner/runner.ts < fixture-frame.bin` +- [ ] `pnpm lint:fix` passes + +#### Phase 3: Agent Verification +- [ ] Agent re-runs an S2 forge attempt against the built `runner.ts`; confirms the forged frame is **not** emitted on the control channel +- [ ] Agent confirms the deny-belt blocks remote import, npm import, update check, FS write, env read, subprocess spawn, FFI, and network (each attempt fails closed) +- [ ] Agent reviews `runner.ts` to confirm realm lockdown happens **before** the first untrusted `runPythonAsync` + +### Phase 3: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 3._ + +--- + +## Phase 4: Node-side `PyodideSandbox` manager + IPC client + +The trusted Node half: spawn/own the Deno subprocess, frame the protocol with full integrity checks, serialize execs, enforce throw-not-return cancellation and respawn. No session wiring yet — unit-testable in isolation. **Gated by S2.** + +### Changes + +#### 1. IPC framing + integrity +**File**: `src/api/pyodide/ipc.ts` (new) +**Action**: create — imports types from `../../pyodide-runner/protocol.js`. + +- `encodeFrame(obj): Buffer` / `decodeFrames(chunk: Buffer): { frames: Frame[]; rest: Buffer }` (length-prefixed; mirror the protocol helpers, Node `Buffer` variant). +- `validateInbound(frame, expected): void` — **schema-validate + enforce integrity on every inbound frame.** + - **`result`/`error` frames:** match `requestId`, monotonic `seq`, exact `type`, current `generation`, and **exactly one response per request**. + - **`ready` handshake (explicit exception):** carries `generation` only (no `requestId`/`seq`). It is valid **exactly once**, **before any `result`/`error`**, and **only with the current `generation`**. A second `ready`, a `ready` after the first response, or a stale/wrong-generation `ready` is an integrity violation → kill the child. (The handshake marks the `starting → idle` transition.) +- **Size caps measured on the base64-encoded wire size** (accounts for ~33% expansion): `PYODIDE_MAX_FRAME_BYTES` per frame + `PYODIDE_MAX_AGGREGATE_BYTES` aggregate per response. +- Any malformed / oversized / duplicate / out-of-sequence / wrong-generation / unexpected frame → throw a typed `IpcIntegrityError` that the manager turns into **kill-the-child**. + +#### 2. Manager / worker state machine +**File**: `src/api/pyodide/manager.ts` (new) +**Action**: create + +```ts +export type WorkerState = "cold" | "starting" | "idle" | "busy" | "terminating" | "dead"; + +export class PyodideSandbox { + readonly state: WorkerState; + readonly generation: number; + run(req: RunRequest, signal: AbortSignal): Promise; + dispose(): Promise; +} +``` + +- Spawns the child via `child_process.spawn` (**no shell** — so `$VAR` is NOT expanded). **Node resolves `PYODIDE_ASSET_DIR` and `DENO_BIN_PATH` from its own parent config/env BEFORE spawn**, then passes the **absolute** asset dir **literally** in both `--allow-read=` and as a runner argv (never via the child env — it is scrubbed). The child env is **only** `{ DENO_NO_UPDATE_CHECK: "1" }` (no `AUTH_SECRET`/`DATABASE_URL`; Node does NOT inherit the parent env when `env` is given). Sketch: + ```ts + const assetDir = process.env.PYODIDE_ASSET_DIR!; // resolved by Node, absolute + const denoBin = process.env.DENO_BIN_PATH ?? "deno"; + spawn(denoBin, [ + "run", ...COMMITTED_FLAGS, `--allow-read=${assetDir}`, + "dist/pyodide-runner/runner.ts", assetDir, // asset dir via ARGV, not env + ], { env: { DENO_NO_UPDATE_CHECK: "1" }, stdio: ["pipe", "pipe", "pipe"] }); + ``` +- **Per-subprocess mutex/queue** serializes `run()` (reuse the `SemaphoreWaiter`-style abort cleanup pattern from `session-manager.ts:1180-1219`). +- **Cancellation is state-dependent — never kill the child out from under an innocent active request:** + - **Abort while still queued (before this call acquires the mutex):** remove only this waiter and reject it with `AbortError`. **Do NOT kill the child** — another `run()` may be actively executing on it; killing would terminate the wrong request and corrupt its generation. The active call is unaffected. + - **Abort after this call acquires the mutex** (it now owns the child — during init/preload or mid-run), **or internal runtime timeout** (`PYODIDE_RUNTIME_TIMEOUT_MS`): `child.kill("SIGKILL")` and **retire the generation**. +- **Cancellation = throw, never return** (for the kill cases above): route abort (external `signal`) → reject with `AbortError` (`name:"AbortError"`, `code:"ABORTED"`); internal timeout → throw a typed `PyodideTimeoutError`. Returning a normal `{exitCode}` from a timed-out/aborted run is forbidden. +- On unexpected child exit (or any `IpcIntegrityError`): mark `dead`, reject the in-flight `run()`, and **respawn lazily with an incremented `generation`** on the next `run()`. + +#### 3. Unit tests +**File**: `src/api/pyodide/tests/unit/manager.test.ts`, `src/api/pyodide/tests/unit/ipc.test.ts` (new) +**Action**: create — use a **fake child** (a stub `runner` script or a mock process that echoes frames) so no real Deno/Pyodide is needed. + +Cover (each names the design decision it protects): +- serialization order of two overlapping `run()` calls; +- **abort while still queued (before acquire)** → removes only that waiter, rejects it with `AbortError`, **does NOT kill the child**, and a concurrently-active `run()` still completes normally; +- **abort after acquiring the mutex / abort during init/preload** (this call owns the child) → kills the child, retires the generation, rejects/throws (never returns a normal result); +- malformed / oversized / duplicate / out-of-sequence / **stale-generation** / forged frame → each kills the child; +- **`ready`-handshake integrity** — a duplicate `ready`, a `ready` arriving after the first `result`, or a wrong-generation `ready` kills the child; +- **base64 expansion counted against the size cap** (a payload whose raw bytes are under cap but base64 wire size is over → rejected); +- respawn-on-exit increments `generation`. + +### Phase 4: Success Criteria + +#### Phase 4: Programmatic Verification +- [ ] `pnpm typecheck && pnpm lint:fix` pass +- [ ] `pnpm test -- src/api/pyodide/tests/unit/manager.test.ts src/api/pyodide/tests/unit/ipc.test.ts` pass: serialization; abort-while-queued (waiter removed, child survives, active call unaffected); abort-after-acquire/during-init (kills child, retires generation); malformed/oversized/duplicate/stale-generation/forged-frame and `ready`-handshake violations each kill the child; base64-aware cap; respawn bumps generation +- [ ] `pnpm test:unit` passes (no regressions) + +#### Phase 4: Agent Verification +- [ ] Agent exercises `manager.run()` with two overlapping calls against the fake child and confirms: they serialize; aborting the **queued** call removes its waiter and rejects only it while the active call completes (the child is NOT killed); aborting the **active** call (or an internal timeout) kills the child and retires the generation +- [ ] Agent reviews `manager.ts` to confirm the spawn uses a scrubbed `env` (no secrets) and the committed flag set verbatim + +### Phase 4: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 4._ + +--- + +## Phase 5: `pyodide` custom commands + file staging drain (core requirement) + +Wire `python_runtime: "pyodide"` to register `python3`+`python` custom commands backed by a per-session `PyodideSandbox` owned as a first-class `Session` field, with cwd-scoped diff-and-drain into SqlFs. **Delivers the issue's core requirement.** + +### Changes + +#### 1. Pyodide custom commands +**File**: `src/api/commands/pyodide-command.ts` (new) +**Action**: create — modeled on `node-command.ts` (`defineCommand`). + +```ts +export function createPyodideCommands(session: Session): CustomCommand[] { + const handler = async (args: string[], ctx: CommandContext): Promise => { /* shared */ }; + return [defineCommand("python3", handler), defineCommand("python", handler)]; +} +``` + +- Parse the built-in `python3` surface: `-c CODE`, script `FILE`, `-` / stdin, `--version`/`-V`, `-m MODULE` (reject with a clear stderr if unsupported), bare → exit hint. +- Resolve the script path via `ctx.fs.resolvePath(ctx.cwd, arg)`; `--version` → `"Python 3.x (Pyodide)\n"`. +- Build the `RunRequest`: stage the **cwd subtree** + the **resolved script path** (even if outside cwd, for `python3 FILE` parity) into `files`; carry `argv`, `stdin` (base64), `cwd`. +- Call `session.pyodideSandbox.run(req, ctx.signal!)`. On a successful `result`, **drain** (below) and return `{ stdout, stderr, exitCode }` (decode base64). On timeout/abort → the manager throws; let it propagate (so `bash.exec` rejects → script-tx rollback). + +#### 2. First-class session ownership + wiring +**File**: `src/api/session-manager.ts` +**Action**: modify `Session` interface (:148-209), `getOrCreate` (:462-540), teardown paths (destroy :1039, reaper :1146-1153, shutdown :1098-1118, failed-create :545-551) + +- Add `pyodideSandbox?: PyodideSandbox;` to `Session`. +- In `getOrCreate`, when `resolvedRuntime.pythonRuntime === "pyodide"`: construct the manager (lazily-spawning; capacity reserved via the Phase 6 residency registry), assign `session.pyodideSandbox`, and push `...createPyodideCommands(session)` into `customCommands` (keep `python: undefined`). The `session` object is built before the commands need it — capture it in the closure (build commands right after the `session` literal, then `bash` already constructed... note ordering: `Bash` is constructed at :487 before the `session` object at :519). **Resolution:** construct the `PyodideSandbox` and the commands using a forward reference object, OR build the command list with a late-bound `getSession` thunk. Simplest: create a small holder `const sessionRef: { current?: Session } = {}`, build commands closing over `sessionRef`, then set `sessionRef.current = session` after the literal. Document this in the file. +- **Teardown — kill the child on every path.** Add a helper `disposePyodide(session)` (best-effort `await session.pyodideSandbox?.dispose()`) and call it alongside `disconnectFs` in: `destroy` (`finally` block ~:1038), `runReaper` (`.finally` ~:1151), `shutdown` (per-session cleanup ~:1112), and the `getOrCreate` failure `catch` (~:545, dispose any partially-built manager before rethrow). + +#### 3. Diff-and-drain into SqlFs +**File**: `src/api/commands/pyodide-command.ts` (drain helper) +**Action**: create (within the command module) + +- Drain runs **only on a successful run**, inside the existing script transaction (the command runs within `execWithRuntimeThrottle`'s `scriptTx` scope, so writes to `ctx.fs` are atomic and roll back if the handler later throws). **Never drain on timeout/abort/protocol-invalid** (the manager threw → no `result`). +- For each entry in `created`/`modified`: **validate the path stays under `ctx.cwd`** (reject `..`, absolute-outside-cwd, null bytes via a normalize-and-prefix check); reject symlinks (SqlFs default-deny); apply dirs-before-files; `ctx.fs.writeFile(path, bytes)` (0644 default), `ctx.fs.chmod` only if a non-default exec bit is needed. For `deleted`: delete depth-first. +- **Per-file + total byte caps both directions** (`PYODIDE_MAX_FILE_BYTES`, `PYODIDE_MAX_TOTAL_BYTES`) — enforced on staging (Node→Deno) and on drain (Deno→Node). +- A **read-only exec** that produced MEMFS mutations must be rejected: when running under a read-only scope, `ctx.fs.writeFile` already throws `EREADONLY` → surfaces as `EREADONLY_VIOLATION` (existing remap in `withSessionReadEntry`). The drain must attempt the write (not silently drop) so the violation fails closed. + +#### 4. Integration test +**File**: `src/api/tests/integration/pyodide.integration.test.ts` (new) +**Action**: create — `describe.skipIf(!process.env.DATABASE_URL)`. Requires the vendored assets + Deno present (skip with a clear message if `PYODIDE_ASSET_DIR`/`DENO_BIN_PATH` missing). + +- Create a `pyodide` sandbox; write `data.csv`; `python3 analyze.py` does `import pandas`, reads the CSV, writes `out.xlsx`; assert `out.xlsx` is **retrievable via the files API** and is a valid xlsx. +- A drain path containing `..`/absolute is rejected. +- A **read-only exec** whose script mutates MEMFS is rejected with `EREADONLY_VIOLATION`. +- An **abort after the child responds but before drain completes** drains nothing (assert no partial files). +- **reaper / destroy / shutdown / failed-create** each kill the child (assert via a dispose spy / process-liveness check). + +### Phase 5: Success Criteria + +#### Phase 5: Programmatic Verification +- [ ] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass +- [ ] `pnpm test -- src/api/tests/integration/pyodide.integration.test.ts` passes (with assets + Deno present): CSV→`python3 analyze.py`→`out.xlsx` retrievable; `..`/absolute drain rejected; read-only MEMFS mutation → `EREADONLY_VIOLATION`; abort-before-drain drains nothing; all teardown paths kill the child +- [ ] Full `pnpm test:integration` green with both containers up + +#### Phase 5: Agent Verification +_(Dev-server protocol applies.)_ +- [ ] Against the running dev server, in a `pyodide` sandbox: run a `-c` one-liner (`python3 -c "import pandas; print(pandas.__version__)"`) and a script-file form; confirm stdout, exit code, and that a written file persists via the files API +- [ ] Agent reviews `pyodide-command.ts` drain to confirm path validation (reject `..`/absolute/null-byte) runs before any `ctx.fs` write and that drain is skipped when the manager throws + +### Phase 5: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 5._ + +--- + +## Phase 6: Concurrency semaphore + atomic-admission residency LRU + memory posture + +Bound in-flight `pyodide` execs and resident subprocesses independently (both required per design §5), with atomic admission over the worker state machine. **Gated by S3.** + +### Changes + +#### 1. Dedicated concurrency semaphore +**File**: `src/api/session-manager.ts` +**Action**: modify — add `pyodideSem` alongside `pythonSem`/`jsSem` (:313-314, built :357-374), route in `execWithRuntimeThrottle` (:1235-1308) + +- New `pyodideSem`: `MAX_CONCURRENT_PYODIDE` (default **2**), `MAX_PYODIDE_QUEUE` (default 100), `PYODIDE_QUEUE_TIMEOUT_MS` (default 60000) — mirror the python set; add corresponding `SessionManagerOptions` (`maxConcurrentPyodide?`). +- In `execWithRuntimeThrottle`: `const usesPyodide = session.runtimeOptions.pythonRuntime === "pyodide" && PYTHON_INVOCATION_REGEX.test(script);`. Acquire/release `pyodideSem` for `usesPyodide` (independent of `usesPython`, which stays `stdlib`-only). Preserve deadlock-avoidance acquire order + rollback-on-failure (extend the existing python→js pattern to pyodide). + +#### 2. Residency registry (atomic admission) +**File**: `src/api/pyodide/residency.ts` (new) +**Action**: create + +```ts +export class PyodideResidency { + constructor(opts: { maxResident: number; idleMs: number }); + // Atomic critical section: reserve slot → select idle eviction victim → + // (caller) spawn → rollback on failed init. Never evicts starting/busy. + admit(spawn: () => Promise): Promise; + release(worker: PyodideSandbox): void; +} +``` + +- Global cap `MAX_RESIDENT_PYODIDE` (default **2**). An **admission mutex** (reuse `async-mutex`, already a dependency) wraps *reserve → select victim → spawn → rollback-on-fail* as one critical section so concurrent cold starts cannot both observe a free slot. +- Eviction targets only `idle` workers; **`starting`/`busy` are never evictable.** +- `PYODIDE_IDLE_MS` (default e.g. **120000**, must be `< SESSION_IDLE_MS`) idle-kills resident subprocesses on a timer. +- **Startup invariant:** assert `MAX_RESIDENT_PYODIDE >= MAX_CONCURRENT_PYODIDE` — throw on violation (fail boot). +- Wire `getOrCreate` (Phase 5) to obtain the manager **through** `residency.admit(...)` and `dispose` paths to call `residency.release(...)`. + +#### 3. Memory posture (design D5, accepted availability risk) +**File**: `src/api/pyodide/manager.ts` (best-effort), `CLAUDE.md` (env table + posture note) +**Action**: modify + +- Best-effort cgroup `memory.max` per child **only where supported** (gated on S3 finding); otherwise no-op. The **container memory limit is the documented real guard** — no per-child OOM isolation guarantee. +- Document all new env vars in the `CLAUDE.md` Environment Variables table: `MAX_CONCURRENT_PYODIDE`, `MAX_PYODIDE_QUEUE`, `PYODIDE_QUEUE_TIMEOUT_MS`, `MAX_RESIDENT_PYODIDE`, `PYODIDE_IDLE_MS`, `PYODIDE_RUNTIME_TIMEOUT_MS`, `PYODIDE_MAX_FILE_BYTES`, `PYODIDE_MAX_TOTAL_BYTES`, `PYODIDE_MAX_FRAME_BYTES`, `PYODIDE_MAX_AGGREGATE_BYTES`, `PYODIDE_ASSET_DIR`, `DENO_BIN_PATH`, and the memory-posture/`MAX_RESIDENT × per-proc ceiling` sizing guidance. + +#### 4. Tests +**File**: `src/api/pyodide/tests/unit/residency.test.ts` (new), `src/api/tests/unit/session-manager.test.ts` (extend) +**Action**: create / modify + +- `residency.test.ts`: concurrent admissions **never exceed** `MAX_RESIDENT_PYODIDE`; LRU **never evicts a `busy` or `starting` worker**; idle-kill fires after `PYODIDE_IDLE_MS`; failed init rolls back the reserved slot. +- `session-manager.test.ts`: N concurrent `pyodide` execs queue + `queue_full`/`wait_timeout`; the startup invariant violation fails construction; `stdlib` routing still uses `pythonSem` (unaffected). + +### Phase 6: Success Criteria + +#### Phase 6: Programmatic Verification +- [ ] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass +- [ ] `pnpm test -- src/api/pyodide/tests/unit/residency.test.ts` passes: concurrent admissions ≤ cap, LRU spares busy/starting, idle-kill, rollback +- [ ] `session-manager.test.ts` passes: pyodide queue full / wait timeout; invariant violation fails startup; `stdlib` routing unchanged + +#### Phase 6: Agent Verification +_(Dev-server protocol applies.)_ +- [ ] Agent drives more than `MAX_RESIDENT_PYODIDE` concurrent `pyodide` sessions; confirms an idle subprocess is killed and an evicted session cold-starts on its next exec +- [ ] Agent reviews `residency.ts` to confirm the admission mutex covers reserve→select→spawn→rollback as one critical section + +### Phase 6: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 6._ + +--- + +## Phase 7: Adversarial escape suite (first-class security acceptance — merge gate) + +Prove the boundary holds. This suite is the security sign-off gate (design Open Risk: C1 reversal). + +### Changes + +#### 1. Escape + deny-belt suite +**File**: `src/api/pyodide/tests/integration/escape.integration.test.ts` (new) +**Action**: create — `describe.skipIf(!assetsPresent)`. + +- Each escape **fails closed** (no secret read, no network, no host-FS reach — assert capability denial, not just a thrown error): + `import js; js.process.env`, `js.fetch(...)`, `pyodide.code.run_js(...)`, `ctypes.CDLL(None)`, `import('node:child_process')`. +- **Deny-belt coverage:** remote import, npm import, update check, filesystem write, env read, subprocess spawn, FFI (`Deno.dlopen`), network — each denied. +- **Fresh-globals isolation:** two execs in one session do not share Python `globals` (design Open Risk: intra-session state) — assert a variable set in exec 1 is undefined in exec 2. + +#### 2. Frame-forgery suite +**File**: `src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` (new) +**Action**: create + +- Escaped JS attempting `Deno.stdout.write`/`console.log`/raw-fd **cannot** forge a control frame, replay a stale-generation frame, forge or replay a `ready` handshake (duplicate / post-response / wrong-generation), nor redirect a drain write outside cwd (assert the Node side kills the child and drains nothing). + +### Phase 7: Success Criteria + +#### Phase 7: Programmatic Verification +- [ ] `pnpm test -- src/api/pyodide/tests/integration/escape.integration.test.ts src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` pass (every escape blocked, every deny-belt item denied, fresh-globals isolation holds, control-frame + `ready`-handshake forgery/replay and drain-redirect blocked) +- [ ] Full `pnpm test:integration` green with both containers up + assets present + +#### Phase 7: Agent Verification +- [ ] Agent reviews each escape assertion and confirms it proves **capability denial** (no secret/net/FS), not merely a thrown error +- [ ] Agent confirms S3 memory behaviour is exercised or explicitly documented as the accepted availability risk + +### Phase 7: Discoveries and Notable Information +_Placeholder — filled by the implementing agent during/after Phase 7._ + +--- + +## Human Verification +_Performed once by a person after the **entire plan** is complete — the only human-testing step; not per phase._ + +- [ ] **Security sign-off:** a maintainer/security reviewer reads the escape suite, confirms it covers the C1-reversal surface, and **accepts the single-layer threat model** (Deno/V8 runtime escape out of scope) for the deployment +- [ ] **Memory under load on the target host:** run `MAX_RESIDENT_PYODIDE` heavy scripts concurrently on the real container; confirm behaviour matches the accepted availability risk (no surprise parent OOM beyond what's documented) and that defaults (+`PYODIDE_IDLE_MS`) are sane +- [ ] **Cold-start UX:** time a first `pyodide` exec (Deno spawn + Pyodide init + package load); confirm the exec timeout default accommodates it for the LibreChat loop +- [ ] **Image portability:** confirm the vendored Deno + Pyodide assets build and run on a clean self-hosted install (image size acceptable) +- [ ] **End-to-end issue acceptance:** a human runs the real workflow (upload CSV → analyze → download xlsx) through the product UI/LibreChat and confirms the output file is correct diff --git a/thoughts/issue-118-pyodide-runtime/questions.md b/thoughts/issue-118-pyodide-runtime/questions.md new file mode 100644 index 0000000..66eee8b --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/questions.md @@ -0,0 +1,34 @@ +# Research Questions + +## Context + +Focus on the sandbox runtime layer of this repository and the surfaces that describe a sandbox's capabilities: + +- The sandbox-create flow: HTTP route + request validation, the `SandboxMeta` / `SandboxListEntry` / `RuntimeOptions` types, and how the `python` / `javascript` / `network` capability flags travel from request to persistence to response (`src/api/routes/sandboxes.ts`, `src/sql-fs/types.ts`, `src/api/session-manager.ts`). +- The session manager's integration with the `just-bash` library: how `new Bash({...})` is constructed, how `customCommands` are registered, and the existing custom command under `src/api/commands/`. +- Concurrency and execution controls: the per-runtime semaphores, their env-var limits, and exec timeout/cancellation. +- The `sandboxes` database table and its migrations across SQL dialects (`src/sql-fs/dialects/`, `src/sql-fs/migrations/`). +- The capability-field surface across the Python SDK (`clients/python/`), TypeScript SDK (`clients/typescript/`), the MCP tools (`src/api/mcp/`), the OpenAPI spec (`src/api/openapi-spec.ts`), and the plugin skill docs (`plugins/sql-fs/skills/`). +- The WASM Python execution path and how sandbox files are read by executing code. + +Three questions target external-technology research (web) — Q7 (Pyodide runtime), Q8 (Pyodide security posture), and Q9 (isolation architecture / Deno); the rest are codebase-grounded. (Q8–Q9 were added during the design phase, when the security-isolation dimension surfaced as a gap in the original set — see `design.md`.) + +## Questions + +1. Trace the full lifecycle of a sandbox's runtime capability flags (`python`, `javascript`, `network`): how they are validated on the create request, represented in `SandboxMeta` / `SandboxListEntry` / `RuntimeOptions`, persisted to and read back from the database, rehydrated when a warm session is rebuilt, and echoed in create / get / list responses. Which files and functions form this chain, and which tests cover the persistence and session-rehydrate paths? + +2. How does the session manager construct a `just-bash` `Bash` instance, and how does the `customCommands` mechanism work? Specifically: how is the existing custom command in `src/api/commands/` defined (`defineCommand`), in what order are custom commands registered relative to built-ins, can a custom command shadow a built-in of the same name, and what does the `CommandContext` / `ctx.fs` (`IFileSystem`) interface expose to a command at execution time (e.g. `readFile`, `readdir`, `stat`, `resolvePath`, path/cwd, stdin, and any `getAllPaths` extension)? Also document the command I/O contract a command must satisfy: how arguments are received and parsed (a script-path argument versus an inline `-c "code"` form), how stdin is forwarded, how the working directory is set, and how stdout, stderr, and the exit code are returned — and how the existing built-in `python3` (and its `python` alias) handles these forms today. + +3. How are concurrent and long-running executions controlled? Document the per-runtime semaphores (limits, env vars such as `MAX_CONCURRENT_PYTHON` / `MAX_CONCURRENT_JS`, queueing/wait-timeout behaviour), how a given exec is matched to the correct semaphore (e.g. the invocation regex), and how an exec's wall-clock timeout and cancellation (AbortController / client disconnect) are enforced around `bash.exec`. Also note which tests exercise semaphore routing and queueing. + +4. How is the `sandboxes` table defined and migrated? Document where the table schema and its `python` / `javascript` / `network` columns live, how migrations are structured and applied per SQL dialect, which dialect migration sets actually exist in the repo today (postgres / mysql / azure-sql), and how DDL changes are normally introduced and verified (including the integration migration test suite). + +5. Across the Python SDK, TypeScript SDK, MCP `sandbox_create` / `sandbox_list` tools, OpenAPI spec, and plugin skill docs, how is the sandbox `python` capability currently represented end-to-end — the request field, the response/record field, type definitions, tool/spec descriptions, and documentation examples? Enumerate every location that would have to stay consistent if that field's shape changed, and identify the test files that exercise each layer (route/handler unit tests, MCP tool tests, and the Python/TypeScript SDK client tests). + +6. When executing code reads or writes sandbox files, how do files move between SqlFs and the runtime, in both directions, and what size limits apply? On the read side: trace how the current WASM `python3` path obtains file contents, what file-read/-list APIs `SqlFs` and the `CommandContext` expose, where any per-file or per-read size caps are defined (and what each cap actually governs — e.g. `REDIS_BLOB_MAX_BYTES`), and whether there is a true `readFile` size limit versus a caching threshold. On the write side: how are files that executing code creates, modifies, or deletes persisted back to SqlFs after a run — what write/delete/mkdir APIs does the `IFileSystem` expose, do they handle binary content, directory creation, and symlinks, and how is such write-back currently detected and tested (e.g. with a mock `ctx.fs`)? + +7. (Web research.) What is the current stable release of the `pyodide` npm package — distinguishing the npm `latest`/stable tag from any prerelease/alpha versions the documentation may point at — and how is it documented to run inside a server-side Node.js process? Cover: the `loadPyodide()` initialization (including offline/pre-bundled `indexURL` usage without network fetches), the `pyodide.FS` Emscripten filesystem API for writing inputs and reading back outputs, how packages like `numpy` / `pandas` / `scipy` / `openpyxl` are loaded and whether they are available offline, `runPythonAsync` and stdout/stderr capture, and the documented memory footprint, cold-start time, and package/bundle size. Also note `just-bash`'s current documented behaviour for its built-in `python3` (CPython WASM) command and whether `customCommands` are documented to override built-ins. + +8. (Web research; added during design.) What is the security posture of running `pyodide` server-side under Node.js for **untrusted / LLM-generated** Python — does Pyodide isolate that code, or can it reach the host? Enumerate the Python→JS escape vectors (`import js` → `globalThis`/`process.env`/`fetch`, `pyodide.code.run_js`, the `pyodide.ffi`/`JsProxy` prototype chain, `ctypes`, `object.__subclasses__()`, `_pyodide._base.eval_code`), and determine whether configuration-level hardening (`jsglobals`, `unregisterJsModule("js")`, a restricted `runPython` globals dict) can fully sever them. What is the project's official/maintainer stance on untrusted code, and what real-world sandbox-escape CVEs exist? How does `worker_threads` env-scrubbing and network access factor into the boundary? + +9. (Web research; added during design.) Given that in-process Pyodide is not a security boundary, what isolation architecture is recommended for running untrusted Pyodide server-side, and is a **Deno subprocess** a viable boundary? Cover: the OS-level options used in production (gVisor, Firecracker microVM, separate process + `seccomp`/namespaces) and who uses them; the Deno permission model (`--no-prompt`, the `--deny-*` flags, scoped `--allow-read`, env scrubbing via `clearEnv`, and how unix-socket vs `stdin`/`stdout` transports interact with those permissions); Pyodide-on-Deno feasibility and prior art; offline package staging under those constraints (the built-in numpy/pandas/scipy wheels vs. openpyxl + `et_xmlfile` via lockfile tooling); and whether per-child memory limits (`prlimit`/cgroup) are achievable in a slim, non-root Node image. diff --git a/thoughts/issue-118-pyodide-runtime/research.md b/thoughts/issue-118-pyodide-runtime/research.md new file mode 100644 index 0000000..fa2ce11 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/research.md @@ -0,0 +1,127 @@ +# Research Findings + +Scope: sandbox runtime layer of `sql-fs-api` and the surfaces describing a sandbox's capabilities. `just-bash` is pinned at **3.0.1** (consumed as a dependency; not modified). All `file:line` refs are relative to repo root unless noted. + +--- + +## Q1: Lifecycle of the runtime capability flags (`python`, `javascript`, `network`) + +### Findings +- **Validation** — `src/api/routes/sandboxes.ts:15-23` defines an inline Zod `createBodySchema` with `python/javascript/network: z.boolean().optional()`. Parsed at `:43` (`safeParse`), 400 on failure at `:46`. Defaults applied at `:50-52` (`?? false`); absent body leaves all `false` (`:37-39`). The generic `validateBody` middleware in `src/api/validation.ts` is **not** used here. +- **Types** — `SandboxMeta` (`src/sql-fs/types.ts:61-70`) and `SandboxListEntry` (`:73-82`) both hold all three as non-optional `readonly boolean`. `RuntimeOptions` lives in `src/api/session-manager.ts:110-120`; `DEFAULT_RUNTIME_OPTIONS = {python:false, javascript:false, network:false}` at `:122`. +- **Persistence** — `dialect.createSandbox()` (`src/sql-fs/dialects/postgres.ts:263`) inserts only `id/root_inode/owner`; flags default `false` in DB. Route then calls `sessionManager.persistSandboxMeta()` (`sandboxes.ts:104-111`) → `persistSandboxMetaFn` (`session-manager.ts:961-965`) → wired in `server.ts:112-120` → `PostgresDialect.updateSandboxMeta()` (`postgres.ts:345-362`, `UPDATE sandboxes SET owner,name,python,javascript,network`). Read back via `getSandboxMeta()` (`postgres.ts:316-343`, `SELECT ... python, javascript, network`). +- **Rehydrate** — On cold start both `rehydrateAndExec()` (`session-manager.ts:930-959`) and `rehydrateAndExecRead()` (`:883-906`) call `getSandboxMetaFn`, then build `resolvedRuntime` from `meta` (`:945-947`) and pass to `getOrCreate()` (`:438-560`), which threads flags into `new Bash(...)` at `:487-497`. Stored on `Session.runtimeOptions` (`:522`) for later throttle decisions. +- **Responses** — Create 201 echoes all three (`sandboxes.ts:134`). List maps all three per entry (`:142-159`, backed by `postgres.ts:364-402`). **Single GET (`:179-196`) does NOT echo the flags** in either warm-hit or cold branch. +- **Tests** — Persistence: `src/sql-fs/dialects/tests/unit/postgres.advisory-lock.test.ts:131-170`; `src/api/tests/unit/sandboxes.test.ts:64-101` (network flag). Rehydrate: `src/api/tests/unit/session-manager.rehydrate.test.ts:63-71, 114-150` (asserts rehydrated `runtimeOptions` carry `python:true`). + +--- + +## Q2: `Bash` construction, `customCommands`, `CommandContext`, built-in `python3` + +### Findings +- **Bash construction** — `src/api/session-manager.ts:487-497`: `new Bash({ fs, python: resolvedRuntime.python || undefined, javascript: ...|| undefined, network: resolvedRuntime.network ? {dangerouslyAllowFullInternetAccess:true} : undefined, defenseInDepth, customCommands: len>0 ? customCommands : undefined })`. Falsy flags coerce to `undefined` (built-in not registered). `BashOptions` type at `node_modules/just-bash/dist/Bash.d.ts:52-182`. +- **customCommands precedence** — `customCommands: CustomCommand[]` (`Bash.d.ts:110-126`). Registry is a single `Map` (`types.d.ts:241`). Built-ins loaded first, then python/js commands, then customCommands via `registerCommand` (`Bash.d.ts:247`, `this.commands.set(name, cmd)`). **A custom command of the same name overwrites a built-in** — JSDoc at `Bash.d.ts:111` states "These take precedence over built-ins with the same name." So a custom `python3`/`python` would shadow the built-in. +- **Existing custom command** — `src/api/commands/node-command.ts:48` `nodeCommand = defineCommand("node", async (args, ctx) => {...})`. Registered only when `resolvedRuntime.javascript` is true (`session-manager.ts:478-485`). Delegates to `js-exec` via `ctx.exec("js-exec ...")` for `-e CODE`, `FILE`, `--version`; bare/help → exit 127/0 hint. Uses `shellQuote` for safe embedding. +- **CommandContext / ctx.fs** — `CommandContext` (`types.d.ts:123-229`) exposes `fs: IFileSystem`, `cwd: string`, `env: Map`, `stdin: ByteString`, optional `exec?`, `fetch?`, `signal?`, etc. `ctx.fs` (`fs/interface.d.ts:96-231`) exposes async `readFile/readFileBuffer/readFileBytes?/writeFile/appendFile/exists/stat/lstat/mkdir/readdir/readdirWithFileTypes?/rm/cp/mv/chmod/symlink/link/readlink/realpath/utimes` plus **synchronous** `resolvePath(base,path)` and `getAllPaths()`. No `cwd` on `fs`; working dir is `ctx.cwd`. +- **I/O contract** — `args: string[]` excludes the command name; the handler parses script-path vs `-c CODE` itself. `ctx.stdin` is a `ByteString` (decode via `decodeBytesToUtf8`/`latin1FromBytes`). Working dir is `ctx.cwd`; resolve args via `ctx.fs.resolvePath(ctx.cwd, arg)`. Handler returns `Promise` = `{stdout, stderr, exitCode, stdoutKind?, env?}` (`types.d.ts:12-40`). +- **Built-in python3** — `node_modules/just-bash/dist/bin/chunks/python3-BQWDPUBM.js:3-13`. Arg parser handles `-c CODE`, `-m MODULE`, `--version`/`-V`, `--`, script FILE, `-` (stdin), bare. `--version` returns `"Python 3.13.2 (Emscripten)\n"`. Script-file form: `fs.resolvePath` → `fs.exists` (ENOENT→exit 2) → `fs.readFile`. Then `Y(code, ctx, scriptTag, scriptArgs)` spawns a Node `worker_threads` Worker (`./worker.js`) via `DefenseInDepthBox.runTrusted`, with a `SharedArrayBuffer` bridge for FS access; timeout `limits?.maxPythonTimeoutMs ?? 10000` (60000 if `fetch` present); per-`fs` queue serializes Python exec (`WeakMap`). stdout/stderr collected via the SharedArrayBuffer bridge. `pythonCommand` (`python`) is a thin alias. + +--- + +## Q3: Concurrency and long-running execution controls + +### Findings +- **Semaphores** — `Semaphore`/`SemaphoreWaiter` structs at `src/api/session-manager.ts:268-283`; instances `pythonSem`/`jsSem` (`:313-314`); built at `:357-374`. Limits: `MAX_CONCURRENT_PYTHON`/`MAX_CONCURRENT_JS` (default **5**); queue depth `MAX_PYTHON_QUEUE`/`MAX_JS_QUEUE` (default **100**); wait timeout `PYTHON_QUEUE_TIMEOUT_MS`/`JS_QUEUE_TIMEOUT_MS` (default **60000**). Constructor opts override env (used by tests). +- **Acquire/queue** — `acquireSlot()` (`:1169-1220`): immediate if `inFlight` via admin conn (`:33-43`), drops in `afterAll` (`:45-58`); runs `runMigrations`, asserts 4 tables exist (`:73`) and ≥1 `fs_resolve` proc (`:81`), then re-runs to assert idempotency (`:86`). Does **not** assert the capability columns individually. + +--- + +## Q5: `python` capability represented end-to-end (consistency surface) + +### Findings +- **Python SDK** — `clients/python/src/sqlfs/models.py:22-23` (`SandboxRecord.python/.javascript: bool`; **no `network`**), `:32-33` (`from_api` parse). `client.py:109-111` (`create()` params `python/javascript/network=False`), `:137-142` (conditional body). Tests `clients/python/tests/test_client.py:82-160`. Example `examples/quickstart.py:30`. +- **TypeScript SDK** — `clients/typescript/src/models.ts:9-10` (`SandboxRecord.python/javascript`; **no `network`**), `:87-88` (`sandboxRecordFromApi`). `client.ts:14-16` (`CreateSandboxOptions.python?/javascript?/network?`), `:72-80` (body). Tests `tests/sandboxes.test.ts:15-38`. Examples `examples/quickstart.ts:10`, `e2e-local.ts:32, 76-79`. +- **MCP tools** — `src/api/mcp/tools.ts`: `sandbox_create` input schema `python/javascript: z.boolean().optional()` (`:37-38`) — **`network` not exposed, hardcoded `false`** (`:46`); handler persists + echoes `python/javascript` (`:54-62`). `sandbox_list` returns `python/javascript` per entry (`:98-99`, **no `network`**). `bash_exec` description prose (`:175-179`). Tool tests `mcp-tools.test.ts`/`mcp.test.ts` exist but contain **no** python/js/network assertions. +- **OpenAPI** — `src/api/openapi-spec.ts`: `sandboxRecordSchema` has `python`/`javascript` with `required:[...,"python","javascript"]` — **`network` absent from both record and create-request schema** (`:23-26`). Create body `python`/`javascript` with descriptions (`:321-322`). +- **Plugin docs** — `plugins/sql-fs/skills/api/ref/endpoints.md:119-168` (request/response tables, examples, and the note **"python/javascript/network must be set at creation. They cannot be changed later"** at `:148`; `network:true` requires `javascript:true` at `:150-152`). `api/ref/bash.md:48-78`, `api/SETUP.md:61`, `py-sdk/ref/models.md:25-37`, `py-sdk/ref/client.md:113-142`, `py-sdk/SKILL.md:204-208`. +- **Route/handler tests** — `src/api/tests/unit/sandboxes.test.ts:64-101` (network flag; **no python-only HTTP test**), `exec.test.ts:597-599`, `session-manager.test.ts:338-650`, `session-manager.rehydrate.test.ts:36-149`. +- **Asymmetry to note**: `network` is request-only — present in HTTP route, server types, DB, TS-SDK request, Python-SDK request, and DB-list response, but **absent** from MCP tools, OpenAPI schemas, and both SDK `SandboxRecord` response types. + +--- + +## Q6: File movement between SqlFs and runtime; size limits + +### Findings +- **Read path (WASM)** — `python3` pre-reads a script file on the Node side via `ctx.fs.resolvePath`→`exists`→`readFile` (before the worker starts). During execution, a `SharedArrayBuffer` bridge (`FsProtocolBridge`, `node_modules/.pnpm/just-bash@3.0.1/.../chunks/chunk-5H5SCKJM.js`) services WASM FS ops by calling `ctx.fs`: `READ_FILE→readFileBuffer`, `STAT→stat`, `READDIR→readdir`, `LSTAT/EXISTS/REALPATH/READLINK`. WASM blocks on `Atomics.wait`; Node services async FS between wait points. +- **SqlFs read APIs** (`src/sql-fs/sql-fs.ts`): `readFile:1066` (decode), `readFileBuffer:1070`, private `#readBytes:1043` (contentCache → `getBlobNoTx`), `exists:1074`, `stat:1079`, `lstat:1101`, `readdir:1116`, `readdirWithFileTypes:1124`, `getAllPaths:732` (sync), `resolvePath:1324` (sync, pure), `readlink:1385`, `realpath:1393` (hits DB). +- **Size caps** (three, none a true `readFile` hard limit at the SqlFs layer): + 1. **SharedArrayBuffer `DATA_BUFFER` = 8,388,608 B (8 MB)** — `chunk-5H5SCKJM.js`. `setResult()` throws "Result too large" beyond this → a single WASM `READ_FILE` (and write) is hard-capped at 8 MB. This is the only true hard limit, and it applies only **through the WASM bridge**. + 2. **`DEFAULT_CONTENT_CACHE_MAX_BYTES = 50 MB`** (`sql-fs.ts:66`, LRU `maxSize` at `:188`) — per-session content-cache byte budget; a >50 MB file simply bypasses the cache and reads from Postgres each time. Caching threshold, not a read limit. + 3. **`REDIS_BLOB_MAX_BYTES` default 8 MB** (`src/sql-fs/redis-blob-cache.ts:12`, guard `:84`; env at `index.ts:138`) — max blob size written to Redis; larger blobs skip Redis and fall through to Postgres. Caching threshold, not a read limit. + `#readBytes` (`:1043-1063`) has **no size guard** — a direct `ctx.fs.readFile()` (non-WASM, e.g. `cat`) is uncapped at the SqlFs layer. +- **Write path** — `IFileSystem` writes implemented in `sql-fs.ts`: `writeFile:738`, `appendFile:793`, `mkdir:863` (recursive at `:869-906`), `rm:941`, `cp:1142`, `mv:1241`, `chmod:1013`, `utimes:1027`, `symlink:1330`, `link:1365`. Each calls `#assertWritable` (`:610-616`, throws `EREADONLY` in read-only depth) + `validatePath` (rejects null bytes), sets `#dirty=true`. +- **Binary / dirs / symlinks** — `writeFile` encodes string or uses `Uint8Array` directly (`:746-747`); WASM `handleWriteFile` passes a `Uint8Array` slice — binary preserved end-to-end. Recursive mkdir walks segments (`:870-906`). **Symlinks default-deny**: `symlink()` throws `EPERM` unless `#allowSymlinks` (default `false`, `:182`; factory `index.ts:75` doesn't pass it) — `:1335`. +- **Persistence model** — writes persist synchronously inside `#withTx`/`#withBareTx`; no deferred queue. A `bash.exec` shares one lazy script-tx (`beginScriptScope:579`/`endScriptScope:618`); on throw `abortScriptScope:653` rolls back + `reload()` discards cache mutations. +- **Tests** — `src/sql-fs/tests/unit/sql-fs.write.test.ts` (asserts mock-dialect calls + `getAllPaths()`/`stat()` visibility, incl. recursive mkdir/rm), `sql-fs.symlink.test.ts:61-68` (EPERM + no DB calls), `sql-fs.read-file.test.ts:112-127` (cache-miss→`getBlobNoTx`, cache-hit no DB), `session-manager.test.ts:122-138` (write→`pathCacheBytes` refresh, InMemoryFs), integration `tests/integration/postgres.test.ts`. + +--- + +## External Research + +### pyodide npm package (server-side Node.js) +- **Versions** (checked 2026-06-08 via `npm view pyodide`): `latest` dist-tag = **0.29.4** (released 2026-05-07; present in the npm versions list and "Latest" on GitHub releases). The prerelease lives on the **`next`** dist-tag = **`314.0.0-alpha.2`** (npm package-version format; Pyodide docs render the same release as `314.0.0a2`; prior `314.0.0-alpha.1`) — a CPython-aligned major versioning shift, with `pyodide/pyodide-recipes` decoupling packages from core. (npm also carries a legacy `beta` dist-tag at `0.19.0-alpha.1`.) When referring to npm, use `314.0.0-alpha.2`; when quoting the docs, `314.0.0a2`. Sources: [npm versions](https://www.npmjs.com/package/pyodide?activeTab=versions), [GitHub releases](https://github.com/pyodide/pyodide/releases), [0.29.4 changelog](https://pyodide.org/en/stable/project/changelog.html). +- **Node init / offline `indexURL`** — `import { loadPyodide }`; `npm install pyodide` auto-resolves assets from `node_modules/pyodide/` (no explicit `indexURL` needed). `indexURL` set to a **local directory path** loads `pyodide.asm.wasm` + `python_stdlib.zip` + packages from disk (no CDN). `packageCacheDir` (Node-only) caches distribution packages to disk for offline reuse; `lockFileURL` points at `pyodide-lock.json`. Node 18+ required (since 0.25.0). Using an `https://` URL as `indexURL` in Node can throw `ERR_MODULE_NOT_FOUND` (issue [#3550](https://github.com/pyodide/pyodide/issues/3550)) — use a path string. Sources: [Using Pyodide](https://pyodide.org/en/stable/usage/index.html), [JS API](https://pyodide.org/en/stable/usage/api/js-api.html), [Downloading & deploying](https://pyodide.org/en/stable/usage/downloading-and-deploying.html). +- **`pyodide.FS` (Emscripten)** — default MEMFS, which lives **inside the WASM heap and is non-persistent**: files Python writes there do **not** propagate to any external `IFileSystem`/host store on their own. Crossing that boundary is explicit — either copy via `FS.writeFile(path, data, {encoding})` / `FS.readFile(path, {encoding})` (both directions) + `FS.mkdirTree` for nested dirs, or mount the host FS with **NODEFS** (Node-only): `FS.mount(FS.filesystems.NODEFS, {root:"."}, "/mnt")` / helper `pyodide.mountNodeFS(...)`, where changes reflect both ways. Source: [Dealing with the file system](https://pyodide.org/en/stable/usage/file-system.html). +- **Packages** — `loadPackage(name)` fetches Pyodide-built binary wheels from `indexURL`; `micropip.install(name)` does PyPI dependency resolution. **numpy / pandas / scipy** are built in Pyodide (bundled binary WASM wheels → fully offline when `indexURL` is local). **openpyxl** is pure-Python, **not** in the built-in list → `micropip.install` from PyPI, or a pre-downloaded local `.whl` for offline. Caveat: micropip PyPI downloads are **not cached** in Node. Sources: [Loading packages](https://pyodide.org/en/stable/usage/loading-packages.html), [Packages built in Pyodide](https://pyodide.org/en/stable/usage/packages-in-pyodide.html). +- **Execution / output** — `runPythonAsync(code, {globals, locals, filename})`; since 0.18.0 does NOT auto-load packages-from-imports. Capture via `loadPyodide({stdout, stderr})` (line callbacks, catches startup output) or `pyodide.setStdout/setStderr({batched|raw|write})`; Node default writes to `process.stdout/stderr`. Sources: [JS API](https://pyodide.org/en/stable/usage/api/js-api.html), [Redirecting streams](https://pyodide.org/en/stable/usage/streams.html). +- **Footprint** — `pyodide-core-0.29.4.tar.bz2` = minimal Node set (what npm installs); full `pyodide-0.29.4.tar.bz2` ≈ **200+ MB** with all packages; first browser load ≈ 6.4 MB core. Init ≈ ~2 s on a browser (docs; community 4–5 s real-world; <1 s with memory snapshot). WASM memory capped at **2 GB** by default; unreleased `PyProxy` objects leak. Runtime ≈ 3–5× slower than native CPython. Per-package numpy/pandas sizes not on official docs. Sources: [Downloading & deploying](https://pyodide.org/en/stable/usage/downloading-and-deploying.html), [wasm-constraints](https://pyodide.org/en/stable/usage/wasm-constraints.html), [Cloudflare Workers + Pyodide](https://blog.cloudflare.com/python-workers/). + +### just-bash built-in `python3` and customCommands (docs) +- Docs ([justbash.dev](https://justbash.dev/), [README](https://github.com/vercel-labs/just-bash/blob/main/packages/just-bash/README.md), npm **3.0.1**, checked 2026-06-08) state Python is "CPython compiled to WASM" (vendor dir `vendor/cpython-emscripten/`), **opt-in** via `new Bash({python:true})`, **Node-only**. No documented Python memory cap, timeout default, or stdlib/package list (only `js-exec`/QuickJS has documented 64 MB + configurable timeout). Issue [#94](https://github.com/vercel-labs/just-bash/issues/94) shows it loading assets from a `/pyodide/` path, hinting Pyodide internally — but this is inferred, not stated. +- **customCommands override behaviour**: documented in the **distributed package types** — `node_modules/.../just-bash/dist/Bash.d.ts:111` JSDoc states custom commands "take precedence over built-ins with the same name" (matching the Q2 code finding that `registerCommand` overwrites the registry entry). It is **not** covered in the **README / website docs / CHANGELOG** (no precedence table or conflict-resolution prose there). So the override is contractually documented in the shipped types, just not in the narrative docs. + +### Pyodide server-side security & sandbox escape — untrusted code (Q8, added during design) +- **`import js` exposes the host scope.** Pyodide's `js` module proxies `jsglobals` (default `globalThis`); under Node that surfaces `process`/`process.env` (any secrets the process holds), `fetch`, `eval`, `Function`, dynamic `import('node:*')` module-loading (and `require` in CommonJS contexts), `Buffer`. In a `worker_threads` worker the worker's own `globalThis` is still a full Node environment. Source: [Pyodide JS API](https://pyodide.org/en/stable/usage/api/js-api.html) (checked 2026-06-08). +- **Multiple escape paths survive JS-module removal.** (a) `pyodide.code.run_js(code)` is `eval()` in the *real* global scope and is **not** gated by `jsglobals` ([code module docs](https://pyodide.org/en/stable/usage/api/python-api/code.html)); (b) the `JsProxy` prototype chain `{}.constructor.constructor → Function("return globalThis")()` reaches host scope with reads only ([CERT/CC VU#414811](https://kb.cert.org/vuls/id/414811)); (c) `ctypes.CDLL(None)` reaches the Emscripten symbol table → `system()` / `emscripten_run_script_string` (runs JS in the host) ([n8n CVE-2025-68668](https://www.cyera.com/research/n8scape-pyodide-sandbox-escape-9-9-critical-post-auth-rce-in-n8n-cve-2025-68668), [Grist GHSA-7xvx-8pf2-pv5g](https://github.com/gristlabs/grist-core/security/advisories/GHSA-7xvx-8pf2-pv5g)); (d) `object.__subclasses__()` → `__builtins__`; (e) `_pyodide._base.eval_code()` bypasses caller namespace patches. No complete official enumeration exists ([pyodide #4120](https://github.com/pyodide/pyodide/issues/4120)). +- **Config-level hardening is insufficient.** `jsglobals` (does not affect `run_js`; a plain object literal still carries `Object.prototype`→`Function`, so requires `Object.create(null)`), `unregisterJsModule("js")` (does not disable `run_js`), and a restricted `runPython` globals dict (does not stop `import js`/`run_js`/`ctypes`/`__subclasses__`) each close only part of the surface. Severing **all** paths via Pyodide config alone is **not achievable** — an OS/container boundary is mandatory. +- **Official-adjacent stance.** No formal "not a sandbox" page, but: Grist advisory — *"pyodide on node does not have a useful sandbox barrier"* (CVE-2026-24002, CVSS 9.0); n8n advisory — *"Pyodide was not designed as a security sandbox … fundamentally unsuitable for executing untrusted user code without additional hardening layers"* (CVE-2025-68668, CVSS 9.9); Pyodide [#4120] treats one user running another user's Python as an open problem. The browser limits exposure (no `require`/`child_process`/fs); a bare Node process has no containing sandbox. Sources: [GHSA-7xvx-8pf2-pv5g](https://github.com/gristlabs/grist-core/security/advisories/GHSA-7xvx-8pf2-pv5g), [GHSA-62r4-hw23-cc8v](https://github.com/n8n-io/n8n/security/advisories/GHSA-62r4-hw23-cc8v). +- **`worker_threads` is a concurrency boundary, not a security one.** You *can* scrub the worker's env (`new Worker(file, { env: {} })` — the default is a *copy* of the parent env, not `SHARE_ENV`), but you **cannot** block outbound network (the worker shares the process network stack) or dynamic `import('node:fs')`/`import('node:child_process')`. Node's `--permission` model does not inherit to workers and explicitly "does not protect against malicious code." Sources: [Node worker_threads](https://nodejs.org/api/worker_threads.html), [Node permissions](https://nodejs.org/api/permissions.html) (checked 2026-06-08). + +### Isolating untrusted Pyodide server-side; the Deno permission model (Q9, added during design) +- **An OS-level boundary is required; production systems use one.** Cloudflare Python Workers (V8 isolates + `seccomp` + Linux namespaces); Grist moved Pyodide to Deno's permission model and recommends gVisor (`GRIST_SANDBOX_FLAVOR=gvisor`); n8n v2 moved Python to an out-of-process task runner; Modal / Gemini code-exec use gVisor; E2B uses Firecracker microVMs. Sources: [Cloudflare Workers security model](https://developers.cloudflare.com/workers/reference/security-model/), [shayon.dev — sandbox isolation](https://www.shayon.dev/post/2026/52/lets-discuss-sandbox-isolation/). +- **Deno as the boundary.** Deno is deny-by-default; for an adversarial runner be explicit: `--no-prompt` + `--deny-net --deny-run --deny-write --deny-env --deny-ffi --deny-sys --deny-import`, with `--allow-read` scoped to the asset dir only. Deno gates the *JS layer itself*, so even a full Python→JS escape lands capability-less; `--deny-ffi` is critical (`Deno.dlopen` would be host-native code). Spawn the child with a scrubbed env (`clearEnv`). A unix-socket transport needs `--allow-read`/`--allow-write` on the socket path (conflicts with `--deny-write`); `stdin`/`stdout` std streams need no permission. Source: [Deno security & permissions](https://docs.deno.com/runtime/fundamentals/security/) (checked 2026-06-08). +- **Pyodide-on-Deno feasibility & prior art.** The npm package documents Node.js; Deno support has been tracked separately, but the combination is proven in the wild — Grist (post-1.7.9), [LangChain Sandbox](https://github.com/langchain-ai/langchain-sandbox), and [Simon Willison's deno+pyodide writeup](https://til.simonwillison.net/deno/pyodide-sandbox). The *exact* offline + restrictive-flags + numpy/pandas/scipy/openpyxl combination is unverified and warrants a spike. +- **Offline packaging under these constraints.** numpy/pandas/scipy are built-in Pyodide wheels (offline from a local `indexURL`); **openpyxl is not built-in and pulls `et_xmlfile`** (+ transitive) — stage via Pyodide lockfile tooling (`micropip.freeze` → a custom `pyodide-lock.json`), not hand-edited JSON. Loader option casing is `packageBaseUrl`. Sources: [Loading packages](https://pyodide.org/en/stable/usage/loading-packages.html), [micropip](https://micropip.pyodide.org/en/stable/project/api.html). +- **Per-child memory limiting.** Pyodide's ~2 GB WASM heap is a per-instance ceiling, **not** an OOM guard — WASM growth consumes real container memory and adjacent Deno/V8/package allocations can OOM first. `prlimit`/cgroup per-child limits may be unavailable in a slim, non-root Node image (no `util-linux`, no cgroup write access); the **container memory limit is the real guard**. Source: [Pyodide wasm-constraints](https://pyodide.org/en/stable/usage/wasm-constraints.html). + +--- + +## Cross-Cutting Observations +- **Capability flags are immutable post-create** (docs `endpoints.md:148`) and only consumed when (re)building the `Bash` instance (`session-manager.ts:487-497`) — there is no update/patch path. +- **`network` is a second-class field**: present in route/types/DB/SDK-requests but absent from MCP tools, OpenAPI, and both SDK `SandboxRecord` response types. Any change to capability shape must reconcile this asymmetry. +- **Single source of read truth** for command dispatch is one `Map` registry; the same `SqlFs` instance is shared as `ctx.fs` to every command, so a custom command has the full async `IFileSystem` (plus sync `getAllPaths`/`resolvePath`). +- **8 MB recurs** as the WASM bridge transfer cap and the Redis-cache default — but only the bridge cap is a hard ceiling on what executing Python can read/write per file. +- **Postgres is the only realized dialect** (migrations, runner discovery, integration tests all hardcode `postgres/`); mysql/azure-sql are described but not present. +- **Per-`fs` Python execution is already serialized** inside just-bash (the `WeakMap` queue) in addition to the app-level `pythonSem` (5-wide, env-tunable). +- **Write-back to SqlFs is bridge-mediated, not an automatic FS sync.** Files created by the current `python3` path reach SqlFs only because just-bash's SharedArrayBuffer bridge calls `ctx.fs.writeFile`/`mkdir`/`rm` synchronously per op (Q6). This wiring is specific to that bridge — pyodide's MEMFS (Q7) has no equivalent connection to `ctx.fs`, so any MEMFS-based path's output would have to be drained across the WASM/`ctx.fs` boundary explicitly (FS copy or a live NODEFS-style mount) rather than persisting on its own. +- **`stdlib` vs `pyodide` is a *security* distinction, not only a feature one.** just-bash's `python3` is CPython-emscripten with **no** JS↔Python bridge — genuinely air-gapped (only the `ctx.fs` bridge + stdout). Pyodide ships the `js` bridge by design, so it is **not** a security boundary on Node (Q8), and the `pyodide` runtime must be wrapped in an OS-level sandbox (Q9) — unlike `stdlib`, which inherits just-bash's existing isolation. + +## Open Areas +- **CPython-emscripten vs Pyodide in just-bash 3.0.1**: the bundled `python3` reports `Python 3.13.2 (Emscripten)` and spawns a `worker_threads` worker with a SharedArrayBuffer FS bridge (Q2/Q6), while just-bash docs/issue #94 reference a `/pyodide/` asset path. The two descriptions aren't fully reconciled from available sources; exact engine identity inside just-bash is not authoritatively documented. +- **Per-package offline sizes** (numpy/pandas/scipy/openpyxl wheel byte sizes) are not published on pyodide's official docs — would need inspection of `pyodide-lock.json` / the distribution. +- **just-bash python timeout/memory defaults** are undocumented upstream; the codebase observes a 10 s (60 s with fetch) default via `maxPythonTimeoutMs`, but no memory cap was located for the CPython worker. diff --git a/thoughts/issue-118-pyodide-runtime/structure.md b/thoughts/issue-118-pyodide-runtime/structure.md new file mode 100644 index 0000000..1f09f3e --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/structure.md @@ -0,0 +1,280 @@ +# Structure Outline + +## Approach + +Replace boolean `python` with `python_runtime: "stdlib" | "pyodide" | null` end-to-end (rolling-safe +expand/contract migration), then build the `pyodide` runtime as an **OS-isolated Deno subprocess** with zero +permissions, a **committed** stdin/stdout + realm-lockdown IPC with fixed frame-integrity invariants, cwd-scoped +file diff-and-drain, dedicated concurrency + atomic-admission residency caps, and a first-class adversarial +escape suite. Field slices (1–2) ship `stdlib`/`null` on their own; runtime slices (3–7) build the Deno boundary +bottom-up. Three POC **spikes gate** the runtime phases. + +> **Coordinated-release note (design Open Risk).** The boolean→enum break and the `pyodide` runtime land in +> **one breaking release**. Phases are *implementation* slices on one branch, not separate releases. Between +> Phase 2 and Phase 5 a `pyodide` sandbox creates fine and `stdlib` works, but `python3` in a `pyodide` sandbox +> is "not yet registered" — an intermediate state that never reaches a cut release. The DB layer is +> rolling-safe (Phase 1); the API/SDK break still requires clients to upgrade. + +--- + +## Phase 0: Spikes (merge gates — throwaway POCs, not shippable) + +Three POCs from design §"Required Spikes". Each must pass before its dependent phase; fail any → revisit the +architecture with the user. + +**Files**: `thoughts/issue-118-pyodide-runtime/spikes/` (scratch scripts + a short findings note per spike) +**Key checks**: +- **S1 — Pyodide-on-Deno offline** (gates P3): a Deno subprocess under the **committed flags** (`--no-prompt + --deny-net --deny-run --deny-write --deny-env --deny-ffi --deny-sys --deny-import --no-remote --no-npm + --cached-only --no-config`, `--allow-read=`, env `{DENO_NO_UPDATE_CHECK:1}`) can `loadPyodide` from a + **local** `indexURL`, `loadPackage` numpy/pandas/scipy from disk, install the frozen openpyxl+et_xmlfile lock, + run a pandas→openpyxl round-trip, capture stdout/stderr — **zero network**. +- **S2 — IPC integrity** (gates P3/P4): **confirm the committed design** (stdin/stdout + realm-lockdown), not + choose a transport. Length-prefixed JSON framing works under the deny flags with binary payloads; after realm + lockdown an adversarial `Deno.stdout.write`/`console.log`/raw-fd attempt **cannot forge, interleave, or replay** + a control frame; a stale-generation message is rejected. +- **S3 — Per-child memory** (gates P6): confirm `node:22-slim` non-root **cannot** reliably set cgroup + `memory.max` (and that `prlimit --as` is unusable with V8's vaddr reservation); confirm the container-limit + guard + **accepted availability risk** (design D5) is the operating model. + +**Verify**: +- Programmatic: each spike script exits 0 and prints its asserted result (round-trip bytes, rejected forged + frame, memory-limit probe). +- Agent: review the three findings notes; confirm each gate's pass/fail is explicit before starting P3/P6. + +--- + +## Phase 1: `python_runtime` field — server-side break, rolling-safe (stdlib + null end-to-end) + +Replace boolean `python` with the nullable enum across DB, types, persistence, validation, runtime resolution, +and HTTP responses, **safely under a mixed-version rolling deploy**. `stdlib`→`new Bash({ python: true })`; +`null`→no Python; `pyodide` is a valid stored value but Python stays unregistered until P5. + +**Files**: `src/sql-fs/migrations/postgres/0006_python_runtime.sql` (new), `src/sql-fs/types.ts`, +`src/sql-fs/dialects/postgres.ts`, `src/api/routes/sandboxes.ts`, `src/api/session-manager.ts`, +`src/api/tests/integration/migrations.integration.test.ts` +**Key changes**: +- Migration `0006`: idempotent `ADD COLUMN IF NOT EXISTS python_runtime TEXT` + `CHECK (python_runtime IN + ('stdlib','pyodide'))`; backfill **only `WHERE python_runtime IS NULL`**, guarded by a `DO` block checking + `pg_attribute` for `python` (`python=true → 'stdlib'`) so it survives the later `python`-drop release. +- `type PythonRuntime = "stdlib" | "pyodide" | null` — new; `SandboxMeta`/`SandboxListEntry` gain + `readonly python_runtime: PythonRuntime` (`types.ts:61-82`). +- `RuntimeOptions { pythonRuntime: PythonRuntime; … }` + `DEFAULT_RUNTIME_OPTIONS` (`session-manager.ts:110-122`); + `getOrCreate` maps `pythonRuntime==="stdlib"` → `python:true` in the `new Bash({…})` block (`:487-497`); + `pyodide` → leave `python:undefined` (commands added P5). +- **Rolling-safe persistence** (design D2): `getSandboxMeta`/list **read** + `COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END)`; `updateSandboxMeta` **dual-writes** `python` + (`stdlib→true`, `pyodide/null→false`) alongside `python_runtime` (`postgres.ts:316-402`). +- `createBodySchema`: `python_runtime: z.enum(["stdlib","pyodide"]).nullable().optional()`; reject legacy + `python: bool` (`sandboxes.ts:15-23`). Create-201, list, **and GET** echo `python_runtime` — **fix the GET + omission** (`:134,142-159,179-196`). + +**Verify**: +- Programmatic: `pnpm typecheck && pnpm lint:fix && pnpm test:unit`; integration test re-runs `0006` idempotently, + asserts the column + CHECK exist, asserts an **old-replica-style `python=true` (python_runtime NULL) row reads + back as `stdlib`** (COALESCE), and asserts re-running the migration **after a simulated `python`-drop** does not + error (pg_attribute guard). +- Agent: create a `stdlib` and a `null` sandbox against the dev server; confirm create/list/GET all echo + `python_runtime` and a `stdlib` sandbox runs `python3 -c "print(1)"`. + +--- + +## Phase 2: `python_runtime` — client & contract surfaces (SDKs, MCP, OpenAPI, docs) + +Propagate the enum to every external representation (research Q5) and reconcile the second-class `network` field. +One coordinated break across the consistency surface. + +**Files**: `clients/python/src/sqlfs/models.py`+`client.py`, `clients/typescript/src/models.ts`+`client.ts`, +`src/api/mcp/tools.ts`, `src/api/openapi-spec.ts`, `plugins/sql-fs/skills/api/ref/endpoints.md` (+ `bash.md`, +`py-sdk/ref/*`), changeset. +**Key changes**: +- `SandboxRecord.python_runtime: PythonRuntime` + `CreateSandboxOptions.python_runtime?` in both SDKs; drop + `python: bool`; add `network` to the response records (reconcile asymmetry). +- MCP `sandbox_create` input `python_runtime` enum; `sandbox_list`/handler echo it (`tools.ts:37-99`). +- OpenAPI `sandboxRecordSchema` + create-body `python_runtime` enum, `required` updated; add `network` + (`openapi-spec.ts:23-26,321-322`). Version bump via changeset (no manual edits). +- Docs: replace `python` with `python_runtime` (values + immutability note `endpoints.md:148`). + +**Verify**: +- Programmatic: `pnpm typecheck && pnpm lint:fix && pnpm test:unit`; SDK suites (py + ts), `mcp-tools.test.ts` + pass with new assertions; OpenAPI spec validates. +- Agent: diff every Q5 surface against a checklist; confirm no remaining boolean `python` and that `network` + now appears in both SDK records + OpenAPI. + +--- + +## Phase 3: Offline assets + Deno harness (the untrusted side) + +Vendor the runtime assets and write the Deno-side harness that loads Pyodide offline, runs untrusted Python, +locks down its realm, and speaks the committed IPC. Hardens S1/S2 into product. Standalone-testable without Node. + +**Files**: `Dockerfile`, `vendor/deno/` + `vendor/pyodide/` (baked; gitignore/LFS decided in plan), +`scripts/build-pyodide-lock.mjs` (new — `micropip.freeze`), `src/pyodide-runner/runner.ts` (new, Deno entry), +`src/pyodide-runner/protocol.ts` (shared frame types). +**Key changes**: +- Pin pyodide **0.29.x**; build step bakes Deno binary + full distribution + **custom lock** (openpyxl + + et_xmlfile + transitive) generated by tooling, **not** hand-edited. +- `runner.ts`: `loadPyodide({ indexURL, lockFileURL, packageBaseUrl })` from local paths; preload + {numpy,pandas,scipy,openpyxl}; **realm lockdown** (capture the stdout writer, then delete `Deno`/`console`/other + write primitives from `globalThis`) **before** any untrusted Python; IPC read→eval→respond loop; stdout/stderr + via Pyodide stream callbacks into JSON fields; stage input files into MEMFS; report `{created,modified,deleted}` + diff. +- `protocol.ts`: `RunRequest`/`RunResponse` frame schemas carrying `requestId` (random), `seq` (monotonic), + `type`, `generation` (shared by P4). + +**Verify**: +- Programmatic: build script produces the lock + asset tree; `deno run --allow-read= + runner.ts < fixture-frame` returns a valid response frame with pandas→openpyxl output bytes, **zero network**. +- Agent: re-run an S2 forge attempt against the built `runner.ts`; confirm the forged frame is not emitted on the + control channel; confirm the deny-belt blocks remote/npm imports, update check, FS, env, subprocess, FFI, net. + +--- + +## Phase 4: Node-side `PyodideSandbox` manager + IPC client + +The trusted Node half: spawn/own the Deno subprocess, frame the protocol with full integrity checks, serialize +execs, enforce **throw-not-return** cancellation and respawn. No session wiring yet — unit-testable in isolation. + +**Files**: `src/api/pyodide/manager.ts` (new), `src/api/pyodide/ipc.ts` (new), +`src/api/pyodide/tests/unit/*.test.ts` (new) +**Key changes**: +- `type WorkerState = "cold"|"starting"|"idle"|"busy"|"terminating"|"dead"`; + `class PyodideSandbox { readonly state: WorkerState; readonly generation: number; + run(req: RunRequest, signal: AbortSignal): Promise; dispose(): Promise }` — spawns + `deno run --allow-read=` with **scrubbed `env:{DENO_NO_UPDATE_CHECK:"1"}`**. +- `ipc.ts`: `encodeFrame(obj): Buffer` / `decodeFrames(chunk): Frame[]` (length-prefixed); **schema-validate + + enforce integrity on every inbound frame** — match `requestId`, monotonic `seq`, exact `type`, current + `generation`, **single response per request**; **per-frame + aggregate size caps measured on base64-encoded + wire size**. Any malformed/oversized/duplicate/stale-generation/unexpected frame → **kill the child**. +- **Per-subprocess mutex/queue** serializes `run()` (readOnly execs are concurrent — design §Patterns); a queued + waiter is removed on `signal` abort. +- **Cancellation = throw, never return**: `run()` observes `signal` + own timer → `child.kill("SIGKILL")` → + **route abort rejects `AbortError`; internal timeout throws a typed timeout error**; **retire the generation**. + Manager reports error + respawns (new `generation`) on unexpected child exit. + +**Verify**: +- Programmatic: `pnpm typecheck && pnpm lint:fix`; new unit tests cover serialization order; **abort while + waiting on the mutex**, **abort during init/preload** — each kills the child and rejects/throws (never returns + a normal result); malformed/oversized/duplicate/stale-generation/forged frame each kill the child; **base64 + expansion counted against the size cap**; respawn-on-exit increments `generation`. +- Agent: exercise `manager.run()` with two overlapping calls; confirm they serialize and an aborted call kills + the child within the timeout. + +--- + +## Phase 5: `pyodide` custom commands + file staging drain (core requirement) + +Wire `python_runtime: "pyodide"` to register `python3`+`python` custom commands backed by a per-session +`PyodideSandbox` owned as a **first-class `Session` field**, with cwd-scoped diff-and-drain into SqlFs. +**Delivers the issue's core requirement.** + +**Files**: `src/api/commands/pyodide-command.ts` (new — `createPyodideCommands(session)` factory), +`src/api/session-manager.ts`, `src/api/tests/integration/pyodide.integration.test.ts` (new) +**Key changes**: +- `createPyodideCommands(session: Session): CustomCommand[]` — two commands, one shared handler; parse `-c CODE` / + script FILE / stdin / `--version` matching the built-in `python3` surface; resolve paths via + `ctx.fs.resolvePath(ctx.cwd, arg)`; return `ExecResult`. +- **First-class ownership** (design finding 7): add `Session.pyodideSandbox?: PyodideSandbox`. In `getOrCreate` + (`:438-560`), when `pythonRuntime==="pyodide"` build/attach the manager on the session, push its commands into + `customCommands`, keep `python:undefined`. **Teardown kills the child on every path** — session destroy, + **reaper eviction, shutdown** (`:1038,1088`), and **failed `getOrCreate`** — extending the `session.fs`-only + cleanup. +- **Staging**: ship cwd subtree + resolved script path to Deno; on a **successful** run, drain + `{created,modified,deleted}` into `ctx.fs` **inside the script transaction** (atomic rollback). **Never drain on + timeout/abort/protocol-invalid** (design finding 6). **Validate every drain path stays under cwd** (reject + `..`/absolute/null-byte); reject symlinks; dirs-before-files, delete depth-first; exec-bit via `chmod`; per-file + + total byte caps both directions. + +**Verify**: +- Programmatic: `pnpm typecheck && pnpm lint:fix && pnpm test:unit`; integration test: create `pyodide` sandbox, + write a CSV, `python3 analyze.py` does `import pandas` + writes `out.xlsx`, assert `out.xlsx` **retrievable via + files API**; a drain path with `..`/absolute is rejected; a **read-only exec's MEMFS mutation is rejected + (`EREADONLY_VIOLATION`)**; an **abort after the child responds but before drain completes drains nothing**; + **reaper/destroy/shutdown/failed-create each kill the child**. +- Agent: run a `-c` one-liner and a script-file form in a `pyodide` sandbox on the dev server; confirm stdout, + exit code, and written-file persistence. + +--- + +## Phase 6: Concurrency semaphore + atomic-admission residency LRU + memory posture + +Bound in-flight `pyodide` execs and resident subprocesses independently (both required per design §5), with +**atomic admission** over the worker state machine. + +**Files**: `src/api/session-manager.ts`, `src/api/pyodide/residency.ts` (new), +`src/api/tests/unit/session-manager.test.ts` (extend), `src/api/pyodide/tests/unit/residency.test.ts` (new) +**Key changes**: +- New `pyodideSem` (`MAX_CONCURRENT_PYODIDE` default **2**) + `MAX_PYODIDE_QUEUE`/`PYODIDE_QUEUE_TIMEOUT_MS` + mirroring the python set; route by `python_runtime==="pyodide"` in `execWithRuntimeThrottle` (`:1235-1307`) so + `stdlib` keeps `MAX_CONCURRENT_PYTHON=5`. +- `class PyodideResidency` — global cap `MAX_RESIDENT_PYODIDE` (default **2**). An **atomic admission mutex** + covers *reserve slot → select eviction victim → spawn → roll back on failed init* as one critical section. + Eviction only targets `idle` workers; **`starting`/`busy` are never evictable**. `PYODIDE_IDLE_MS` + (< `SESSION_IDLE_MS`) idle-kills subprocesses. **Invariant `MAX_RESIDENT_PYODIDE >= MAX_CONCURRENT_PYODIDE`** + enforced at startup. +- **Memory posture (design D5, accepted availability risk):** best-effort cgroup `memory.max` per child only + where supported (gated on S3); **container memory limit is the documented real guard** — no per-child OOM + isolation guarantee. + +**Verify**: +- Programmatic: `pnpm typecheck && pnpm lint:fix && pnpm test:unit`; new tests: N concurrent pyodide execs queue + + `queue_full`/`wait_timeout`; **concurrent resident admissions never exceed the cap**; **LRU never evicts a + `busy` or `starting` worker**; invariant violation fails startup; `stdlib` routing unaffected. +- Agent: drive >`MAX_RESIDENT` concurrent `pyodide` sessions; confirm an idle subprocess is killed and an evicted + session cold-starts on next exec. + +--- + +## Phase 7: Adversarial escape suite (first-class security acceptance — merge gate) + +Prove the boundary holds. This suite is the security sign-off gate (design Open Risk: C1 reversal). + +**Files**: `src/api/pyodide/tests/integration/escape.integration.test.ts` (new), +`src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` (new) +**Key changes**: +- Escape cases each **fail closed**: `import js; js.process.env`, `js.fetch(...)`, `pyodide.code.run_js(...)`, + `ctypes.CDLL(None)`, `import('node:child_process')` — none read secrets, reach network, or touch host FS. +- **Deny-belt coverage**: remote import, npm import, update check, filesystem write, env read, subprocess spawn, + FFI (`Deno.dlopen`), network — each denied. +- **Frame-forgery**: escaped JS attempting `Deno.stdout.write`/`console.log`/raw-fd cannot forge a control frame, + replay a stale-generation frame, nor redirect a drain write outside cwd. +- Fresh-globals isolation between a session's execs (design Open Risk: intra-session state). + +**Verify**: +- Programmatic: escape + deny-belt + forgery + isolation suites pass (every escape blocked); full + `pnpm test:integration` green with both containers up. +- Agent: review each escape's assertion proves *capability denial* (no secret/net/FS), not just a thrown error; + confirm S3 memory behavior is exercised or documented. + +--- + +## Testing Checkpoints + +- **After P0**: three spike gates explicitly pass (or architecture revisited with user). +- **After P1**: `python_runtime` column + CHECK exist; `stdlib`/`null` work end-to-end via HTTP; GET fixed; + old-replica `python=true` reads as `stdlib`; migration re-runs idempotently after a simulated `python`-drop. +- **After P2**: every Q5 surface uses `python_runtime`; no boolean `python` remains; `network` reconciled. +- **After P3**: `runner.ts` loads Pyodide offline under the committed flags, round-trips pandas→openpyxl, blocks + the deny-belt; a forged frame is rejected. +- **After P4**: manager serializes execs; aborts (waiting / init / preload) throw-not-return and kill the child; + malformed/oversized/duplicate/stale-generation/forged frames kill the child (base64-aware cap); respawn bumps + generation — unit-proven. +- **After P5**: CSV→`python3 analyze.py`→`out.xlsx` retrievable via files API; drain path-validation holds; + read-only mutation rejected; abort-before-drain drains nothing; all teardown paths kill the child. +- **After P6**: concurrency + atomic-admission residency caps + invariant enforced; LRU spares busy/starting; + `stdlib` routing unchanged. +- **After P7**: all escape/deny-belt/forgery/isolation cases fail closed; security sign-off can proceed. + +## Human Verification (end of plan) + +- **Security sign-off**: a maintainer/security reviewer reads the escape suite, confirms it covers the C1-reversal + surface, and **accepts the single-layer threat model** (Deno/V8 runtime escape out of scope) for the deployment. +- **Memory under load on the target host**: run `MAX_RESIDENT_PYODIDE` heavy scripts concurrently on the real + container; confirm behaviour matches the **accepted availability risk** (no surprise parent OOM beyond what's + documented) and that defaults (+`PYODIDE_IDLE_MS`) are sane. +- **Cold-start UX**: time a first `pyodide` exec (Deno spawn + Pyodide init + package load) and confirm the exec + timeout default accommodates it acceptably for the LibreChat loop. +- **Image portability**: confirm the vendored Deno + Pyodide assets build and run on a clean self-hosted install + (image size acceptable). +- **End-to-end issue acceptance**: a human runs the real workflow (upload CSV → analyze → download xlsx) through + the product UI/LibreChat and confirms the output file is correct. diff --git a/thoughts/issue-118-pyodide-runtime/task.md b/thoughts/issue-118-pyodide-runtime/task.md new file mode 100644 index 0000000..d1a2cfc --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/task.md @@ -0,0 +1,5 @@ +# Task — Pyodide Python runtime for in-sandbox data analysis + +Add an opt-in Pyodide-backed Python runtime to sql-fs so sandboxed code can `import numpy`, `pandas`, `scipy`, and `openpyxl` entirely in WASM, preserving the air-gapped, no-host-process security posture. This replaces the existing boolean `python` capability flag with a `python_runtime: "stdlib" | "pyodide" | null` field that spans the API, both SDKs, the MCP tool, the OpenAPI spec, and a DB migration — a breaking, major-version change. The driving use case is AI agents analysing user-uploaded CSV/Excel files behind a self-hosted LibreChat deployment, which additionally requires that **output files written by the script are drained back to SqlFs and retrievable**, not just input loading. + +Source: [Hazzng/sql-fs#118](https://github.com/Hazzng/sql-fs/issues/118) (issue body + the maintainer's accepted implementation analysis in the first comment). From 1ecbfda94b5cf3f98500b71718fffdb5caeb652d Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 15:39:28 +0930 Subject: [PATCH 02/16] =?UTF-8?q?Phase=200:=20Spikes=20(merge=20gates)=20?= =?UTF-8?q?=E2=80=94=20Pyodide-on-Deno,=20IPC=20integrity,=20per-child=20m?= =?UTF-8?q?emory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validate the three architecture gates for the Pyodide runtime (issue #118) before building product phases. All three spike scripts pass (exit 0): - S1: Pyodide 0.29.4 loads fully offline on Deno v2.8.2 under the committed deny-belt and runs a pandas->openpyxl xlsx round-trip with zero network. - S2: realm lockdown + frame-integrity validation; surfaced that lockdown is not stdout containment (import(node:fs).writeSync reaches stdout), so Node-side validation with secret requestId/seq/generation is load-bearing. - S3: non-root cannot set cgroup memory.max and prlimit --as is unusable for RSS (V8/WASM vaddr >> RSS) -> container memory limit is the guard. Records findings per spike and propagates the S2 finding into the Phase 3/4/7 plan and design wording (no false stdout-containment claims). Assets are git-ignored and reproduced by the spike scripts. --- thoughts/issue-118-pyodide-runtime/design.md | 16 +- thoughts/issue-118-pyodide-runtime/plan.md | 43 ++- .../spikes/.gitignore | 4 + .../spikes/S1-findings.md | 88 ++++++ .../spikes/S2-findings.md | 103 +++++++ .../spikes/S3-findings.md | 66 +++++ .../spikes/s1-pyodide-deno.sh | 100 +++++++ .../spikes/s1_runner.ts | 120 ++++++++ .../spikes/s2-ipc.ts | 275 ++++++++++++++++++ .../spikes/s3-memory.sh | 109 +++++++ 10 files changed, 908 insertions(+), 16 deletions(-) create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/.gitignore create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/S1-findings.md create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/S2-findings.md create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md create mode 100755 thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/s1_runner.ts create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/s2-ipc.ts create mode 100644 thoughts/issue-118-pyodide-runtime/spikes/s3-memory.sh diff --git a/thoughts/issue-118-pyodide-runtime/design.md b/thoughts/issue-118-pyodide-runtime/design.md index c845edd..f25b7f2 100644 --- a/thoughts/issue-118-pyodide-runtime/design.md +++ b/thoughts/issue-118-pyodide-runtime/design.md @@ -60,8 +60,11 @@ threat model** — it would expose the host under the same uid. Compensating con scoped to read-only Pyodide assets only. We **recommend** (do not require) operators add a gVisor/seccomp layer. **Security acceptance is first-class:** an adversarial suite proves `import js; js.process.env`, `js.fetch(...)`, `pyodide.code.run_js(...)`, `ctypes.CDLL(None)`, `import('node:child_process')` **fail closed** (no secret read, -no network, no host-FS reach) **and** that escaped JS **cannot forge, interleave, or replay an IPC control -frame** — proving capability denial, not merely a thrown error. +no network, no host-FS reach) **and** that escaped JS **cannot produce an _accepted_ IPC control frame** +(spike S2 finding A: it *can* write raw bytes to stdout via `import("node:fs").writeSync(1,…)`, but the +Node-side validator rejects every forged/interleaved/replayed/stale-generation frame and kills the child — +the secret `requestId`/`seq`/`generation` are never exposed to untrusted code) — proving capability denial, +not merely a thrown error. ## Patterns to Follow @@ -215,9 +218,12 @@ Each must pass a POC before its dependent phase is built; fail any → revisit t numpy/pandas/scipy from disk, install the **frozen openpyxl+et_xmlfile** lock, run a pandas→openpyxl round-trip, and capture stdout/stderr — all with **zero network**. 2. **IPC integrity (gates P3/P4) — confirm the *committed* design, do not re-choose transport.** The - stdin/stdout + realm-lockdown framing works under the deny flags with binary payloads, and after lockdown an - adversarial `Deno.stdout.write`/`console.log`/raw-fd attempt **cannot forge, interleave, or replay** a control - frame; verify generation-id rejection of a killed-generation message. + stdin/stdout + realm-lockdown framing works under the deny flags with binary payloads. Lockdown blocks the + deletable primitives (`Deno.stdout.write`/`console.log`/`require`), but **finding A:** `import("node:fs").writeSync(1,…)` + still reaches stdout — so the provable invariant is that an adversary **cannot produce an _accepted_** forged / + interleaved / replayed / stale-generation control frame (the Node-side validator rejects each and kills the + child; the secret `requestId`/`seq`/`generation` are never exposed to untrusted code). Verify generation-id + rejection of a killed-generation message. 3. **Per-child memory behavior (gates P6).** Confirm `node:22-slim` non-root cannot reliably set cgroup `memory.max` (and that `prlimit --as` is unusable with V8); validate the container-limit guard + accepted availability risk (Decision 5) is the operating model. diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index f443bd9..dab567d 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -29,22 +29,40 @@ Three POCs from `design.md` §"Required Spikes". Each must pass before its depen --cached-only --no-config --allow-read= s1_runner.ts ``` `s1_runner.ts` reads `` from `Deno.args` (**not** `Deno.env`), then `loadPyodide({ indexURL, lockFileURL, packageBaseUrl })` from local paths, `loadPackage(["numpy","pandas","scipy"])`, install the frozen openpyxl+et_xmlfile lock, run a pandas→openpyxl round-trip (DataFrame → `out.xlsx` bytes → read back), and print the round-trip byte length. **Assert zero network** (the deny flags + a packet-capture or a deliberate offline run prove it). -- **S2 — IPC integrity** (gates Phase 3/4). Script `spikes/s2-ipc.ts`. Implement length-prefixed JSON framing over stdin/stdout with a binary payload; after realm lockdown (capture stdout writer, delete `Deno`/`console`/write primitives from `globalThis`), run an adversarial snippet that tries `Deno.stdout.write`/`console.log`/raw-fd to **forge, interleave, or replay** a control frame, and a stale-`generation` message. Assert all are rejected. +- **S2 — IPC integrity** (gates Phase 3/4). Script `spikes/s2-ipc.ts`. Implement length-prefixed JSON framing with a binary payload; model the real runner realm (install **then** delete `Deno`/`console`/`require`/`__dirname`/`__filename` from `globalThis`), and run an adversarial snippet that tries to **forge, interleave, or replay** a control frame, a stale-`generation` message, and a forged `ready` handshake. **Result (finding A): realm lockdown is not stdout containment** — `Deno.stdout.write`/`console.log`/`require` are blocked, but `import("node:fs").writeSync(1,…)` reaches stdout. The provable invariant is narrower: a frame validator rejects every forged/interleaved/replayed/stale-generation/bad-handshake frame, so escaped JS **cannot produce an _accepted_ frame** (it can't guess the secret `requestId`/`seq`/`generation`). Assert each is rejected and exit 0. - **S3 — Per-child memory** (gates Phase 6). Script `spikes/s3-memory.sh`. In a `node:22-slim` non-root context, attempt to set cgroup v2 `memory.max` for a child and attempt `prlimit --as`; confirm both are unavailable/unusable (V8 vaddr reservation). Document that the **container memory limit + accepted availability risk (design D5)** is the operating model. ### Phase 0: Success Criteria #### Phase 0: Programmatic Verification -- [ ] `spikes/s1-pyodide-deno.sh` exits 0 and prints the pandas→openpyxl round-trip byte length with zero network access -- [ ] `spikes/s2-ipc.ts` exits 0 and prints PASS for each of: forged-frame rejected, interleave rejected, replay rejected, stale-generation rejected -- [ ] `spikes/s3-memory.sh` exits 0 and prints the memory-limit probe result (cgroup write denied / prlimit unusable) +- [x] `spikes/s1-pyodide-deno.sh` exits 0 and prints the pandas→openpyxl round-trip byte length with zero network access — `S1 PASS pyodide=0.29.4 roundtrip_xlsx_bytes=~4970` (varies ±1; openpyxl embeds timestamps), exit 0 +- [x] `spikes/s2-ipc.ts` exits 0 and prints PASS for each of: forged-frame rejected, interleave rejected, replay rejected, stale-generation rejected — all four PASS (+ realm-lockdown, baseline, oversized), exit 0 +- [x] `spikes/s3-memory.sh` exits 0 and prints the memory-limit probe result (cgroup write denied / prlimit unusable) — `cgroup_write_denied=1 rlimit_as_unusable=1`, exit 0 #### Phase 0: Agent Verification -- [ ] Agent reviews `S1-findings.md`, `S2-findings.md`, `S3-findings.md` and confirms each gate's pass/fail is **explicit** before Phase 3/Phase 6 begin -- [ ] Agent confirms the exact Deno version and pyodide version validated in S1 match what Phase 3 pins +- [x] Agent reviews `S1-findings.md`, `S2-findings.md`, `S3-findings.md` and confirms each gate's pass/fail is **explicit** before Phase 3/Phase 6 begin — each note opens with `GATE: ✅ PASS` +- [x] Agent confirms the exact Deno version and pyodide version validated in S1 match what Phase 3 pins — Phase 3 names no specific patch yet ("pins versions in a constant"); S1 is the source of truth → Phase 3 must pin **Deno v2.8.2** + **Pyodide 0.29.4** (recorded in `S1-findings.md`) ### Phase 0: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 0._ + +**Validated pins (Phase 3 MUST adopt these — see `spikes/S1-findings.md`):** Deno **v2.8.2**, Pyodide **0.29.4** (full dist `pyodide-0.29.4.tar.bz2`, 408 MB, ships CPython 3.12), openpyxl **3.1.5** (`py2.py3-none-any`), et_xmlfile **2.0.0** (`py3-none-any`). The plan's `fetch-pyodide-assets.mjs` ("pins versions in a constant") names no patch yet — no conflict; S1 is the source of truth. + +**Surprises (carry into Phase 3 `runner.ts`):** +- **Deno is detected as Node by Pyodide** (`process.versions.node` is populated), so it uses the **Node-fs load path** — Pyodide 0.29.4 has **no `Deno.readFile` branch** (its only alternative is browser `fetch`, which needs network). The Node-fs path is therefore the correct & only offline path. +- **Emscripten (`pyodide.asm.js`) needs CommonJS globals the Deno ESM realm lacks.** Before importing `pyodide.mjs`, `runner.ts` MUST set on `globalThis`: `require = createRequire(import.meta.url)` (from `node:module`), `__dirname = `, `__filename = /pyodide.asm.js`. Otherwise: `ReferenceError: require is not defined` / `__dirname is not defined` during `loadPyodide`. These node-builtin imports are NOT blocked by `--deny-import`/`--no-npm` (remote/npm only). +- **The npm `import("ws")` is never reached** — Pyodide's `initNodeModules` returns early (`typeof A < "u"`), so `--no-npm` causes no failure. Only `node:url/fs/fs-promises/vm/path` are imported. + +**Adaptations / plan mismatches to fix in Phase 3:** +- **openpyxl + et_xmlfile are NOT in the Pyodide distribution** (absent from `pyodide-lock.json` and disk; numpy/pandas/scipy/**micropip** ARE present). Plan line 354's `loadPackage([...,"openpyxl"])` **by name throws** `No known package with name 'openpyxl'` against the stock lock. It only works once `build-pyodide-lock.mjs` produces the **custom lock** naming openpyxl+et_xmlfile (plan line 303). S1 instead vendors the two pure-python wheels and loads them by **direct `file://` URL** — also valid. Phase 3 should do one or the other deliberately. +- **`packageBaseUrl` is unnecessary** (plan lines 31/354): S1 used only `loadPyodide({ indexURL, lockFileURL, stdout, stderr })` and `loadPackage` resolved wheels relative to `indexURL`. Drop it or set `= indexURL`. + +**Gotchas:** `indexURL` must end with `/` and be absolute; asset dir passed as **argv** (never `Deno.env` — `--deny-env`); `DENO_NO_UPDATE_CHECK=1` in the **spawn env** (read by the Deno runtime, not the program); vendored wheels must live **inside** the `--allow-read` asset dir. **Zero-network proof = success under `--deny-net`** (any network attempt throws). Cold load (loadPyodide + packages) takes several seconds → informs Phase 6 exec-timeout default. + +**S2 finding A (HIGH) — realm lockdown is NOT stdout containment.** S2 models the real runner realm (installs + then deletes `require`/`__dirname`/`__filename` alongside `Deno`/`console`). The deletable primitives are blocked, but **`(await import("node:fs")).writeSync(1, bytes)` still reaches stdout under the full deny-belt** (`import` is syntax; `node:` builtins aren't gated by `--deny-import`/`--no-npm`; `--deny-write` doesn't block the open stdout fd; a dedicated control fd doesn't help — `fs.writeSync(anyFd,…)` works). ⇒ **Node-side frame validation with secret `requestId`/`seq`/`generation` is the LOAD-BEARING control, not defense-in-depth.** A stdout-writing attacker still cannot forge an *accepted* frame (it must guess unguessable secrets; can't read its own stdout to replay) — worst case is a corrupt/forged frame → kill-the-child. **Phase 3 `runner.ts` must (1) delete `require`/`__dirname`/`__filename` in lockdown and (2) NEVER expose `requestId`/`seq`/`generation` to untrusted Python.** Plan updated at Phase 3 (runner lockdown) + Phase 4 (`validateInbound` is primary control). The plan's `ready` handshake rules (plan.md:415, valid-once/pre-response/current-gen) were already correct; S2 now matches them exactly. + +**S3 numbers (Phase 6 operating model):** non-root **cannot** write cgroup `memory.max` (read-only mount) nor create a child cgroup. RLIMIT_AS (`prlimit --as`/`ulimit -v`) is **unusable for RSS**: a 2 GiB-max WASM heap reserves **VmSize ≈ 10,712 MB** vs **VmRSS ≈ 41 MB**, and a 2 GiB RLIMIT_AS makes the allocation fail (`RangeError: could not allocate memory`). ⇒ **container memory limit is the only guard**; manager must respawn-on-exit (incl. OOM-kill); per-child OOM isolation is an accepted risk. + +**Spike hygiene:** all downloaded assets live under `spikes/assets/` and are git-ignored (`spikes/.gitignore` covers `assets/` + `*.log`) — the ~408 MB Pyodide dist + Deno binary + wheels are reproducible from the script, never committed. To re-run: `bash spikes/s1-pyodide-deno.sh` (cached after first download); S2 via the bootstrapped `spikes/assets/deno-v2.8.2/deno` under the deny-belt; `bash spikes/s3-memory.sh` (needs Docker). --- @@ -352,7 +370,8 @@ export function decodeFrames(buf: Uint8Array): { frames: Frame[]; rest: Uint8Arr - Resolve asset paths (`indexURL`, `lockFileURL` (the custom lock), `packageBaseUrl` — all local, derived from the absolute asset dir) from `Deno.args` (**argv only**, passed by Node). **Never `Deno.env`** — it is blocked by `--deny-env`. (`DENO_NO_UPDATE_CHECK` is set in Node's spawn env and read by the Deno runtime, not by the program.) - `const pyodide = await loadPyodide({ indexURL, lockFileURL, packageBaseUrl, stdout, stderr })`; preload `await pyodide.loadPackage(["numpy","pandas","scipy","openpyxl"])`. -- **Realm lockdown BEFORE any untrusted Python:** capture the raw stdout writer (`Deno.stdout.write` bound), then delete `Deno`, `console`, and other write primitives from `globalThis`. All control-frame writes go through the captured writer only. +- **Realm lockdown BEFORE any untrusted Python:** capture the raw stdout writer (`Deno.stdout.write` bound), then delete from `globalThis`: `Deno`, `console`, **and the Node-compat globals S1 installs for Pyodide — `require`, `__dirname`, `__filename`** (spike S2 finding A: these are live write primitives — `require("fs").writeSync(1,…)` — and must be deleted too). All control-frame writes go through the captured writer only. + - **S2 finding A (HIGH) — lockdown is NOT stdout containment.** `(await import("node:fs")).writeSync(1, bytes)` still reaches stdout under the deny-belt (`import` is syntax; `node:` builtins aren't gated by `--deny-import`/`--no-npm`; `--deny-write` doesn't block the open stdout fd; a dedicated fd doesn't help — `fs.writeSync(anyFd,…)` works). The **Node-side frame validation with secret `requestId`/`seq`/`generation` is therefore LOAD-BEARING, not defense-in-depth** (Phase 4). To keep that guarantee, **`runner.ts` MUST NOT expose `requestId`/`seq`/`generation` to untrusted Python** — pass only `code`/`argv`/`stdin`/`files` into Pyodide; keep the integrity fields in JS closure. - IPC loop: read length-prefixed frames from stdin (`Deno.stdin`), for each `run` frame: stage `files` into MEMFS (`FS.mkdirTree` + `FS.writeFile`), reset Python `globals`, run `pyodide.runPythonAsync(code, ...)`, capture stdout/stderr via Pyodide stream callbacks into base64 fields, compute the `{created,modified,deleted}` diff against the staged input set, and emit exactly one `result`/`error` frame carrying the matching `requestId`/`seq`/`generation`. - Between execs: fresh `globals` + wipe staged MEMFS paths (bounds variable scope + staged files; `sys.modules`/package globals persist within a session — same trust boundary, per design D3). @@ -373,7 +392,7 @@ export function decodeFrames(buf: Uint8Array): { frames: Frame[]; rest: Uint8Arr - [ ] `pnpm lint:fix` passes #### Phase 3: Agent Verification -- [ ] Agent re-runs an S2 forge attempt against the built `runner.ts`; confirms the forged frame is **not** emitted on the control channel +- [ ] Agent re-runs an S2 forge attempt against the built `runner.ts`; confirms the forged frame is **not accepted** — escaped JS may write bytes to stdout (S2 finding A: `import("node:fs").writeSync(1,…)`), but the Node side rejects it, kills the child, and drains nothing - [ ] Agent confirms the deny-belt blocks remote import, npm import, update check, FS write, env read, subprocess spawn, FFI, and network (each attempt fails closed) - [ ] Agent reviews `runner.ts` to confirm realm lockdown happens **before** the first untrusted `runPythonAsync` @@ -398,6 +417,7 @@ The trusted Node half: spawn/own the Deno subprocess, frame the protocol with fu - **`ready` handshake (explicit exception):** carries `generation` only (no `requestId`/`seq`). It is valid **exactly once**, **before any `result`/`error`**, and **only with the current `generation`**. A second `ready`, a `ready` after the first response, or a stale/wrong-generation `ready` is an integrity violation → kill the child. (The handshake marks the `starting → idle` transition.) - **Size caps measured on the base64-encoded wire size** (accounts for ~33% expansion): `PYODIDE_MAX_FRAME_BYTES` per frame + `PYODIDE_MAX_AGGREGATE_BYTES` aggregate per response. - Any malformed / oversized / duplicate / out-of-sequence / wrong-generation / unexpected frame → throw a typed `IpcIntegrityError` that the manager turns into **kill-the-child**. +- **This validation is the PRIMARY, load-bearing security control — not defense-in-depth (spike S2 finding A).** Realm lockdown in `runner.ts` cannot contain stdout (`(await import("node:fs")).writeSync(1,…)` reaches it under the deny-belt), so untrusted code *can* write raw bytes to the channel. It still cannot produce an **accepted** frame: `requestId`/`seq`/`generation` are unguessable secrets never exposed to the child's Python (Phase 3 requirement), and a process cannot read its own stdout to replay. Treat `validateInbound` as security-critical; the worst an attacker achieves is corrupting/forging a frame → kill-the-child (self-DoS), never a drain of forged files. #### 2. Manager / worker state machine **File**: `src/api/pyodide/manager.ts` (new) @@ -608,12 +628,13 @@ Prove the boundary holds. This suite is the security sign-off gate (design Open **File**: `src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` (new) **Action**: create -- Escaped JS attempting `Deno.stdout.write`/`console.log`/raw-fd **cannot** forge a control frame, replay a stale-generation frame, forge or replay a `ready` handshake (duplicate / post-response / wrong-generation), nor redirect a drain write outside cwd (assert the Node side kills the child and drains nothing). +- **Threat model (per spike S2 finding A):** escaped JS **can** write arbitrary bytes to stdout — `Deno.stdout.write`/`console.log`/raw-fd are blocked by realm lockdown, but `(await import("node:fs")).writeSync(1, …)` is **not** (node: builtins aren't on the deny-belt; `--deny-write` doesn't gate the open stdout fd). The invariant is therefore **narrower**: escaped JS **cannot produce an _accepted_ control frame**, because `requestId`/`seq`/`generation` are unguessable secrets never exposed to untrusted Python (Phase 3 requirement) and a process can't read its own stdout to replay a real frame. +- Assert, for each attempt — a guessed/forged control frame, an interleaved/out-of-sequence frame, a replayed frame, a stale/wrong-`generation` frame, a forged/duplicate/post-response/wrong-generation `ready` handshake, **and** a frame injected specifically via `import("node:fs").writeSync(1, …)` — that the **Node side kills the child and drains nothing**. Also assert a drain write resolved outside cwd is rejected. **Do NOT assert "stdout cannot be written"** (it can) — assert "no forged frame is ever _accepted_, and any forgery attempt → kill-the-child + zero drain." ### Phase 7: Success Criteria #### Phase 7: Programmatic Verification -- [ ] `pnpm test -- src/api/pyodide/tests/integration/escape.integration.test.ts src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` pass (every escape blocked, every deny-belt item denied, fresh-globals isolation holds, control-frame + `ready`-handshake forgery/replay and drain-redirect blocked) +- [ ] `pnpm test -- src/api/pyodide/tests/integration/escape.integration.test.ts src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` pass (every escape blocked, every deny-belt item denied, fresh-globals isolation holds; no forged/guessed/interleaved/replayed/stale-generation control frame or forged `ready` handshake is ever **accepted** — each forgery attempt, including one injected via `import("node:fs").writeSync(1,…)`, **kills the child and drains nothing** — and drain-redirect outside cwd is rejected) - [ ] Full `pnpm test:integration` green with both containers up + assets present #### Phase 7: Agent Verification diff --git a/thoughts/issue-118-pyodide-runtime/spikes/.gitignore b/thoughts/issue-118-pyodide-runtime/spikes/.gitignore new file mode 100644 index 0000000..3802da3 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/.gitignore @@ -0,0 +1,4 @@ +# Spike S1/S2 download artifacts — large, reproducible from the pinned manifest. +# Never commit the Deno binary or the ~400 MB Pyodide distribution. +assets/ +*.log diff --git a/thoughts/issue-118-pyodide-runtime/spikes/S1-findings.md b/thoughts/issue-118-pyodide-runtime/spikes/S1-findings.md new file mode 100644 index 0000000..f1c1a44 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/S1-findings.md @@ -0,0 +1,88 @@ +# S1 — Pyodide-on-Deno offline (gates Phase 3) + +## GATE: ✅ PASS + +`s1-pyodide-deno.sh` exits **0** and prints (the byte count varies ±1 between +runs — openpyxl embeds document created/modified timestamps in the `.xlsx`, so +the zip size is not bit-deterministic; the gate is a **non-zero** round-trip): + +``` +S1 PASS pyodide=0.29.4 roundtrip_xlsx_bytes=4970 # ~4969–4970 +``` + +A Deno subprocess, under the **exact committed deny-belt**, loaded Pyodide from +local disk, loaded numpy/pandas/scipy + openpyxl/et_xmlfile from local wheels, +and ran a pandas → openpyxl `.xlsx` round-trip (write 4970 bytes → read back → +structural assertions) with **zero network**. + +## Pinned versions validated (Phase 3 MUST pin exactly these) + +| Component | Version | Source | +|---|---|---| +| Deno | **v2.8.2** (`aarch64-apple-darwin` locally; pick the matching Linux target in the image) | `dl.deno.land/release/v2.8.2/...` | +| Pyodide full distribution | **0.29.4** | `github.com/pyodide/pyodide/releases/download/0.29.4/pyodide-0.29.4.tar.bz2` (408 MB) | +| openpyxl | **3.1.5** (`openpyxl-3.1.5-py2.py3-none-any.whl`, pure-python) | PyPI files.pythonhosted.org | +| et_xmlfile | **2.0.0** (`et_xmlfile-2.0.0-py3-none-any.whl`, pure-python) | PyPI files.pythonhosted.org | + +> The plan (Phase 3, `fetch-pyodide-assets.mjs`) says "pins versions in a constant +> at the top of the file" but names no specific patch. **No conflicting pin +> exists yet** — S1 is the source of truth: Phase 3 should adopt Deno **v2.8.2** +> and Pyodide **0.29.4**. (Pyodide 0.29.4 ships CPython 3.12; both wheels are +> `requires_python >=3.8`, compatible.) + +## Zero-network proof + +The run uses the full belt: `--no-prompt --deny-net --deny-run --deny-write +--deny-env --deny-ffi --deny-sys --deny-import --no-remote --no-npm +--cached-only --no-config --allow-read=`. Any network attempt by +Pyodide would throw a Deno permission error and fail the run. **Success under +`--deny-net` is the proof.** The only network is in the BUILD step (downloading +Deno + Pyodide + the two wheels), which is the intended `fetch-pyodide-assets` +behavior and is wholly separate from runtime. + +## Surprises / corrections (carry into Phase 3) + +1. **Deno is detected as Node, not Deno.** Pyodide's loader computes `IN_NODE` + from `process.versions.node` — which Deno populates — so it takes the + **Node-fs load path**, not a Deno path. Pyodide 0.29.4 has **no + `Deno.readFile` branch**; its only non-Node path is browser `fetch` (would + need network). So the Node-fs path is the *correct and only* offline path. +2. **Emscripten needs CommonJS globals the Deno ESM realm lacks.** `pyodide.asm.js` + (Emscripten output) calls bare `require("fs"/"path"/"crypto")`, `__dirname`, + `__filename`. In a Deno ES module these are undefined → `ReferenceError`. **Fix + (required in `runner.ts`):** before importing `pyodide.mjs`, set on `globalThis`: + - `require = createRequire(import.meta.url)` (from `node:module`), + - `__dirname = `, `__filename = /pyodide.asm.js`. + Bare identifiers fall through to `globalThis`, so this satisfies Emscripten. + These node-builtin imports are **not** blocked by `--deny-import`/`--no-npm` + (those gate remote/npm only). +3. **The npm `import("ws")` is never reached.** Pyodide's `initNodeModules` + short-circuits (`typeof A < "u"`, its bundler require-shim) and returns before + the `await import("ws")` line. So `--no-npm` causes **no** failure. Confirmed: + only `node:url/fs/fs-promises/vm/path` are imported (all Deno-supported). +4. **openpyxl + et_xmlfile are NOT in the Pyodide distribution.** Not in + `pyodide-lock.json`, not on disk. numpy/pandas/scipy/**micropip** ARE present. + - S1 vendors the two pure-python wheels and loads them via **direct file:// URL** + `loadPackage(["file://…/et_xmlfile….whl","file://…/openpyxl….whl"])`. + - **Phase 3 mismatch to fix:** plan line 354 preloads + `loadPackage(["numpy","pandas","scipy","openpyxl"])` **by name** — that throws + `No known package with name 'openpyxl'` against the stock lock. It only works + once `build-pyodide-lock.mjs` (plan line 303) produces a **custom lock** that + names openpyxl+et_xmlfile. Either generate the custom lock (preferred, per + plan) OR load the wheels by URL as S1 does. +5. **`packageBaseUrl` is unnecessary.** Plan lines 31/354 pass + `loadPyodide({ indexURL, lockFileURL, packageBaseUrl })`. S1 used only + `{ indexURL, lockFileURL }` and `loadPackage` resolved distribution wheels + relative to `indexURL` correctly. `packageBaseUrl` can be dropped (or kept + equal to `indexURL`) — it is not load-bearing here. + +## Gotchas for Phase 3 + +- `indexURL` **must end in a trailing slash**; pass the **absolute** asset dir. +- Asset dir is passed as **argv** (`Deno.args[0]`), never `Deno.env` (blocked by + `--deny-env`). `DENO_NO_UPDATE_CHECK=1` goes in the **spawn env** (read by the + Deno runtime itself, not by the program). +- `--allow-read` must be scoped to the asset dir **and the vendored wheels must + live inside it** (S1 places them in the pyodide dir) so reads stay in scope. +- Cold load (loadPyodide + loadPackage of numpy/pandas/scipy/openpyxl) is several + seconds — informs the Phase 6 cold-start exec-timeout default. diff --git a/thoughts/issue-118-pyodide-runtime/spikes/S2-findings.md b/thoughts/issue-118-pyodide-runtime/spikes/S2-findings.md new file mode 100644 index 0000000..f43b370 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/S2-findings.md @@ -0,0 +1,103 @@ +# S2 — IPC integrity (gates Phase 3/4) + +## GATE: ✅ PASS — with a load-bearing finding (see "Finding A") + +`s2-ipc.ts`, run with the bootstrapped Deno **under the committed deny-belt**, +exits **0** and prints: + +``` + PASS realm-lockdown blocks deletable primitives [Deno=blocked,console=blocked,require=blocked] +NOTE lockdown is NOT sufficient: import("node:fs").writeSync(1,…) reaches stdout under the deny-belt +NOTE => Node-side frame validation with secret generation/requestId is LOAD-BEARING (design.md D1) +PASS forge-resistance: guessed-secret frame rejected +PASS ready handshake accepted once +PASS duplicate-ready rejected +PASS ready-after-response rejected +PASS wrong-generation-ready rejected +PASS baseline ready + result accepted +PASS forged-frame rejected +PASS interleave rejected +PASS replay rejected +PASS stale-generation rejected +PASS oversized rejected +S2 ALL PASS +``` + +> The **leading space** on the first line is not cosmetic — it is the single +> `0x20` byte the adversary wrote to fd 1 via `import("node:fs")`. Its presence +> on the wire IS the evidence for Finding A. + +The spike now models the **actual post-S1 runner realm**: it installs the same +Node-compat globals the real `runner.ts` must (`require` via `createRequire`, +`__dirname`, `__filename`) and locks them down, so the adversary faces the realm +untrusted code will truly see. **This confirms the committed design** (design.md +Decision 1) — it does not re-choose a transport. + +Invocation (no `--allow-read` needed — and, per Finding A, the node:fs import + +stdout write need no permission either): +``` +DENO_NO_UPDATE_CHECK=1 deno run --no-prompt --deny-net --deny-run --deny-write \ + --deny-env --deny-ffi --deny-sys --deny-import --no-remote --no-npm \ + --cached-only --no-config s2-ipc.ts +``` + +## Finding A (HIGH) — realm lockdown is hardening, NOT stdout containment + +After capturing the writer and deleting `Deno`, `console`, `require`, +`__dirname`, `__filename` from `globalThis`: +- `Deno.stdout.write` → **blocked** (Deno deleted) +- `console.log` → **blocked** (console deleted) +- `require("fs")` → **blocked** (require deleted) +- **`(await import("node:fs")).writeSync(1, bytes)` → SUCCEEDS — writes to stdout.** + +Why the last one cannot be stopped by lockdown: +1. `import(...)` is **syntax**, not a deletable global. +2. `node:` builtins are **not** gated by `--deny-import`/`--no-npm` (those gate + remote/npm only). +3. `--deny-write` does **not** block writes to the **already-open stdout fd**. + +A dedicated control fd (e.g. fd 3) does **not** fix this either: `fs.writeSync(N,…)` +works for **any** open fd. The correct, sufficient defense is therefore the +**Node-side frame validation with secret integrity fields**, which design.md +Decision 1 already mandates. **This makes Node-side validation LOAD-BEARING, not +"defense-in-depth."** + +## Finding B — why the bypass cannot actually forge an accepted frame + +Every frame carries a random **requestId**, monotonic **seq**, and +child-**generation** id. These are held only by Node and the runner's closure and +are **never exposed to untrusted Python** (the runner passes only `code` to +`runPythonAsync`). An attacker who can write bytes to stdout therefore: +- **cannot forge an accepted frame** — it must GUESS the secrets; the validator + KILLS the child on any mismatch (spike: `forge-resistance: guessed-secret frame + rejected`); +- **cannot replay a real frame** — a process cannot read its own stdout pipe; +- **can only corrupt/interleave** → a malformed/oversized/out-of-sequence frame → + Node kills the child (self-DoS of that one session, no escalation). + +## What the validator enforces (kill on ANY anomaly) + +**One-time `ready` handshake** — `{ type:"ready", generation }`, NO requestId/seq +(matches plan.md:357). Valid **once**, **before any response**, **current +generation only**. Rejected (→ kill): `duplicate-ready`, `ready-after-response`, +`wrong-generation-ready`. + +**Per-request responses** — `result|error` with `{ requestId, seq, generation }`. +Rejected (→ kill): `oversized` (cap measured on **encoded wire size**), +`stale-generation`, `forged-type`, `forged-requestId` (never issued), +`duplicate-response` (replay — one response per requestId), `out-of-sequence` +(interleave — seq must strictly increase per generation). + +## Mandatory requirements for Phase 3/4 (carry forward) + +1. **Phase 3 `runner.ts` lockdown must delete `require`, `__dirname`, `__filename` + in addition to `Deno`/`console`** (plan.md:371 said only "Deno, console, and + other write primitives" — these ARE the "other" ones, now named explicitly). +2. **`runner.ts` MUST NOT expose `requestId`/`seq`/`generation` to untrusted + Python** (keep them in JS closure; pass only `code`/`argv`/`stdin`/`files` into + Pyodide). If they leak, Finding B's guarantee collapses. +3. **Phase 4 Node side MUST validate every frame and kill on the first anomaly** — + this is the primary control, not optional hardening. Re-enforce cwd + path-validation + size caps on drain regardless of frame contents. +4. The integrity fields must be **unguessable** (random requestId; monotonic seq; + generation bumped on every respawn). diff --git a/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md b/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md new file mode 100644 index 0000000..c5202bf --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md @@ -0,0 +1,66 @@ +# S3 — Per-child memory behavior (gates Phase 6) + +## GATE: ✅ PASS + +`s3-memory.sh` exits **0** and confirms the operating model in design.md +Decision 5: as the non-root `node` user in `node:22-slim`, **per-child memory +limiting is unavailable/unusable**, so the **operator-set container memory limit ++ accepted availability risk is the guard**. + +``` +cgroup_write_denied=1 rlimit_as_unusable=1 (vaddr_decoupled=1 rlimit_breaks_wasm=1) +S3 PASS: non-root cannot set cgroup memory.max; prlimit --as is unusable for RSS + (V8/WASM vaddr >> RSS). Container memory limit is the real guard. +``` + +(Run in a real Linux container because the host is macOS — cgroup v2 and the V8 +virtual-address-space semantics only exist Linux-side, which is the prod target.) + +## Probe 1 — cgroup v2 `memory.max` write: DENIED (expected) + +As uid 1000 (`node`) with the default Docker cgroup mount: +- `echo … > /sys/fs/cgroup/memory.max` → **Read-only file system** (denied). +- `mkdir /sys/fs/cgroup/s3probe` (child cgroup) → **Read-only file system** (denied). + +The cgroup v2 hierarchy is mounted **read-only** for the unprivileged container, +so a non-root (and indeed any in-container) process cannot set a per-child +`memory.max`. Confirms: no per-child cgroup memory cap without `--privileged` / +host cgroup delegation, which this deployment does not assume. + +## Probe 2 — `prlimit --as` / `ulimit -v` (RLIMIT_AS): UNUSABLE for RSS (expected) + +RLIMIT_AS caps **virtual** address space, not **resident** memory, and V8 + a +WASM heap reserve enormous virtual space while staying tiny resident: + +- **(2a)** Creating `WebAssembly.Memory({initial:16, maximum:32768})` (~2 GiB max, + Pyodide-like) → **VmSize ≈ 10,712 MB** but **VmRSS ≈ 41 MB**. The virtual + reservation is ~260× the resident size — RLIMIT_AS cannot track RSS. +- **(2b)** Under `ulimit -v 2097152` (2 GiB — a limit that *would* be a useful RSS + cap), the same allocation **fails**: `RangeError: WebAssembly.Memory(): could + not allocate memory`. So any RLIMIT_AS low enough to bound RSS **breaks the + workload**, and any value letting it run (≥ ~10.7 GiB here) is far too high to + bound RSS. ⇒ unusable as a per-child RAM cap. + +## Operating model for Phase 6 (carry forward) + +- **Do NOT** attempt per-child `cgroup memory.max` or `prlimit --as` as the RAM + guard — both are confirmed unavailable/unusable in this posture. +- The **container memory limit** covers Node + **all** Deno children together. + Operators size it as `MAX_RESIDENT_PYODIDE × per-proc ceiling` (`MAX_RESIDENT=1` + on small hosts). A runaway child can OOM-kill the whole container — **accepted + availability risk** (Decision 5), mitigated by multi-replica + restart. +- Phase 6's manager must **report an error + respawn (new generation) on child + exit** (including OOM-kill), since it cannot prevent the OOM itself. +- The Pyodide ~2 GiB WASM cap is only a per-instance **heap** ceiling — note the + full process vaddr reservation is ~5× that (~10 GiB here), but RSS is what the + container limit actually constrains. + +## Notable + +- The `bash: …memory.max: Read-only file system` line is the shell's own redirect + error (emitted before the command runs, so not captured by `2>/tmp/cgerr`); + it is cosmetic and corroborates the denial — the probe still classifies it + correctly. +- cgroup controllers present in-container: `cpuset cpu io memory hugetlb pids rdma` + (so cgroup v2 with the memory controller **is** active — the limit just isn't + writable by the container). diff --git a/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh b/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh new file mode 100755 index 0000000..254ac19 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Spike S1 — Pyodide-on-Deno offline (gates Phase 3). +# +# Bootstraps a PINNED Deno binary + the PINNED Pyodide full distribution into +# ./assets/ (cached; re-runs skip downloads), then runs s1_runner.ts under the +# EXACT committed deny-belt flags from design.md Decision 1. A successful run +# under --deny-net is the proof of zero network access. +# +# Usage: bash s1-pyodide-deno.sh +# Exit 0 => round-trip byte length printed, zero-network confirmed. + +set -euo pipefail + +# --- Pinned versions (must match what Phase 3 pins) ------------------------- +DENO_VERSION="v2.8.2" +PYODIDE_VERSION="0.29.4" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ASSETS="${SCRIPT_DIR}/assets" +DENO_DIR="${ASSETS}/deno-${DENO_VERSION}" +DENO_BIN="${DENO_DIR}/deno" +PYODIDE_DIR="${ASSETS}/pyodide-${PYODIDE_VERSION}" # extracted dist root (contains pyodide.mjs) +mkdir -p "${ASSETS}" + +# --- Resolve Deno download target for this host ----------------------------- +OS="$(uname -s)"; ARCH="$(uname -m)" +case "${OS}-${ARCH}" in + Darwin-arm64) DENO_TARGET="aarch64-apple-darwin" ;; + Darwin-x86_64) DENO_TARGET="x86_64-apple-darwin" ;; + Linux-aarch64) DENO_TARGET="aarch64-unknown-linux-gnu" ;; + Linux-x86_64) DENO_TARGET="x86_64-unknown-linux-gnu" ;; + *) echo "S1 FAIL: unsupported host ${OS}-${ARCH}" >&2; exit 2 ;; +esac + +# --- 1. Fetch pinned Deno (cached) ------------------------------------------ +if [[ ! -x "${DENO_BIN}" ]]; then + echo "[s1] downloading Deno ${DENO_VERSION} (${DENO_TARGET})…" >&2 + mkdir -p "${DENO_DIR}" + deno_zip="${DENO_DIR}/deno.zip" + curl -fSL -o "${deno_zip}" \ + "https://dl.deno.land/release/${DENO_VERSION}/deno-${DENO_TARGET}.zip" + unzip -oq "${deno_zip}" -d "${DENO_DIR}" + rm -f "${deno_zip}" + chmod +x "${DENO_BIN}" + # Strip any quarantine xattr (no-op on Linux / when absent). + xattr -d com.apple.quarantine "${DENO_BIN}" 2>/dev/null || true +else + echo "[s1] Deno ${DENO_VERSION} already present" >&2 +fi +echo "[s1] deno version: $("${DENO_BIN}" --version | head -1)" >&2 + +# --- 2. Fetch + extract pinned Pyodide full distribution (cached) ----------- +if [[ ! -f "${PYODIDE_DIR}/pyodide.mjs" ]]; then + echo "[s1] downloading Pyodide ${PYODIDE_VERSION} full distribution (~408 MB)…" >&2 + tarball="${ASSETS}/pyodide-${PYODIDE_VERSION}.tar.bz2" + if [[ ! -f "${tarball}" ]]; then + curl -fSL -o "${tarball}" \ + "https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-${PYODIDE_VERSION}.tar.bz2" + fi + echo "[s1] extracting…" >&2 + tmp_extract="${ASSETS}/_extract-${PYODIDE_VERSION}" + rm -rf "${tmp_extract}"; mkdir -p "${tmp_extract}" + tar -xjf "${tarball}" -C "${tmp_extract}" + # The tarball unpacks to a top-level "pyodide/" dir. + rm -rf "${PYODIDE_DIR}" + mv "${tmp_extract}/pyodide" "${PYODIDE_DIR}" + rm -rf "${tmp_extract}" "${tarball}" +else + echo "[s1] Pyodide ${PYODIDE_VERSION} already extracted" >&2 +fi +echo "[s1] pyodide assets: $(ls "${PYODIDE_DIR}" | wc -l | tr -d ' ') files in ${PYODIDE_DIR}" >&2 + +# --- 2b. Vendor openpyxl + et_xmlfile wheels (NOT in the pyodide dist) ------- +# FINDING (gates Phase 3): openpyxl + et_xmlfile are absent from pyodide +# 0.29.4's distribution + pyodide-lock.json. Both are PURE-PYTHON wheels; we +# vendor pinned copies into the asset dir (cached) so the round-trip is fully +# offline. Phase 3/7 must bake these the same way and generate a custom lock +# (micropip.freeze) rather than relying on the stock distribution. +OPENPYXL_WHL="openpyxl-3.1.5-py2.py3-none-any.whl" +ETXML_WHL="et_xmlfile-2.0.0-py3-none-any.whl" +OPENPYXL_URL="https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/${OPENPYXL_WHL}" +ETXML_URL="https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/${ETXML_WHL}" +for pair in "${ETXML_WHL}|${ETXML_URL}" "${OPENPYXL_WHL}|${OPENPYXL_URL}"; do + whl="${pair%%|*}"; url="${pair#*|}" + if [[ ! -f "${PYODIDE_DIR}/${whl}" ]]; then + echo "[s1] vendoring ${whl}…" >&2 + curl -fSL --no-progress-meter -o "${PYODIDE_DIR}/${whl}" "${url}" + fi +done + +# --- 3. Run under the EXACT committed deny-belt ----------------------------- +# DENO_NO_UPDATE_CHECK is read by the Deno runtime from the spawn env (NOT via +# Deno.env, which --deny-env blocks). The asset dir is passed as ARGV. +echo "[s1] running s1_runner.ts under the committed deny-belt…" >&2 +DENO_NO_UPDATE_CHECK=1 "${DENO_BIN}" run \ + --no-prompt \ + --deny-net --deny-run --deny-write --deny-env --deny-ffi --deny-sys --deny-import \ + --no-remote --no-npm --cached-only --no-config \ + --allow-read="${PYODIDE_DIR}" \ + "${SCRIPT_DIR}/s1_runner.ts" "${PYODIDE_DIR}" "${ETXML_WHL}" "${OPENPYXL_WHL}" diff --git a/thoughts/issue-118-pyodide-runtime/spikes/s1_runner.ts b/thoughts/issue-118-pyodide-runtime/spikes/s1_runner.ts new file mode 100644 index 0000000..d650a09 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/s1_runner.ts @@ -0,0 +1,120 @@ +// Spike S1 — Pyodide-on-Deno, fully offline, under the committed deny-belt. +// +// Gates Phase 3. Run via s1-pyodide-deno.sh, which spawns Deno with: +// --no-prompt --deny-net --deny-run --deny-write --deny-env --deny-ffi +// --deny-sys --deny-import --no-remote --no-npm --cached-only --no-config +// --allow-read= +// +// The asset dir is passed as ARGV (Deno.args[0]) — NOT via Deno.env, which +// --deny-env blocks. DENO_NO_UPDATE_CHECK=1 is set in the parent spawn env and +// is read by the Deno runtime itself (not via Deno.env). +// +// Success criterion: load Pyodide + numpy/pandas/scipy/openpyxl from local disk, +// run a pandas -> openpyxl .xlsx round-trip, and print the round-trip byte +// length — all with ZERO network (proven by --deny-net: any network attempt +// would throw a Deno permission error and fail the run). + +import { createRequire } from "node:module"; + +const assetDir = Deno.args[0]; +if (!assetDir) { + console.error("S1 FAIL: asset dir not provided as argv[0]"); + Deno.exit(2); +} + +// indexURL must end with a trailing slash; Pyodide resolves pyodide.asm.wasm, +// python_stdlib.zip, package wheels, and pyodide-lock.json relative to it. +const indexURL = assetDir.endsWith("/") ? assetDir : `${assetDir}/`; +const assetRoot = indexURL.replace(/\/$/, ""); // dir, no trailing slash + +// --- Deno-runs-Pyodide compatibility shims (install BEFORE importing Pyodide) - +// Deno exposes `process.versions.node`, so Pyodide's loader detects IN_NODE and +// takes the Node-fs path (it has NO Deno.readFile branch — it would otherwise +// need network `fetch`). That path works offline, but Emscripten's pyodide.asm.js +// is generated for CommonJS Node and uses BARE `require(...)` / `__dirname` / +// `__filename`, none of which exist in a Deno ESM module. Provide them as globals +// (bare identifiers fall through to globalThis) so Emscripten resolves them: +// - require -> Deno node-compat builtins (fs/path/crypto/...) +// - __dirname/__filename -> the asset dir, where pyodide.asm.wasm lives. +// Pyodide's own initNodeModules returns early (its bundler require-shim is +// defined), so the npm `import("ws")` line is never reached — no --no-npm break. +// deno-lint-ignore no-explicit-any +const g = globalThis as any; +g.require = createRequire(import.meta.url); +g.__dirname = assetRoot; +g.__filename = `${assetRoot}/pyodide.asm.js`; + +function log(msg: string): void { + console.error(`[s1] ${msg}`); // diagnostics on stderr; the asserted result goes to stdout +} + +try { + log(`assetDir=${assetDir}`); + log("importing pyodide.mjs from local disk…"); + // Local dynamic import under --allow-read. --deny-import blocks REMOTE imports + // only; a local file:// import within the read scope is permitted. + const pyodideModule = await import(`file://${indexURL}pyodide.mjs`); + const loadPyodide = pyodideModule.loadPyodide; + + log("loadPyodide({ indexURL, lockFileURL }) — offline…"); + const pyodide = await loadPyodide({ + indexURL, + lockFileURL: `${indexURL}pyodide-lock.json`, + // Keep stdout clean — Pyodide's own banner/print goes to our handlers. + stdout: (s: string) => log(`py.stdout: ${s}`), + stderr: (s: string) => log(`py.stderr: ${s}`), + }); + log(`Pyodide ${pyodide.version} loaded`); + + // numpy/pandas/scipy ship in the pyodide distribution → load by name. + log("loadPackage([numpy, pandas, scipy]) — from local distribution wheels…"); + await pyodide.loadPackage(["numpy", "pandas", "scipy"]); + + // openpyxl + et_xmlfile are NOT in the distribution; load the vendored + // pure-python wheels by local file:// URL (argv[1..]). loadPackage reads them + // via node:fs under --allow-read — no network, no PyPI. + const wheelArgs = Deno.args.slice(1); + if (wheelArgs.length === 0) { + console.error("S1 FAIL: no vendored wheels passed (expected et_xmlfile + openpyxl)"); + Deno.exit(2); + } + const wheelUrls = wheelArgs.map((w) => `file://${indexURL}${w}`); + log(`loadPackage(vendored wheels) — ${wheelArgs.join(", ")}…`); + await pyodide.loadPackage(wheelUrls); + log("packages loaded"); + + // pandas -> openpyxl round-trip entirely in-memory (no FS write needed for the spike). + const roundTrip = ` +import io +import pandas as pd +import numpy as np +import scipy # prove scipy imports + +df = pd.DataFrame({"a": np.arange(3), "b": ["x", "y", "z"]}) +buf = io.BytesIO() +df.to_excel(buf, index=False, engine="openpyxl") +xlsx_bytes = buf.getvalue() + +# Read it back and assert structural equality. +df2 = pd.read_excel(io.BytesIO(xlsx_bytes), engine="openpyxl") +assert list(df2.columns) == ["a", "b"], df2.columns +assert df2.shape == (3, 2), df2.shape +assert df2["b"].tolist() == ["x", "y", "z"], df2["b"].tolist() + +len(xlsx_bytes) +`; + const byteLen = await pyodide.runPythonAsync(roundTrip); + + if (typeof byteLen !== "number" || byteLen <= 0) { + console.error(`S1 FAIL: round-trip produced no bytes (got ${byteLen})`); + Deno.exit(1); + } + + // The single asserted result line on stdout. + console.log(`S1 PASS pyodide=${pyodide.version} roundtrip_xlsx_bytes=${byteLen}`); + Deno.exit(0); +} catch (err) { + console.error("S1 FAIL: exception during offline Pyodide run:"); + console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); + Deno.exit(1); +} diff --git a/thoughts/issue-118-pyodide-runtime/spikes/s2-ipc.ts b/thoughts/issue-118-pyodide-runtime/spikes/s2-ipc.ts new file mode 100644 index 0000000..6f7d58d --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/s2-ipc.ts @@ -0,0 +1,275 @@ +// Spike S2 — IPC integrity (gates Phase 3/4). +// +// Confirms the COMMITTED design (design.md Decision 1) — it does NOT re-choose a +// transport. It models the ACTUAL post-S1 runner realm (which installs Node-compat +// globals `require`/`__dirname`/`__filename` so Emscripten/Pyodide can load) and +// proves what the design's IPC integrity actually rests on. +// +// THREE things are proven/characterised in a real Deno realm under the deny-belt: +// +// (A) Realm lockdown — necessary but NOT sufficient. Capturing the stdout writer +// into a closure and deleting Deno/console/require/__dirname/__filename from +// globalThis blocks the EASY write primitives. But `import("node:fs")` is +// SYNTAX (undeletable), node: builtins are not gated by --deny-import/--no-npm, +// and --deny-write does NOT block writes to the already-open stdout fd — so +// untrusted JS CAN still `(await import("node:fs")).writeSync(1, bytes)`. +// => Lockdown is hardening; it cannot contain raw stdout writes. +// +// (B) Forge-resistance is the REAL guarantee. Every frame carries mandatory +// integrity fields — random requestId, monotonic seq, child-generation id — +// held only by Node and the runner's closure, NEVER exposed to untrusted code. +// So an attacker who CAN write bytes to stdout still cannot produce a frame +// Node accepts: it must GUESS the secret fields, and the validator KILLS the +// child on any mismatch. It also cannot read its own stdout, so it cannot +// replay a real frame. Node-side validation is LOAD-BEARING, not optional. +// +// (C) Frame validation rejects forged / interleaved / replayed / stale-generation +// per-request frames AND enforces the one-time `ready` handshake (no +// requestId/seq; valid once, before any response, current generation only; +// duplicate / post-response / wrong-generation all kill the child). +// +// Run under the committed deny-belt (no --allow-read needed; node:fs import + the +// stdout write below need NO permission — that is the whole point of finding A): +// deno run --no-prompt --deny-net --deny-run --deny-write --deny-env --deny-ffi \ +// --deny-sys --deny-import --no-remote --no-npm --cached-only --no-config s2-ipc.ts +// +// Exit 0 only if every committed-design assertion passes; prints "PASS " per +// case and "NOTE " for the lockdown-insufficiency characterisation. + +// deno-lint-ignore-file no-explicit-any +import { createRequire } from "node:module"; + +const enc = new TextEncoder(); +const MAX_FRAME_BYTES = 4096; // wire-size cap for the spike + +// --- Capture the legitimate writer BEFORE lockdown -------------------------- +// A synchronous writer is captured into a module-local closure. This is the ONLY +// sanctioned path to the real stdout and is NOT reachable from globalThis. +const _denoRef = (globalThis as any).Deno; +const writeSync: (b: Uint8Array) => number = _denoRef.stdout.writeSync.bind(_denoRef.stdout); +function out(line: string): void { + writeSync(enc.encode(`${line}\n`)); +} + +// --- Length-prefixed JSON framing ------------------------------------------- +function encodeFrame(obj: unknown): Uint8Array { + const json = enc.encode(JSON.stringify(obj)); + const buf = new Uint8Array(4 + json.byteLength); + new DataView(buf.buffer).setUint32(0, json.byteLength, false); // big-endian length prefix + buf.set(json, 4); + return buf; +} + +// `ready` handshake frame: { type, generation } — NO requestId/seq (plan.md:357). +interface ReadyFrame { + type: "ready"; + generation: number; +} +// Per-request response frame: result|error with the secret integrity fields. +interface ResponseFrame { + type: "result" | "error"; + requestId: string; + seq: number; + generation: number; + payload?: string; // base64 stand-in for stdout/created-files bytes +} + +type Verdict = { ok: true } | { kill: string }; + +// Parent-side validator for one child generation. Mirrors the production +// "kill the child on ANY anomaly" rule: any non-ok verdict ⇒ kill the child. +class SessionValidator { + #lastSeq = -1; + #readySeen = false; + #anyResponse = false; + readonly #responded = new Set(); + readonly #generation: number; + readonly #inflight: ReadonlySet; // requestIds Node has actually issued + + constructor(generation: number, inflight: Iterable) { + this.#generation = generation; + this.#inflight = new Set(inflight); + } + + // One-time pre-run handshake. Valid once, before any response, current generation. + acceptReady(wire: Uint8Array, f: ReadyFrame): Verdict { + if (wire.byteLength > MAX_FRAME_BYTES) return { kill: "oversized" }; + if (f.type !== "ready") return { kill: "not-a-ready-frame" }; + if (f.generation !== this.#generation) return { kill: "wrong-generation-ready" }; + if (this.#anyResponse) return { kill: "ready-after-response" }; + if (this.#readySeen) return { kill: "duplicate-ready" }; + this.#readySeen = true; + return { ok: true }; + } + + // Per-request response. Exactly one result|error per issued requestId, in + // strictly-increasing seq, current generation only. + acceptResponse(wire: Uint8Array, f: ResponseFrame): Verdict { + if (wire.byteLength > MAX_FRAME_BYTES) return { kill: "oversized" }; + if (f.generation !== this.#generation) return { kill: "stale-generation" }; + if (f.type !== "result" && f.type !== "error") return { kill: "forged-type" }; + if (!this.#inflight.has(f.requestId)) return { kill: "forged-requestId" }; + if (this.#responded.has(f.requestId)) return { kill: "duplicate-response" }; + if (typeof f.seq !== "number" || f.seq <= this.#lastSeq) return { kill: "out-of-sequence" }; + this.#lastSeq = f.seq; + this.#responded.add(f.requestId); + this.#anyResponse = true; + return { ok: true }; + } +} + +let failures = 0; +function check(label: string, ok: boolean, detail = ""): void { + if (ok) out(`PASS ${label}`); + else { + failures++; + out(`FAIL ${label} ${detail}`); + } +} + +// ============================================================================ +// Model the ACTUAL runner realm: install the Node-compat globals S1 requires, +// then lock down. (This is the realm untrusted code actually sees in prod.) +// ============================================================================ +const g = globalThis as any; +g.require = createRequire(import.meta.url); +g.__dirname = "/spike"; +g.__filename = "/spike/runner.ts"; + +// Closure-held control writer; never placed on globalThis. +const capturedWriter = { emit: (f: Uint8Array) => writeSync(f) }; +void capturedWriter; + +// Lockdown: delete EVERY deletable host/Node-compat write primitive. +delete g.Deno; +delete g.console; +delete g.require; +delete g.__dirname; +delete g.__filename; + +// ============================================================================ +// (A) Realm lockdown — deletable primitives blocked; import("node:fs") is NOT. +// ============================================================================ +// Untrusted adversary: an async `new Function` body sees ONLY globalThis + import(). +const adversary = new Function(`return (async () => { + const r = {}; + try { Deno.stdout.write(new Uint8Array([1])); r.deno = "SUCCEEDED"; } catch { r.deno = "blocked"; } + try { console.log("x"); r.console = "SUCCEEDED"; } catch { r.console = "blocked"; } + try { require("fs"); r.require = "SUCCEEDED"; } catch { r.require = "blocked"; } + try { + const fs = await import("node:fs"); + // Write a single harmless byte to fd 1 to detect whether this path reaches stdout. + fs.writeSync(1, new Uint8Array([0x20])); // a space; benign on the wire + r.importFsWrite = "SUCCEEDED"; + } catch (e) { r.importFsWrite = "blocked:" + (e.code || e.name); } + return r; +})();`) as () => Promise>; + +const a = await adversary(); +// The deletable primitives MUST be blocked. +check( + `realm-lockdown blocks deletable primitives [Deno=${a.deno},console=${a.console},require=${a.require}]`, + a.deno === "blocked" && a.console === "blocked" && a.require === "blocked", + JSON.stringify(a), +); +// The import("node:fs") stdout write is EXPECTED to succeed — this is the finding. +if (a.importFsWrite === "SUCCEEDED") { + out(`NOTE lockdown is NOT sufficient: import("node:fs").writeSync(1,…) reaches stdout under the deny-belt`); + out(`NOTE => Node-side frame validation with secret generation/requestId is LOAD-BEARING (design.md D1)`); +} else { + out(`NOTE import("node:fs") stdout write was ${a.importFsWrite} (Deno behaviour differs from spike host)`); +} + +// ============================================================================ +// (B) Forge-resistance: an attacker WITH stdout access still cannot pass Node's +// validator, because it must guess the secret generation/requestId. +// ============================================================================ +const GEN = 7; +const INFLIGHT = ["req-A", "req-B", "req-X"]; // requestIds Node actually issued +{ + const v = new SessionValidator(GEN, INFLIGHT); + // Attacker writes a "result" with GUESSED secrets (it cannot read the real ones). + const guessed: ResponseFrame = { type: "result", requestId: "req-GUESS", seq: 0, generation: 999 }; + const verdict = v.acceptResponse(encodeFrame(guessed), guessed); + check("forge-resistance: guessed-secret frame rejected", "kill" in verdict, JSON.stringify(verdict)); +} + +// ============================================================================ +// (C1) One-time `ready` handshake — valid once / duplicate / post-response / +// wrong-generation. +// ============================================================================ +{ + const v = new SessionValidator(GEN, INFLIGHT); + const ready: ReadyFrame = { type: "ready", generation: GEN }; + check("ready handshake accepted once", "ok" in v.acceptReady(encodeFrame(ready), ready)); + // duplicate ready + check("duplicate-ready rejected", "kill" in v.acceptReady(encodeFrame(ready), ready)); +} +{ + const v = new SessionValidator(GEN, INFLIGHT); + const resp: ResponseFrame = { type: "result", requestId: "req-A", seq: 0, generation: GEN }; + v.acceptResponse(encodeFrame(resp), resp); // first response + const lateReady: ReadyFrame = { type: "ready", generation: GEN }; + check("ready-after-response rejected", "kill" in v.acceptReady(encodeFrame(lateReady), lateReady)); +} +{ + const v = new SessionValidator(GEN, INFLIGHT); + const staleReady: ReadyFrame = { type: "ready", generation: GEN - 1 }; + check("wrong-generation-ready rejected", "kill" in v.acceptReady(encodeFrame(staleReady), staleReady)); +} + +// ============================================================================ +// (C2) Per-request frames — forged / interleave / replay / stale-generation / +// oversized. (Required gate labels — keep exact text.) +// ============================================================================ +// baseline: ready then a valid result accepted. +{ + const v = new SessionValidator(GEN, INFLIGHT); + const ready: ReadyFrame = { type: "ready", generation: GEN }; + const r: ResponseFrame = { type: "result", requestId: "req-A", seq: 1, generation: GEN }; + check( + "baseline ready + result accepted", + "ok" in v.acceptReady(encodeFrame(ready), ready) && "ok" in v.acceptResponse(encodeFrame(r), r), + ); +} +// forged: a result for a requestId Node never issued. +{ + const v = new SessionValidator(GEN, INFLIGHT); + const forged: ResponseFrame = { type: "result", requestId: "req-NEVER", seq: 0, generation: GEN }; + check("forged-frame rejected", "kill" in v.acceptResponse(encodeFrame(forged), forged)); +} +// interleave: after accepting seq=5 (req-A), an earlier-numbered seq=3 (req-B). +{ + const v = new SessionValidator(GEN, INFLIGHT); + const a5: ResponseFrame = { type: "result", requestId: "req-A", seq: 5, generation: GEN }; + v.acceptResponse(encodeFrame(a5), a5); + const b3: ResponseFrame = { type: "result", requestId: "req-B", seq: 3, generation: GEN }; + check("interleave rejected", "kill" in v.acceptResponse(encodeFrame(b3), b3)); +} +// replay: a second response for an already-responded requestId (higher seq, so +// only the per-request single-response rule catches it). +{ + const v = new SessionValidator(GEN, INFLIGHT); + const x2: ResponseFrame = { type: "result", requestId: "req-X", seq: 2, generation: GEN }; + v.acceptResponse(encodeFrame(x2), x2); + const x4: ResponseFrame = { type: "result", requestId: "req-X", seq: 4, generation: GEN }; + check("replay rejected", "kill" in v.acceptResponse(encodeFrame(x4), x4)); +} +// stale-generation: a response from a killed/old child generation. +{ + const v = new SessionValidator(GEN, INFLIGHT); + const stale: ResponseFrame = { type: "result", requestId: "req-A", seq: 0, generation: GEN - 1 }; + check("stale-generation rejected", "kill" in v.acceptResponse(encodeFrame(stale), stale)); +} +// oversized: a frame above the wire-size cap. +{ + const v = new SessionValidator(GEN, INFLIGHT); + const big: ResponseFrame = { type: "result", requestId: "req-A", seq: 0, generation: GEN, payload: "A".repeat(MAX_FRAME_BYTES) }; + check("oversized rejected", "kill" in v.acceptResponse(encodeFrame(big), big)); +} + +if (failures > 0) { + out(`S2 FAIL: ${failures} case(s) failed`); + throw new Error(`S2 FAIL: ${failures} case(s) failed`); // Deno exits 1 on uncaught throw +} +out("S2 ALL PASS"); diff --git a/thoughts/issue-118-pyodide-runtime/spikes/s3-memory.sh b/thoughts/issue-118-pyodide-runtime/spikes/s3-memory.sh new file mode 100644 index 0000000..9380e79 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/spikes/s3-memory.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Spike S3 — Per-child memory behavior (gates Phase 6). +# +# Confirms the operating model in design.md Decision 5: on `node:22-slim` as the +# non-root `node` user, per-PROCESS memory limiting is NOT reliably available, so +# the OPERATOR-SET CONTAINER MEMORY LIMIT + accepted availability risk is the +# guard — NOT per-child cgroup `memory.max` or `prlimit --as`. +# +# Runs inside a real Linux container (this host is macOS; the cgroup v2 / V8 +# vaddr semantics only exist in the Linux container — exactly the prod target): +# +# Probe 1 — cgroup v2 memory.max write: a non-root user attempts to write +# memory.max / create a child cgroup. EXPECTED: denied (read-only fs). +# Probe 2 — RLIMIT_AS (== `prlimit --as` / `ulimit -v`) caps VIRTUAL address +# space, not RSS. V8 + a WASM heap (Pyodide's model) RESERVE a huge +# virtual region while resident memory stays tiny, so: +# (2a) VmSize >> VmRSS — the limit cannot track real RSS; and +# (2b) an RLIMIT_AS low enough to bound RSS (2 GiB) makes the WASM +# reservation FAIL — the workload can't even start. +# => RLIMIT_AS / prlimit --as is unusable as a per-child RAM cap. +# +# Exit 0 => both probes confirm the EXPECTED (limiting unavailable/unusable). +# Exit 1 => a probe behaved unexpectedly (Decision 5 would need revisiting). + +set -euo pipefail + +IMAGE="node:22-slim" + +if ! command -v docker >/dev/null 2>&1; then + echo "S3 FAIL: docker not available — required to exercise Linux cgroup v2 / V8 vaddr semantics" >&2 + exit 2 +fi + +echo "[s3] pulling ${IMAGE} if needed…" >&2 +docker pull -q "${IMAGE}" >/dev/null + +# In-container probe. Runs as the image's built-in non-root `node` user (uid 1000) +# with the default Docker cgroup mount (read-only for unprivileged containers — +# exactly the prod posture). +read -r -d '' PROBE <<'PROBE_EOF' || true +set -u +echo "=== identity ===" +id +echo "=== cgroup controllers ===" +cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null || echo "(no unified cgroup.controllers)" + +echo "=== PROBE 1: write cgroup v2 memory.max as non-root ===" +MEMMAX_DENIED=0 +if [ -e /sys/fs/cgroup/memory.max ]; then + CUR="$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo '?')" + if echo 104857600 > /sys/fs/cgroup/memory.max 2>/tmp/cgerr; then + echo "RESULT: memory.max write SUCCEEDED (unexpected)" + else + echo "RESULT: memory.max write DENIED (current=${CUR}) -> $(cat /tmp/cgerr 2>/dev/null)" + MEMMAX_DENIED=1 + fi +else + echo "memory.max not present at cgroup root" +fi +if mkdir /sys/fs/cgroup/s3probe 2>/tmp/cgmk; then + echo "RESULT: child cgroup mkdir SUCCEEDED (unexpected)"; rmdir /sys/fs/cgroup/s3probe 2>/dev/null || true +else + echo "RESULT: child cgroup mkdir DENIED -> $(cat /tmp/cgmk 2>/dev/null)" + [ "$MEMMAX_DENIED" -eq 0 ] && MEMMAX_DENIED=1 # fall back to mkdir denial if memory.max was absent +fi + +echo "=== PROBE 2: RLIMIT_AS (prlimit --as / ulimit -v) caps VIRTUAL memory, not RSS ===" +# 2a — V8 + a large-max WASM heap reserve vaddr >> RSS. +read VMSIZE VMRSS < <(node -e ' + const fs=require("fs"); + new WebAssembly.Memory({initial:16, maximum:32768}); // ~2 GiB max heap (Pyodide-like) + const s=fs.readFileSync("/proc/self/status","utf8"); + const g=k=>Math.round(+(new RegExp(k+":\\s+(\\d+) kB").exec(s)[1])/1024); + console.log(g("VmSize"), g("VmRSS")); +') +echo "RESULT(2a): with a large-max WASM heap -> VmSize=${VMSIZE:-?}MB VmRSS=${VMRSS:-?}MB" +VADDR_DECOUPLED=0 +if [ "${VMSIZE:-0}" -gt 4096 ] && [ "${VMRSS:-999999}" -lt 512 ]; then + echo " => virtual reservation (${VMSIZE}MB) >> resident (${VMRSS}MB): RLIMIT_AS cannot track RSS" + VADDR_DECOUPLED=1 +fi + +# 2b — an RLIMIT_AS low enough to be a useful RSS cap (2 GiB) BREAKS the reservation. +RLIMIT_BREAKS=0 +( ulimit -v 2097152; node -e ' + try { new WebAssembly.Memory({initial:16, maximum:32768}); console.log("WASM_RESERVE_OK"); } + catch (e) { console.log("WASM_RESERVE_FAILED: "+e.constructor.name+": "+e.message); process.exit(7); } +' ); RC=$? +if [ "$RC" -ne 0 ]; then + echo "RESULT(2b): under 2 GiB RLIMIT_AS the WASM reservation FAILED (rc=$RC) => a limit usable for RSS breaks the workload" + RLIMIT_BREAKS=1 +else + echo "RESULT(2b): WASM reserved under 2 GiB RLIMIT_AS (unexpected)" +fi + +RLIMIT_UNUSABLE=0 +[ "$VADDR_DECOUPLED" -eq 1 ] && [ "$RLIMIT_BREAKS" -eq 1 ] && RLIMIT_UNUSABLE=1 + +echo "=== SUMMARY ===" +echo "cgroup_write_denied=${MEMMAX_DENIED} rlimit_as_unusable=${RLIMIT_UNUSABLE} (vaddr_decoupled=${VADDR_DECOUPLED} rlimit_breaks_wasm=${RLIMIT_BREAKS})" +if [ "$MEMMAX_DENIED" -eq 1 ] && [ "$RLIMIT_UNUSABLE" -eq 1 ]; then + echo "S3 PASS: non-root cannot set cgroup memory.max; prlimit --as is unusable for RSS (V8/WASM vaddr >> RSS). Container memory limit is the real guard." + exit 0 +fi +echo "S3 FAIL: a memory-limit probe behaved unexpectedly — revisit Decision 5" +exit 1 +PROBE_EOF + +docker run --rm --user node "${IMAGE}" bash -c "${PROBE}" From e8caa69da150304e71904cc732cbc3a0fd8673af Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 16:02:52 +0930 Subject: [PATCH 03/16] =?UTF-8?q?Phase=201:=20python=5Fruntime=20field=20?= =?UTF-8?q?=E2=80=94=20server-side=20break,=20rolling-safe=20(stdlib=20+?= =?UTF-8?q?=20null=20end-to-end)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- biome.json | 2 +- src/api/mcp/tools.ts | 16 +++-- src/api/routes/sandboxes.ts | 44 ++++++++----- src/api/session-manager.ts | 20 ++++-- .../migrations.integration.test.ts | 36 ++++++++++ .../tests/session-manager.script-tx.test.ts | 4 +- src/api/tests/unit/exec-batch.test.ts | 2 +- src/api/tests/unit/exec.test.ts | 2 +- src/api/tests/unit/files.test.ts | 2 +- src/api/tests/unit/ingest.test.ts | 2 +- src/api/tests/unit/sandboxes.test.ts | 2 +- .../unit/session-manager.rehydrate.test.ts | 24 ++++--- src/api/tests/unit/session-manager.test.ts | 66 +++++++++++++++---- src/sql-fs/dialects/postgres.ts | 39 ++++++++--- .../tests/unit/postgres.advisory-lock.test.ts | 4 +- .../postgres/0006_python_runtime.sql | 34 ++++++++++ src/sql-fs/types.ts | 7 +- thoughts/issue-118-pyodide-runtime/plan.md | 36 +++++++--- 18 files changed, 263 insertions(+), 79 deletions(-) create mode 100644 src/sql-fs/migrations/postgres/0006_python_runtime.sql diff --git a/biome.json b/biome.json index 0fe7c9a..1bdc69e 100644 --- a/biome.json +++ b/biome.json @@ -22,6 +22,6 @@ "lineWidth": 120 }, "files": { - "ignore": ["node_modules", "dist", "*.sql", "scripts", "clients/python/.venv"] + "ignore": ["node_modules", "dist", "*.sql", "scripts", "clients/python/.venv", "thoughts"] } } diff --git a/src/api/mcp/tools.ts b/src/api/mcp/tools.ts index b966122..5725c1c 100644 --- a/src/api/mcp/tools.ts +++ b/src/api/mcp/tools.ts @@ -40,8 +40,11 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, async (args) => { const id = randomUUID(); const name = args.name ?? null; + // Phase 1: keep the boolean `python` MCP input/output contract; map it + // internally onto the new pythonRuntime/python_runtime fields. Phase 2 + // migrates this surface to the `python_runtime` enum. const runtimeOptions = { - python: args.python ?? false, + pythonRuntime: ((args.python ?? false) ? "stdlib" : null) as "stdlib" | null, javascript: args.javascript ?? false, network: false, }; @@ -51,7 +54,7 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, await sessionManager.persistSandboxMeta(tenant, id, { owner, name, - python: runtimeOptions.python, + python_runtime: runtimeOptions.pythonRuntime, javascript: runtimeOptions.javascript, network: runtimeOptions.network, }); @@ -59,7 +62,12 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, content: [ { type: "text" as const, - text: JSON.stringify({ id, name, python: runtimeOptions.python, javascript: runtimeOptions.javascript }), + text: JSON.stringify({ + id, + name, + python: runtimeOptions.pythonRuntime === "stdlib", + javascript: runtimeOptions.javascript, + }), }, ], }; @@ -95,7 +103,7 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, name: s.name, owner: s.owner, createdAt: s.createdAt.toISOString(), - python: s.python, + python: s.python_runtime === "stdlib", javascript: s.javascript, })), }), diff --git a/src/api/routes/sandboxes.ts b/src/api/routes/sandboxes.ts index 4a430d3..6f0c667 100644 --- a/src/api/routes/sandboxes.ts +++ b/src/api/routes/sandboxes.ts @@ -8,19 +8,24 @@ import { Hono } from "hono"; import type { ContentfulStatusCode } from "hono/utils/http-status"; import { z } from "zod"; +import type { PythonRuntime } from "../../sql-fs/types.js"; import type { AuthVariables } from "../auth.js"; import { forbiddenResponse, isForbiddenError, isOwnedBy, withOwnedSessionOrRehydrate } from "../ownership.js"; import type { SessionManager } from "../session-manager.js"; -const createBodySchema = z.object({ - name: z.string().max(255).optional(), - env: z.record(z.string()).optional(), - files: z.record(z.string()).optional(), - python: z.boolean().optional(), - javascript: z.boolean().optional(), - /** When true, js-exec fetch() is granted unrestricted outbound HTTPS access. */ - network: z.boolean().optional(), -}); +const createBodySchema = z + .object({ + name: z.string().max(255).optional(), + env: z.record(z.string()).optional(), + files: z.record(z.string()).optional(), + /** Python runtime: "stdlib" (CPython WASM) or "pyodide" (numpy/pandas/scipy/openpyxl). */ + python_runtime: z.enum(["stdlib", "pyodide"]).nullable().optional(), + javascript: z.boolean().optional(), + /** When true, js-exec fetch() is granted unrestricted outbound HTTPS access. */ + network: z.boolean().optional(), + }) + // Reject the legacy `python: bool` field (and any other unknown key) with a 400. + .strict(); // Audit H11 (#2): bound the optional initial-files map so a single create can't // buffer an unbounded number of files / bytes. Shares the bulk-write env knobs. @@ -33,7 +38,7 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: router.post("/", async (c) => { let name: string | null = null; let files: Record | undefined; - let python = false; + let pythonRuntime: PythonRuntime = null; let javascript = false; let network = false; @@ -47,7 +52,7 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: } name = result.data.name ?? null; files = result.data.files; - python = result.data.python ?? false; + pythonRuntime = result.data.python_runtime ?? null; javascript = result.data.javascript ?? false; network = result.data.network ?? false; } catch { @@ -104,7 +109,7 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: await sessionManager.persistSandboxMeta(tenant, sandboxId, { owner, name, - python, + python_runtime: pythonRuntime, javascript, network, createdAt, @@ -127,11 +132,14 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: } } }, - { python, javascript, network }, + { pythonRuntime, javascript, network }, owner, ); - return c.json({ id: sandboxId, name, owner, createdAt, python, javascript, network }, 201 as ContentfulStatusCode); + return c.json( + { id: sandboxId, name, owner, createdAt, python_runtime: pythonRuntime, javascript, network }, + 201 as ContentfulStatusCode, + ); }); router.get("/", async (c) => { @@ -145,7 +153,7 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: name: s.name, owner: s.owner, createdAt: s.createdAt.toISOString(), - python: s.python, + python_runtime: s.python_runtime, javascript: s.javascript, network: s.network, })), @@ -182,6 +190,9 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: owner: meta.owner, createdAt: meta.createdAt ?? null, lastUsedAt: null, + python_runtime: meta.python_runtime, + javascript: meta.javascript, + network: meta.network, }); } if (!isOwnedBy(session.owner, caller)) { @@ -193,6 +204,9 @@ export function sandboxRoutes(sessionManager: SessionManager): Hono<{ Variables: owner: session.owner, createdAt: session.createdAt, lastUsedAt: new Date(session.lastUsed).toISOString(), + python_runtime: session.runtimeOptions.pythonRuntime, + javascript: session.runtimeOptions.javascript, + network: session.runtimeOptions.network, }); }); diff --git a/src/api/session-manager.ts b/src/api/session-manager.ts index 4f1f997..b40c24b 100644 --- a/src/api/session-manager.ts +++ b/src/api/session-manager.ts @@ -20,7 +20,7 @@ import type { RedisBlobCache } from "../sql-fs/redis-blob-cache.js"; import { type RedisPathSnapshot, versionKey } from "../sql-fs/redis-path-snapshot.js"; import { SessionScopedFs } from "../sql-fs/session-scoped-fs.js"; import type { ICoherentFs, IReadOnlyScopeFs, IScriptTxFs } from "../sql-fs/sql-fs.js"; -import type { PathCacheEntry, SandboxListEntry, SandboxMeta } from "../sql-fs/types.js"; +import type { PathCacheEntry, PythonRuntime, SandboxListEntry, SandboxMeta } from "../sql-fs/types.js"; import { nodeCommand } from "./commands/node-command.js"; import { execLockKey, withDistributedLock } from "./distributed-lock.js"; import { type DistributedRWLockOptions, rwLockKeys, withDistributedRWLock } from "./distributed-rw-lock.js"; @@ -108,7 +108,8 @@ function asScriptTxFs(fs: IFileSystem): IScriptTxFs | undefined { * because `just-bash` decides which commands to register when the `Bash` instance is built. */ export interface RuntimeOptions { - readonly python: boolean; + /** Python runtime: "stdlib" (CPython WASM), "pyodide", or null (no Python). */ + readonly pythonRuntime: PythonRuntime; readonly javascript: boolean; /** * When true, the `js-exec` runtime is given a permissive `NetworkConfig` that @@ -119,7 +120,7 @@ export interface RuntimeOptions { readonly network: boolean; } -const DEFAULT_RUNTIME_OPTIONS: RuntimeOptions = { python: false, javascript: false, network: false }; +const DEFAULT_RUNTIME_OPTIONS: RuntimeOptions = { pythonRuntime: null, javascript: false, network: false }; /** * Syntactically valid sandbox id: UUIDs and dashed/underscored slugs, 1–128 @@ -486,7 +487,10 @@ export class SessionManager { const bash = new Bash({ fs, - python: resolvedRuntime.python || undefined, + // "stdlib" → just-bash's WASM python3; "pyodide" leaves python + // unregistered here (Phase 5 adds the custom python3 commands); + // null → no Python. + python: resolvedRuntime.pythonRuntime === "stdlib" || undefined, javascript: resolvedRuntime.javascript || undefined, // When network is enabled, grant js-exec unrestricted outbound HTTPS. // Bash itself remains air-gapped — no curl/wget/DNS — because @@ -896,7 +900,7 @@ export class SessionManager { throw Object.assign(new Error(`ENOENT: sandbox ${sandboxId} not found`), { code: "ENOENT" }); } const resolvedRuntime: RuntimeOptions = meta - ? { python: meta.python, javascript: meta.javascript, network: meta.network } + ? { pythonRuntime: meta.python_runtime, javascript: meta.javascript, network: meta.network } : (runtimeOptions ?? DEFAULT_RUNTIME_OPTIONS); const session = await this.getOrCreate(tenantId, sandboxId, resolvedRuntime, meta?.owner ?? ""); if (meta?.owner) session.owner = meta.owner; @@ -943,7 +947,7 @@ export class SessionManager { throw Object.assign(new Error(`ENOENT: sandbox ${sandboxId} not found`), { code: "ENOENT" }); } const resolvedRuntime: RuntimeOptions = meta - ? { python: meta.python, javascript: meta.javascript, network: meta.network } + ? { pythonRuntime: meta.python_runtime, javascript: meta.javascript, network: meta.network } : (runtimeOptions ?? DEFAULT_RUNTIME_OPTIONS); const session = await this.getOrCreate(tenantId, sandboxId, resolvedRuntime, meta?.owner ?? ""); if (meta?.owner) { @@ -1233,7 +1237,9 @@ export class SessionManager { } async execWithRuntimeThrottle(session: Session, script: string, opts?: ExecOptions): Promise { - const usesPython = session.runtimeOptions.python && PYTHON_INVOCATION_REGEX.test(script); + // Only "stdlib" (WASM python3) routes through pythonSem; "pyodide" gets its + // own semaphore in Phase 6. + const usesPython = session.runtimeOptions.pythonRuntime === "stdlib" && PYTHON_INVOCATION_REGEX.test(script); const usesJs = session.runtimeOptions.javascript && JS_INVOCATION_REGEX.test(script); // Apply the session-tracked cwd as the starting directory for this exec, diff --git a/src/api/tests/integration/migrations.integration.test.ts b/src/api/tests/integration/migrations.integration.test.ts index fbe2047..791ec55 100644 --- a/src/api/tests/integration/migrations.integration.test.ts +++ b/src/api/tests/integration/migrations.integration.test.ts @@ -79,10 +79,46 @@ describe.skipIf(SKIP)("runMigrations (integration)", () => { WHERE n.nspname = 'public' AND p.proname = 'fs_resolve' `; expect(Number(procs[0]?.n)).toBeGreaterThanOrEqual(1); + + // 0006: python_runtime column + CHECK exist. + const col = await sql<{ n: string }[]>` + SELECT count(*)::text AS n FROM information_schema.columns + WHERE table_name = 'sandboxes' AND column_name = 'python_runtime'`; + expect(col[0]?.n).toBe("1"); + + // Old-replica-style row (python=true, python_runtime NULL) reads back as stdlib via COALESCE. + await sql`INSERT INTO sandboxes (id, python, python_runtime) VALUES ('mig-legacy', true, NULL)`; + const legacy = await sql<{ pr: string | null }[]>` + SELECT COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END) AS pr + FROM sandboxes WHERE id = 'mig-legacy'`; + expect(legacy[0]?.pr).toBe("stdlib"); } finally { await sql.end({ timeout: 5 }); } await expect(runMigrations(cfg)).resolves.toBeUndefined(); }); + + it("re-runs idempotently after a simulated python-column drop (N+1)", async () => { + const base = process.env.DATABASE_URL; + if (!base) { + throw new Error("DATABASE_URL required for this suite"); + } + // own ephemeral DB, mirroring beforeAll's create/teardown pattern + const dropDb = `vfs_mig_drop_${randomBytes(8).toString("hex")}`; + await admin!.unsafe(`CREATE DATABASE ${dropDb}`); + const dropUrl = withDatabase(base, dropDb); + const dropCfg = loadTenantConfig({ TENANT_DATABASES: JSON.stringify({ default: dropUrl }) }); + const s = postgres(dropUrl, { prepare: false, max: 1 }); + try { + await runMigrations(dropCfg); + await s`ALTER TABLE sandboxes DROP COLUMN IF EXISTS python`; // simulate the N+1 drop + await expect(runMigrations(dropCfg)).resolves.toBeUndefined(); // pg_attribute guard → no error + } finally { + await s.end({ timeout: 5 }); + await admin!`SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = ${dropDb} AND pid <> pg_backend_pid()`; + await admin!.unsafe(`DROP DATABASE IF EXISTS ${dropDb}`); + } + }); }); diff --git a/src/api/tests/session-manager.script-tx.test.ts b/src/api/tests/session-manager.script-tx.test.ts index 9ef1f4d..60ba313 100644 --- a/src/api/tests/session-manager.script-tx.test.ts +++ b/src/api/tests/session-manager.script-tx.test.ts @@ -148,7 +148,7 @@ describe("execWithRuntimeThrottle script-tx wrapping", () => { const abortScope = vi.fn(async () => {}); const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPython: 1 }); - const session = await sm.getOrCreate(T, "wrap-py", { python: true, javascript: false, network: false }); + const session = await sm.getOrCreate(T, "wrap-py", { pythonRuntime: "stdlib", javascript: false, network: false }); stubBashExec(session, async () => ({ stdout: "py", stderr: "", exitCode: 0, env: {} })); const mockScriptTx = { @@ -176,7 +176,7 @@ describe("execWithRuntimeThrottle script-tx wrapping", () => { const abortScope = vi.fn(async () => {}); const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentJs: 1 }); - const session = await sm.getOrCreate(T, "wrap-js", { python: false, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "wrap-js", { pythonRuntime: null, javascript: true, network: false }); stubBashExec(session, async () => ({ stdout: "js", stderr: "", exitCode: 0, env: {} })); const mockScriptTx = { diff --git a/src/api/tests/unit/exec-batch.test.ts b/src/api/tests/unit/exec-batch.test.ts index a92ca55..f6db8ae 100644 --- a/src/api/tests/unit/exec-batch.test.ts +++ b/src/api/tests/unit/exec-batch.test.ts @@ -252,7 +252,7 @@ describe("POST /v1/sandboxes/:id/exec-sync-batch", () => { await mgr.persistSandboxMeta("default", SANDBOX_ID, { owner: "agent-1", name: null, - python: false, + python_runtime: null, javascript: false, network: false, }); diff --git a/src/api/tests/unit/exec.test.ts b/src/api/tests/unit/exec.test.ts index f284e97..6ec5758 100644 --- a/src/api/tests/unit/exec.test.ts +++ b/src/api/tests/unit/exec.test.ts @@ -594,7 +594,7 @@ describe("POST /v1/sandboxes/:id/exec (SSE streaming)", () => { await ownerManager.persistSandboxMeta("default", SANDBOX_ID, { owner: "agent-1", name: null, - python: false, + python_runtime: null, javascript: false, network: false, }); diff --git a/src/api/tests/unit/files.test.ts b/src/api/tests/unit/files.test.ts index f054f15..c4992ad 100644 --- a/src/api/tests/unit/files.test.ts +++ b/src/api/tests/unit/files.test.ts @@ -189,7 +189,7 @@ describe("PUT /v1/sandboxes/:id/files/*path", () => { await ownerManager.persistSandboxMeta("default", SANDBOX_ID, { owner: "agent-1", name: null, - python: false, + python_runtime: null, javascript: false, network: false, }); diff --git a/src/api/tests/unit/ingest.test.ts b/src/api/tests/unit/ingest.test.ts index bf69f4b..9a28852 100644 --- a/src/api/tests/unit/ingest.test.ts +++ b/src/api/tests/unit/ingest.test.ts @@ -269,7 +269,7 @@ describe("POST /v1/sandboxes/:id/ingest-files", () => { await ownerManager.persistSandboxMeta("default", SANDBOX_ID, { owner: "agent-1", name: null, - python: false, + python_runtime: null, javascript: false, network: false, }); diff --git a/src/api/tests/unit/sandboxes.test.ts b/src/api/tests/unit/sandboxes.test.ts index c54167d..cfb6f58 100644 --- a/src/api/tests/unit/sandboxes.test.ts +++ b/src/api/tests/unit/sandboxes.test.ts @@ -267,7 +267,7 @@ describe("GET /v1/sandboxes/:id", () => { meta.set(sandboxId, { owner: "owner-evict", name: null, - python: false, + python_runtime: null, javascript: false, network: false, createdAt: knownCreatedAt, diff --git a/src/api/tests/unit/session-manager.rehydrate.test.ts b/src/api/tests/unit/session-manager.rehydrate.test.ts index 4957108..7bef981 100644 --- a/src/api/tests/unit/session-manager.rehydrate.test.ts +++ b/src/api/tests/unit/session-manager.rehydrate.test.ts @@ -10,7 +10,7 @@ import { InMemoryFs } from "just-bash"; import type { IFileSystem } from "just-bash"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { SandboxMeta } from "../../../sql-fs/types.js"; -import { SessionManager } from "../../session-manager.js"; +import { type RuntimeOptions, SessionManager } from "../../session-manager.js"; let pgSandboxes: Map; @@ -33,7 +33,7 @@ function makeSessionManager(): SessionManager { }); } -const DEFAULT_META: SandboxMeta = { owner: null, name: null, python: false, javascript: false, network: false }; +const DEFAULT_META: SandboxMeta = { owner: null, name: null, python_runtime: null, javascript: false, network: false }; describe("SessionManager.withSessionOrRehydrate()", () => { let sm: SessionManager; @@ -102,7 +102,13 @@ describe("SessionManager.withSessionOrRehydrate()", () => { }); it("restores owner from PG metadata on rehydration", async () => { - pgSandboxes.set("sb-owned", { owner: "user-a", name: null, python: false, javascript: false, network: false }); + pgSandboxes.set("sb-owned", { + owner: "user-a", + name: null, + python_runtime: null, + javascript: false, + network: false, + }); let capturedOwner = ""; await sm.withSessionOrRehydrate("default", "sb-owned", async (session) => { @@ -112,13 +118,13 @@ describe("SessionManager.withSessionOrRehydrate()", () => { }); it("restores runtime options from PG metadata on rehydration", async () => { - pgSandboxes.set("sb-py", { owner: null, name: null, python: true, javascript: false, network: false }); + pgSandboxes.set("sb-py", { owner: null, name: null, python_runtime: "stdlib", javascript: false, network: false }); - let capturedRuntime = { python: false, javascript: false, network: false }; + let capturedRuntime: RuntimeOptions = { pythonRuntime: null, javascript: false, network: false }; await sm.withSessionOrRehydrate("default", "sb-py", async (session) => { capturedRuntime = session.runtimeOptions; }); - expect(capturedRuntime).toEqual({ python: true, javascript: false, network: false }); + expect(capturedRuntime).toEqual({ pythonRuntime: "stdlib", javascript: false, network: false }); }); it("persistSandboxMeta writes to store and is readable on rehydration", async () => { @@ -127,7 +133,7 @@ describe("SessionManager.withSessionOrRehydrate()", () => { await sm.persistSandboxMeta("default", "sb-meta", { owner: "creator", name: null, - python: true, + python_runtime: "stdlib", javascript: false, network: false, }); @@ -140,12 +146,12 @@ describe("SessionManager.withSessionOrRehydrate()", () => { }); let owner = ""; - let runtime = { python: false, javascript: false, network: false }; + let runtime: RuntimeOptions = { pythonRuntime: null, javascript: false, network: false }; await sm2.withSessionOrRehydrate("default", "sb-meta", async (session) => { owner = session.owner; runtime = session.runtimeOptions; }); expect(owner).toBe("creator"); - expect(runtime).toEqual({ python: true, javascript: false, network: false }); + expect(runtime).toEqual({ pythonRuntime: "stdlib", javascript: false, network: false }); }); }); diff --git a/src/api/tests/unit/session-manager.test.ts b/src/api/tests/unit/session-manager.test.ts index c2fa8c6..2a20073 100644 --- a/src/api/tests/unit/session-manager.test.ts +++ b/src/api/tests/unit/session-manager.test.ts @@ -335,15 +335,23 @@ describe("SessionManager runtime options + Python semaphore (US-080a)", () => { it("warm session ignores subsequent runtimeOptions (cache-hit path)", async () => { const sm = new SessionManager({ createFs: makeCreateFs() }); - const first = await sm.getOrCreate(T, "sandbox-warm", { python: true, javascript: false, network: false }); - const second = await sm.getOrCreate(T, "sandbox-warm", { python: false, javascript: true, network: false }); + const first = await sm.getOrCreate(T, "sandbox-warm", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); + const second = await sm.getOrCreate(T, "sandbox-warm", { pythonRuntime: null, javascript: true, network: false }); expect(second).toBe(first); - expect(second.runtimeOptions).toEqual({ python: true, javascript: false, network: false }); + expect(second.runtimeOptions).toEqual({ pythonRuntime: "stdlib", javascript: false, network: false }); }); it("non-Python script bypasses semaphore entirely", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPython: 1 }); - const session = await sm.getOrCreate(T, "sandbox-no-py", { python: true, javascript: false, network: false }); + const session = await sm.getOrCreate(T, "sandbox-no-py", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); stubBashExec(session, async () => ({ stdout: "hi", stderr: "", exitCode: 0, env: {} })); await sm.execWithRuntimeThrottle(session, "echo hi"); @@ -376,7 +384,11 @@ describe("SessionManager runtime options + Python semaphore (US-080a)", () => { it("regex does not match mypython_script or python-config (word boundary)", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPython: 1 }); - const session = await sm.getOrCreate(T, "sandbox-regex", { python: true, javascript: false, network: false }); + const session = await sm.getOrCreate(T, "sandbox-regex", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); let running = 0; let peak = 0; @@ -400,7 +412,11 @@ describe("SessionManager runtime options + Python semaphore (US-080a)", () => { it("semaphore allows up to N concurrent Python executions, queues the (N+1)th until a slot frees", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPython: 2 }); - const session = await sm.getOrCreate(T, "sandbox-sem", { python: true, javascript: false, network: false }); + const session = await sm.getOrCreate(T, "sandbox-sem", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); const releasers: Array<() => void> = []; let started = 0; @@ -433,7 +449,11 @@ describe("SessionManager runtime options + Python semaphore (US-080a)", () => { it("slot is released even when bash.exec throws", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPython: 1 }); - const session = await sm.getOrCreate(T, "sandbox-throw", { python: true, javascript: false, network: false }); + const session = await sm.getOrCreate(T, "sandbox-throw", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); let execCount = 0; stubBashExec(session, async () => { @@ -481,7 +501,11 @@ describe("SessionManager JavaScript semaphore (MAX_CONCURRENT_JS)", () => { it("JS regex does not match mynode/nodejs_tool/js-exec-helper etc (word boundary)", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentJs: 1 }); - const session = await sm.getOrCreate(T, "sandbox-js-regex", { python: false, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "sandbox-js-regex", { + pythonRuntime: null, + javascript: true, + network: false, + }); let running = 0; let peak = 0; @@ -503,7 +527,7 @@ describe("SessionManager JavaScript semaphore (MAX_CONCURRENT_JS)", () => { it("12 parallel js-exec scripts with cap=4 run in 3 batches of 4", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentJs: 4 }); - const session = await sm.getOrCreate(T, "sandbox-js-12", { python: false, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "sandbox-js-12", { pythonRuntime: null, javascript: true, network: false }); let running = 0; let peak = 0; @@ -528,7 +552,11 @@ describe("SessionManager JavaScript semaphore (MAX_CONCURRENT_JS)", () => { it("semaphore allows up to N concurrent JS executions, queues the (N+1)th until a slot frees", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentJs: 2 }); - const session = await sm.getOrCreate(T, "sandbox-js-sem", { python: false, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "sandbox-js-sem", { + pythonRuntime: null, + javascript: true, + network: false, + }); const releasers: Array<() => void> = []; let started = 0; @@ -558,7 +586,11 @@ describe("SessionManager JavaScript semaphore (MAX_CONCURRENT_JS)", () => { it("JS slot is released even when bash.exec throws", async () => { const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentJs: 1 }); - const session = await sm.getOrCreate(T, "sandbox-js-throw", { python: false, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "sandbox-js-throw", { + pythonRuntime: null, + javascript: true, + network: false, + }); let execCount = 0; stubBashExec(session, async () => { @@ -591,7 +623,11 @@ describe("SessionManager combined python + js semaphores", () => { maxConcurrentPython: 1, maxConcurrentJs: 1, }); - const session = await sm.getOrCreate(T, "sandbox-both", { python: true, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "sandbox-both", { + pythonRuntime: "stdlib", + javascript: true, + network: false, + }); const releasers: Array<() => void> = []; stubBashExec(session, async () => { @@ -630,7 +666,11 @@ describe("SessionManager combined python + js semaphores", () => { maxConcurrentPython: 1, maxConcurrentJs: 1, }); - const session = await sm.getOrCreate(T, "sandbox-deadlock", { python: true, javascript: true, network: false }); + const session = await sm.getOrCreate(T, "sandbox-deadlock", { + pythonRuntime: "stdlib", + javascript: true, + network: false, + }); let peak = 0; let running = 0; diff --git a/src/sql-fs/dialects/postgres.ts b/src/sql-fs/dialects/postgres.ts index 2edadcf..d10c17d 100644 --- a/src/sql-fs/dialects/postgres.ts +++ b/src/sql-fs/dialects/postgres.ts @@ -17,6 +17,7 @@ import { type InodeKind, type InodeRow, type PathCacheEntry, + type PythonRuntime, type SandboxListEntry, type SandboxMeta, type SqlDialect, @@ -332,24 +333,30 @@ export class PostgresDialect implements SqlDialect { async getSandboxMeta(tx: PgTx, sandboxId: string): Promise { try { + // Rolling-safe read: an old replica may have written python=true with + // python_runtime still NULL. COALESCE the legacy column so such a row + // reads back as 'stdlib' until the N+1 release drops `python`. const rows = await tx< { owner: string | null; name: string | null; - python: boolean; + python_runtime: string | null; javascript: boolean; network: boolean; created_at: Date; }[] >` - SELECT owner, name, python, javascript, network, created_at FROM sandboxes WHERE id = ${sandboxId} + SELECT owner, name, + COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END) AS python_runtime, + javascript, network, created_at + FROM sandboxes WHERE id = ${sandboxId} `; if (rows.length === 0) return null; const r = rows[0]!; return { owner: r.owner, name: r.name, - python: r.python, + python_runtime: r.python_runtime as PythonRuntime, javascript: r.javascript, network: r.network, createdAt: r.created_at.toISOString(), @@ -362,9 +369,15 @@ export class PostgresDialect implements SqlDialect { async updateSandboxMeta(tx: PgTx, sandboxId: string, meta: SandboxMeta): Promise { let rows: Array<{ id: string }>; try { + // Dual-write both columns so old replicas (which still read `python`) + // keep working until the N+1 release. stdlib → python=true; + // pyodide/null → python=false. rows = await tx<{ id: string }[]>` UPDATE sandboxes - SET owner = ${meta.owner}, name = ${meta.name}, python = ${meta.python}, javascript = ${meta.javascript}, network = ${meta.network ?? false} + SET owner = ${meta.owner}, name = ${meta.name}, + python_runtime = ${meta.python_runtime}, + python = ${meta.python_runtime === "stdlib"}, + javascript = ${meta.javascript}, network = ${meta.network ?? false} WHERE id = ${sandboxId} RETURNING id `; @@ -380,6 +393,8 @@ export class PostgresDialect implements SqlDialect { async listSandboxes(tx: PgTx, owner?: string): Promise { try { + // Same rolling-safe COALESCE as getSandboxMeta: an old replica's + // python=true / python_runtime NULL row lists as 'stdlib'. const rows = owner !== undefined ? await tx< @@ -388,28 +403,34 @@ export class PostgresDialect implements SqlDialect { name: string | null; owner: string | null; created_at: Date; - python: boolean; + python_runtime: string | null; javascript: boolean; network: boolean; }[] - >`SELECT id, name, owner, created_at, python, javascript, network FROM sandboxes WHERE owner = ${owner} ORDER BY created_at DESC` + >`SELECT id, name, owner, created_at, + COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END) AS python_runtime, + javascript, network + FROM sandboxes WHERE owner = ${owner} ORDER BY created_at DESC` : await tx< { id: string; name: string | null; owner: string | null; created_at: Date; - python: boolean; + python_runtime: string | null; javascript: boolean; network: boolean; }[] - >`SELECT id, name, owner, created_at, python, javascript, network FROM sandboxes ORDER BY created_at DESC`; + >`SELECT id, name, owner, created_at, + COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END) AS python_runtime, + javascript, network + FROM sandboxes ORDER BY created_at DESC`; return rows.map((r) => ({ id: r.id, name: r.name, owner: r.owner, createdAt: r.created_at, - python: r.python, + python_runtime: r.python_runtime as PythonRuntime, javascript: r.javascript, network: r.network, })); diff --git a/src/sql-fs/dialects/tests/unit/postgres.advisory-lock.test.ts b/src/sql-fs/dialects/tests/unit/postgres.advisory-lock.test.ts index 6f6ef80..2c91705 100644 --- a/src/sql-fs/dialects/tests/unit/postgres.advisory-lock.test.ts +++ b/src/sql-fs/dialects/tests/unit/postgres.advisory-lock.test.ts @@ -145,7 +145,7 @@ describe("PostgresDialect metadata helpers — SQL error translation", () => { dialect.updateSandboxMeta(tx, "sandbox-missing", { owner: null, name: null, - python: false, + python_runtime: null, javascript: false, network: false, }), @@ -160,7 +160,7 @@ describe("PostgresDialect metadata helpers — SQL error translation", () => { dialect.updateSandboxMeta(tx, "sandbox-meta", { owner: null, name: null, - python: false, + python_runtime: null, javascript: false, network: false, }), diff --git a/src/sql-fs/migrations/postgres/0006_python_runtime.sql b/src/sql-fs/migrations/postgres/0006_python_runtime.sql new file mode 100644 index 0000000..b493359 --- /dev/null +++ b/src/sql-fs/migrations/postgres/0006_python_runtime.sql @@ -0,0 +1,34 @@ +-- Migration 0006: replace boolean `python` with nullable `python_runtime` enum. +-- Expand/contract step N (this release). Rolling-deploy-safe: reads COALESCE the +-- legacy column, writes dual-write it (see postgres.ts). Step N+1 (later release) +-- drops `python` and removes the COALESCE/dual-write. + +ALTER TABLE sandboxes + ADD COLUMN IF NOT EXISTS python_runtime TEXT; + +-- CHECK constraint (idempotent: add only if absent). +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'sandboxes_python_runtime_check' + ) THEN + ALTER TABLE sandboxes + ADD CONSTRAINT sandboxes_python_runtime_check + CHECK (python_runtime IN ('stdlib','pyodide')); + END IF; +END $$; + +-- Backfill ONLY rows not yet migrated, and ONLY while the legacy `python` column +-- still exists (so this is a no-op after the N+1 drop release — never errors). +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = 'sandboxes'::regclass + AND attname = 'python' AND NOT attisdropped + ) THEN + UPDATE sandboxes + SET python_runtime = CASE WHEN python THEN 'stdlib' END + WHERE python_runtime IS NULL; + END IF; +END $$; diff --git a/src/sql-fs/types.ts b/src/sql-fs/types.ts index ac3ee41..be4e826 100644 --- a/src/sql-fs/types.ts +++ b/src/sql-fs/types.ts @@ -57,11 +57,14 @@ export interface PathCacheEntry { readonly symlinkTarget: string | null; } +/** Python runtime selection. null = no Python. */ +export type PythonRuntime = "stdlib" | "pyodide" | null; + /** Persisted sandbox metadata needed for session rehydration on cold replicas. */ export interface SandboxMeta { readonly owner: string | null; readonly name: string | null; - readonly python: boolean; + readonly python_runtime: PythonRuntime; readonly javascript: boolean; /** When true, js-exec fetch() can reach external HTTP endpoints (60 s timeout). */ readonly network: boolean; @@ -75,7 +78,7 @@ export interface SandboxListEntry { readonly name: string | null; readonly owner: string | null; readonly createdAt: Date; - readonly python: boolean; + readonly python_runtime: PythonRuntime; readonly javascript: boolean; /** When true, js-exec fetch() can reach external HTTP endpoints (60 s timeout). */ readonly network: boolean; diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index dab567d..5c81f68 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -225,21 +225,37 @@ it("re-runs idempotently after a simulated python-column drop (N+1)", async () = ### Phase 1: Success Criteria #### Phase 1: Programmatic Verification -- [ ] `pnpm typecheck` passes -- [ ] `pnpm lint:fix` passes (no remaining `python:` boolean in server code) -- [ ] `pnpm test:unit` passes (updated rehydrate/advisory-lock/sandboxes assertions green) -- [ ] `pnpm test:integration` migration test passes: `python_runtime` column + CHECK exist; legacy `python=true` row reads back `stdlib`; migration re-runs cleanly after a simulated `python` drop -- [ ] Server starts (`pnpm dev`) without migration errors +- [x] `pnpm typecheck` passes +- [x] `pnpm lint:fix` passes (no remaining `python:` boolean in server code) — clean after adding `thoughts` to biome `files.ignore` (Phase 0 vendored Pyodide `.d.ts` assets were polluting the lint; biome doesn't read `.gitignore`) +- [x] `pnpm test:unit` passes (875 pass / 4 skip; updated rehydrate/advisory-lock/sandboxes/session-manager + extra meta-literal assertions green) +- [x] `pnpm test:integration` migration test passes: `python_runtime` column + CHECK exist; legacy `python=true` row reads back `stdlib`; migration re-runs cleanly after a simulated `python` drop (full integration suite 106 pass) +- [x] Server starts (`pnpm dev`) without migration errors — booted on :8080; `migration_ok` logged for `0006_python_runtime.sql` (idempotent "already exists, skipping" NOTICE is not an error), then `server_start`. Agent-managed boot; server shut down after checks. #### Phase 1: Agent Verification _(Dev-server protocol: see Success Criteria Guidelines — if no server is running for this worktree, ask the user to start `pnpm dev` or authorise the agent to manage it.)_ -- [ ] Against the running dev server, create a `stdlib` sandbox and a `null` (no `python_runtime`) sandbox; confirm **create-201, list, AND GET** all echo `python_runtime` -- [ ] In the `stdlib` sandbox, `bash_exec` `python3 -c "print(1)"` returns `1`, exit 0 -- [ ] In the `null` sandbox, `python3 -c "print(1)"` reports command-not-found (Python not registered) -- [ ] Agent reviews `postgres.ts` `getSandboxMeta`/`updateSandboxMeta` to confirm COALESCE read + dual-write are both present +- [x] Against the running dev server, create a `stdlib` sandbox and a `null` (no `python_runtime`) sandbox; confirm **create-201, list, AND GET** all echo `python_runtime` — stdlib: all three echo `"stdlib"`; null: create+GET echo `null`. (Bonus: legacy `{"python":true}` body → 400 via `.strict()`.) +- [x] In the `stdlib` sandbox, `bash_exec` `python3 -c "print(1)"` returns `1`, exit 0 — `{"stdout":"1\n","exitCode":0}` +- [x] In the `null` sandbox, `python3 -c "print(1)"` reports command-not-found (Python not registered) — exit 127, `bash: python3: command not available…` +- [x] Agent reviews `postgres.ts` `getSandboxMeta`/`updateSandboxMeta` to confirm COALESCE read + dual-write are both present — `getSandboxMeta`/`listSandboxes` select `COALESCE(python_runtime, CASE WHEN python THEN 'stdlib' END)`; `updateSandboxMeta` sets both `python_runtime = ${meta.python_runtime}` and `python = ${meta.python_runtime === "stdlib"}` ### Phase 1: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 1._ + +**Type-rename blast radius (plan under-listed the files to touch).** Renaming the shared types (`SandboxMeta.python`→`python_runtime`, `SandboxListEntry.python`→`python_runtime`, `RuntimeOptions.python`→`pythonRuntime`) breaks the **whole-program** `tsc` typecheck, so every consumer had to change in Phase 1 — not just the files the plan named. Beyond the plan's step-6 list, these also required edits to compile: +- **`src/api/mcp/tools.ts`** (plan defers MCP to **Phase 2**). It consumes `RuntimeOptions`/`SandboxMeta`/`SandboxListEntry`, so it can't be deferred for typecheck. **Adaptation:** kept its existing **boolean `python` MCP wire contract** (input `python: z.boolean()`, output `python: …`) and only rewired it internally onto the new fields (`pythonRuntime: (args.python ?? false) ? "stdlib" : null`; persist `python_runtime`; echo `python: runtimeOptions.pythonRuntime === "stdlib"` / `s.python_runtime === "stdlib"`). **Phase 2 still owns the real MCP migration** (enum input schema, `python_runtime` echo, `network` field, descriptions). The MCP wire contract is therefore unchanged by Phase 1. +- **Extra `SandboxMeta` mock literals** the plan didn't list: `src/api/tests/unit/ingest.test.ts`, `exec.test.ts`, `exec-batch.test.ts`, `files.test.ts` (all `python: false`→`python_runtime: null`). +- **Extra `RuntimeOptions` literals** the plan didn't list: `src/api/tests/unit/session-manager.test.ts` (many `getOrCreate(..., {python,…})` args + a `toEqual` assertion) and `src/api/tests/session-manager.script-tx.test.ts` (2). The `toEqual({ python: true, … })` on `runtimeOptions` shared the exact literal string with the `getOrCreate` args, so a single `replace_all` fixed both. + +**`openapi-spec.ts` is plain `as const` data, NOT typed against `SandboxMeta`** → its `python: { type: "boolean" }` does NOT break typecheck and was correctly **left for Phase 2**. `clients/` SDKs are outside `tsconfig` `include: ["src"]` → not in the main typecheck either (Phase 2). + +**Gotcha for any future migration — the dev DB must be migrated before integration tests.** Integration tests like `defense-in-depth.integration.test.ts` connect straight to `DATABASE_URL` and **assume the schema is already migrated** (they never call `runMigrations`). Because `getSandboxMeta`/`listSandboxes` now reference `python_runtime`, running integration tests against the local `sqlfs` DB failed with `column "python_runtime" does not exist` until 0006 was applied. **Fix:** apply the new migration to the dev DB once (`docker exec -i sqlfs-postgres psql -U sqlfs_app -d sqlfs < src/sql-fs/migrations/postgres/0006_python_runtime.sql`, idempotent) — in prod the boot-time runner does this automatically. Any later phase that adds a migration must re-apply it to the dev DB before `pnpm test:integration`. + +**Biome lints git-ignored vendored assets (deviation: edited `biome.json`).** Biome 1.9.0 has no `vcs.useIgnoreFile` here, so `pnpm lint:fix` scanned the **Phase 0** git-ignored Pyodide `.d.ts` assets under `thoughts/.../spikes/assets/` and reported ~870 errors in third-party code. **Adaptation (outside the plan's file list):** added `"thoughts"` to `biome.json` `files.ignore` (matching the existing `scripts`/`clients/python/.venv` exclusions). Needed for the Phase 1 lint gate to pass honestly. **Note for Phase 3:** the new `vendor/deno/` + `vendor/pyodide/` asset dirs will need the same biome ignore (they're git-ignored too). + +**`.strict()` on `createBodySchema`** implements the design's "reject legacy `python: bool`": an unknown key (incl. legacy `python`) now returns 400 `INVALID_INPUT`. Confirmed **no existing test sends `python:` in a request body** (all `python:` test literals were `SandboxMeta`/`RuntimeOptions` mocks, not HTTP bodies), so this is safe. + +**Test count moved 105→106** integration tests (new Test B "re-runs idempotently after a simulated python-column drop (N+1)"). Unit tests: 875 pass / 4 skip. + +**Env note for running integration locally:** the suite does NOT auto-load `.env` (`vitest.setup.ts` only seeds `TENANT_DATABASES`); pass `DATABASE_URL=postgres://sqlfs_app:sqlfs_app@localhost:5433/sqlfs` (and `REDIS_URL`) explicitly on the command line. --- From 1967f5604a286d43c9dda8c84068b20f6b4edb22 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 16:36:45 +0930 Subject: [PATCH 04/16] Phase 2: python_runtime client & contract surfaces (SDKs, MCP, OpenAPI, docs) --- .changeset/python-runtime-enum.md | 13 +++++ clients/python/CHANGELOG.md | 14 ++++++ clients/python/README.md | 4 +- clients/python/examples/quickstart.py | 2 +- clients/python/pyproject.toml | 2 +- clients/python/src/sqlfs/__init__.py | 4 +- clients/python/src/sqlfs/_version.py | 2 +- clients/python/src/sqlfs/client.py | 13 +++-- clients/python/src/sqlfs/models.py | 10 +++- clients/python/tests/test_client.py | 24 +++++++--- clients/python/uv.lock | 56 +++++++++++----------- clients/typescript/CHANGELOG.md | 14 ++++++ clients/typescript/README.md | 4 +- clients/typescript/examples/e2e-local.ts | 2 +- clients/typescript/examples/quickstart.ts | 2 +- clients/typescript/package.json | 2 +- clients/typescript/src/client.ts | 14 ++++-- clients/typescript/src/index.ts | 1 + clients/typescript/src/models.ts | 9 +++- clients/typescript/src/version.ts | 2 +- clients/typescript/tests/sandboxes.test.ts | 16 ++++--- plugins/sql-fs/skills/api/SETUP.md | 2 +- plugins/sql-fs/skills/api/ref/bash.md | 7 ++- plugins/sql-fs/skills/api/ref/endpoints.md | 10 ++-- plugins/sql-fs/skills/py-sdk/SKILL.md | 2 +- plugins/sql-fs/skills/py-sdk/ref/client.md | 2 +- plugins/sql-fs/skills/py-sdk/ref/models.md | 11 +++-- src/api/mcp/tools.ts | 19 ++++---- src/api/openapi-spec.ts | 18 +++++-- src/api/tests/unit/mcp-tools.test.ts | 44 +++++++++++++++++ thoughts/issue-118-pyodide-runtime/plan.md | 29 ++++++++--- 31 files changed, 253 insertions(+), 101 deletions(-) create mode 100644 .changeset/python-runtime-enum.md diff --git a/.changeset/python-runtime-enum.md b/.changeset/python-runtime-enum.md new file mode 100644 index 0000000..dc3ad56 --- /dev/null +++ b/.changeset/python-runtime-enum.md @@ -0,0 +1,13 @@ +--- +"sql-fs-api": major +--- + +**Breaking:** replace the boolean `python` sandbox capability with a nullable `python_runtime` enum (`"stdlib" | "pyodide" | null`) across the HTTP API, MCP tools, OpenAPI spec, and both SDKs. + +- `POST /v1/sandboxes` now accepts `python_runtime` instead of `python`; the legacy `python` key is rejected with `400 INVALID_INPUT`. Create/get/list responses echo `python_runtime` (and now consistently include `network`). +- MCP `sandbox_create` takes `python_runtime` and `sandbox_list` echoes it. +- TypeScript SDK (`SandboxRecord`, `CreateSandboxOptions`) and Python SDK (`SandboxRecord`, `client.create(...)`) use `python_runtime`; both `SandboxRecord` types now also expose `network`. + +`python_runtime: "stdlib"` is the air-gapped CPython-WASM runtime (the previous `python: true`). `python_runtime: "pyodide"` adds a numpy/pandas/scipy/openpyxl runtime in an OS-isolated Deno subprocess. The DB layer migrates rolling-deploy-safe (migration 0006 dual-writes/back-reads the legacy column). + +Clients must migrate `python: true` → `python_runtime: "stdlib"` and `python: false` → omit (or `null`). diff --git a/clients/python/CHANGELOG.md b/clients/python/CHANGELOG.md index 90ec10e..1a9b576 100644 --- a/clients/python/CHANGELOG.md +++ b/clients/python/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to the SQL-FS Python SDK are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2026-06-08 + +### Changed + +- **Breaking:** replaced the boolean `python` argument of `client.sandboxes.create()` + with `python_runtime` (`"stdlib" | "pyodide" | None`). `SandboxRecord.python` is now + `SandboxRecord.python_runtime`. Migrate `python=True` → `python_runtime="stdlib"` and + `python=False` → omit (or `None`). + +### Added + +- `SandboxRecord.network` is now surfaced (previously server-only). +- Exported the `PythonRuntime` type alias. + ## [0.3.1] - 2026-06-08 ### Added diff --git a/clients/python/README.md b/clients/python/README.md index ad6309a..87b73a5 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -22,7 +22,7 @@ pip install -e clients/python from sqlfs import Client with Client(base_url="https://api.example.com", auth_secret="", sub="agent-001") as fs: - sb = fs.sandboxes.create(name="demo", python=True) + sb = fs.sandboxes.create(name="demo", python_runtime="stdlib") # Bash execution result = sb.exec("echo hello && ls /home/user") @@ -81,7 +81,7 @@ fs = Client(base_url="...", auth_secret="...", sub="agent", max_file_size=0) | Method | HTTP | Notes | |---|---|---| | `client.sandboxes.list()` | `GET /v1/sandboxes` | → `list[SandboxRecord]` | -| `client.sandboxes.create(name=, env=, files=, python=, javascript=)` | `POST /v1/sandboxes` | → `Sandbox` | +| `client.sandboxes.create(name=, env=, files=, python_runtime=, javascript=, network=)` | `POST /v1/sandboxes` | → `Sandbox` | | `client.sandboxes.get(id)` | `GET /v1/sandboxes/{id}` | → `SandboxInfo` | | `client.sandboxes.attach(id)` | _(no network)_ | → `Sandbox` for an existing id | | `client.sandboxes.delete(id)` | `DELETE /v1/sandboxes/{id}` | | diff --git a/clients/python/examples/quickstart.py b/clients/python/examples/quickstart.py index 6242e37..e3b5c1a 100644 --- a/clients/python/examples/quickstart.py +++ b/clients/python/examples/quickstart.py @@ -27,7 +27,7 @@ def main() -> int: token=token, sub="quickstart", ) as fs: - sb = fs.sandboxes.create(name="quickstart", python=False) + sb = fs.sandboxes.create(name="quickstart") print(f"created sandbox {sb.id}") try: # Bulk write — single round-trip for many files. diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index a8c0686..6cf272c 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sql-fs-sdk" -version = "0.3.1" +version = "0.4.0" description = "Python SDK for the SQL-FS API — persistent bash sandboxes for AI agents" readme = "README.md" requires-python = ">=3.9" diff --git a/clients/python/src/sqlfs/__init__.py b/clients/python/src/sqlfs/__init__.py index c0bc751..baf61b2 100644 --- a/clients/python/src/sqlfs/__init__.py +++ b/clients/python/src/sqlfs/__init__.py @@ -5,7 +5,7 @@ from sqlfs import Client with Client(base_url="https://...", auth_secret="...", sub="my-agent") as c: - sb = c.sandboxes.create(name="demo", python=True) + sb = c.sandboxes.create(name="demo", python_runtime="stdlib") result = sb.exec("echo hello") print(result.stdout, result.exit_code) sb.delete() @@ -28,6 +28,7 @@ BatchExecResult, ExecResult, FileStat, + PythonRuntime, ReadResult, SandboxInfo, SandboxRecord, @@ -46,6 +47,7 @@ "FileStat", "FilesAPI", "NotFoundError", + "PythonRuntime", "RateLimitError", "ReadResult", "SQLFSError", diff --git a/clients/python/src/sqlfs/_version.py b/clients/python/src/sqlfs/_version.py index 260c070..6a9beea 100644 --- a/clients/python/src/sqlfs/_version.py +++ b/clients/python/src/sqlfs/_version.py @@ -1 +1 @@ -__version__ = "0.3.1" +__version__ = "0.4.0" diff --git a/clients/python/src/sqlfs/client.py b/clients/python/src/sqlfs/client.py index 091615e..c2b5e04 100644 --- a/clients/python/src/sqlfs/client.py +++ b/clients/python/src/sqlfs/client.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional import httpx @@ -106,7 +106,7 @@ def create( name: Optional[str] = None, env: Optional[Mapping[str, str]] = None, files: Optional[Mapping[str, str]] = None, - python: bool = False, + python_runtime: Optional[Literal["stdlib", "pyodide"]] = None, javascript: bool = False, network: bool = False, ) -> Sandbox: @@ -116,7 +116,10 @@ def create( name: Optional human-readable label for the sandbox. env: Environment variables exposed to processes inside the sandbox. files: Initial files to seed the sandbox filesystem with, keyed by path. - python: Enable the CPython WASM runtime. + python_runtime: Python runtime to enable. ``"stdlib"`` registers the + air-gapped CPython WASM `python3`; ``"pyodide"`` registers a + numpy/pandas/scipy/openpyxl-capable Python in an OS-isolated Deno + subprocess. ``None`` (default) means no Python. javascript: Enable the QuickJS / `js-exec` runtime. network: Opt-in to outbound network access. When enabled, `fetch()` inside `js-exec` can reach external HTTP endpoints (timeout @@ -134,8 +137,8 @@ def create( body["env"] = dict(env) if files is not None: body["files"] = dict(files) - if python: - body["python"] = True + if python_runtime is not None: + body["python_runtime"] = python_runtime if javascript: body["javascript"] = True if network: diff --git a/clients/python/src/sqlfs/models.py b/clients/python/src/sqlfs/models.py index 3f483ed..9fe1636 100644 --- a/clients/python/src/sqlfs/models.py +++ b/clients/python/src/sqlfs/models.py @@ -10,6 +10,9 @@ from dataclasses import dataclass from typing import Any, Dict, List, Literal, Optional, cast +# Python runtime selection. None = no Python. +PythonRuntime = Literal["stdlib", "pyodide"] + @dataclass(frozen=True) class SandboxRecord: @@ -19,8 +22,9 @@ class SandboxRecord: name: Optional[str] owner: str created_at: str - python: bool + python_runtime: Optional[PythonRuntime] javascript: bool + network: bool @classmethod def from_api(cls, payload: Dict[str, Any]) -> SandboxRecord: @@ -29,8 +33,9 @@ def from_api(cls, payload: Dict[str, Any]) -> SandboxRecord: name=payload.get("name"), owner=payload["owner"], created_at=payload["createdAt"], - python=bool(payload.get("python", False)), + python_runtime=payload.get("python_runtime"), javascript=bool(payload.get("javascript", False)), + network=bool(payload.get("network", False)), ) @@ -200,6 +205,7 @@ def text(self, encoding: str = "utf-8") -> str: "BatchExecResult", "ExecResult", "FileStat", + "PythonRuntime", "ReadResult", "SandboxInfo", "SandboxRecord", diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 980e531..6ab3bc0 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -79,8 +79,9 @@ def test_list_sandboxes(): "name": "a", "owner": "alice", "createdAt": "2026-01-01T00:00:00Z", - "python": True, + "python_runtime": "stdlib", "javascript": False, + "network": False, } ] }, @@ -89,12 +90,13 @@ def test_list_sandboxes(): sandboxes = make_client().sandboxes.list() assert len(sandboxes) == 1 assert sandboxes[0].id == "id-1" - assert sandboxes[0].python is True + assert sandboxes[0].python_runtime == "stdlib" + assert sandboxes[0].network is False @respx.mock def test_create_sandbox_returns_handle(): - respx.post(f"{BASE_URL}/v1/sandboxes").mock( + route = respx.post(f"{BASE_URL}/v1/sandboxes").mock( return_value=httpx.Response( 201, json={ @@ -102,14 +104,20 @@ def test_create_sandbox_returns_handle(): "name": "demo", "owner": "alice", "createdAt": "2026-01-01T00:00:00Z", - "python": False, + "python_runtime": "stdlib", "javascript": False, + "network": False, }, ) ) - sb = make_client().sandboxes.create(name="demo", env={"FOO": "bar"}) + sb = make_client().sandboxes.create(name="demo", env={"FOO": "bar"}, python_runtime="stdlib") assert sb.id == "sb-9" assert sb.record is not None and sb.record.name == "demo" + # python_runtime round-trips: request body carries it, record reflects it. + sent = json.loads(route.calls.last.request.content) + assert sent.get("python_runtime") == "stdlib" + assert sb.record.python_runtime == "stdlib" + assert sb.record.network is False @respx.mock @@ -123,8 +131,9 @@ def test_create_sandbox_with_network_passes_flag(): "name": None, "owner": "alice", "createdAt": "2026-01-01T00:00:00Z", - "python": False, + "python_runtime": None, "javascript": True, + "network": True, }, ) ) @@ -146,8 +155,9 @@ def test_create_sandbox_network_default_omitted(): "name": None, "owner": "alice", "createdAt": "2026-01-01T00:00:00Z", - "python": False, + "python_runtime": None, "javascript": False, + "network": False, }, ) ) diff --git a/clients/python/uv.lock b/clients/python/uv.lock index f4886c7..1089cb0 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -522,6 +522,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, ] +[[package]] +name = "sql-fs-sdk" +version = "0.4.0" +source = { editable = "." } +dependencies = [ + { name = "httpx" }, +] + +[package.optional-dependencies] +dev = [ + { name = "mypy", version = "1.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "mypy", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pytest", version = "9.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "respx" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "httpx", specifier = ">=0.27,<1.0" }, + { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, + { name = "respx", marker = "extra == 'dev'", specifier = ">=0.21" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5" }, +] +provides-extras = ["dev"] + [[package]] name = "tomli" version = "2.4.1" @@ -584,31 +612,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac8 wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] - -[[package]] -name = "sql-fs-sdk" -version = "0.2.3" -source = { editable = "." } -dependencies = [ - { name = "httpx" }, -] - -[package.optional-dependencies] -dev = [ - { name = "mypy", version = "1.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "mypy", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "9.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "respx" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "httpx", specifier = ">=0.27,<1.0" }, - { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, - { name = "respx", marker = "extra == 'dev'", specifier = ">=0.21" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5" }, -] -provides-extras = ["dev"] diff --git a/clients/typescript/CHANGELOG.md b/clients/typescript/CHANGELOG.md index febd10d..dc2b764 100644 --- a/clients/typescript/CHANGELOG.md +++ b/clients/typescript/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to the TypeScript SDK are documented here. +## [0.4.0] - 2026-06-08 + +### Changed + +- **Breaking:** replaced the boolean `python` create option with the + `python_runtime` enum (`"stdlib" | "pyodide" | null`). `SandboxRecord.python` + is now `SandboxRecord.python_runtime`. Migrate `python: true` → + `python_runtime: "stdlib"` and `python: false` → omit (or `null`). + +### Added + +- `SandboxRecord.network` is now surfaced (previously server-only). +- Exported the `PythonRuntime` type. + ## [0.3.1] - 2026-06-08 ### Added diff --git a/clients/typescript/README.md b/clients/typescript/README.md index a319660..cbae4be 100644 --- a/clients/typescript/README.md +++ b/clients/typescript/README.md @@ -25,7 +25,7 @@ const client = new Client({ sub: "agent-001", }); -const sandbox = await client.sandboxes.create({ name: "demo", python: true }); +const sandbox = await client.sandboxes.create({ name: "demo", python_runtime: "stdlib" }); const result = await sandbox.exec("echo hello && ls /home/user"); console.log(result.stdout, result.exitCode, result.ok); @@ -58,7 +58,7 @@ const targetJwt = await adminClient.getToken(); ## API Surface - `client.sandboxes.list()` -- `client.sandboxes.create({ name, env, files, python, javascript, network })` +- `client.sandboxes.create({ name, env, files, python_runtime, javascript, network })` - `client.sandboxes.get(id)` - `client.sandboxes.attach(id)` - `client.sandboxes.delete(id)` diff --git a/clients/typescript/examples/e2e-local.ts b/clients/typescript/examples/e2e-local.ts index 794f684..69e8b30 100644 --- a/clients/typescript/examples/e2e-local.ts +++ b/clients/typescript/examples/e2e-local.ts @@ -29,7 +29,7 @@ async function setup(): Promise { const sandbox = await client.sandboxes.create({ name: "typescript-sdk-e2e", - python: true, + python_runtime: "stdlib", javascript: true, files: { "/home/user/seed.txt": "seeded" }, }); diff --git a/clients/typescript/examples/quickstart.ts b/clients/typescript/examples/quickstart.ts index 6f264a4..65846bf 100644 --- a/clients/typescript/examples/quickstart.ts +++ b/clients/typescript/examples/quickstart.ts @@ -7,7 +7,7 @@ const client = new Client({ sub: process.env.SQLFS_SUB ?? "typescript-quickstart", }); -const sandbox = await client.sandboxes.create({ name: "typescript-quickstart", python: true }); +const sandbox = await client.sandboxes.create({ name: "typescript-quickstart", python_runtime: "stdlib" }); try { const result = await sandbox.exec("echo hello from sql-fs && pwd"); console.log(result.stdout); diff --git a/clients/typescript/package.json b/clients/typescript/package.json index 530468d..1cf7204 100644 --- a/clients/typescript/package.json +++ b/clients/typescript/package.json @@ -1,6 +1,6 @@ { "name": "sql-fs-sdk", - "version": "0.3.1", + "version": "0.4.0", "description": "TypeScript SDK for the SQL-FS API - persistent bash sandboxes for AI agents", "type": "module", "license": "MIT", diff --git a/clients/typescript/src/client.ts b/clients/typescript/src/client.ts index 83635f4..77f5742 100644 --- a/clients/typescript/src/client.ts +++ b/clients/typescript/src/client.ts @@ -1,6 +1,12 @@ import { Transport, type TransportOptions } from "./http.js"; import { defaultMaxFileSize, isRecord, readJsonObject } from "./internal.js"; -import { type SandboxInfo, type SandboxRecord, sandboxInfoFromApi, sandboxRecordFromApi } from "./models.js"; +import { + type PythonRuntime, + type SandboxInfo, + type SandboxRecord, + sandboxInfoFromApi, + sandboxRecordFromApi, +} from "./models.js"; import { Sandbox } from "./sandbox.js"; export interface ClientOptions extends TransportOptions { @@ -11,7 +17,7 @@ export interface CreateSandboxOptions { name?: string; env?: Record; files?: Record; - python?: boolean; + python_runtime?: PythonRuntime; javascript?: boolean; network?: boolean; } @@ -69,8 +75,8 @@ export class SandboxesResource { if (options.files !== undefined) { body.files = { ...options.files }; } - if (options.python !== undefined) { - body.python = options.python; + if (options.python_runtime !== undefined) { + body.python_runtime = options.python_runtime; } if (options.javascript !== undefined) { body.javascript = options.javascript; diff --git a/clients/typescript/src/index.ts b/clients/typescript/src/index.ts index 72657d1..5c65b02 100644 --- a/clients/typescript/src/index.ts +++ b/clients/typescript/src/index.ts @@ -15,6 +15,7 @@ export { type ExecResult, type FileKind, type FileStat, + type PythonRuntime, ReadResult, type SandboxInfo, type SandboxRecord, diff --git a/clients/typescript/src/models.ts b/clients/typescript/src/models.ts index 305bd67..42e5634 100644 --- a/clients/typescript/src/models.ts +++ b/clients/typescript/src/models.ts @@ -1,13 +1,17 @@ export type FileKind = "file" | "dir" | "symlink"; export type StreamEventType = "stdout" | "stderr" | "exit"; +/** Python runtime selection. null = no Python. */ +export type PythonRuntime = "stdlib" | "pyodide" | null; + export interface SandboxRecord { id: string; name: string | null; owner: string; createdAt: string; - python: boolean; + python_runtime: PythonRuntime; javascript: boolean; + network: boolean; } export interface SandboxInfo { @@ -84,8 +88,9 @@ export function sandboxRecordFromApi(payload: ApiObject): SandboxRecord { name: payload.name == null ? null : String(payload.name), owner: String(payload.owner), createdAt: String(payload.createdAt), - python: Boolean(payload.python), + python_runtime: (payload.python_runtime ?? null) as PythonRuntime, javascript: Boolean(payload.javascript), + network: Boolean(payload.network), }; } diff --git a/clients/typescript/src/version.ts b/clients/typescript/src/version.ts index 50dde9e..6034731 100644 --- a/clients/typescript/src/version.ts +++ b/clients/typescript/src/version.ts @@ -1 +1 @@ -export const version = "0.3.1"; +export const version = "0.4.0"; diff --git a/clients/typescript/tests/sandboxes.test.ts b/clients/typescript/tests/sandboxes.test.ts index 9d7c640..1f9b6ca 100644 --- a/clients/typescript/tests/sandboxes.test.ts +++ b/clients/typescript/tests/sandboxes.test.ts @@ -12,8 +12,9 @@ describe("TypeScript SQL-FS SDK sandboxes", () => { name: "a", owner: "alice", createdAt: "2026-01-01T00:00:00Z", - python: true, + python_runtime: "stdlib", javascript: false, + network: false, }, ], }), @@ -22,20 +23,23 @@ describe("TypeScript SQL-FS SDK sandboxes", () => { name: null, owner: "alice", createdAt: "2026-01-01T00:00:00Z", - python: false, + python_runtime: "pyodide", javascript: false, + network: true, }), ]); const client = makeClient(fetchMock); const sandboxes = await client.sandboxes.list(); expect(sandboxes[0]?.id).toBe("id-1"); - expect(sandboxes[0]?.python).toBe(true); + expect(sandboxes[0]?.python_runtime).toBe("stdlib"); + expect(sandboxes[0]?.network).toBe(false); - const sandbox = await client.sandboxes.create({ python: false, javascript: false, network: false }); + const sandbox = await client.sandboxes.create({ python_runtime: "pyodide", javascript: false, network: true }); expect(sandbox.id).toBe("sb-net"); - expect(sandbox.record?.javascript).toBe(false); - expect(captured[1]?.body).toEqual({ python: false, javascript: false, network: false }); + expect(sandbox.record?.python_runtime).toBe("pyodide"); + expect(sandbox.record?.network).toBe(true); + expect(captured[1]?.body).toEqual({ python_runtime: "pyodide", javascript: false, network: true }); }); it("sends an empty object when creating a default sandbox", async () => { diff --git a/plugins/sql-fs/skills/api/SETUP.md b/plugins/sql-fs/skills/api/SETUP.md index 4ee66a9..a1be542 100644 --- a/plugins/sql-fs/skills/api/SETUP.md +++ b/plugins/sql-fs/skills/api/SETUP.md @@ -58,7 +58,7 @@ curl -fsS -X POST "$BASE_URL/v1/sandboxes" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{}' | jq -# → {"id":"...","owner":"admin","createdAt":"...","python":false,"javascript":false} +# → {"id":"...","owner":"admin","createdAt":"...","python_runtime":null,"javascript":false,"network":false} ``` --- diff --git a/plugins/sql-fs/skills/api/ref/bash.md b/plugins/sql-fs/skills/api/ref/bash.md index 33456d3..f143e99 100644 --- a/plugins/sql-fs/skills/api/ref/bash.md +++ b/plugins/sql-fs/skills/api/ref/bash.md @@ -45,9 +45,12 @@ It is NOT a real Linux shell. The following is the authoritative list of what wo ## Optional runtimes (must be enabled at sandbox creation) -### Python — `python: true` +### Python — `python_runtime: "stdlib"` or `"pyodide"` -Runs **CPython compiled to WASM** as `python3` (also aliased as `python`). Stdlib only — no `pip`, no network, no `os.system`/`subprocess`. Each invocation is a fresh, isolated interpreter (clean globals, clean `sys.modules`); interpreter state does **not** persist across calls. +Enabled by setting `python_runtime` at sandbox creation (`null` = no Python): + +- **`"stdlib"`** — runs **CPython compiled to WASM** as `python3` (also aliased as `python`). Stdlib only — no `pip`, no network, no `os.system`/`subprocess`. Each invocation is a fresh, isolated interpreter (clean globals, clean `sys.modules`); interpreter state does **not** persist across calls. Air-gapped. +- **`"pyodide"`** — runs Python with **numpy / pandas / scipy / openpyxl** preloaded, inside an **OS-isolated Deno subprocess** (zero network, no host filesystem reach). Use this for data-analysis workloads (e.g. read a CSV, write an `.xlsx`). Files written under the cwd are drained back into the sandbox filesystem. ```bash python3 -c "print(1 + 1)" diff --git a/plugins/sql-fs/skills/api/ref/endpoints.md b/plugins/sql-fs/skills/api/ref/endpoints.md index 3377469..3f32aef 100644 --- a/plugins/sql-fs/skills/api/ref/endpoints.md +++ b/plugins/sql-fs/skills/api/ref/endpoints.md @@ -116,7 +116,7 @@ SB=$(curl -s -X POST "$BASE_URL/v1/sandboxes" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "python": false, + "python_runtime": "stdlib", "javascript": false, "files": { "/home/user/hello.txt": "hello world" }, "env": { "NODE_ENV": "test" } @@ -127,7 +127,7 @@ All body fields are **optional**: | Field | Type | Default | Description | |---|---|---|---| -| `python` | boolean | false | Enable CPython WASM — registers `python3` (and `python` alias), stdlib only, isolated per call | +| `python_runtime` | `"stdlib" \| "pyodide" \| null` | null | Python runtime. `stdlib` = CPython WASM (`python3`/`python`, stdlib only, isolated per call, air-gapped). `pyodide` = numpy/pandas/scipy/openpyxl in an OS-isolated Deno subprocess. `null` = no Python | | `javascript` | boolean | false | Enable QuickJS WASM (`js-exec`/`node`) | | `network` | boolean | false | Enable outbound `fetch()` from `js-exec` (see note below) | | `files` | `Record` | — | Seed files (absolute path → plain text) | @@ -139,13 +139,13 @@ Response `201`: "id": "550e8400-...", "owner": "admin", "createdAt": "2026-04-25T...", - "python": false, + "python_runtime": "stdlib", "javascript": false, "network": false } ``` -**Important:** `python`/`javascript`/`network` must be set at creation. They cannot be changed later. +**Important:** `python_runtime`/`javascript`/`network` must be set at creation. They cannot be changed later. **`network: true` — outbound fetch() from js-exec** @@ -165,7 +165,7 @@ Response `200`: ```json { "id": "...", "owner": "admin", "createdAt": "...", "lastUsedAt": "...", - "python": false, "javascript": false + "python_runtime": null, "javascript": false, "network": false } ``` diff --git a/plugins/sql-fs/skills/py-sdk/SKILL.md b/plugins/sql-fs/skills/py-sdk/SKILL.md index c4953fc..945f73d 100644 --- a/plugins/sql-fs/skills/py-sdk/SKILL.md +++ b/plugins/sql-fs/skills/py-sdk/SKILL.md @@ -206,7 +206,7 @@ These come from real benchmarks (`clients/python/examples/perf_benchmark.py`): process — cwd, env vars, and shell functions persist within that window. - **Batch multi-step Python into one `python3` script, not many `python3 -c` calls.** - On `python=True` sandboxes, `python3` is CPython-on-WASM, stdlib only, and each + On `python_runtime="stdlib"` sandboxes, `python3` is CPython-on-WASM, stdlib only, and each call cold-boots a fresh isolated interpreter (~1.4 s, no shared state). Write your logic to a file and run `python3 script.py` once rather than looping `python3 -c '...'` — one script with a loop avoids paying the cold-boot N times. diff --git a/plugins/sql-fs/skills/py-sdk/ref/client.md b/plugins/sql-fs/skills/py-sdk/ref/client.md index b1df8bb..68f5a1d 100644 --- a/plugins/sql-fs/skills/py-sdk/ref/client.md +++ b/plugins/sql-fs/skills/py-sdk/ref/client.md @@ -115,7 +115,7 @@ sb = fs.sandboxes.create( name="my-project", # human label, optional env={"GREETING": "hi"}, # initial sandbox env vars files={"/home/user/seed.txt": "..."}, # text-only seed (use ingest_files for many/binary) - python=False, # enable CPython WASM runtime + python_runtime="stdlib", # "stdlib" (CPython WASM) | "pyodide" (numpy/pandas/scipy/openpyxl) | None javascript=False, # enable QuickJS runtime network=False, # enable outbound fetch() from js-exec (opt-in) ) diff --git a/plugins/sql-fs/skills/py-sdk/ref/models.md b/plugins/sql-fs/skills/py-sdk/ref/models.md index 595310f..e4730bb 100644 --- a/plugins/sql-fs/skills/py-sdk/ref/models.md +++ b/plugins/sql-fs/skills/py-sdk/ref/models.md @@ -22,20 +22,21 @@ from sqlfs import ( | `name` | `str \| None` | Human label, optional at creation | | `owner` | `str` | The `sub` claim of the creating token | | `created_at` | `str` | ISO-8601 UTC | -| `python` | `bool` | CPython WASM runtime enabled? | +| `python_runtime` | `"stdlib" \| "pyodide" \| None` | Python runtime: `stdlib` (CPython WASM), `pyodide` (numpy/pandas/scipy/openpyxl), or `None` | | `javascript` | `bool` | QuickJS runtime enabled? | +| `network` | `bool` | Outbound `fetch()` from `js-exec` enabled? | ```python -sb = fs.sandboxes.create(name="demo", python=True) -print(sb.record.id, sb.record.python) +sb = fs.sandboxes.create(name="demo", python_runtime="stdlib") +print(sb.record.id, sb.record.python_runtime) ``` --- ## `SandboxInfo` — returned by `sandboxes.get()` -Same as `SandboxRecord` minus the `python`/`javascript` flags, plus -`last_used_at`. +Same as `SandboxRecord` minus the `python_runtime`/`javascript`/`network` flags, +plus `last_used_at`. | Field | Type | Notes | |---|---|---| diff --git a/src/api/mcp/tools.ts b/src/api/mcp/tools.ts index 5725c1c..eb2fb2e 100644 --- a/src/api/mcp/tools.ts +++ b/src/api/mcp/tools.ts @@ -31,20 +31,17 @@ const MAX_EXPORT_CONCURRENCY = positiveIntEnv(process.env.MAX_EXPORT_CONCURRENCY export function registerTools(server: McpServer, sessionManager: SessionManager, owner: string, tenant: string): void { server.tool( "sandbox_create", - "Create an isolated bash sandbox with a virtual filesystem. Optional runtime flags opt in to python3/python (CPython WASM, stdlib only) and js-exec/node (QuickJS WASM) commands. Optional name for human-readable identification.", + 'Create an isolated bash sandbox with a virtual filesystem. `python_runtime` opts into python3/python: "stdlib" = CPython WASM (stdlib only, air-gapped) or "pyodide" = numpy/pandas/scipy/openpyxl in an OS-isolated Deno subprocess. `javascript` opts into js-exec/node (QuickJS WASM). Optional name for human-readable identification.', { name: z.string().max(255).optional().describe("Human-readable name for the sandbox"), - python: z.boolean().optional(), + python_runtime: z.enum(["stdlib", "pyodide"]).nullable().optional(), javascript: z.boolean().optional(), }, async (args) => { const id = randomUUID(); const name = args.name ?? null; - // Phase 1: keep the boolean `python` MCP input/output contract; map it - // internally onto the new pythonRuntime/python_runtime fields. Phase 2 - // migrates this surface to the `python_runtime` enum. const runtimeOptions = { - pythonRuntime: ((args.python ?? false) ? "stdlib" : null) as "stdlib" | null, + pythonRuntime: args.python_runtime ?? null, javascript: args.javascript ?? false, network: false, }; @@ -65,7 +62,7 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, text: JSON.stringify({ id, name, - python: runtimeOptions.pythonRuntime === "stdlib", + python_runtime: runtimeOptions.pythonRuntime, javascript: runtimeOptions.javascript, }), }, @@ -103,7 +100,7 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, name: s.name, owner: s.owner, createdAt: s.createdAt.toISOString(), - python: s.python_runtime === "stdlib", + python_runtime: s.python_runtime, javascript: s.javascript, })), }), @@ -180,8 +177,10 @@ export function registerTools(server: McpServer, sessionManager: SessionManager, "/proc /sys /dev (no special filesystems), ln -s (symlinks off by default),", "gcc/make/rustc (no compilers), network access of any kind.", "", - "Optional runtimes (only if sandbox was created with python:true or javascript:true):", - "- python3 / python — CPython WASM, stdlib only (no pip, no network, no os.system).", + "Optional runtimes (only if the sandbox was created with python_runtime set or javascript:true):", + '- python3 / python (python_runtime: "stdlib") — CPython WASM, stdlib only (no pip, no network, no os.system).', + '- python3 / python (python_runtime: "pyodide") — numpy/pandas/scipy/openpyxl available, runs in an', + " OS-isolated Deno subprocess (no network, no host FS). Use for data-analysis workloads.", " Concurrent python3 executions across the server are capped to prevent OOM; excess", " scripts queue until a slot frees.", "- js-exec / node — QuickJS WASM. TypeScript supported. No npm, no network.", diff --git a/src/api/openapi-spec.ts b/src/api/openapi-spec.ts index 4623d94..0f0daf6 100644 --- a/src/api/openapi-spec.ts +++ b/src/api/openapi-spec.ts @@ -20,10 +20,11 @@ const sandboxSchema = { name: { type: "string", nullable: true, example: "my-project-sandbox" }, owner: { type: "string", example: "alice" }, createdAt: { type: "string", format: "date-time" }, - python: { type: "boolean", example: false }, + python_runtime: { type: "string", enum: ["stdlib", "pyodide"], nullable: true, example: null }, javascript: { type: "boolean", example: false }, + network: { type: "boolean", example: false }, }, - required: ["id", "name", "owner", "createdAt", "python", "javascript"], + required: ["id", "name", "owner", "createdAt", "python_runtime", "javascript", "network"], } as const; const sandboxInfoSchema = { @@ -318,8 +319,19 @@ export const openapiSpec = { additionalProperties: { type: "string" }, description: "Initial files to write (path → content)", }, - python: { type: "boolean", default: false, description: "Enable CPython WASM runtime" }, + python_runtime: { + type: "string", + enum: ["stdlib", "pyodide"], + nullable: true, + description: + "Python runtime: stdlib (CPython WASM) or pyodide (numpy/pandas/scipy/openpyxl, OS-isolated)", + }, javascript: { type: "boolean", default: false, description: "Enable QuickJS runtime" }, + network: { + type: "boolean", + default: false, + description: "Enable outbound fetch() from js-exec (requires javascript)", + }, }, }, }, diff --git a/src/api/tests/unit/mcp-tools.test.ts b/src/api/tests/unit/mcp-tools.test.ts index c2e9072..a5c8b38 100644 --- a/src/api/tests/unit/mcp-tools.test.ts +++ b/src/api/tests/unit/mcp-tools.test.ts @@ -106,3 +106,47 @@ describe("MCP tool — bash_exec_batch disconnect signal", () => { } }); }); + +describe("MCP tools — python_runtime echo", () => { + beforeEach(() => { + process.env.AUTH_SECRET = "test-secret-mcp-tools-at-least-32bytes!!"; + }); + + afterEach(() => { + process.env.AUTH_SECRET = ""; + vi.restoreAllMocks(); + }); + + it("sandbox_create echoes the python_runtime it was given", async () => { + const sessionManager = new SessionManager({ createFs: async () => new InMemoryFs() }); + const { server, getHandler } = captureToolHandler("sandbox_create"); + registerTools(server, sessionManager, "test-owner", "default"); + + const result = (await getHandler()({ python_runtime: "pyodide" }, {})) as { content: { text: string }[] }; + const body = JSON.parse(result.content[0]!.text); + expect(body.python_runtime).toBe("pyodide"); + }); + + it("sandbox_list echoes python_runtime per sandbox", async () => { + const sessionManager = new SessionManager({ + createFs: async () => new InMemoryFs(), + listSandboxesFn: async () => [ + { + id: "sb-1", + name: null, + owner: "test-owner", + createdAt: new Date("2026-01-01T00:00:00Z"), + python_runtime: "stdlib", + javascript: false, + network: false, + }, + ], + }); + const { server, getHandler } = captureToolHandler("sandbox_list"); + registerTools(server, sessionManager, "test-owner", "default"); + + const result = (await getHandler()({}, {})) as { content: { text: string }[] }; + const body = JSON.parse(result.content[0]!.text); + expect(body.sandboxes[0].python_runtime).toBe("stdlib"); + }); +}); diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index 5c81f68..9f6739c 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -309,17 +309,32 @@ Propagate the enum to every external representation (research Q5) and reconcile ### Phase 2: Success Criteria #### Phase 2: Programmatic Verification -- [ ] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass -- [ ] Python SDK suite passes (`cd clients/python && `); TS SDK suite passes (`cd clients/typescript && `) -- [ ] `mcp-tools.test.ts` passes with the new `python_runtime` assertions -- [ ] OpenAPI spec still serializes (server boots; `GET /openapi.json` valid JSON) and `.changeset/*.md` exists +- [x] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass (877 pass / 4 skip — +2 new MCP `python_runtime` echo tests) +- [x] Python SDK suite passes (`uv run --extra dev pytest` — 37 pass; also `mypy --strict` + `ruff` clean); TS SDK suite passes (`pnpm install && pnpm typecheck && pnpm test` — 19 pass) +- [x] `mcp-tools.test.ts` passes with the new `python_runtime` assertions (sandbox_create echoes `python_runtime`; sandbox_list echoes per-sandbox `python_runtime`) +- [x] OpenAPI spec still serializes (route is `c.json(openapiSpec)`; spec JSON-round-trips, 24 KB, Sandbox + create-body carry `python_runtime`+`network`, `python` removed) and `.changeset/python-runtime-enum.md` exists (major) #### Phase 2: Agent Verification -- [ ] Agent diffs every research-Q5 surface against a checklist (Python SDK model+client, TS SDK model+client, MCP create+list, OpenAPI record+create, all docs) and confirms **no remaining boolean `python`** -- [ ] Agent confirms `network` now appears in both SDK `SandboxRecord` types and the OpenAPI record + create schemas +- [x] Agent diffs every research-Q5 surface against a checklist (Python SDK model+client, TS SDK model+client, MCP create+list, OpenAPI record+create, all docs) and confirms **no remaining boolean `python`** — also caught + fixed 3 surfaces the plan's docs list omitted (both SDK READMEs + the Python `__init__.py` module docstring) +- [x] Agent confirms `network` now appears in both SDK `SandboxRecord` types (TS `models.ts:14`/`:93`, Py `models.py:27`/`:38`) and the OpenAPI record (`Sandbox.properties.network` + `required`) + create schemas ### Phase 2: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 2._ + +**Surfaces the plan's docs list (step 6) under-counted.** Beyond the six `plugins/sql-fs/skills/*` docs, three more user-facing SDK doc surfaces carried boolean `python` and had to change for the "no remaining boolean `python`" gate: `clients/python/README.md`, `clients/python/src/sqlfs/__init__.py` (the module-level quick-start docstring), and `clients/typescript/README.md`. Sweep the READMEs + module docstrings, not just the plugin skill docs. + +**SDK packages are standalone (no pnpm workspace).** There is **no `pnpm-workspace.yaml`** — `clients/typescript` is its own package (`sql-fs-sdk@0.3.0`) and is NOT covered by the root `pnpm typecheck`/`test:unit` (root `tsconfig` is `include: ["src"]`). To run the TS SDK suite you must `cd clients/typescript && pnpm install` first (its `node_modules` was absent), then `pnpm typecheck && pnpm test`. The **root** `biome` (`pnpm lint:fix`) DOES lint `clients/typescript/src` (only `clients/python/.venv` is biome-ignored), so SDK TS style is enforced centrally. + +**Python SDK runner:** no `.venv` committed; use `uv run --extra dev pytest` (creates `.venv`, builds the editable pkg). 37 tests pass. `mypy --strict` + `ruff` also clean. `pyproject.toml` pins `[tool.mypy] python_version = "3.9"` which newer mypy warns is unsupported (must be ≥3.10) but still runs clean — pre-existing, not ours to fix here. **`uv run` rewrites `clients/python/uv.lock`** (syncs the `sql-fs-sdk` self-package version). When NOT bumping the version it just re-normalizes/re-sorts (spurious → revert); when bumping (this phase) it's a legitimate version sync (keep). Note: the committed lock self-version was **stale at `0.2.3`** vs pyproject `0.3.0` — a pre-existing drift now corrected to `0.4.0`. + +**Changeset (`pnpm changeset` TUI is non-interactive here, so hand-authored):** mirrors existing `.changeset/*.md` — front-matter `"sql-fs-api": major` + body. Changesets track ONLY `sql-fs-api` (the root). ⚠️ **The SDKs are NOT changeset-managed and need their own version bumps** (see below) — a Codex review correctly flagged that the root changeset alone would never publish the breaking SDK change. + +**SDK release model + version bumps (Codex review, post-implementation fixes).** Each SDK has its own release pipeline — `.github/workflows/ts-sdk-release.yml` / `python-sdk-release.yml` — that fires on a push to `main` touching `clients//**` and **publishes only when the version is new** (TS: `pnpm check:version` requires `package.json` ↔ `src/version.ts` ↔ `CHANGELOG.md` to all agree; Python: detects version from `CHANGELOG.md`, cross-checks `pyproject.toml` + `src/sqlfs/_version.py`, and skips publish if the git tag already exists). **Consequence:** a breaking SDK change with no version bump is silently *skipped at publish* (0.3.0 is immutable on npm/PyPI) → the change never reaches users. **Fix applied:** bumped both SDKs **0.3.0 → 0.4.0** (conventional 0.x breaking bump — minor) with `## [0.4.0] - 2026-06-08` CHANGELOG entries, across all version files each (TS: `package.json` + `src/version.ts` + `CHANGELOG.md`, verified by `pnpm check:version`; Python: `pyproject.toml` + `_version.py` + `CHANGELOG.md`, verified by `uv lock --check`). Picked 0.4.0 not 1.0.0 to keep the SDKs pre-1.0 (initial release was 0.3.0). **Also (Codex finding 2):** two SDK README API-surface lines still advertised boolean `python` (TS `README.md:61` signature list, Py `README.md:84` method table) — both migrated to `python_runtime` (+`network` on the Py row). + +**OpenAPI verification without a live server:** the route is literally `app.get("/openapi.json", (c) => c.json(openapiSpec))`, so `JSON.stringify(openapiSpec)` round-tripping (via a one-shot `npx tsx -e`) deterministically proves `GET /openapi.json` returns valid JSON — no dev server needed for this programmatic check. + +**SDK public type export (minor addition beyond the plan):** exported `PythonRuntime` from both SDKs (`clients/typescript/src/index.ts`, Python `__init__.py` + `models.py __all__`) so users assigning `python_runtime` can name the type — parity with the new field. **`network` was newly added to both `SandboxRecord` types** (it existed server-side but was never surfaced in the SDK record — the plan's "reconcile asymmetry"). + +**SandboxInfo unchanged:** the SDKs' `.get()` model (`SandboxInfo`) was deliberately NOT given `python_runtime` — Phase 2 only touched `SandboxRecord` (create/list). The server's GET now returns the capability fields (Phase 1) but the SDK `SandboxInfo` model still doesn't parse them (pre-existing, out of scope). --- From 47a93105d469c842631fd6a9f9011c67b1be19ac Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 17:31:44 +0930 Subject: [PATCH 05/16] Phase 3: offline assets + Deno harness (the untrusted side) --- .dockerignore | 5 + .gitignore | 6 + Dockerfile | 14 + biome.json | 12 +- package.json | 2 +- scripts/build-pyodide-lock.mjs | 84 ++++++ scripts/copy-pyodide-runner.mjs | 20 ++ scripts/fetch-pyodide-assets.mjs | 171 +++++++++++ src/pyodide-runner/protocol.ts | 137 +++++++++ src/pyodide-runner/runner.ts | 318 +++++++++++++++++++++ thoughts/issue-118-pyodide-runtime/plan.md | 66 ++++- tsconfig.json | 2 +- 12 files changed, 821 insertions(+), 16 deletions(-) create mode 100644 scripts/build-pyodide-lock.mjs create mode 100644 scripts/copy-pyodide-runner.mjs create mode 100644 scripts/fetch-pyodide-assets.mjs create mode 100644 src/pyodide-runner/protocol.ts create mode 100644 src/pyodide-runner/runner.ts diff --git a/.dockerignore b/.dockerignore index 5663b6f..9e5fc21 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,8 @@ dist tasks/ .dockerignore Dockerfile +# Vendored Pyodide assets are regenerated fresh in the builder stage; never copy +# a (possibly stale) local copy into the build context. +vendor +# Design records + ~408 MB of spike scratch assets — not part of the image. +thoughts diff --git a/.gitignore b/.gitignore index 5e2e4db..2340b38 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ dist/ *.tsbuildinfo .claude/worktrees/ docs/ +docker-compose.local.yml +.tmp/ + +# Vendored Pyodide runtime assets — reproduced by scripts/fetch-pyodide-assets.mjs +# (pinned Deno binary + Pyodide distribution + wheels + download cache). Never committed. +vendor/ diff --git a/Dockerfile b/Dockerfile index 234b925..1456c3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,10 @@ FROM node:22-bookworm AS builder WORKDIR /app RUN corepack enable +# curl/unzip/bzip2 are needed by scripts/fetch-pyodide-assets.mjs (curl download, +# unzip the Deno binary, tar -xj the Pyodide .tar.bz2). +RUN apt-get update && apt-get install -y --no-install-recommends curl unzip bzip2 \ + && rm -rf /var/lib/apt/lists/* COPY pnpm-lock.yaml package.json ./ # Remove the `prepare` lifecycle hook before install. lefthook install panics # in Docker Desktop's Linux VM (Go taggedPointerPack runtime bug). Git hooks @@ -12,16 +16,26 @@ RUN node -e "const fs=require('fs'),p=JSON.parse(fs.readFileSync('package.json', RUN pnpm install --frozen-lockfile COPY . . RUN pnpm build +# Vendor the Pyodide runtime assets (pinned Deno binary + Pyodide distribution + +# wheels) and build the supplementary custom lock. Reproduced fresh here — never +# committed (see .gitignore / .dockerignore). +RUN node scripts/fetch-pyodide-assets.mjs && node scripts/build-pyodide-lock.mjs # ------- Runtime ------- FROM node:22-slim AS runtime ENV NODE_ENV=production ENV PORT=8080 +# Pyodide runtime: Node resolves these before spawning the Deno child (Phase 4). +ENV PYODIDE_ASSET_DIR=/app/vendor/pyodide +ENV DENO_BIN_PATH=/app/vendor/deno/deno WORKDIR /app RUN groupadd -r app && useradd -r -g app app COPY --from=builder --chown=app:app /app/dist ./dist COPY --from=builder --chown=app:app /app/node_modules ./node_modules COPY --from=builder --chown=app:app /app/package.json ./package.json +# dist/pyodide-runner (raw .ts, copied by the build) ships inside ./dist above; +# vendor/ carries the Deno binary + Pyodide distribution + custom lock. +COPY --from=builder --chown=app:app /app/vendor ./vendor USER app EXPOSE 8080 HEALTHCHECK --interval=15s --timeout=3s --start-period=10s \ diff --git a/biome.json b/biome.json index 1bdc69e..26f453d 100644 --- a/biome.json +++ b/biome.json @@ -22,6 +22,16 @@ "lineWidth": 120 }, "files": { - "ignore": ["node_modules", "dist", "*.sql", "scripts", "clients/python/.venv", "thoughts"] + "ignore": [ + "node_modules", + "dist", + ".tmp", + "*.sql", + "scripts", + "clients/python/.venv", + "thoughts", + "vendor", + "src/pyodide-runner/runner.ts" + ] } } diff --git a/package.json b/package.json index f567c35..288a769 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "scripts": { "dev": "tsx watch --env-file .env src/api/server.ts", "dev:portless": "portless", - "build": "tsc && node scripts/copy-postgres-migrations.mjs", + "build": "tsc && node scripts/copy-postgres-migrations.mjs && node scripts/copy-pyodide-runner.mjs", "start": "node dist/api/server.js", "bench:remote-bash": "python3 scripts/benchmark_remote_bash.py", "typecheck": "tsc --noEmit", diff --git a/scripts/build-pyodide-lock.mjs b/scripts/build-pyodide-lock.mjs new file mode 100644 index 0000000..e6f74a0 --- /dev/null +++ b/scripts/build-pyodide-lock.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +/** + * Produce vendor/pyodide/pyodide-lock.custom.json — the stock Pyodide lock + * augmented with the two vendored pure-python packages (openpyxl + et_xmlfile) + * that the distribution omits. Run after fetch-pyodide-assets.mjs. + * + * MECHANISM (deliberate deviation from the plan's "micropip.freeze", per Phase 0 + * Discoveries' "do one or the other deliberately"): a DETERMINISTIC, fully + * offline MERGE — read the stock lock, append entries for the two wheels (name / + * version parsed from the filename, sha256 computed from the file), write the + * custom lock. This is more robust than running micropip.freeze offline and + * produces an equivalent artifact: a lock in which `loadPackage(["openpyxl"])` + * resolves by name. + * + * This lock is SUPPLEMENTARY. runner.ts deliberately loads numpy/pandas/scipy by + * name from the STOCK lock and openpyxl/et_xmlfile by direct file:// URL — the + * spike-S1-proven offline path — so it does NOT depend on this custom lock. The + * custom lock is a canonical dependency manifest and a future path to + * loadPackage-by-name. NEVER hand-edit it; regenerate via this script. + */ + +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const ASSET_DIR = process.env.PYODIDE_ASSET_DIR ?? join(ROOT, "vendor", "pyodide"); +const STOCK_LOCK = join(ASSET_DIR, "pyodide-lock.json"); +const CUSTOM_LOCK = join(ASSET_DIR, "pyodide-lock.custom.json"); + +function log(msg) { + console.error(`[build-pyodide-lock] ${msg}`); +} + +if (!existsSync(STOCK_LOCK)) { + throw new Error(`stock lock not found at ${STOCK_LOCK} — run scripts/fetch-pyodide-assets.mjs first`); +} + +// The two pure-python packages absent from the distribution, with their +// dependency edges and import names (deterministic — known set). +const VENDORED = { + et_xmlfile: { match: /^et_xmlfile-.*\.whl$/, imports: ["et_xmlfile"], depends: [] }, + openpyxl: { match: /^openpyxl-.*\.whl$/, imports: ["openpyxl"], depends: ["et_xmlfile"] }, +}; + +function findWheel(re) { + const hit = readdirSync(ASSET_DIR).find((f) => re.test(f)); + if (!hit) throw new Error(`wheel matching ${re} not found in ${ASSET_DIR} — run fetch-pyodide-assets.mjs`); + return hit; +} + +function parseNameVersion(fileName) { + // PEP 427 wheel: {name}-{version}-{pytag}-{abitag}-{platformtag}.whl + const [name, version] = fileName.split("-"); + return { name, version }; +} + +function sha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +const lock = JSON.parse(readFileSync(STOCK_LOCK, "utf8")); + +for (const [key, spec] of Object.entries(VENDORED)) { + const fileName = findWheel(spec.match); + const { version } = parseNameVersion(fileName); + lock.packages[key] = { + name: key, + version, + file_name: fileName, + install_dir: "site", + sha256: sha256(join(ASSET_DIR, fileName)), + package_type: "package", + imports: spec.imports, + depends: spec.depends, + unvendored_tests: false, + shared_library: false, + }; + log(`added ${key}@${version} (${fileName})`); +} + +writeFileSync(CUSTOM_LOCK, `${JSON.stringify(lock, null, 1)}\n`); +log(`wrote ${CUSTOM_LOCK} (${Object.keys(lock.packages).length} packages)`); diff --git a/scripts/copy-pyodide-runner.mjs b/scripts/copy-pyodide-runner.mjs new file mode 100644 index 0000000..72eb8ca --- /dev/null +++ b/scripts/copy-pyodide-runner.mjs @@ -0,0 +1,20 @@ +/** + * Copy the Pyodide runner sources to dist/ as RAW .ts. + * + * Deno runs `dist/pyodide-runner/runner.ts` directly (it uses Deno globals and is + * excluded from the tsc build). `runner.ts` imports `./protocol.ts`, so the raw + * protocol source must sit beside it. tsc separately emits + * `dist/pyodide-runner/protocol.js` for the Node side (src/api/pyodide/ipc.ts) — + * both coexist; Deno resolves the explicit `.ts`, Node imports the `.js`. + */ +import { copyFileSync, mkdirSync, readdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const src = join(root, "src/pyodide-runner"); +const dest = join(root, "dist/pyodide-runner"); +mkdirSync(dest, { recursive: true }); +for (const name of readdirSync(src)) { + if (name.endsWith(".ts")) copyFileSync(join(src, name), join(dest, name)); +} diff --git a/scripts/fetch-pyodide-assets.mjs b/scripts/fetch-pyodide-assets.mjs new file mode 100644 index 0000000..449cb07 --- /dev/null +++ b/scripts/fetch-pyodide-assets.mjs @@ -0,0 +1,171 @@ +#!/usr/bin/env node +/** + * Fetch the vendored Pyodide runtime assets into ./vendor/ (git-ignored). + * + * Downloads, under PINNED versions, the exact artifacts validated by spike S1: + * - the Deno binary → vendor/deno/deno + * - the Pyodide full distribution → vendor/pyodide/ (wasm + python_stdlib.zip + * + numpy/pandas/scipy wheels + stock lock) + * - openpyxl + et_xmlfile wheels → vendor/pyodide/ (absent from the dist) + * + * Idempotent: skips any artifact already present whose checksum still matches. + * + * INTEGRITY. Platform-independent runtime bytes are SHA-256-pinned to the exact + * artifacts spike S1 validated (pyodide.mjs / pyodide.asm.wasm / python_stdlib.zip + * and the two pure-python wheels). The Deno binary is pinned by version + the + * official dl.deno.land URL only — its bytes are platform-specific, so a single + * cross-arch checksum is impossible; we verify it extracted and is executable. + * + * Requires `curl`, `unzip`, and `tar` (with bzip2) on PATH — present on macOS and + * installed in the Docker builder stage. Mirrors the proven spike + * `s1-pyodide-deno.sh` curl/unzip/tar flow. + */ + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync } from "node:fs"; +import { arch, platform } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +// ── Pinned versions (must match spike S1 / Phase 0 Discoveries) ────────────── +const PINS = { + deno: "v2.8.2", + pyodide: "0.29.4", + openpyxlWheel: "openpyxl-3.1.5-py2.py3-none-any.whl", + etXmlfileWheel: "et_xmlfile-2.0.0-py3-none-any.whl", +}; + +const WHEEL_URLS = { + [PINS.openpyxlWheel]: + "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", + [PINS.etXmlfileWheel]: + "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", +}; + +// SHA-256 of the platform-independent artifacts, pinned to the exact bytes +// spike S1 validated. Mismatch ⇒ hard failure (supply-chain / corruption guard). +const SHA256 = { + "pyodide.mjs": "c8dffeefeb6f9c4bf635baf0cdb51f4da06df0e3aab4fe1a99b8ad3570065461", + "pyodide.asm.wasm": "10090fe41e019ae669d512e1f747021a8db2aaab0f6dd6f85fa9368c55d681e3", + "python_stdlib.zip": "92cb24faa546818f3ef4050fd5bd2b6487bd2042efed2113af141d035f30efb4", + [PINS.openpyxlWheel]: "5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", + [PINS.etXmlfileWheel]: "7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", +}; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const VENDOR = join(ROOT, "vendor"); +const DENO_DIR = join(VENDOR, "deno"); +const DENO_BIN = join(DENO_DIR, "deno"); +const PYODIDE_DIR = join(VENDOR, "pyodide"); +const CACHE = join(VENDOR, ".cache"); + +function log(msg) { + console.error(`[fetch-pyodide] ${msg}`); +} + +function sha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +function verify(path, name) { + const expected = SHA256[name]; + if (!expected) return; + const actual = sha256(path); + if (actual !== expected) { + throw new Error(`checksum mismatch for ${name}: expected ${expected}, got ${actual}`); + } + log(`verified ${name} (sha256 ok)`); +} + +function curl(url, dest) { + execFileSync("curl", ["-fSL", "--retry", "3", "-o", dest, url], { stdio: ["ignore", "ignore", "inherit"] }); +} + +function denoTarget() { + const key = `${platform()}-${arch()}`; + const map = { + "darwin-arm64": "aarch64-apple-darwin", + "darwin-x64": "x86_64-apple-darwin", + "linux-arm64": "aarch64-unknown-linux-gnu", + "linux-x64": "x86_64-unknown-linux-gnu", + }; + const target = map[key]; + if (!target) throw new Error(`unsupported host ${key} for Deno download`); + return target; +} + +function fetchDeno() { + if (existsSync(DENO_BIN)) { + log(`Deno ${PINS.deno} already present`); + return; + } + const target = denoTarget(); + mkdirSync(DENO_DIR, { recursive: true }); + mkdirSync(CACHE, { recursive: true }); + const zip = join(CACHE, `deno-${PINS.deno}-${target}.zip`); + if (!existsSync(zip)) { + log(`downloading Deno ${PINS.deno} (${target})…`); + curl(`https://dl.deno.land/release/${PINS.deno}/deno-${target}.zip`, zip); + } + execFileSync("unzip", ["-oq", zip, "-d", DENO_DIR], { stdio: "inherit" }); + chmodSync(DENO_BIN, 0o755); + // Strip macOS quarantine xattr if present (no-op elsewhere). + try { + execFileSync("xattr", ["-d", "com.apple.quarantine", DENO_BIN], { stdio: "ignore" }); + } catch { + /* not macOS / no xattr */ + } + if (!existsSync(DENO_BIN)) throw new Error("Deno binary not present after extraction"); + log(`Deno ${PINS.deno} ready at ${DENO_BIN}`); +} + +function fetchPyodide() { + const marker = join(PYODIDE_DIR, "pyodide.mjs"); + if (existsSync(marker)) { + log(`Pyodide ${PINS.pyodide} already extracted`); + } else { + mkdirSync(CACHE, { recursive: true }); + const tarball = join(CACHE, `pyodide-${PINS.pyodide}.tar.bz2`); + if (!existsSync(tarball)) { + log(`downloading Pyodide ${PINS.pyodide} full distribution (~408 MB)…`); + curl( + `https://github.com/pyodide/pyodide/releases/download/${PINS.pyodide}/pyodide-${PINS.pyodide}.tar.bz2`, + tarball, + ); + } + log("extracting Pyodide distribution…"); + const tmp = join(VENDOR, `_extract-${PINS.pyodide}`); + rmSync(tmp, { recursive: true, force: true }); + mkdirSync(tmp, { recursive: true }); + execFileSync("tar", ["-xjf", tarball, "-C", tmp], { stdio: "inherit" }); + // The tarball unpacks to a top-level "pyodide/" dir. + rmSync(PYODIDE_DIR, { recursive: true, force: true }); + renameSync(join(tmp, "pyodide"), PYODIDE_DIR); + rmSync(tmp, { recursive: true, force: true }); + } + // Verify the platform-independent runtime bytes regardless of skip/extract. + for (const name of ["pyodide.mjs", "pyodide.asm.wasm", "python_stdlib.zip"]) { + verify(join(PYODIDE_DIR, name), name); + } +} + +function fetchWheels() { + mkdirSync(PYODIDE_DIR, { recursive: true }); + for (const wheel of [PINS.etXmlfileWheel, PINS.openpyxlWheel]) { + const dest = join(PYODIDE_DIR, wheel); + if (existsSync(dest) && sha256(dest) === SHA256[wheel]) { + log(`${wheel} already present`); + continue; + } + log(`downloading ${wheel}…`); + curl(WHEEL_URLS[wheel], dest); + verify(dest, wheel); + } +} + +mkdirSync(VENDOR, { recursive: true }); +fetchDeno(); +fetchPyodide(); +fetchWheels(); +log(`done — vendor/ populated (deno=${PINS.deno}, pyodide=${PINS.pyodide})`); diff --git a/src/pyodide-runner/protocol.ts b/src/pyodide-runner/protocol.ts new file mode 100644 index 0000000..9e73915 --- /dev/null +++ b/src/pyodide-runner/protocol.ts @@ -0,0 +1,137 @@ +/** + * Shared IPC protocol contract for the Pyodide runner. + * + * RUNTIME-AGNOSTIC by design: uses only `Uint8Array` / `DataView` / + * `TextEncoder` / `TextDecoder` — NO `Buffer`, NO `Deno` globals. `tsc` compiles + * this file to `dist/pyodide-runner/protocol.js` for the Node side (see + * `src/api/pyodide/ipc.ts`, Phase 4), and the Deno entry `runner.ts` imports the + * raw `.ts` directly. Keep it free of any host-specific API. + * + * Wire format: length-prefixed JSON frames — a 4-byte big-endian uint32 byte + * length followed by that many bytes of UTF-8 JSON. + * + * SECURITY (spike S2 finding A): the integrity fields `requestId` / `seq` / + * `generation` are unguessable secrets held by Node (and the runner's JS + * closure) and are NEVER exposed to untrusted Python. Node-side frame validation + * keyed on these fields is the load-bearing control — realm lockdown in the + * child cannot contain raw stdout writes, so an escaped process can emit bytes + * but can never produce an *accepted* frame. + */ + +export const PROTOCOL_VERSION = 1; + +export type FrameType = "run" | "result" | "error" | "ready"; + +/** + * A staged-in / drained-out filesystem entry. `kind` distinguishes regular files + * from directories so the manager's drain (Phase 5) can apply dirs-before-files + * and represent script-created EMPTY directories (a files-only shape could not). + * `data` is base64 file contents for `kind: "file"` and `""` for `kind: "dir"`. + */ +export interface FsEntry { + readonly path: string; + readonly kind: "file" | "dir"; + readonly mode: number; + readonly data: string; // base64 file contents; "" for dirs +} + +/** Node → child: run untrusted Python. */ +export interface RunRequest { + readonly type: "run"; + readonly requestId: string; // random, set by Node + readonly seq: number; // monotonic per child + readonly generation: number; // child generation id + readonly code: string; // resolved script or -c body + readonly argv: readonly string[]; + readonly stdin: string; // base64 + readonly files: readonly FsEntry[]; // cwd subtree staged into MEMFS (files + dirs) + readonly cwd: string; +} + +/** child → Node: the single response to a `run`. */ +export interface RunResponse { + readonly type: "result" | "error"; + readonly requestId: string; + readonly seq: number; + readonly generation: number; + readonly stdout: string; // base64 + readonly stderr: string; // base64 + readonly exitCode: number; + // `created` is ordered dirs-before-files (dirs shallow→deep) so the drain can + // apply it directly. `modified` carries changed files only. `deleted` is + // depth-first (deepest paths first) so children are removed before parents. + readonly created: readonly FsEntry[]; + readonly modified: readonly FsEntry[]; + readonly deleted: readonly string[]; +} + +/** + * child → Node: a ONE-TIME pre-run handshake (no requestId/seq), validated + * separately from per-request frames — see `ipc.ts` integrity rules. It carries + * `generation` only, and marks the `starting → idle` transition. + */ +export interface ReadyFrame { + readonly type: "ready"; + readonly generation: number; +} + +export type Frame = RunRequest | RunResponse | ReadyFrame; + +const HEADER_BYTES = 4; + +/** + * Hard ceiling on a single frame's JSON-body byte length. This is a framing-level + * safety net against a malformed/hostile 4-byte length prefix (e.g. 0xFFFFFFFF) + * triggering a multi-GB allocation while reassembling. It is intentionally + * generous; the *policy* per-frame / per-response caps live in `ipc.ts` (Phase 4). + */ +export const MAX_FRAME_BYTES = 256 * 1024 * 1024; + +/** Raised by `decodeFrames` when a length prefix exceeds {@link MAX_FRAME_BYTES}. */ +export class FrameTooLargeError extends Error { + constructor(declared: number) { + super(`frame length ${declared} exceeds MAX_FRAME_BYTES ${MAX_FRAME_BYTES}`); + this.name = "FrameTooLargeError"; + } +} + +const encoder = new TextEncoder(); +const decoder = new TextDecoder("utf-8", { fatal: true }); + +/** Encode one frame to a length-prefixed byte buffer. */ +export function encodeFrame(obj: Frame): Uint8Array { + const body = encoder.encode(JSON.stringify(obj)); + const buf = new Uint8Array(HEADER_BYTES + body.byteLength); + new DataView(buf.buffer).setUint32(0, body.byteLength, false); // big-endian + buf.set(body, HEADER_BYTES); + return buf; +} + +/** + * Parse every complete frame at the front of `buf`. Returns the decoded frames + * and the unconsumed tail (a partial frame still being received). The caller + * accumulates the tail and re-invokes as more bytes arrive. + * + * Throws {@link FrameTooLargeError} on an oversized length prefix and a + * `SyntaxError`/`TypeError` on malformed JSON or invalid UTF-8 — both of which + * the Node side treats as an integrity violation (kill the child). + */ +export function decodeFrames(buf: Uint8Array): { frames: Frame[]; rest: Uint8Array } { + const frames: Frame[] = []; + let offset = 0; + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + + while (buf.byteLength - offset >= HEADER_BYTES) { + const len = view.getUint32(offset, false); + if (len > MAX_FRAME_BYTES) throw new FrameTooLargeError(len); + const start = offset + HEADER_BYTES; + const end = start + len; + if (end > buf.byteLength) break; // incomplete frame; wait for more bytes + const json = decoder.decode(buf.subarray(start, end)); + frames.push(JSON.parse(json) as Frame); + offset = end; + } + + const rest = offset === 0 ? buf : buf.subarray(offset); + return { frames, rest }; +} diff --git a/src/pyodide-runner/runner.ts b/src/pyodide-runner/runner.ts new file mode 100644 index 0000000..0902af1 --- /dev/null +++ b/src/pyodide-runner/runner.ts @@ -0,0 +1,318 @@ +// Pyodide runner — the UNTRUSTED side. Deno entry point, EXCLUDED from the tsc +// build (uses Deno globals) and shipped to dist/ as raw .ts (Deno runs it +// directly). Hardens spikes S1 (offline Pyodide-on-Deno) and S2 (IPC integrity) +// into product. +// +// Launched by the Node manager (Phase 4) with the committed deny-belt, e.g.: +// DENO_NO_UPDATE_CHECK=1 deno run --no-prompt --deny-net --deny-run \ +// --deny-write --deny-env --deny-ffi --deny-sys --deny-import --no-remote \ +// --no-npm --cached-only --no-config --allow-read= \ +// runner.ts +// +// argv: [assetDir, generation]. The asset dir is passed as ARGV (never Deno.env, +// which --deny-env blocks). DENO_NO_UPDATE_CHECK lives in the spawn env, read by +// the Deno runtime itself. +// +// SECURITY (spike S2 finding A): realm lockdown is NOT stdout containment — +// (await import("node:fs")).writeSync(1,…) still reaches stdout under the +// deny-belt. The load-bearing control is Node-side frame validation keyed on the +// secret requestId/seq/generation, which this runner NEVER exposes to untrusted +// Python (only code/argv/stdin/files cross into Pyodide; integrity fields stay +// in JS closure). Lockdown is hardening that blocks the easy write primitives. + +import { Buffer } from "node:buffer"; +import { createRequire } from "node:module"; +import { + type Frame, + type FsEntry, + type ReadyFrame, + type RunRequest, + type RunResponse, + decodeFrames, + encodeFrame, +} from "./protocol.ts"; + +// ── Capture host primitives BEFORE lockdown (closure-held, never on globalThis) ─ +// deno-lint-ignore no-explicit-any +const denoRef = (globalThis as any).Deno; +const stdoutWriteSync: (b: Uint8Array) => number = denoRef.stdout.writeSync.bind(denoRef.stdout); +const stdinReadable: ReadableStream = denoRef.stdin.readable; +const denoExit: (code: number) => never = denoRef.exit.bind(denoRef); +const denoArgs: string[] = denoRef.args; + +const assetDir: string | undefined = denoArgs[0]; +const generation = Number(denoArgs[1] ?? "0"); +if (!assetDir) { + stdoutWriteSync(new TextEncoder().encode("RUNNER FATAL: asset dir not provided as argv[0]\n")); + denoExit(2); +} + +// indexURL must be absolute and end with "/"; Pyodide resolves the wasm, +// python_stdlib.zip, wheels and stock lock relative to it. +const indexURL = assetDir.endsWith("/") ? assetDir : `${assetDir}/`; +const assetRoot = indexURL.replace(/\/$/, ""); + +// ── Node-compat globals required for Emscripten/Pyodide under Deno's ESM realm ─ +// Deno populates process.versions.node, so Pyodide takes the Node-fs load path +// (its only offline path — there is no Deno.readFile branch). Emscripten's +// pyodide.asm.js uses bare require/__dirname/__filename, absent in a Deno ESM +// module; provide them as globals so it resolves. These node-builtin requires are +// NOT blocked by --deny-import/--no-npm (those gate remote/npm only). +// deno-lint-ignore no-explicit-any +const g = globalThis as any; +g.require = createRequire(import.meta.url); +g.__dirname = assetRoot; +g.__filename = `${assetRoot}/pyodide.asm.js`; + +function emit(frame: Frame): void { + const bytes = encodeFrame(frame); + let written = 0; + while (written < bytes.byteLength) { + written += stdoutWriteSync(bytes.subarray(written)); // loop: a pipe write may be partial + } +} + +// ── Load Pyodide + packages (spike S1 proven, fully offline) ──────────────── +// deno-lint-ignore no-explicit-any +const pyodideModule = await import(`file://${indexURL}pyodide.mjs`); +const loadPyodide = pyodideModule.loadPyodide; + +const pyodide = await loadPyodide({ + indexURL, + lockFileURL: `${indexURL}pyodide-lock.json`, + // Discard Pyodide's own load-time banner/print; per-run capture is wired below. + stdout: () => {}, + stderr: () => {}, +}); + +// numpy/pandas/scipy ship in the distribution → load by name. +await pyodide.loadPackage(["numpy", "pandas", "scipy"]); +// openpyxl + et_xmlfile are NOT in the distribution; load the vendored pure-python +// wheels by local file:// URL (discovered in the asset dir). loadPackage reads +// them via node:fs under --allow-read — no network. (Phase 0 Discoveries: the +// stock lock has no openpyxl, so loadPackage-by-name would throw; file:// wheels +// are the S1-proven offline path. The supplementary custom lock from +// build-pyodide-lock.mjs is not required by this runner.) +const wheelNames: string[] = []; +for (const entry of denoRef.readDirSync(assetRoot)) { + if (entry.isFile && (/^openpyxl-.*\.whl$/.test(entry.name) || /^et_xmlfile-.*\.whl$/.test(entry.name))) { + wheelNames.push(entry.name); + } +} +// et_xmlfile before openpyxl (dependency order). +wheelNames.sort((a, b) => (a.startsWith("et_xmlfile") ? -1 : b.startsWith("et_xmlfile") ? 1 : 0)); +await pyodide.loadPackage(wheelNames.map((w) => `file://${indexURL}${w}`)); + +const FS = pyodide.FS; + +// ── REALM LOCKDOWN — before any untrusted runPythonAsync (spike S2) ───────── +// Delete every deletable host / Node-compat write primitive. import("node:fs") +// cannot be deleted (it is syntax), so this is hardening, not containment — the +// Node-side validator is the real guarantee (see file header). +delete g.Deno; +delete g.console; +delete g.require; +delete g.__dirname; +delete g.__filename; + +// ── MEMFS helpers ─────────────────────────────────────────────────────────── +function mkdirTree(dir: string): void { + FS.mkdirTree(dir); +} + +interface TreeNode { + path: string; + kind: "file" | "dir"; + mode: number; + bytes?: Uint8Array; // present only for files +} + +/** Walk the subtree under `root` (excluding `root` itself), returning every dir + * and file with its mode (and bytes for files). */ +function walkTree(root: string): TreeNode[] { + const out: TreeNode[] = []; + const walk = (dir: string): void => { + let names: string[]; + try { + names = FS.readdir(dir) as string[]; + } catch { + return; + } + for (const name of names) { + if (name === "." || name === "..") continue; + const full = dir === "/" ? `/${name}` : `${dir}/${name}`; + const { mode } = FS.stat(full); + if (FS.isDir(mode)) { + out.push({ path: full, kind: "dir", mode: mode & 0o777 }); + walk(full); + } else if (FS.isFile(mode)) { + out.push({ path: full, kind: "file", mode: mode & 0o777, bytes: readFileBytes(full) }); + } + } + }; + walk(root); + return out; +} + +function readFileBytes(path: string): Uint8Array { + return FS.readFile(path, { encoding: "binary" }) as Uint8Array; +} + +function depth(path: string): number { + return path.split("/").length; +} + +function sameBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) if (a[i] !== b[i]) return false; + return true; +} + +// ── Run one request ───────────────────────────────────────────────────────── +async function runOne(req: RunRequest): Promise { + const cwd = req.cwd && req.cwd.startsWith("/") ? req.cwd : "/home/pyodide"; + mkdirTree(cwd); + + // Stage the input subtree (dirs + files) into MEMFS. + for (const f of req.files) { + if (f.kind === "dir") { + mkdirTree(f.path); + } else { + const dir = f.path.slice(0, f.path.lastIndexOf("/")) || "/"; + mkdirTree(dir); + FS.writeFile(f.path, new Uint8Array(Buffer.from(f.data, "base64"))); + } + if (typeof f.mode === "number") FS.chmod(f.path, f.mode); + } + + // Snapshot the cwd subtree AFTER staging, BEFORE running user code — this is + // the diff baseline (excludes staging infrastructure dirs, which pre-exist + // from the caller's SqlFs tree). + const baseFiles = new Map(); + const basePaths = new Set(); + for (const node of walkTree(cwd)) { + basePaths.add(node.path); + if (node.kind === "file" && node.bytes) baseFiles.set(node.path, node.bytes); + } + + // Prelude: argv + cwd + stdin, plus redirect sys.stdout/sys.stderr to StringIO + // buffers. We read those buffers' getvalue() after the run — this captures ALL + // Python output regardless of trailing newlines / flushing (Pyodide's batched + // JS stdout only flushes per-line, dropping a final unterminated line). + // argv/stdin/cwd are the user's own inputs — safe to expose; the secret + // integrity fields are NEVER passed into Python. + pyodide.globals.set("__sqlfs_argv", JSON.stringify(req.argv ?? [])); + pyodide.globals.set("__sqlfs_stdin", req.stdin ? Buffer.from(req.stdin, "base64").toString("utf-8") : ""); + pyodide.globals.set("__sqlfs_cwd", cwd); + await pyodide.runPythonAsync(` +import sys, os, io, json as __json +sys.argv = __json.loads(__sqlfs_argv) or [""] +os.chdir(__sqlfs_cwd) +sys.stdin = io.StringIO(__sqlfs_stdin) +__sqlfs_out = io.StringIO() +__sqlfs_err = io.StringIO() +sys.stdout = __sqlfs_out +sys.stderr = __sqlfs_err +`); + + let exitCode = 0; + let jsError = ""; + try { + // User code runs in a FRESH namespace each call (bounds variable scope; + // sys.modules / package globals persist within the session — design D3). + const ns = pyodide.globals.get("dict")(); + try { + await pyodide.runPythonAsync(req.code, { globals: ns }); + } finally { + ns.destroy(); + } + } catch (err) { + exitCode = 1; + jsError = err instanceof Error ? (err.message ?? String(err)) : String(err); + } + + // Read the captured buffers, then restore the real streams. + const outProxy = pyodide.globals.get("__sqlfs_out"); + const errProxy = pyodide.globals.get("__sqlfs_err"); + const capturedOut = (outProxy.getvalue() as string) ?? ""; + let capturedErr = (errProxy.getvalue() as string) ?? ""; + outProxy.destroy(); + errProxy.destroy(); + await pyodide.runPythonAsync("sys.stdout = sys.__stdout__\nsys.stderr = sys.__stderr__"); + if (jsError) capturedErr = capturedErr && !capturedErr.endsWith("\n") ? `${capturedErr}\n${jsError}` : capturedErr + jsError; + + // ── Diff the cwd subtree (dirs + files) against the staged baseline ─────── + const after = walkTree(cwd); + const createdDirs: FsEntry[] = []; + const createdFiles: FsEntry[] = []; + const modified: FsEntry[] = []; + const afterPaths = new Set(); + for (const node of after) { + afterPaths.add(node.path); + if (node.kind === "dir") { + if (!basePaths.has(node.path)) createdDirs.push({ path: node.path, kind: "dir", mode: node.mode, data: "" }); + continue; + } + const bytes = node.bytes ?? new Uint8Array(0); + const before = baseFiles.get(node.path); + const entry: FsEntry = { path: node.path, kind: "file", mode: node.mode, data: Buffer.from(bytes).toString("base64") }; + if (before === undefined) createdFiles.push(entry); + else if (!sameBytes(before, bytes)) modified.push(entry); + } + // dirs-before-files, dirs shallow→deep, so the drain can apply created in order. + createdDirs.sort((a, b) => depth(a.path) - depth(b.path)); + const created: FsEntry[] = [...createdDirs, ...createdFiles]; + // deleted: any baseline path gone, deepest-first (children before parents). + const deleted = [...basePaths].filter((p) => !afterPaths.has(p)).sort((a, b) => depth(b) - depth(a)); + + // Wipe the ENTIRE cwd subtree (files + dirs, deepest-first) so the next exec + // in this warm child starts from a clean cwd — no leftover dirs leak across + // execs. (sys.modules / package globals still persist — design D3.) + for (const node of walkTree(cwd).sort((a, b) => depth(b.path) - depth(a.path))) { + try { + if (node.kind === "file") FS.unlink(node.path); + else FS.rmdir(node.path); + } catch { + /* already gone */ + } + } + + return { + type: exitCode === 0 ? "result" : "error", + requestId: req.requestId, + seq: req.seq, + generation: req.generation, + stdout: Buffer.from(capturedOut, "utf-8").toString("base64"), + stderr: Buffer.from(capturedErr, "utf-8").toString("base64"), + exitCode, + created, + modified, + deleted, + }; +} + +// ── IPC loop ───────────────────────────────────────────────────────────────── +// Announce readiness (one-time handshake; carries generation only). +const ready: ReadyFrame = { type: "ready", generation }; +emit(ready); + +let buf = new Uint8Array(0); +const reader = stdinReadable.getReader(); +for (;;) { + const { value, done } = await reader.read(); + if (done) break; + const merged = new Uint8Array(buf.byteLength + value.byteLength); + merged.set(buf, 0); + merged.set(value, buf.byteLength); + const { frames, rest } = decodeFrames(merged); + buf = rest; + for (const frame of frames) { + if (frame.type === "run") { + const resp = await runOne(frame); + emit(resp); + } + // Non-run inbound frames are ignored — Node only sends `run`. + } +} + +denoExit(0); diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index 9f6739c..ad7374e 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -356,10 +356,26 @@ Vendor the runtime assets and write the Deno-side harness that loads Pyodide off **File**: `src/pyodide-runner/protocol.ts` (new) **Action**: create — runtime-agnostic (`Uint8Array`/`DataView`, no `Buffer`/`Deno`). +> **Note (as implemented in Phase 3 — supersedes the original files-only sketch).** +> FS entries carry a `kind` so the manager/drain (Phase 5) can apply dirs-before-files +> and represent script-created **empty directories**. `created` is ordered +> dirs-before-files (dirs shallow→deep); `deleted` is depth-first (deepest first). +> See Phase 3 Discoveries for why (resolves a plan inconsistency vs the Phase 5 +> "dirs-before-files" / "delete depth-first" drain spec). + ```ts export const PROTOCOL_VERSION = 1; export type FrameType = "run" | "result" | "error" | "ready"; +// A staged-in / drained-out filesystem entry. `data` is base64 file contents for +// `kind:"file"` and "" for `kind:"dir"`. +export interface FsEntry { + readonly path: string; + readonly kind: "file" | "dir"; + readonly mode: number; + readonly data: string; // base64 file contents; "" for dirs +} + export interface RunRequest { readonly type: "run"; readonly requestId: string; // random, set by Node @@ -368,7 +384,7 @@ export interface RunRequest { readonly code: string; // resolved script or -c body readonly argv: readonly string[]; readonly stdin: string; // base64 - readonly files: ReadonlyArray<{ path: string; mode: number; data: string /*base64*/ }>; + readonly files: readonly FsEntry[]; // cwd subtree staged into MEMFS (files + dirs) readonly cwd: string; } @@ -380,9 +396,9 @@ export interface RunResponse { readonly stdout: string; // base64 readonly stderr: string; // base64 readonly exitCode: number; - readonly created: ReadonlyArray<{ path: string; mode: number; data: string /*base64*/ }>; - readonly modified: ReadonlyArray<{ path: string; mode: number; data: string /*base64*/ }>; - readonly deleted: readonly string[]; + readonly created: readonly FsEntry[]; // dirs-before-files (dirs shallow→deep) + readonly modified: readonly FsEntry[]; // changed files + readonly deleted: readonly string[]; // depth-first (deepest first) } // `ready` is a ONE-TIME pre-run handshake (no requestId/seq), validated separately @@ -416,19 +432,43 @@ export function decodeFrames(buf: Uint8Array): { frames: Frame[]; rest: Uint8Arr ### Phase 3: Success Criteria #### Phase 3: Programmatic Verification -- [ ] `node scripts/fetch-pyodide-assets.mjs && node scripts/build-pyodide-lock.mjs` produce `vendor/pyodide/` + the custom lock -- [ ] `pnpm typecheck` passes (protocol.ts compiles; runner.ts excluded) -- [ ] Running the built `runner.ts` under the committed flags with a fixture `run` frame on stdin returns a valid `result` frame whose pandas→openpyxl output bytes decode correctly — **zero network**: - `deno run --allow-read=$PYODIDE_ASSET_DIR dist/pyodide-runner/runner.ts < fixture-frame.bin` -- [ ] `pnpm lint:fix` passes +- [x] `node scripts/fetch-pyodide-assets.mjs && node scripts/build-pyodide-lock.mjs` produce `vendor/pyodide/` + the custom lock — fetch verifies pinned SHA256 of the dist + downloads/verifies both wheels; build-lock wrote `pyodide-lock.custom.json` (381 packages, openpyxl+et_xmlfile added) +- [x] `pnpm typecheck` passes (protocol.ts compiles; runner.ts excluded via tsconfig `exclude`) +- [x] Running the built `runner.ts` under the committed flags with a fixture `run` frame on stdin returns a valid `result` frame whose pandas→openpyxl output bytes decode correctly — **zero network** (ran under `--deny-net`): `out.xlsx` drained 4970 bytes (PK zip), exit 0, ready+result frames, integrity fields echoed. Extended (post-review) to a 2-frame fixture: `created` carries `kind:"dir"` entries (empty dir + nested dir) ordered dirs-before-files, and a second frame confirms the cwd subtree is fully wiped between execs +- [x] `pnpm lint:fix` passes (runner.ts biome-ignored — Deno-realm patterns; see Discoveries) #### Phase 3: Agent Verification -- [ ] Agent re-runs an S2 forge attempt against the built `runner.ts`; confirms the forged frame is **not accepted** — escaped JS may write bytes to stdout (S2 finding A: `import("node:fs").writeSync(1,…)`), but the Node side rejects it, kills the child, and drains nothing -- [ ] Agent confirms the deny-belt blocks remote import, npm import, update check, FS write, env read, subprocess spawn, FFI, and network (each attempt fails closed) -- [ ] Agent reviews `runner.ts` to confirm realm lockdown happens **before** the first untrusted `runPythonAsync` +- [x] Agent re-runs an S2 forge attempt against the built `runner.ts`; confirms the forged frame is **not accepted** — re-ran `s2-ipc.ts` under the committed flags via the vendored Deno (ALL PASS: forged/interleave/replay/stale-generation + ready-handshake violations rejected; finding A reconfirmed). Code review: `runner.ts` passes ONLY `argv`/`stdin`/`cwd` into Python — `requestId`/`seq`/`generation` never cross into Pyodide, so a forger can't produce an accepted frame. (The reject+kill-the-child wiring is Phase 4's `validateInbound`, which the S2 validator models — there is no Node manager yet in Phase 3.) +- [x] Agent confirms the deny-belt blocks remote import, npm import, FS write, env read, subprocess spawn, FFI, sys-info, and network (deny-belt probe under the committed flags: all 8 fail closed). Update-check is suppressed by `DENO_NO_UPDATE_CHECK=1` (spawn env) + `--deny-net`. +- [x] Agent reviews `runner.ts` to confirm realm lockdown happens **before** the first untrusted `runPythonAsync` — `delete g.Deno/console/require/__dirname/__filename` (lines ~111-115) run after the trusted `loadPyodide`/`loadPackage` (79-103) and before the first untrusted `runPythonAsync(req.code)` (line ~200, reached only when a `run` frame arrives in the IPC loop) ### Phase 3: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 3._ + +**Design fork resolved (Phase-0-authorized "do one or the other deliberately").** Two coherent paths existed for openpyxl/et_xmlfile (absent from the stock dist): (A) a custom lock + `loadPackage`-by-name, or (B) the **S1-proven** `file://` wheel load. **runner.ts uses path B** — it's the only end-to-end-proven offline path (lowest risk for the verification gate). `build-pyodide-lock.mjs` still produces `pyodide-lock.custom.json` (the plan's deliverable) but the runner does NOT depend on it. The custom lock is a supplementary manifest / future path to loadPackage-by-name. + +**`build-pyodide-lock.mjs` uses a deterministic offline MERGE, not `micropip.freeze`** (deliberate deviation). Offline `micropip.freeze` is unproven/fragile; the merge (read stock lock → append openpyxl+et_xmlfile entries with sha256 from the vendored wheels → write custom lock) is deterministic, fast, no runtime spawn, and produces an equivalent artifact. Schema mirrored from a stock pure-python entry (`affine`): `{name, version, file_name, install_dir:"site", sha256, package_type:"package", imports, depends, unvendored_tests:false, shared_library:false}`. + +**Checksum pinning is platform-aware.** `fetch-pyodide-assets.mjs` SHA256-pins the platform-INDEPENDENT bytes — `pyodide.mjs`, `pyodide.asm.wasm`, `python_stdlib.zip`, and the two wheels — to the exact S1-validated artifacts (hard-fail on mismatch). The **Deno binary is pinned by version + official dl.deno.land URL only**: its bytes are platform-specific (arm64-darwin locally vs linux-x64 in Docker), so a single cross-arch checksum is impossible. Pinned wheel sha256: openpyxl-3.1.5 `5282c12b…`, et_xmlfile-2.0.0 `7a91720b…`. + +**Pyodide 0.29.4 ships CPython 3.13, not 3.12** (Phase 0 note said 3.12). The dist wheels are tagged `cp313-cp313-pyemscripten_2025_0_wasm32`; lock `info.platform = emscripten_4_0_9`. Immaterial to behaviour — noted for accuracy. + +**Stdout capture must NOT use `pyodide.setStdout({batched})`** — batched only flushes per-line, so a final `print(..., end="")` (no newline) is silently dropped. The runner instead **redirects Python `sys.stdout`/`sys.stderr` to `io.StringIO`** in the prelude and reads `getvalue()` after the run (then restores `sys.__stdout__`/`__stderr__`). Captures everything regardless of newlines/flush. (Found via the fixture test: first attempt drained the xlsx fine but `result.stdout` was empty.) + +**`runner.ts` is excluded from BOTH tsc and biome.** tsc: `tsconfig.json` `exclude` (Deno globals won't compile under the Node config). biome: added `src/pyodide-runner/runner.ts` to `files.ignore` — its Deno-realm patterns (`as any` on Deno globals, the 5 realm-lockdown `delete` statements) trip `noExplicitAny`/`noDelete`, which are wrong for this file (the deletes are one-time security lockdown, not a perf concern). `protocol.ts` stays fully linted+typed (clean, runtime-agnostic). Also added `vendor` to biome `files.ignore` (Phase 1 Discoveries flagged this). + +**Build wiring:** `dist/pyodide-runner/` ends up with `protocol.js` (tsc, Node side) + `protocol.ts` + `runner.ts` (raw, copied by `scripts/copy-pyodide-runner.mjs`, wired into `pnpm build`). `runner.ts` imports `./protocol.ts` (explicit `.ts`) — Deno resolves the raw `.ts`; Node imports `protocol.js`. Both coexist. + +**`.dockerignore` needed two adds** (the plan's Dockerfile change implied them): `vendor` (builder regenerates it fresh; never copy a stale local copy) and `thoughts` (holds ~408 MB of spike scratch assets — would bloat the build context). Dockerfile builder also `apt-get install`s `curl unzip bzip2` (needed by the fetch script). + +**Verification economy:** to avoid a redundant 408 MB Pyodide download, `vendor/` was seeded from the byte-identical S1 spike cache, then the two wheels were deleted to force a REAL download+SHA256-verify of small artifacts. The fetch script's skip-if-present + checksum-verify paths ran against the real pinned bytes; the 408 MB download path is identical curl/tar logic to the proven `s1-pyodide-deno.sh`. In Docker/CI the script does a full fresh download. + +**Gotcha for Phase 4/5:** the runner reads `[assetDir, generation]` from `Deno.args`; emits a `ready` frame (generation only) before reading any `run` frame; per `run` it stages files → fresh-namespace `runPythonAsync` → diffs the cwd subtree → emits exactly one `result`/`error` (echoing `requestId`/`seq`/`generation` from the request, held in JS). Manager spawn (Phase 4) must pass `--allow-read=`, the assetDir + generation as argv, and `DENO_NO_UPDATE_CHECK=1` in the (scrubbed) env. Cold load (loadPyodide + numpy/pandas/scipy + 2 wheels) takes several seconds — informs the exec-timeout default. + +**Post-review protocol extension (Codex review — fixed in Phase 3, resolves a plan internal-inconsistency).** The plan's Phase 3 protocol modelled `created`/`modified` as files-only (`{path, mode, data}`), but the plan's Phase 5 drain says "apply **dirs-before-files**" and "for `deleted`: delete **depth-first**" — which is dir-aware. Two valid review findings followed: (1) the files-only protocol can't represent script-created **empty directories** or the dir ordering Phase 5 demands; (2) the wipe unlinked only files (`baseline ∪ seen`, `seen` excluded dirs), so script-created **dirs leaked** into the next exec in the same warm child. **Fix (cheapest here, not retrofitted in Phase 5):** +- `protocol.ts` now has a shared `FsEntry { path; kind: "file" | "dir"; mode; data }` (`data: ""` for dirs), used by `RunRequest.files` (input staging — can carry empty dirs) and `RunResponse.created`/`modified` (output). `created` is ordered **dirs-before-files (dirs shallow→deep)**; `deleted` is **depth-first (deepest first)** — both drain-ready for Phase 5. +- `runner.ts`: `walkTree(cwd)` now walks dirs + files; staging handles `kind`; the baseline is snapshotted AFTER staging (so staging-infra dirs that pre-exist in the caller's tree aren't mis-reported as created); the diff emits created dirs + files / modified files / deleted; the wipe now **recursively clears the entire cwd subtree depth-first** (files via `unlink`, dirs via `rmdir`) so no dir leaks across execs. (`sys.modules`/package globals still persist — design D3 unchanged.) +- Verified: the extended fixture creates a file + empty dir + nested-dir-with-file → `created` carries `kind:"dir"` entries ordered dirs-before-files; a second frame's `os.listdir(".")` is empty → cwd fully wiped between execs. +- `.tmp` added to `biome.json` `files.ignore` (it's gitignored ephemeral scratch; biome was linting the throwaway verification probes). --- diff --git a/tsconfig.json b/tsconfig.json index df2a801..a2eb6c8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,5 +19,5 @@ "noUnusedParameters": true }, "include": ["src"], - "exclude": ["node_modules", "dist"] + "exclude": ["node_modules", "dist", "src/pyodide-runner/runner.ts"] } From 9e3a8f5e06b1d1afb09931264c5e4e899a94d5f6 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 18:13:49 +0930 Subject: [PATCH 06/16] Phase 4: Node-side PyodideSandbox manager + IPC client --- src/api/pyodide/ipc.ts | 219 +++++++++ src/api/pyodide/manager.ts | 508 +++++++++++++++++++++ src/api/pyodide/tests/unit/fake-child.ts | 148 ++++++ src/api/pyodide/tests/unit/ipc.test.ts | 234 ++++++++++ src/api/pyodide/tests/unit/manager.test.ts | 433 ++++++++++++++++++ thoughts/issue-118-pyodide-runtime/plan.md | 29 +- 6 files changed, 1565 insertions(+), 6 deletions(-) create mode 100644 src/api/pyodide/ipc.ts create mode 100644 src/api/pyodide/manager.ts create mode 100644 src/api/pyodide/tests/unit/fake-child.ts create mode 100644 src/api/pyodide/tests/unit/ipc.test.ts create mode 100644 src/api/pyodide/tests/unit/manager.test.ts diff --git a/src/api/pyodide/ipc.ts b/src/api/pyodide/ipc.ts new file mode 100644 index 0000000..5933029 --- /dev/null +++ b/src/api/pyodide/ipc.ts @@ -0,0 +1,219 @@ +/** + * Node-side IPC framing + integrity for the Pyodide runner. + * + * This is the trusted half of the channel to the untrusted Deno subprocess. It + * mirrors the wire format defined in `../../pyodide-runner/protocol.ts` + * (4-byte big-endian uint32 length prefix + UTF-8 JSON body) using Node + * `Buffer`, and adds the *load-bearing* security control: `validateInbound`. + * + * SECURITY (spike S2 finding A): realm lockdown in the child CANNOT contain raw + * stdout — `(await import("node:fs")).writeSync(1, …)` reaches it under the full + * deny-belt. Untrusted code therefore *can* emit arbitrary bytes on the channel. + * It still cannot produce an *accepted* frame: `requestId`/`seq`/`generation` are + * unguessable secrets the manager assigns and NEVER exposes to the child's Python + * (Phase 3 requirement), and a process cannot read its own stdout to replay a + * real frame. `validateInbound` enforces those invariants; any violation is an + * `IpcIntegrityError` the manager turns into kill-the-child. Treat this file as + * security-critical — the worst an attacker achieves is a corrupt/forged frame → + * kill-the-child (self-DoS), never a drain of forged files. + */ + +import { Buffer } from "node:buffer"; +import type { Frame, RunResponse } from "../../pyodide-runner/protocol.js"; + +const HEADER_BYTES = 4; + +/** + * Default per-frame wire cap (the declared JSON-body byte length). Because file + * payloads are base64 in the body, this naturally measures the ~33%-expanded + * size. Generous default; Phase 6 wires the `PYODIDE_MAX_FRAME_BYTES` env var. + * Stays well below the protocol-level {@link protocol.MAX_FRAME_BYTES} ceiling. + */ +export const PYODIDE_MAX_FRAME_BYTES_DEFAULT = 64 * 1024 * 1024; + +/** + * Default aggregate cap: total bytes the manager will buffer from the child for a + * single response (reset on each accepted `ready`/`result`/`error`). Bounds a + * slowloris-style stream that never forms a complete/valid frame. Must be ≥ the + * per-frame cap. Phase 6 wires the `PYODIDE_MAX_AGGREGATE_BYTES` env var. + */ +export const PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT = 96 * 1024 * 1024; + +/** A framing / integrity violation. The manager turns this into kill-the-child. */ +export class IpcIntegrityError extends Error { + readonly code = "EIPC_INTEGRITY"; + constructor(message: string) { + super(`EIPC_INTEGRITY: ${message}`); + this.name = "IpcIntegrityError"; + } +} + +/** Raised when a declared frame length exceeds the configured per-frame cap. */ +export class IpcFrameTooLargeError extends IpcIntegrityError { + constructor(declared: number, cap: number) { + super(`frame length ${declared} exceeds cap ${cap}`); + this.name = "IpcFrameTooLargeError"; + } +} + +/** Encode one frame to a length-prefixed Node `Buffer`. */ +export function encodeFrame(obj: Frame): Buffer { + const body = Buffer.from(JSON.stringify(obj), "utf8"); + const buf = Buffer.allocUnsafe(HEADER_BYTES + body.byteLength); + buf.writeUInt32BE(body.byteLength, 0); + body.copy(buf, HEADER_BYTES); + return buf; +} + +// Fatal decoder: invalid UTF-8 in a frame body is an integrity violation, not a +// silent U+FFFD replacement (matches the protocol's strictness). +const utf8 = new TextDecoder("utf-8", { fatal: true }); + +/** + * Parse every complete frame at the front of `buf`. Returns the decoded frames + * and the unconsumed tail (a partial frame still arriving). The caller + * accumulates the tail and re-invokes as more bytes arrive. + * + * Throws {@link IpcFrameTooLargeError} on an oversized declared length and + * {@link IpcIntegrityError} on malformed JSON / invalid UTF-8 — both of which the + * manager treats as kill-the-child. Note: frames are returned UNVALIDATED; the + * caller MUST run {@link validateInbound} on each before trusting it. + */ +export function decodeFrames( + buf: Buffer, + maxFrameBytes: number = PYODIDE_MAX_FRAME_BYTES_DEFAULT, +): { frames: Frame[]; rest: Buffer } { + const frames: Frame[] = []; + let offset = 0; + + while (buf.byteLength - offset >= HEADER_BYTES) { + const len = buf.readUInt32BE(offset); + if (len > maxFrameBytes) throw new IpcFrameTooLargeError(len, maxFrameBytes); + const start = offset + HEADER_BYTES; + const end = start + len; + if (end > buf.byteLength) break; // incomplete frame; wait for more bytes + let json: string; + try { + json = utf8.decode(buf.subarray(start, end)); + } catch { + throw new IpcIntegrityError("invalid UTF-8 in frame body"); + } + let parsed: unknown; + try { + parsed = JSON.parse(json); + } catch (e) { + throw new IpcIntegrityError(`malformed frame JSON: ${(e as Error).message}`); + } + frames.push(parsed as Frame); + offset = end; + } + + const rest = offset === 0 ? buf : buf.subarray(offset); + return { frames, rest }; +} + +/** The manager's current expectation, against which an inbound frame is checked. */ +export interface InboundContext { + /** The current (live) child generation. Stale-generation frames are rejected. */ + readonly generation: number; + /** Whether the one-time `ready` handshake has already been accepted. */ + readonly ready: boolean; + /** The in-flight request awaiting its single response, or null if none. */ + readonly pending: { readonly requestId: string; readonly seq: number } | null; +} + +function isObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +/** + * Validate a single drain entry against the {@link protocol.FsEntry} shape. + * `data` is base64 file contents for `kind:"file"` (`""` is a legal empty file) + * and MUST be `""` for `kind:"dir"`. Throws {@link IpcIntegrityError} on any + * mismatch so a protocol-invalid `created`/`modified` entry kills the child + * rather than reaching the Phase 5 drain. + */ +function assertFsEntry(entry: unknown, label: string): void { + if (!isObject(entry)) throw new IpcIntegrityError(`${label}: entry is not an object`); + if (typeof entry.path !== "string" || entry.path.length === 0) { + throw new IpcIntegrityError(`${label}: missing/invalid path`); + } + if (entry.kind !== "file" && entry.kind !== "dir") throw new IpcIntegrityError(`${label}: invalid kind`); + if (typeof entry.mode !== "number" || !Number.isInteger(entry.mode) || entry.mode < 0) { + throw new IpcIntegrityError(`${label}: missing/invalid mode`); + } + if (typeof entry.data !== "string") throw new IpcIntegrityError(`${label}: missing/invalid data`); + if (entry.kind === "dir" && entry.data !== "") + throw new IpcIntegrityError(`${label}: dir entry must have empty data`); +} + +/** + * Schema-validate AND enforce integrity on a single inbound frame. Throws + * {@link IpcIntegrityError} on any violation; on success narrows `frame` to + * {@link Frame} for the caller. + * + * - `ready` (one-time handshake, no requestId/seq): valid EXACTLY once, only with + * the current generation, and only before any response / outside an in-flight + * request. A duplicate `ready`, a post-response `ready`, a `ready` during an + * in-flight request, or a wrong-generation `ready` is a violation. + * - `result` / `error` (the single response to a `run`): must arrive after the + * handshake, while a request is in-flight, and match its `requestId`, `seq`, and + * the current `generation`. A second response (none in-flight) is a violation. + * - `run` / unknown types inbound are always violations (Node never receives them). + */ +export function validateInbound(frame: unknown, ctx: InboundContext): asserts frame is Frame { + if (!isObject(frame)) throw new IpcIntegrityError("frame is not an object"); + const type = frame.type; + + if (type === "ready") { + if (typeof frame.generation !== "number") throw new IpcIntegrityError("ready: missing/invalid generation"); + if (ctx.ready) throw new IpcIntegrityError("ready: duplicate handshake"); + if (ctx.pending !== null) throw new IpcIntegrityError("ready: arrived during an in-flight request"); + if (frame.generation !== ctx.generation) { + throw new IpcIntegrityError(`ready: wrong generation ${frame.generation} (expected ${ctx.generation})`); + } + return; + } + + if (type === "result" || type === "error") { + // Schema: a response carries the full RunResponse shape. + if (typeof frame.requestId !== "string") throw new IpcIntegrityError("response: missing/invalid requestId"); + if (typeof frame.seq !== "number") throw new IpcIntegrityError("response: missing/invalid seq"); + if (typeof frame.generation !== "number") throw new IpcIntegrityError("response: missing/invalid generation"); + if (typeof frame.stdout !== "string" || typeof frame.stderr !== "string") { + throw new IpcIntegrityError("response: missing/invalid stdout/stderr"); + } + if (typeof frame.exitCode !== "number") throw new IpcIntegrityError("response: missing/invalid exitCode"); + if (!Array.isArray(frame.created) || !Array.isArray(frame.modified) || !Array.isArray(frame.deleted)) { + throw new IpcIntegrityError("response: missing/invalid created/modified/deleted"); + } + // Element schema: created/modified are FsEntry[]; deleted is string[]. A + // malformed element is a protocol violation (kill-the-child), not something + // to defer to the Phase 5 drain. + for (const e of frame.created) assertFsEntry(e, "response.created"); + for (const e of frame.modified) assertFsEntry(e, "response.modified"); + for (const p of frame.deleted) { + if (typeof p !== "string" || p.length === 0) { + throw new IpcIntegrityError("response: deleted contains a missing/invalid path"); + } + } + // Integrity: ordering + secret match. + if (!ctx.ready) throw new IpcIntegrityError("response: arrived before the ready handshake"); + if (ctx.pending === null) throw new IpcIntegrityError("response: no in-flight request"); + if (frame.requestId !== ctx.pending.requestId) { + throw new IpcIntegrityError("response: requestId mismatch"); + } + if (frame.seq !== ctx.pending.seq) throw new IpcIntegrityError("response: seq mismatch / out of sequence"); + if (frame.generation !== ctx.generation) { + throw new IpcIntegrityError(`response: wrong/stale generation ${frame.generation} (expected ${ctx.generation})`); + } + return; + } + + throw new IpcIntegrityError(`unexpected inbound frame type: ${String(type)}`); +} + +/** Narrow an already-validated frame to a {@link RunResponse}. */ +export function asRunResponse(frame: Frame): RunResponse { + return frame as RunResponse; +} diff --git a/src/api/pyodide/manager.ts b/src/api/pyodide/manager.ts new file mode 100644 index 0000000..0297018 --- /dev/null +++ b/src/api/pyodide/manager.ts @@ -0,0 +1,508 @@ +/** + * Node-side `PyodideSandbox` — the TRUSTED half of the Pyodide runtime. + * + * Owns one Deno subprocess (the untrusted `runner.ts`), frames the IPC with full + * integrity validation, serializes `run()` calls, and enforces throw-not-return + * cancellation + lazy respawn. No session wiring yet (Phase 5) — unit-testable in + * isolation against a fake child via the injectable `spawnFn`. + * + * Spawn posture (design + Phase 3 Discoveries): the child is launched with the + * committed deny-belt verbatim. Node resolves the asset dir from its own env + * BEFORE spawn and passes it LITERALLY in both `--allow-read=` and as a + * runner argv — never via the child env, which is scrubbed to ONLY + * `{ DENO_NO_UPDATE_CHECK: "1" }` (no AUTH_SECRET/DATABASE_URL; the child does not + * inherit the parent env). `spawn` uses no shell, so `$VAR` is never expanded. + * + * Cancellation is state-dependent and NEVER kills an innocent active request: + * - abort while still queued (before this call acquires the mutex): remove only + * this waiter and reject it with AbortError; the child is NOT killed and any + * concurrently-active run() is unaffected. + * - abort after acquiring the mutex (this call now owns the child — during + * init/preload or mid-run) OR an internal runtime timeout: SIGKILL the child + * and retire the generation; reject/throw (never return a normal result). + * - unexpected child exit or any IpcIntegrityError: mark dead, reject the + * in-flight run(), and respawn lazily with an incremented generation on the + * next run(). + */ + +import { Buffer } from "node:buffer"; +import { type ChildProcess, type SpawnOptions, spawn as nodeSpawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import type { Frame, RunRequest, RunResponse } from "../../pyodide-runner/protocol.js"; +import { + type InboundContext, + IpcIntegrityError, + PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT, + PYODIDE_MAX_FRAME_BYTES_DEFAULT, + asRunResponse, + decodeFrames, + encodeFrame, + validateInbound, +} from "./ipc.js"; + +/** Worker lifecycle state. */ +export type WorkerState = "cold" | "starting" | "idle" | "busy" | "terminating" | "dead"; + +/** + * A `run()` payload WITHOUT the integrity fields. The manager assigns + * `requestId` / `seq` / `generation` itself — they are unguessable secrets it + * must never accept from a caller (and never expose to the child's Python). + */ +export type RunRequestInput = Omit; + +/** + * The committed deny-belt — MUST match `runner.ts`'s documented flags and the + * Phase 3 verification harness exactly. `--allow-read=` is appended at + * spawn time (the only granted capability). + */ +export const COMMITTED_FLAGS: readonly string[] = [ + "--no-prompt", + "--deny-net", + "--deny-run", + "--deny-write", + "--deny-env", + "--deny-ffi", + "--deny-sys", + "--deny-import", + "--no-remote", + "--no-npm", + "--cached-only", + "--no-config", +]; + +/** Default cap on a single owned run (init/preload + execution). */ +export const PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT = 60_000; + +/** Injectable spawn signature (defaults to `child_process.spawn`). */ +export type SpawnFn = (command: string, args: readonly string[], options: SpawnOptions) => ChildProcess; + +export interface PyodideSandboxOptions { + /** Absolute asset dir (resolved by Node before spawn). Falls back to PYODIDE_ASSET_DIR. */ + readonly assetDir?: string; + /** Deno binary path. Falls back to DENO_BIN_PATH, then "deno". */ + readonly denoBin?: string; + /** Path to the runner entry. Defaults to the vendored `dist/pyodide-runner/runner.ts`. */ + readonly runnerPath?: string; + /** Cap on a single owned run (ms). Default {@link PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT}. */ + readonly runtimeTimeoutMs?: number; + /** Per-frame wire cap (bytes). Default {@link PYODIDE_MAX_FRAME_BYTES_DEFAULT}. */ + readonly maxFrameBytes?: number; + /** Aggregate per-response wire cap (bytes). Default {@link PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT}. */ + readonly maxAggregateBytes?: number; + /** Injected spawn (tests). Defaults to `child_process.spawn`. */ + readonly spawnFn?: SpawnFn; + /** Injected requestId generator (tests). Defaults to `crypto.randomUUID`. */ + readonly randomRequestId?: () => string; +} + +/** Thrown when an owned run exceeds {@link PyodideSandboxOptions.runtimeTimeoutMs}. */ +export class PyodideTimeoutError extends Error { + readonly code = "EPYODIDE_TIMEOUT"; + constructor(ms: number) { + super(`EPYODIDE_TIMEOUT: pyodide run exceeded ${ms}ms`); + this.name = "PyodideTimeoutError"; + } +} + +/** Thrown when the child exits unexpectedly while a run is in flight. */ +export class PyodideChildExitError extends Error { + readonly code = "EPYODIDE_CHILD_EXIT"; + constructor( + readonly exitCode: number | null, + readonly exitSignal: NodeJS.Signals | null, + ) { + super(`EPYODIDE_CHILD_EXIT: pyodide child exited (code=${exitCode}, signal=${exitSignal})`); + this.name = "PyodideChildExitError"; + } +} + +/** Thrown by an in-flight run() when the manager is disposed. */ +export class PyodideDisposedError extends Error { + readonly code = "EPYODIDE_DISPOSED"; + constructor() { + super("EPYODIDE_DISPOSED: pyodide sandbox disposed"); + this.name = "PyodideDisposedError"; + } +} + +function makeAbortError(): Error { + return Object.assign(new Error("ABORTED"), { code: "ABORTED", name: "AbortError" }); +} + +/** A waiter parked in the serialization queue (mirrors session-manager's pattern). */ +interface QueueWaiter { + resolve: () => void; + reject: (err: Error) => void; + readonly signal: AbortSignal | undefined; + onAbort: (() => void) | undefined; + settled: boolean; +} + +/** The single owned operation currently holding the child (init + run). */ +interface OwnedOp { + readonly input: RunRequestInput; + readonly resolve: (r: RunResponse) => void; + readonly reject: (e: Error) => void; + readonly signal: AbortSignal; + readonly onAbort: () => void; + timer: ReturnType | undefined; + stage: "ready" | "response"; + requestId: string; + seq: number; + done: boolean; +} + +const DEFAULT_RUNNER_PATH = fileURLToPath(new URL("../../pyodide-runner/runner.ts", import.meta.url)); + +export class PyodideSandbox { + #state: WorkerState = "cold"; + #generation = 0; + #child: ChildProcess | null = null; + #readBuf: Buffer = Buffer.alloc(0); + #aggregateBytes = 0; + #readyReceived = false; + #seqCounter = 0; + #pending: { requestId: string; seq: number } | null = null; + #current: OwnedOp | null = null; + #disposed = false; + + // Serialization lock (one run owns the child at a time). + #locked = false; + readonly #queue: QueueWaiter[] = []; + + readonly #assetDir: string; + readonly #denoBin: string; + readonly #runnerPath: string; + readonly #runtimeTimeoutMs: number; + readonly #maxFrameBytes: number; + readonly #maxAggregateBytes: number; + readonly #spawnFn: SpawnFn; + readonly #randomRequestId: () => string; + + constructor(opts: PyodideSandboxOptions = {}) { + this.#assetDir = opts.assetDir ?? process.env.PYODIDE_ASSET_DIR ?? ""; + this.#denoBin = opts.denoBin ?? process.env.DENO_BIN_PATH ?? "deno"; + this.#runnerPath = opts.runnerPath ?? DEFAULT_RUNNER_PATH; + this.#runtimeTimeoutMs = opts.runtimeTimeoutMs ?? PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT; + this.#maxFrameBytes = opts.maxFrameBytes ?? PYODIDE_MAX_FRAME_BYTES_DEFAULT; + this.#maxAggregateBytes = opts.maxAggregateBytes ?? PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT; + this.#spawnFn = opts.spawnFn ?? (nodeSpawn as SpawnFn); + this.#randomRequestId = opts.randomRequestId ?? randomUUID; + } + + get state(): WorkerState { + return this.#state; + } + + /** The current (live) child generation; 0 before the first spawn. */ + get generation(): number { + return this.#generation; + } + + /** + * Run untrusted Python on the owned child, serialized behind any prior run(). + * Resolves with the `result`/`error` response (the caller inspects `exitCode`). + * Rejects with AbortError on cancellation, {@link PyodideTimeoutError} on an + * internal timeout, or {@link IpcIntegrityError}/{@link PyodideChildExitError} + * on a channel/child failure. Never returns a normal result for a + * timed-out/aborted run. + */ + async run(input: RunRequestInput, signal: AbortSignal): Promise { + if (this.#disposed) throw new PyodideDisposedError(); + // Abort while still queued → reject only this waiter; do NOT kill the child. + await this.#acquire(signal); + try { + if (this.#disposed) throw new PyodideDisposedError(); + return await this.#executeOwned(input, signal); + } finally { + this.#release(); + } + } + + /** Permanently terminate the child and reject any in-flight run(). Terminal. */ + async dispose(): Promise { + this.#disposed = true; + this.#state = "terminating"; + this.#failOwned(new PyodideDisposedError(), /* respawnable */ false); + // Drain queued waiters so parked run() callers reject rather than hang. + while (this.#queue.length > 0) { + const w = this.#queue.shift(); + if (w && !w.settled) w.reject(new PyodideDisposedError()); + } + this.#killChild(); + this.#state = "dead"; + await Promise.resolve(); + } + + // ── Serialization queue (abort-while-queued = remove waiter, no kill) ───────── + + #acquire(signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(makeAbortError()); + if (!this.#locked) { + this.#locked = true; + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const waiter: QueueWaiter = { + resolve: () => { + if (waiter.settled) return; + waiter.settled = true; + if (waiter.onAbort && waiter.signal) waiter.signal.removeEventListener("abort", waiter.onAbort); + resolve(); + }, + reject: (err: Error) => { + if (waiter.settled) return; + waiter.settled = true; + if (waiter.onAbort && waiter.signal) waiter.signal.removeEventListener("abort", waiter.onAbort); + const idx = this.#queue.indexOf(waiter); + if (idx >= 0) this.#queue.splice(idx, 1); + reject(err); + }, + signal, + onAbort: undefined, + settled: false, + }; + const onAbort = (): void => waiter.reject(makeAbortError()); + waiter.onAbort = onAbort; + signal.addEventListener("abort", onAbort, { once: true }); + this.#queue.push(waiter); + }); + } + + #release(): void { + while (this.#queue.length > 0) { + const next = this.#queue.shift(); + if (next === undefined) break; + if (next.settled) continue; + next.resolve(); + return; + } + this.#locked = false; + } + + // ── Owned section: this call holds the child (init/preload + run) ──────────── + + #executeOwned(input: RunRequestInput, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const op: OwnedOp = { + input, + resolve, + reject, + signal, + onAbort: () => this.#failOwned(makeAbortError(), true), + timer: undefined, + stage: "ready", + requestId: "", + seq: -1, + done: false, + }; + this.#current = op; + + if (signal.aborted) { + this.#failOwned(makeAbortError(), true); + return; + } + signal.addEventListener("abort", op.onAbort, { once: true }); + op.timer = setTimeout( + () => this.#failOwned(new PyodideTimeoutError(this.#runtimeTimeoutMs), true), + this.#runtimeTimeoutMs, + ); + + // Warm reuse if the child is alive + handshaked; otherwise (re)spawn and + // await `ready` (its dispatch will send the run frame). + if (this.#state === "idle" && this.#readyReceived && this.#child !== null) { + this.#sendRunFrame(op); + return; + } + try { + this.#spawnChild(); + } catch (err) { + this.#failOwned(err instanceof Error ? err : new Error(String(err)), true); + } + }); + } + + #sendRunFrame(op: OwnedOp): void { + if (op.done) return; + const frame: RunRequest = { + type: "run", + requestId: this.#randomRequestId(), + seq: ++this.#seqCounter, + generation: this.#generation, + code: op.input.code, + argv: op.input.argv, + stdin: op.input.stdin, + files: op.input.files, + cwd: op.input.cwd, + }; + op.requestId = frame.requestId; + op.seq = frame.seq; + op.stage = "response"; + this.#pending = { requestId: frame.requestId, seq: frame.seq }; + this.#state = "busy"; + const child = this.#child; + if (child?.stdin === null || child?.stdin === undefined) { + this.#failOwned(new PyodideChildExitError(null, null), true); + return; + } + try { + child.stdin.write(encodeFrame(frame)); + } catch (err) { + this.#failOwned(err instanceof Error ? err : new Error(String(err)), true); + } + } + + /** + * Fail the in-flight owned run (if any), SIGKILL the child, and retire its + * generation. `respawnable` distinguishes a recoverable kill (next run() + * respawns) from dispose (terminal). Safe to call with no in-flight op (a + * forged frame while idle still kills the child). + */ + #failOwned(err: Error, respawnable: boolean): void { + const op = this.#current; + this.#killChild(); + if (op && !op.done) { + op.done = true; + if (op.timer !== undefined) clearTimeout(op.timer); + op.signal.removeEventListener("abort", op.onAbort); + this.#current = null; + op.reject(err); + } + if (!respawnable) this.#state = "dead"; + } + + // ── Child lifecycle ────────────────────────────────────────────────────────── + + #spawnChild(): void { + this.#generation += 1; + const gen = this.#generation; + const args = [ + "run", + ...COMMITTED_FLAGS, + `--allow-read=${this.#assetDir}`, + this.#runnerPath, + this.#assetDir, + String(gen), + ]; + const child = this.#spawnFn(this.#denoBin, args, { + // Scrubbed env: NO parent env inheritance, NO secrets — only the + // update-check suppressor the Deno runtime reads itself. + env: { DENO_NO_UPDATE_CHECK: "1" }, + stdio: ["pipe", "pipe", "pipe"], + }); + this.#child = child; + this.#readyReceived = false; + this.#pending = null; + this.#readBuf = Buffer.alloc(0); + this.#aggregateBytes = 0; + this.#state = "starting"; + + // Bind handlers to THIS child so late events from a retired generation are + // ignored after respawn. + child.stdout?.on("data", (chunk: Buffer) => { + if (this.#child !== child) return; + this.#onStdoutData(chunk); + }); + child.on("exit", (code: number | null, sig: NodeJS.Signals | null) => { + if (this.#child !== child) return; + this.#onChildExit(code, sig); + }); + child.on("error", (err: Error) => { + if (this.#child !== child) return; + this.#failOwned(err, true); + }); + } + + #killChild(): void { + const child = this.#child; + this.#child = null; + this.#readyReceived = false; + this.#pending = null; + this.#readBuf = Buffer.alloc(0); + this.#aggregateBytes = 0; + if (this.#state !== "terminating") this.#state = "dead"; + if (child) { + try { + child.kill("SIGKILL"); + } catch { + /* already gone */ + } + } + } + + #onChildExit(code: number | null, sig: NodeJS.Signals | null): void { + // A kill we initiated already set state to dead/terminating. + if (this.#state === "dead" || this.#state === "terminating") return; + this.#failOwned(new PyodideChildExitError(code, sig), true); + } + + // ── Inbound frame processing (the load-bearing security path) ───────────────── + + #isTerminal(): boolean { + return this.#state === "dead" || this.#state === "terminating"; + } + + #onStdoutData(chunk: Buffer): void { + if (this.#isTerminal()) return; + this.#aggregateBytes += chunk.byteLength; + if (this.#aggregateBytes > this.#maxAggregateBytes) { + this.#failOwned(new IpcIntegrityError("aggregate response bytes exceeded cap"), true); + return; + } + this.#readBuf = this.#readBuf.byteLength === 0 ? Buffer.from(chunk) : Buffer.concat([this.#readBuf, chunk]); + + let decoded: { frames: ReturnType["frames"]; rest: Buffer }; + try { + decoded = decodeFrames(this.#readBuf, this.#maxFrameBytes); + } catch (err) { + this.#failOwned(err instanceof Error ? err : new Error(String(err)), true); + return; + } + this.#readBuf = decoded.rest; + + for (const frame of decoded.frames) { + const ctx: InboundContext = { + generation: this.#generation, + ready: this.#readyReceived, + pending: this.#pending ? { requestId: this.#pending.requestId, seq: this.#pending.seq } : null, + }; + try { + validateInbound(frame, ctx); + } catch (err) { + this.#failOwned(err instanceof Error ? err : new Error(String(err)), true); + return; + } + this.#dispatchFrame(frame); + if (this.#isTerminal()) return; + } + } + + #dispatchFrame(frame: Frame): void { + // Reset the aggregate window on each accepted complete frame. + this.#aggregateBytes = this.#readBuf.byteLength; + + if (frame.type === "ready") { + this.#readyReceived = true; + const op = this.#current; + if (op && !op.done && op.stage === "ready") { + this.#sendRunFrame(op); // transitions to busy + } else { + this.#state = "idle"; + } + return; + } + + // result | error — validated as the single response to #pending / #current. + const op = this.#current; + this.#pending = null; + this.#state = "idle"; + if (op && !op.done) { + op.done = true; + if (op.timer !== undefined) clearTimeout(op.timer); + op.signal.removeEventListener("abort", op.onAbort); + this.#current = null; + op.resolve(asRunResponse(frame)); + } + } +} diff --git a/src/api/pyodide/tests/unit/fake-child.ts b/src/api/pyodide/tests/unit/fake-child.ts new file mode 100644 index 0000000..99b6524 --- /dev/null +++ b/src/api/pyodide/tests/unit/fake-child.ts @@ -0,0 +1,148 @@ +/** + * Fake Deno child for PyodideSandbox unit tests — no real Deno/Pyodide needed. + * + * Models the runner's IPC surface: it reads the manager's length-prefixed `run` + * frames off `stdin` and lets the test drive `ready`/`result`/`error`/raw bytes + * onto `stdout`. The fake's `generation` is taken from the last spawn argv + * (exactly as the real runner reads it), so it echoes the generation it was + * launched with unless a test deliberately forges otherwise. + */ + +import { Buffer } from "node:buffer"; +import type { ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; +import type { Frame, RunRequest, RunResponse } from "../../../../pyodide-runner/protocol.js"; +import { decodeFrames, encodeFrame } from "../../ipc.js"; +import type { SpawnFn } from "../../manager.js"; + +/** Minimal stream: `write` delivers `data` on a microtask (deterministic in tests). */ +class FakeStream extends EventEmitter { + write(chunk: Buffer | string): boolean { + const buf = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk); + queueMicrotask(() => this.emit("data", buf)); + return true; + } +} + +export class FakeChild extends EventEmitter { + readonly stdin = new FakeStream(); + readonly stdout = new FakeStream(); + readonly stderr = new FakeStream(); + readonly pid = 4242; + readonly generation: number; + readonly runs: RunRequest[] = []; + + killed = false; + killSignal: string | undefined; + exited = false; + + readonly #pendingRuns: RunRequest[] = []; + readonly #runWaiters: ((r: RunRequest) => void)[] = []; + + constructor(generation: number) { + super(); + this.generation = generation; + this.stdin.on("data", (buf: Buffer) => { + for (const frame of decodeFrames(buf).frames) { + if (frame.type !== "run") continue; + const run = frame as RunRequest; + this.runs.push(run); + const waiter = this.#runWaiters.shift(); + if (waiter) waiter(run); + else this.#pendingRuns.push(run); + } + }); + } + + /** Await the next `run` frame the manager writes to stdin. */ + nextRun(): Promise { + const buffered = this.#pendingRuns.shift(); + if (buffered) return Promise.resolve(buffered); + return new Promise((resolve) => this.#runWaiters.push(resolve)); + } + + sendFrame(frame: Frame): void { + this.stdout.write(encodeFrame(frame)); + } + + /** Emit a raw (possibly malformed) length-prefixed body — for integrity tests. */ + sendRaw(body: Buffer): void { + const buf = Buffer.allocUnsafe(4 + body.byteLength); + buf.writeUInt32BE(body.byteLength, 0); + body.copy(buf, 4); + this.stdout.write(buf); + } + + sendReady(generation: number = this.generation): void { + this.sendFrame({ type: "ready", generation }); + } + + sendResult(run: RunRequest, over: Partial = {}): void { + this.sendFrame({ + type: "result", + requestId: run.requestId, + seq: run.seq, + generation: this.generation, + stdout: "", + stderr: "", + exitCode: 0, + created: [], + modified: [], + deleted: [], + ...over, + }); + } + + /** Simulate an unexpected process exit (crash / OOM-kill). */ + exit(code: number | null = 0, signal: NodeJS.Signals | null = null): void { + if (this.exited) return; + this.exited = true; + this.emit("exit", code, signal); + } + + kill(signal?: NodeJS.Signals | number): boolean { + if (this.killed) return false; + this.killed = true; + this.killSignal = typeof signal === "string" ? signal : "SIGKILL"; + queueMicrotask(() => this.exit(null, (typeof signal === "string" ? signal : "SIGKILL") as NodeJS.Signals)); + return true; + } +} + +export interface Harness { + readonly spawnFn: SpawnFn; + /** Await the next spawned child (resolves in spawn order). */ + nextChild(): Promise; + /** Every child spawned so far, in order. */ + readonly children: FakeChild[]; +} + +export function makeHarness(): Harness { + const children: FakeChild[] = []; + const ready: FakeChild[] = []; + const waiters: ((c: FakeChild) => void)[] = []; + + const spawnFn: SpawnFn = (_cmd, args) => { + // The runner reads its generation from the final argv — the fake does too. + const generation = Number(args[args.length - 1]); + const child = new FakeChild(generation); + children.push(child); + const waiter = waiters.shift(); + if (waiter) waiter(child); + else ready.push(child); + return child as unknown as ChildProcess; + }; + + const nextChild = (): Promise => { + const buffered = ready.shift(); + if (buffered) return Promise.resolve(buffered); + return new Promise((resolve) => waiters.push(resolve)); + }; + + return { spawnFn, nextChild, children }; +} + +/** Flush pending microtasks so queued stream `data` events deliver. */ +export function flush(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} diff --git a/src/api/pyodide/tests/unit/ipc.test.ts b/src/api/pyodide/tests/unit/ipc.test.ts new file mode 100644 index 0000000..622fb15 --- /dev/null +++ b/src/api/pyodide/tests/unit/ipc.test.ts @@ -0,0 +1,234 @@ +/** + * Unit tests for the Node-side IPC framing + integrity (`src/api/pyodide/ipc.ts`). + * + * Each integrity assertion names the spike-S2 invariant it protects: a forged, + * interleaved, replayed, stale-generation, or bad-handshake frame must never be + * *accepted* — `validateInbound` is the load-bearing control (finding A). + */ + +import { Buffer } from "node:buffer"; +import { describe, expect, it } from "vitest"; +import type { Frame, ReadyFrame, RunResponse } from "../../../../pyodide-runner/protocol.js"; +import { + type InboundContext, + IpcFrameTooLargeError, + IpcIntegrityError, + decodeFrames, + encodeFrame, + validateInbound, +} from "../../ipc.js"; + +function readyFrame(generation: number): ReadyFrame { + return { type: "ready", generation }; +} + +function responseFrame(over: Partial = {}): RunResponse { + return { + type: "result", + requestId: "req-1", + seq: 1, + generation: 1, + stdout: "", + stderr: "", + exitCode: 0, + created: [], + modified: [], + deleted: [], + ...over, + }; +} + +const READY_CTX: InboundContext = { generation: 1, ready: false, pending: null }; +const RESPONSE_CTX: InboundContext = { generation: 1, ready: true, pending: { requestId: "req-1", seq: 1 } }; + +describe("encodeFrame / decodeFrames", () => { + it("round-trips a single frame", () => { + const frame = responseFrame({ stdout: Buffer.from("hello", "utf8").toString("base64") }); + const { frames, rest } = decodeFrames(encodeFrame(frame)); + expect(frames).toHaveLength(1); + expect(frames[0]).toEqual(frame); + expect(rest.byteLength).toBe(0); + }); + + it("decodes multiple concatenated frames in order", () => { + const a = readyFrame(1); + const b = responseFrame({ seq: 1 }); + const buf = Buffer.concat([encodeFrame(a), encodeFrame(b)]); + const { frames, rest } = decodeFrames(buf); + expect(frames.map((f) => f.type)).toEqual(["ready", "result"]); + expect(rest.byteLength).toBe(0); + }); + + it("returns the unconsumed tail when the trailing frame is incomplete", () => { + const full = encodeFrame(responseFrame()); + const partial = full.subarray(0, full.byteLength - 3); // drop last 3 bytes + const { frames, rest } = decodeFrames(partial); + expect(frames).toHaveLength(0); + expect(rest.byteLength).toBe(partial.byteLength); + }); + + it("re-assembles a frame split across two decode passes", () => { + const full = encodeFrame(responseFrame({ stdout: "QUJD" })); + const head = full.subarray(0, 5); + const tail = full.subarray(5); + const first = decodeFrames(head); + expect(first.frames).toHaveLength(0); + const second = decodeFrames(Buffer.concat([first.rest, tail])); + expect(second.frames).toHaveLength(1); + expect(second.frames[0]).toEqual(responseFrame({ stdout: "QUJD" })); + }); + + it("throws IpcFrameTooLargeError when the declared length exceeds the cap", () => { + const big = encodeFrame(responseFrame({ stdout: "A".repeat(4096) })); + expect(() => decodeFrames(big, 1024)).toThrow(IpcFrameTooLargeError); + }); + + it("counts base64-expanded wire size against the per-frame cap (S2 size guard)", () => { + // 1 KiB of raw bytes → ~1368 base64 chars in the JSON body. A cap set + // between the raw size and the expanded size must reject it. + const rawBytes = Buffer.alloc(1024, 0x41); + const frame = responseFrame({ stdout: rawBytes.toString("base64") }); + const wire = encodeFrame(frame); + expect(wire.byteLength).toBeGreaterThan(1024 + 4); // base64 expansion is real + expect(() => decodeFrames(wire, 1100)).toThrow(IpcFrameTooLargeError); + }); + + it("throws IpcIntegrityError on malformed JSON", () => { + const body = Buffer.from("{ not json", "utf8"); + const buf = Buffer.allocUnsafe(4 + body.byteLength); + buf.writeUInt32BE(body.byteLength, 0); + body.copy(buf, 4); + expect(() => decodeFrames(buf)).toThrow(IpcIntegrityError); + }); + + it("throws IpcIntegrityError on invalid UTF-8 in the body", () => { + const body = Buffer.from([0xff, 0xfe, 0xfd]); // invalid UTF-8 + const buf = Buffer.allocUnsafe(4 + body.byteLength); + buf.writeUInt32BE(body.byteLength, 0); + body.copy(buf, 4); + expect(() => decodeFrames(buf)).toThrow(IpcIntegrityError); + }); +}); + +describe("validateInbound — ready handshake", () => { + it("accepts a current-generation ready before any request", () => { + expect(() => validateInbound(readyFrame(1), READY_CTX)).not.toThrow(); + }); + + it("rejects a duplicate ready (already handshaked)", () => { + expect(() => validateInbound(readyFrame(1), { ...READY_CTX, ready: true })).toThrow(IpcIntegrityError); + }); + + it("rejects a wrong-generation ready", () => { + expect(() => validateInbound(readyFrame(2), READY_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a ready arriving during an in-flight request", () => { + expect(() => validateInbound(readyFrame(1), { ...READY_CTX, pending: { requestId: "req-1", seq: 1 } })).toThrow( + IpcIntegrityError, + ); + }); + + it("rejects a ready missing the generation field", () => { + expect(() => validateInbound({ type: "ready" }, READY_CTX)).toThrow(IpcIntegrityError); + }); +}); + +describe("validateInbound — response", () => { + it("accepts a matching result/error response", () => { + expect(() => validateInbound(responseFrame(), RESPONSE_CTX)).not.toThrow(); + expect(() => validateInbound(responseFrame({ type: "error", exitCode: 1 }), RESPONSE_CTX)).not.toThrow(); + }); + + it("rejects a response before the ready handshake", () => { + expect(() => validateInbound(responseFrame(), { ...RESPONSE_CTX, ready: false })).toThrow(IpcIntegrityError); + }); + + it("rejects a response with no in-flight request (duplicate / replay)", () => { + expect(() => validateInbound(responseFrame(), { ...RESPONSE_CTX, pending: null })).toThrow(IpcIntegrityError); + }); + + it("rejects a forged requestId", () => { + expect(() => validateInbound(responseFrame({ requestId: "forged" }), RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects an out-of-sequence seq", () => { + expect(() => validateInbound(responseFrame({ seq: 99 }), RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a stale/wrong generation", () => { + expect(() => validateInbound(responseFrame({ generation: 0 }), RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a response missing schema fields", () => { + expect(() => validateInbound({ type: "result", requestId: "req-1", seq: 1, generation: 1 }, RESPONSE_CTX)).toThrow( + IpcIntegrityError, + ); + }); +}); + +describe("validateInbound — drain entry schema (FsEntry[] / string[])", () => { + it("accepts well-formed created/modified/deleted (file, empty file, empty dir)", () => { + const frame = responseFrame({ + created: [ + { path: "/cwd/sub", kind: "dir", mode: 0o755, data: "" }, + { path: "/cwd/sub/out.xlsx", kind: "file", mode: 0o644, data: "UEsD" }, + { path: "/cwd/empty.txt", kind: "file", mode: 0o644, data: "" }, // empty file is legal + ], + modified: [{ path: "/cwd/in.csv", kind: "file", mode: 0o644, data: "YQ==" }], + deleted: ["/cwd/gone.tmp"], + }); + expect(() => validateInbound(frame, RESPONSE_CTX)).not.toThrow(); + }); + + it("rejects a created entry missing path", () => { + const frame = responseFrame({ created: [{ kind: "file", mode: 0o644, data: "" }] as never }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a created entry with an invalid kind", () => { + const frame = responseFrame({ created: [{ path: "/x", kind: "symlink", mode: 0o644, data: "" }] as never }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a dir entry carrying non-empty data", () => { + const frame = responseFrame({ created: [{ path: "/d", kind: "dir", mode: 0o755, data: "QQ==" }] }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a modified entry with a non-numeric mode", () => { + const frame = responseFrame({ modified: [{ path: "/x", kind: "file", mode: "755", data: "" }] as never }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a deleted list containing a non-string path", () => { + const frame = responseFrame({ deleted: [42] as never }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); +}); + +describe("validateInbound — direction / shape", () => { + it("rejects an inbound run frame (Node never receives run)", () => { + const run: Frame = { + type: "run", + requestId: "x", + seq: 1, + generation: 1, + code: "", + argv: [], + stdin: "", + files: [], + cwd: "/", + }; + expect(() => validateInbound(run, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects an unknown frame type", () => { + expect(() => validateInbound({ type: "bogus" }, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a non-object frame", () => { + expect(() => validateInbound(null, RESPONSE_CTX)).toThrow(IpcIntegrityError); + expect(() => validateInbound([1, 2, 3], RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); +}); diff --git a/src/api/pyodide/tests/unit/manager.test.ts b/src/api/pyodide/tests/unit/manager.test.ts new file mode 100644 index 0000000..319f237 --- /dev/null +++ b/src/api/pyodide/tests/unit/manager.test.ts @@ -0,0 +1,433 @@ +/** + * Unit tests for the Node-side PyodideSandbox manager (`src/api/pyodide/manager.ts`). + * + * Driven entirely by a fake child (no real Deno/Pyodide). Each test names the + * design decision it protects. The integrity cases assert the spike-S2 invariant: + * any forged / interleaved / replayed / stale-generation / bad-handshake frame + * kills the child — it is never accepted. + */ + +import { Buffer } from "node:buffer"; +import type { SpawnOptions } from "node:child_process"; +import { afterEach, describe, expect, it } from "vitest"; +import { IpcFrameTooLargeError, IpcIntegrityError } from "../../ipc.js"; +import type { RunRequestInput } from "../../manager.js"; +import { + COMMITTED_FLAGS, + PyodideChildExitError, + PyodideDisposedError, + PyodideSandbox, + PyodideTimeoutError, +} from "../../manager.js"; +import { type FakeChild, type Harness, flush, makeHarness } from "./fake-child.js"; + +const INPUT: RunRequestInput = { code: "print(1)", argv: ["x.py"], stdin: "", files: [], cwd: "/home/pyodide" }; + +let seq = 0; +function makeManager( + harness: Harness, + opts: { runtimeTimeoutMs?: number; maxFrameBytes?: number; maxAggregateBytes?: number } = {}, +): PyodideSandbox { + return new PyodideSandbox({ + assetDir: "/vendor/pyodide", + denoBin: "/vendor/deno/deno", + runnerPath: "/dist/pyodide-runner/runner.ts", + spawnFn: harness.spawnFn, + // Deterministic requestIds so a forged frame can't accidentally match. + randomRequestId: () => `req-${++seq}`, + runtimeTimeoutMs: opts.runtimeTimeoutMs ?? 5_000, + maxFrameBytes: opts.maxFrameBytes, + maxAggregateBytes: opts.maxAggregateBytes, + }); +} + +const sandboxes: PyodideSandbox[] = []; +function track(s: PyodideSandbox): PyodideSandbox { + sandboxes.push(s); + return s; +} + +afterEach(async () => { + for (const s of sandboxes.splice(0)) await s.dispose(); +}); + +/** Spawn + handshake a child, run one request to completion, return [resp, child]. */ +async function warmRun(manager: PyodideSandbox, harness: Harness, signal: AbortSignal): Promise { + const p = manager.run(INPUT, signal); + const child = await harness.nextChild(); + child.sendReady(); + const run = await child.nextRun(); + child.sendResult(run); + await p; + return child; +} + +describe("PyodideSandbox — happy path & serialization", () => { + it("spawns lazily, handshakes, and returns the matching response", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + expect(manager.state).toBe("cold"); + expect(manager.generation).toBe(0); + + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + expect(manager.state).toBe("starting"); + expect(manager.generation).toBe(1); + expect(child.generation).toBe(1); + + child.sendReady(); + const run = await child.nextRun(); + expect(run.requestId).toBe("req-1"); + expect(run.seq).toBe(1); + expect(run.generation).toBe(1); + // Integrity secrets must NEVER cross into the request payload's user fields. + expect(run.code).toBe(INPUT.code); + + child.sendResult(run, { stdout: Buffer.from("1\n").toString("base64"), exitCode: 0 }); + const resp = await p; + expect(resp.type).toBe("result"); + expect(resp.exitCode).toBe(0); + expect(Buffer.from(resp.stdout, "base64").toString()).toBe("1\n"); + expect(manager.state).toBe("idle"); + }); + + it("serializes two overlapping run() calls in submission order and reuses the warm child", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const order: number[] = []; + + const p1 = manager.run(INPUT, new AbortController().signal).then((r) => { + order.push(1); + return r; + }); + const child = await harness.nextChild(); + child.sendReady(); + const run1 = await child.nextRun(); + + // Queue the second call while the first is busy. + const p2 = manager.run(INPUT, new AbortController().signal).then((r) => { + order.push(2); + return r; + }); + await flush(); + expect(child.runs).toHaveLength(1); // run2 is parked behind run1 + + child.sendResult(run1); + await p1; + // run2 now reuses the SAME warm child (no respawn, no second handshake). + const run2 = await child.nextRun(); + expect(run2.seq).toBe(2); + child.sendResult(run2); + await p2; + + expect(order).toEqual([1, 2]); + expect(harness.children).toHaveLength(1); + }); +}); + +describe("PyodideSandbox — cancellation", () => { + it("abort while queued removes only that waiter; the active run is unaffected and the child survives", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + + const p1 = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + const run1 = await child.nextRun(); // run1 active (busy), holds the lock + + const ac2 = new AbortController(); + const p2 = manager.run(INPUT, ac2.signal); + await flush(); + + ac2.abort(); + await expect(p2).rejects.toMatchObject({ name: "AbortError", code: "ABORTED" }); + expect(child.killed).toBe(false); // queued abort must NOT kill the child + expect(manager.state).toBe("busy"); // run1 still active + + child.sendResult(run1); + const resp1 = await p1; + expect(resp1.exitCode).toBe(0); // active run completed normally + }); + + it("abort after acquiring the mutex (during init) kills the child and retires the generation", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + + const ac = new AbortController(); + const p = manager.run(INPUT, ac.signal); + const child = await harness.nextChild(); // spawned, awaiting ready (never sent) + expect(manager.state).toBe("starting"); + + ac.abort(); + await expect(p).rejects.toMatchObject({ name: "AbortError" }); + expect(child.killed).toBe(true); + expect(child.killSignal).toBe("SIGKILL"); + expect(manager.state).toBe("dead"); + + // Generation retired: the next run respawns with an incremented generation. + const p2 = manager.run(INPUT, new AbortController().signal); + const child2 = await harness.nextChild(); + expect(child2.generation).toBe(2); + expect(manager.generation).toBe(2); + child2.sendReady(); + child2.sendResult(await child2.nextRun()); + await p2; + }); + + it("internal timeout during init throws PyodideTimeoutError and kills the child", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness, { runtimeTimeoutMs: 25 })); + + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); // never handshakes + await expect(p).rejects.toBeInstanceOf(PyodideTimeoutError); + expect(child.killed).toBe(true); + expect(manager.state).toBe("dead"); + }); + + it("internal timeout mid-run throws PyodideTimeoutError and kills the child", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness, { runtimeTimeoutMs: 25 })); + + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + await child.nextRun(); // run frame sent, busy, but no response ever comes + await expect(p).rejects.toBeInstanceOf(PyodideTimeoutError); + expect(child.killed).toBe(true); + }); + + it("rejects an already-aborted signal without spawning", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const ac = new AbortController(); + ac.abort(); + await expect(manager.run(INPUT, ac.signal)).rejects.toMatchObject({ name: "AbortError" }); + expect(harness.children).toHaveLength(0); + }); +}); + +describe("PyodideSandbox — respawn on exit", () => { + it("an unexpected child exit marks dead and the next run respawns with an incremented generation", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + + const c1 = await warmRun(manager, harness, new AbortController().signal); + expect(c1.generation).toBe(1); + expect(manager.generation).toBe(1); + + c1.exit(1, null); // crash / OOM-kill while idle + await flush(); + expect(manager.state).toBe("dead"); + + const c2 = await warmRun(manager, harness, new AbortController().signal); + expect(c2.generation).toBe(2); + expect(manager.generation).toBe(2); + expect(harness.children).toHaveLength(2); + }); + + it("an in-flight run rejects when the child exits unexpectedly", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + await child.nextRun(); + child.exit(137, "SIGKILL"); // OOM + await expect(p).rejects.toBeInstanceOf(PyodideChildExitError); + expect(manager.state).toBe("dead"); + }); +}); + +describe("PyodideSandbox — frame integrity (each violation kills the child)", () => { + async function inFlight(opts?: { maxFrameBytes?: number }): Promise<{ + manager: PyodideSandbox; + harness: Harness; + child: FakeChild; + p: Promise; + run: Awaited>; + }> { + const harness = makeHarness(); + const manager = track(makeManager(harness, opts)); + const p = manager.run(INPUT, new AbortController().signal).catch((e) => e); + const child = await harness.nextChild(); + child.sendReady(); + const run = await child.nextRun(); + return { manager, harness, child, p, run }; + } + + it("forged requestId → kill", async () => { + const { child, p, run, manager } = await inFlight(); + child.sendResult(run, { requestId: "forged" }); + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + expect(manager.state).toBe("dead"); + }); + + it("out-of-sequence seq → kill", async () => { + const { child, p, run } = await inFlight(); + child.sendResult(run, { seq: 999 }); + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + }); + + it("stale / wrong generation → kill", async () => { + const { child, p, run } = await inFlight(); + child.sendResult(run, { generation: 0 }); + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + }); + + it("malformed JSON → kill", async () => { + const { child, p } = await inFlight(); + child.sendRaw(Buffer.from("{ not json", "utf8")); + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + }); + + it("a malformed drain entry (integrity-valid frame, bad created[]) → kill", async () => { + const { child, p, run, manager } = await inFlight(); + // Correct requestId/seq/generation, but a created entry that is not an FsEntry. + child.sendResult(run, { created: [{ path: "/x" }] as never }); + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + expect(manager.state).toBe("dead"); + }); + + it("oversized frame (base64 wire size over cap) → kill", async () => { + const { child, p, run } = await inFlight({ maxFrameBytes: 200 }); + // 1 KiB raw → ~1368 base64 chars in the body, well over the 200-byte cap. + child.sendResult(run, { stdout: Buffer.alloc(1024, 0x41).toString("base64") }); + expect(await p).toBeInstanceOf(IpcFrameTooLargeError); + expect(child.killed).toBe(true); + }); + + it("a duplicate / replayed response (none in-flight) → kill", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + const run = await child.nextRun(); + child.sendResult(run); // valid first response + await p; + child.sendResult(run); // replay — no in-flight request + await flush(); + expect(child.killed).toBe(true); + expect(manager.state).toBe("dead"); + }); + + it("duplicate ready handshake → kill", async () => { + const { child, p } = await inFlight(); + child.sendReady(); // second handshake + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + }); + + it("wrong-generation ready handshake → kill", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const p = manager.run(INPUT, new AbortController().signal).catch((e) => e); + const child = await harness.nextChild(); + child.sendReady(99); // forged generation in the handshake + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + }); + + it("a ready arriving after the first response → kill", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + const run = await child.nextRun(); + child.sendResult(run); + await p; + child.sendReady(); // post-response handshake is a violation + await flush(); + expect(child.killed).toBe(true); + expect(manager.state).toBe("dead"); + }); + + it("a never-completing frame stream trips the aggregate cap → kill", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness, { maxAggregateBytes: 4096 })); + const p = manager.run(INPUT, new AbortController().signal).catch((e) => e); + const child = await harness.nextChild(); + child.sendReady(); + await child.nextRun(); + // A header claiming a large (but under per-frame-cap) body, then bytes that + // dribble in below the per-frame cap and never complete the frame → the + // accumulated buffer crosses the aggregate cap before any frame parses. + const header = Buffer.alloc(4); + header.writeUInt32BE(1_000_000, 0); + child.stdout.write(header); + for (let i = 0; i < 10; i++) child.stdout.write(Buffer.alloc(900, 0x20)); + expect(await p).toBeInstanceOf(IpcIntegrityError); + expect(child.killed).toBe(true); + }); +}); + +describe("PyodideSandbox — spawn posture", () => { + it("spawns deno with the committed deny-belt, an asset-dir-scoped allow-read, and a scrubbed env", async () => { + const calls: { cmd: string; args: readonly string[]; opts: SpawnOptions }[] = []; + const harness = makeHarness(); + const recordingSpawn: typeof harness.spawnFn = (cmd, args, opts) => { + calls.push({ cmd, args, opts }); + return harness.spawnFn(cmd, args, opts); + }; + const manager = track( + new PyodideSandbox({ + assetDir: "/vendor/pyodide", + denoBin: "/vendor/deno/deno", + runnerPath: "/dist/pyodide-runner/runner.ts", + spawnFn: recordingSpawn, + randomRequestId: () => "req-x", + }), + ); + + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + child.sendResult(await child.nextRun()); + await p; + + expect(calls).toHaveLength(1); + const call = calls[0]!; + expect(call.cmd).toBe("/vendor/deno/deno"); + // committed flags verbatim, then the only granted capability, then runner + argv. + expect(call.args).toEqual([ + "run", + ...COMMITTED_FLAGS, + "--allow-read=/vendor/pyodide", + "/dist/pyodide-runner/runner.ts", + "/vendor/pyodide", + "1", + ]); + // Scrubbed env: ONLY the update-check suppressor — no AUTH_SECRET/DATABASE_URL. + expect(call.opts.env).toEqual({ DENO_NO_UPDATE_CHECK: "1" }); + expect(call.opts.stdio).toEqual(["pipe", "pipe", "pipe"]); + }); +}); + +describe("PyodideSandbox — dispose", () => { + it("dispose kills the child and rejects an in-flight run; further runs reject", async () => { + const harness = makeHarness(); + const manager = new PyodideSandbox({ + assetDir: "/a", + denoBin: "/d", + runnerPath: "/r", + spawnFn: harness.spawnFn, + randomRequestId: () => "req-d", + }); + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + child.sendReady(); + await child.nextRun(); + + await manager.dispose(); + await expect(p).rejects.toBeInstanceOf(PyodideDisposedError); + expect(child.killed).toBe(true); + await expect(manager.run(INPUT, new AbortController().signal)).rejects.toBeInstanceOf(PyodideDisposedError); + }); +}); diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index ad7374e..23d8b20 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -537,16 +537,33 @@ Cover (each names the design decision it protects): ### Phase 4: Success Criteria #### Phase 4: Programmatic Verification -- [ ] `pnpm typecheck && pnpm lint:fix` pass -- [ ] `pnpm test -- src/api/pyodide/tests/unit/manager.test.ts src/api/pyodide/tests/unit/ipc.test.ts` pass: serialization; abort-while-queued (waiter removed, child survives, active call unaffected); abort-after-acquire/during-init (kills child, retires generation); malformed/oversized/duplicate/stale-generation/forged-frame and `ready`-handshake violations each kill the child; base64-aware cap; respawn bumps generation -- [ ] `pnpm test:unit` passes (no regressions) +- [x] `pnpm typecheck && pnpm lint:fix` pass — both clean (`biome check` reports no fixes); `manager.ts`/`ipc.ts` lint fully (NOT biome-ignored — only the Deno-realm `runner.ts` is) +- [x] `pnpm test -- src/api/pyodide/tests/unit/manager.test.ts src/api/pyodide/tests/unit/ipc.test.ts` pass: serialization; abort-while-queued (waiter removed, child survives, active call unaffected); abort-after-acquire/during-init (kills child, retires generation); malformed/oversized/duplicate/stale-generation/forged-frame and `ready`-handshake violations each kill the child; base64-aware cap; respawn bumps generation — **ipc.test.ts 29 + manager.test.ts 22 = 51 pass** (every listed case covered, incl. the post-review FsEntry-schema cases) +- [x] `pnpm test:unit` passes (no regressions) — **928 pass / 4 skip** (was 877; +51 new) #### Phase 4: Agent Verification -- [ ] Agent exercises `manager.run()` with two overlapping calls against the fake child and confirms: they serialize; aborting the **queued** call removes its waiter and rejects only it while the active call completes (the child is NOT killed); aborting the **active** call (or an internal timeout) kills the child and retires the generation -- [ ] Agent reviews `manager.ts` to confirm the spawn uses a scrubbed `env` (no secrets) and the committed flag set verbatim +- [x] Agent exercises `manager.run()` with two overlapping calls against the fake child and confirms: they serialize; aborting the **queued** call removes its waiter and rejects only it while the active call completes (the child is NOT killed); aborting the **active** call (or an internal timeout) kills the child and retires the generation — verified via the passing `cancellation`/`serialization` suites: queued-abort rejects only p2 with `AbortError` while `child.killed === false` and run1 completes `exitCode 0`; abort-after-acquire and both init/mid-run timeouts set `child.killed === true` + `state "dead"` and the next run respawns at `generation 2` +- [x] Agent reviews `manager.ts` to confirm the spawn uses a scrubbed `env` (no secrets) and the committed flag set verbatim — `#spawnChild` (manager.ts:388-393) passes `env: { DENO_NO_UPDATE_CHECK: "1" }` (no parent inheritance, no AUTH_SECRET/DATABASE_URL) and `args = ["run", ...COMMITTED_FLAGS, "--allow-read=", runnerPath, assetDir, String(gen)]`; `COMMITTED_FLAGS` matches `runner.ts`'s documented deny-belt + the Phase 3 harness verbatim; the `spawn posture` unit test asserts the full argv + env equality ### Phase 4: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 4._ + +**`run()` takes `RunRequestInput`, not the plan's literal `RunRequest` (security-strengthening adaptation).** `RunRequestInput = Omit` — the caller supplies ONLY `{ code, argv, stdin, files, cwd }`. The manager assigns `type:"run"` + the three secret integrity fields itself, so a caller *cannot* inject them. This matches Phase 5's description ("carry argv, stdin, cwd") and makes the S2 finding-A invariant structural (secrets never originate caller-side). **Phase 5 builds a `RunRequestInput`** — it does NOT (and must not) populate requestId/seq/generation. + +**Serialization is a hand-rolled abort-aware lock, NOT `async-mutex`.** `async-mutex`'s `acquire()` returns a promise that can't be cancelled mid-wait, but the abort-while-queued semantics require removing *only* the aborted waiter (rejecting it with `AbortError`) without disturbing the active run. So the manager mirrors `session-manager.ts`'s `SemaphoreWaiter` pattern (`#acquire`/`#release`, settled-flag, splice-on-abort). `async-mutex` remains a dependency for Phase 6's residency admission mutex. + +**Generation increments on SPAWN (cold→1, respawn→2…), which IS "retire the generation".** The `generation` getter returns the current child's gen (0 before any spawn). A kill (timeout/abort/integrity/exit) marks `dead` but does not change the number; the *next* `#spawnChild` increments, so the dead generation is never reused and a stale-generation frame from the old child is rejected by `validateInbound`. The fake child reads its generation from the **last spawn argv** (exactly like the real runner), so it echoes the gen it was launched with unless a test forges otherwise. + +**`result` AND `error` are both valid responses.** The runner sets `type` from the Python exit code (`exitCode===0 ? "result" : "error"`); both carry the full `RunResponse` shape. `validateInbound` accepts either as the single response to a `run`. **Phase 5 must inspect `exitCode`, not `type`,** and drain `created/modified/deleted` regardless (the drain gate is a *resolved* manager promise, i.e. no timeout/abort/integrity failure — not `exitCode===0`). + +**`validateInbound` is the load-bearing control; `ipc.ts` mirrors the protocol with Node `Buffer`.** `decodeFrames` enforces the per-frame cap on the *declared length prefix* — because file payloads are base64 *in the JSON body*, that prefix already measures the ~33%-expanded wire size, so the "base64 wire size over cap → reject" requirement needs no separate accounting (proven by the `1 KiB raw → ~1368 b64` test). The aggregate cap bounds total un-parsed bytes per response (reset to the leftover-buffer size on each accepted frame) to catch a slowloris stream that never completes a valid frame. Invalid UTF-8 uses a `fatal` `TextDecoder` (matches the protocol's strictness) — `Buffer.toString("utf8")` alone is lenient and would silently U+FFFD. + +**Late-event isolation across respawn.** stdout/exit/error handlers are bound per-child and short-circuit if `this.#child !== child`, so buffered `data`/`exit` events from a just-killed (retired) generation cannot contaminate the freshly-spawned child. Combined with the `#isTerminal()` guard at the top of `#onStdoutData`, post-kill bytes are dropped. + +**Deferred to Phase 6 (kept Phase 4 scoped):** the manager reads `process.env` only for `PYODIDE_ASSET_DIR`/`DENO_BIN_PATH` fallbacks; the cap/timeout knobs (`runtimeTimeoutMs`/`maxFrameBytes`/`maxAggregateBytes`) are constructor options with module-default constants — Phase 6 wires their env vars + the semaphore/residency. `runnerPath` default resolves via `fileURLToPath(new URL("../../pyodide-runner/runner.ts", import.meta.url))`, which lands on `src/pyodide-runner/runner.ts` under tsx (dev) and `dist/pyodide-runner/runner.ts` after build. + +**Test harness note:** the shared fake child lives in `src/api/pyodide/tests/unit/fake-child.ts` (a non-`.test.ts` helper — compiled by tsc, never collected as a test) to keep `manager.test.ts` focused on assertions while still being addressable by the plan's two-file verification command. `FakeStream.write` delivers `data` on a `queueMicrotask`, making frame propagation deterministic (`flush()` = one `setImmediate` tick). + +**Post-review fix (Codex — valid contract gap closed).** `validateInbound` is documented as the load-bearing "schema + integrity" boundary, but the response branch originally only checked that `created`/`modified`/`deleted` were *arrays*, not their element shapes. A frame with valid integrity secrets but a malformed drain entry (`created: [{garbage}]`, `deleted: [42]`) would be *accepted* and the child reused — inconsistent with the fail-closed kill-the-child treatment, and it pushed schema enforcement into the Phase 5 drain. Not exploitable as a *forged accepted* frame (the attacker can't guess the integrity secrets), but a real gap. **Fix:** `assertFsEntry` now validates each `created`/`modified` element against `FsEntry` (`path` non-empty string, `kind` ∈ {file,dir}, `mode` non-negative integer, `data` string, and `data === ""` enforced for dirs — note `""` is still a legal *empty file*), and `deleted` is enforced as a non-empty `string[]`. Any malformed element → `IpcIntegrityError` → kill-the-child. Added 6 ipc.test cases (incl. a positive file+empty-file+empty-dir case) + 1 manager case (integrity-valid frame, bad `created[]` → child killed). 44 → 51 pyodide tests. --- From 75ab53e16a3bfd92548bf7fbf46641de483b4c14 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 18:51:15 +0930 Subject: [PATCH 07/16] Phase 5: pyodide custom commands + file staging drain (core requirement) --- src/api/commands/pyodide-command.ts | 341 ++++++++++++++++++ .../tests/unit/pyodide-command.test.ts | 278 ++++++++++++++ src/api/session-manager.ts | 63 +++- .../integration/pyodide.integration.test.ts | 125 +++++++ .../unit/session-manager.pyodide.test.ts | 130 +++++++ src/pyodide-runner/runner.ts | 7 + thoughts/issue-118-pyodide-runtime/plan.md | 29 +- 7 files changed, 965 insertions(+), 8 deletions(-) create mode 100644 src/api/commands/pyodide-command.ts create mode 100644 src/api/pyodide/tests/unit/pyodide-command.test.ts create mode 100644 src/api/tests/integration/pyodide.integration.test.ts create mode 100644 src/api/tests/unit/session-manager.pyodide.test.ts diff --git a/src/api/commands/pyodide-command.ts b/src/api/commands/pyodide-command.ts new file mode 100644 index 0000000..1d2049d --- /dev/null +++ b/src/api/commands/pyodide-command.ts @@ -0,0 +1,341 @@ +/** + * Custom `python3` / `python` commands backed by a per-session + * {@link PyodideSandbox} (the OS-isolated Deno subprocess). Registered only when + * a sandbox's `python_runtime` is `"pyodide"` (see `session-manager.ts`). + * + * Flow per invocation: + * 1. Parse the `python3` surface (`-c CODE`, `FILE`, `-`/stdin, `--version`, + * `-m` → unsupported, bare → hint). + * 2. Stage the **cwd subtree** (+ a resolved script file outside cwd) into the + * request's `files`, base64-encoding contents, under per-file/total caps. + * 3. `sandbox.run(...)` — the manager serializes, validates frames, and THROWS + * on timeout/abort/integrity/child-exit (we let it propagate so `bash.exec` + * rejects → the script transaction rolls back and nothing drains). + * 4. On a resolved response, **drain** `created`/`modified`/`deleted` into + * `ctx.fs` — cwd-scoped + path-validated + capped — inside the same script + * transaction, so the writes are atomic with the rest of the script. + * + * The response may be `type:"error"` (non-zero exit = a normal Python failure); + * that still drains and returns `{stdout, stderr, exitCode}`. The drain gate is a + * *resolved* run, not `exitCode === 0`. + */ + +import { Buffer } from "node:buffer"; +import { + type CommandContext, + type CustomCommand, + type ExecResult, + decodeBytesToUtf8, + defineCommand, + latin1FromBytes, +} from "just-bash"; +import type { IFileSystem } from "just-bash"; +import type { FsEntry, RunResponse } from "../../pyodide-runner/protocol.js"; +import type { PyodideSandbox, RunRequestInput } from "../pyodide/manager.js"; + +/** Default per-file cap on staged-in / drained-out file bytes (32 MiB). */ +export const PYODIDE_MAX_FILE_BYTES_DEFAULT = 32 * 1024 * 1024; +/** Default total cap across all staged-in / drained-out files (128 MiB). */ +export const PYODIDE_MAX_TOTAL_BYTES_DEFAULT = 128 * 1024 * 1024; + +// Pyodide 0.29.4 ships CPython 3.13 (Phase 3 Discoveries). +const VERSION_LINE = "Python 3.13.2 (Pyodide)\n"; + +const HINT = `\ +'python3' here runs in the Pyodide runtime (numpy/pandas/scipy/openpyxl), OS-isolated and air-gapped. + python3 script.py [args…] # run a script file + python3 -c 'CODE' [args…] # run inline code + echo 'CODE' | python3 # run code from stdin + python3 --version # report the runtime version +'-m MODULE' and an interactive REPL are not supported. +`; + +// Never-aborting fallback when a CommandContext has no signal (defensive). +const NEVER_ABORT: AbortSignal = new AbortController().signal; + +interface Caps { + readonly maxFileBytes: number; + readonly maxTotalBytes: number; +} + +export interface PyodideCommandOptions { + readonly maxFileBytes?: number; + readonly maxTotalBytes?: number; +} + +/** A drain / staging policy violation. Surfaces as a non-zero exec, rolling back the script tx. */ +export class PyodideDrainError extends Error { + readonly code = "EPYODIDE_DRAIN"; + constructor(message: string) { + super(`EPYODIDE_DRAIN: ${message}`); + this.name = "PyodideDrainError"; + } +} + +function envInt(name: string, fallback: number): number { + const raw = process.env[name]; + if (raw === undefined) return fallback; + const n = Number(raw); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + +function makeAbortError(): Error { + return Object.assign(new Error("ABORTED"), { code: "ABORTED", name: "AbortError" }); +} + +function errResult(message: string, exitCode: number): ExecResult { + return { stdout: "", stderr: `${message}\n`, exitCode }; +} + +/** byte length of the data a base64 string decodes to, without allocating it. */ +function base64ByteLength(b64: string): number { + if (b64.length === 0) return 0; + const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0; + return Math.floor((b64.length * 3) / 4) - padding; +} + +export function createPyodideCommands(sandbox: PyodideSandbox, opts: PyodideCommandOptions = {}): CustomCommand[] { + const caps: Caps = { + maxFileBytes: opts.maxFileBytes ?? envInt("PYODIDE_MAX_FILE_BYTES", PYODIDE_MAX_FILE_BYTES_DEFAULT), + maxTotalBytes: opts.maxTotalBytes ?? envInt("PYODIDE_MAX_TOTAL_BYTES", PYODIDE_MAX_TOTAL_BYTES_DEFAULT), + }; + const handler = (args: string[], ctx: CommandContext): Promise => runPython(sandbox, caps, args, ctx); + return [defineCommand("python3", handler), defineCommand("python", handler)]; +} + +interface Parsed { + readonly code: string; + readonly argv: string[]; + /** base64 of the program's own stdin ("" when stdin was consumed as the program). */ + readonly stdin: string; + /** Absolute path of a FILE script resolved OUTSIDE cwd that must also be staged + * (python3 FILE parity). Staged in `runPython` under the same caps as the cwd walk. */ + readonly scriptPathOutsideCwd?: string; +} + +async function runPython( + sandbox: PyodideSandbox, + caps: Caps, + args: string[], + ctx: CommandContext, +): Promise { + const first = args[0]; + + if (first === "--version" || first === "-V") return { stdout: VERSION_LINE, stderr: "", exitCode: 0 }; + if (first === "-h" || first === "--help") return { stdout: "", stderr: HINT, exitCode: 0 }; + if (first === "-m") return errResult("python3: the -m option is not supported in the pyodide runtime", 2); + + const parsed = await parseProgram(args, ctx); + if ("error" in parsed) return parsed.error; + + // Stage the cwd subtree (+ a script file outside cwd) into the request, under + // per-file + total byte caps applied to BOTH sources from a shared budget. + let files: FsEntry[]; + try { + const staged = await stageCwd(ctx.fs, ctx.cwd, caps); + files = staged.files; + if (parsed.scriptPathOutsideCwd !== undefined) { + const { entry } = await stageFile(ctx.fs, parsed.scriptPathOutsideCwd, caps, staged.total); + files.push(entry); + } + } catch (err) { + if (err instanceof PyodideDrainError) return errResult(err.message, 1); + throw err; + } + + const input: RunRequestInput = { code: parsed.code, argv: parsed.argv, stdin: parsed.stdin, files, cwd: ctx.cwd }; + + // Manager THROWS on timeout/abort/integrity/child-exit — let it propagate so + // bash.exec rejects and the script transaction rolls back (drains nothing). + const resp = await sandbox.run(input, ctx.signal ?? NEVER_ABORT); + + // Abort that landed after the response but before the drain → drain nothing. + if (ctx.signal?.aborted) throw makeAbortError(); + + await drain(ctx.fs, ctx.cwd, resp, caps); + + return { + stdout: Buffer.from(resp.stdout, "base64").toString("utf8"), + stderr: Buffer.from(resp.stderr, "base64").toString("utf8"), + exitCode: resp.exitCode, + }; +} + +async function parseProgram(args: string[], ctx: CommandContext): Promise { + const first = args[0]; + + if (first === "-c") { + if (args.length < 2) return { error: errResult("python3: argument expected for the -c option", 2) }; + // argv[0]="-c"; the script's own args follow the CODE operand. + return { code: args[1] as string, argv: ["-c", ...args.slice(2)], stdin: stdinBase64(ctx) }; + } + + if (first === "-" || first === undefined) { + // `python3 -` (explicit) or bare `python3` reading a program piped on stdin. + const program = decodeBytesToUtf8(ctx.stdin); + if (first === undefined && program.length === 0) { + // Bare invocation, no piped program → an interactive REPL we don't offer. + return { error: { stdout: "", stderr: HINT, exitCode: 0 } }; + } + const argv = first === "-" ? ["-", ...args.slice(1)] : [""]; + return { code: program, argv, stdin: "" }; + } + + if (first.startsWith("-")) return { error: errResult(`python3: unknown option ${first}`, 2) }; + + // FILE [args…] + if (first.includes("\0")) return { error: errResult("python3: invalid file path", 2) }; + const resolved = ctx.fs.resolvePath(ctx.cwd, first); + let code: string; + try { + code = await ctx.fs.readFile(resolved, "utf8"); + } catch { + return { error: errResult(`python3: can't open file '${first}': [Errno 2] No such file or directory`, 2) }; + } + return { + code, + argv: [first, ...args.slice(1)], + stdin: stdinBase64(ctx), + scriptPathOutsideCwd: isUnderCwd(resolved, ctx.cwd) ? undefined : resolved, + }; +} + +function stdinBase64(ctx: CommandContext): string { + return Buffer.from(latin1FromBytes(ctx.stdin), "latin1").toString("base64"); +} + +function cwdPrefix(cwd: string): string { + return cwd.endsWith("/") ? cwd : `${cwd}/`; +} + +function isUnderCwd(path: string, cwd: string): boolean { + return path === cwd || path.startsWith(cwdPrefix(cwd)); +} + +/** Enforce per-file + running-total staging caps. Throws on violation. */ +function enforceStageCaps(size: number, path: string, caps: Caps, runningTotal: number): void { + if (size > caps.maxFileBytes) { + throw new PyodideDrainError(`'${path}' (${size} bytes) exceeds the per-file stage cap`); + } + if (runningTotal + size > caps.maxTotalBytes) { + throw new PyodideDrainError("staged files exceed the total byte cap"); + } +} + +/** + * Stage a single regular file: refuse symlinks (default-deny), enforce the caps + * against `runningTotal`, capture the real mode, and base64-encode the bytes. + */ +async function stageFile( + fs: IFileSystem, + path: string, + caps: Caps, + runningTotal: number, +): Promise<{ entry: FsEntry; size: number }> { + const st = await fs.lstat(path); + if (st.isSymbolicLink) throw new PyodideDrainError(`refusing to stage a symlink: ${path}`); + const bytes = await fs.readFileBuffer(path); + enforceStageCaps(bytes.byteLength, path, caps, runningTotal); + return { + entry: { path, kind: "file", mode: st.mode & 0o777, data: Buffer.from(bytes).toString("base64") }, + size: bytes.byteLength, + }; +} + +/** Walk the cwd subtree (dirs + files), base64-encoding file bytes, under the caps. */ +async function stageCwd(fs: IFileSystem, cwd: string, caps: Caps): Promise<{ files: FsEntry[]; total: number }> { + const out: FsEntry[] = []; + let total = 0; + if (!(await fs.exists(cwd))) return { files: out, total }; + + const walk = async (dir: string): Promise => { + let names: string[]; + try { + names = await fs.readdir(dir); + } catch { + return; + } + for (const name of names) { + if (name === "." || name === "..") continue; + const full = dir.endsWith("/") ? `${dir}${name}` : `${dir}/${name}`; + const st = await fs.lstat(full); + if (st.isSymbolicLink) continue; // default-deny: never stage symlinks + if (st.isDirectory) { + out.push({ path: full, kind: "dir", mode: st.mode & 0o777, data: "" }); + await walk(full); + } else if (st.isFile) { + const { entry, size } = await stageFile(fs, full, caps, total); + total += size; + out.push(entry); + } + } + }; + await walk(cwd); + return { files: out, total }; +} + +/** + * Drain the cwd-scoped diff into `ctx.fs`, inside the live script transaction. + * Validates EVERY path stays under cwd (rejecting `..`, absolute-outside-cwd, and + * null bytes) and enforces the byte caps BEFORE any write — so a forged/buggy + * runner can never escape the cwd or blow the budget. Applies created + * dirs-before-files (runner-ordered), then modified files, then deleted + * depth-first (runner-ordered). Throws {@link PyodideDrainError} on a violation, + * which rolls the transaction back. + */ +export async function drain(fs: IFileSystem, cwd: string, resp: RunResponse, caps: Caps): Promise { + const assertUnderCwd = (path: string, label: string): string => { + if (path.includes("\0")) throw new PyodideDrainError(`${label} path contains a null byte`); + const resolved = fs.resolvePath(cwd, path); + if (!isUnderCwd(resolved, cwd)) throw new PyodideDrainError(`${label} path escapes cwd: ${path}`); + return resolved; + }; + + // 1. Validate paths + caps for everything before writing anything. + let total = 0; + for (const e of [...resp.created, ...resp.modified]) { + assertUnderCwd(e.path, "drain"); + if (e.kind === "file") { + const n = base64ByteLength(e.data); + if (n > caps.maxFileBytes) throw new PyodideDrainError(`'${e.path}' (${n} bytes) exceeds the per-file drain cap`); + total += n; + } + } + if (total > caps.maxTotalBytes) throw new PyodideDrainError("drained files exceed the total byte cap"); + for (const p of resp.deleted) assertUnderCwd(p, "deleted"); + + // 2. Apply created (dirs shallow→deep, then files), then modified files. + for (const e of resp.created) await applyEntry(fs, cwd, e); + for (const e of resp.modified) await applyEntry(fs, cwd, e); + + // 3. Apply deletions depth-first (runner already orders deepest-first). + for (const p of resp.deleted) { + const resolved = fs.resolvePath(cwd, p); + try { + await fs.rm(resolved, { recursive: true, force: true }); + } catch { + // already gone — idempotent + } + } +} + +async function applyEntry(fs: IFileSystem, cwd: string, entry: FsEntry): Promise { + const resolved = fs.resolvePath(cwd, entry.path); + + // Default-deny: never write through / over an existing symlink at the target. + if (await fs.exists(resolved)) { + const st = await fs.lstat(resolved); + if (st.isSymbolicLink) throw new PyodideDrainError(`refusing to drain over a symlink: ${entry.path}`); + } + + if (entry.kind === "dir") { + await fs.mkdir(resolved, { recursive: true }); + return; + } + + const bytes = new Uint8Array(Buffer.from(entry.data, "base64")); + await fs.writeFile(resolved, bytes); + // writeFile creates with the default 0644; only chmod when the runner reported + // a non-default mode (e.g. an executable bit). + if ((entry.mode & 0o777) !== 0o644) await fs.chmod(resolved, entry.mode & 0o777); +} diff --git a/src/api/pyodide/tests/unit/pyodide-command.test.ts b/src/api/pyodide/tests/unit/pyodide-command.test.ts new file mode 100644 index 0000000..b8deb3e --- /dev/null +++ b/src/api/pyodide/tests/unit/pyodide-command.test.ts @@ -0,0 +1,278 @@ +/** + * Unit tests for the pyodide `python3`/`python` commands + cwd-scoped drain + * (`src/api/commands/pyodide-command.ts`). Driven by a fake sandbox (returns a + * canned RunResponse) over an InMemoryFs — no real Deno/Pyodide. Covers arg + * parsing, staging, and the security-critical drain path validation + caps. + */ + +import { Buffer } from "node:buffer"; +import { type ByteString, EMPTY_BYTES, InMemoryFs, encodeUtf8ToBytes } from "just-bash"; +import type { CommandContext, CustomCommand, ExecResult, IFileSystem } from "just-bash"; +import { describe, expect, it } from "vitest"; +import type { FsEntry, RunResponse } from "../../../../pyodide-runner/protocol.js"; +import { PyodideDrainError, createPyodideCommands } from "../../../commands/pyodide-command.js"; +import type { PyodideSandbox, RunRequestInput } from "../../manager.js"; + +function makeResponse(over: Partial = {}): RunResponse { + return { + type: "result", + requestId: "r", + seq: 1, + generation: 1, + stdout: "", + stderr: "", + exitCode: 0, + created: [], + modified: [], + deleted: [], + ...over, + }; +} + +function fileEntry(path: string, text: string, mode = 0o644): FsEntry { + return { path, kind: "file", mode, data: Buffer.from(text, "utf8").toString("base64") }; +} + +/** A sandbox stub: records the run input, returns `respond(input)`. */ +function fakeSandbox(respond: (input: RunRequestInput) => RunResponse): { + sandbox: PyodideSandbox; + calls: RunRequestInput[]; +} { + const calls: RunRequestInput[] = []; + const sandbox = { + run: (input: RunRequestInput): Promise => { + calls.push(input); + return Promise.resolve(respond(input)); + }, + } as unknown as PyodideSandbox; + return { sandbox, calls }; +} + +function makeCtx( + fs: IFileSystem, + opts: { cwd?: string; stdin?: ByteString; signal?: AbortSignal } = {}, +): CommandContext { + return { + fs, + cwd: opts.cwd ?? "/home/user", + env: new Map(), + stdin: opts.stdin ?? EMPTY_BYTES, + signal: opts.signal, + } as CommandContext; +} + +function run(cmd: CustomCommand, args: string[], ctx: CommandContext): Promise { + return (cmd as { execute: (a: string[], c: CommandContext) => Promise }).execute(args, ctx); +} + +async function freshFs(): Promise { + const fs = new InMemoryFs(); + await fs.mkdir("/home/user", { recursive: true }); + return fs; +} + +describe("pyodide command — argument surface", () => { + it("--version prints the Pyodide CPython version without running the sandbox", async () => { + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + const res = await run(python3 as CustomCommand, ["--version"], makeCtx(await freshFs())); + expect(res.exitCode).toBe(0); + expect(res.stdout).toBe("Python 3.13.2 (Pyodide)\n"); + expect(calls).toHaveLength(0); + }); + + it("registers both python3 and python aliases", () => { + const { sandbox } = fakeSandbox(() => makeResponse()); + const cmds = createPyodideCommands(sandbox); + expect(cmds.map((c) => c.name).sort()).toEqual(["python", "python3"]); + }); + + it("-m MODULE is rejected with a clear message, exit 2, no run", async () => { + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + const res = await run(python3 as CustomCommand, ["-m", "http.server"], makeCtx(await freshFs())); + expect(res.exitCode).toBe(2); + expect(res.stderr).toContain("-m option is not supported"); + expect(calls).toHaveLength(0); + }); + + it("a missing script file reports can't-open, exit 2, no run", async () => { + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + const res = await run(python3 as CustomCommand, ["nope.py"], makeCtx(await freshFs())); + expect(res.exitCode).toBe(2); + expect(res.stderr).toContain("can't open file 'nope.py'"); + expect(calls).toHaveLength(0); + }); + + it("-c CODE passes the code + argv and stages the cwd subtree", async () => { + const fs = await freshFs(); + await fs.writeFile("/home/user/data.csv", "a,b\n1,2\n"); + const { sandbox, calls } = fakeSandbox(() => makeResponse({ stdout: Buffer.from("hi").toString("base64") })); + const [python3] = createPyodideCommands(sandbox); + const res = await run(python3 as CustomCommand, ["-c", "print('hi')", "x"], makeCtx(fs)); + expect(res.exitCode).toBe(0); + expect(res.stdout).toBe("hi"); + expect(calls).toHaveLength(1); + const input = calls[0]!; + expect(input.code).toBe("print('hi')"); + expect(input.argv).toEqual(["-c", "x"]); + expect(input.cwd).toBe("/home/user"); + // data.csv staged (base64 of the file bytes). + const staged = input.files.find((f) => f.path === "/home/user/data.csv"); + expect(staged?.kind).toBe("file"); + expect(Buffer.from(staged?.data ?? "", "base64").toString()).toBe("a,b\n1,2\n"); + }); + + it("reads the program from stdin for the `-` form (and consumes stdin)", async () => { + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + await run(python3 as CustomCommand, ["-"], makeCtx(await freshFs(), { stdin: encodeUtf8ToBytes("print(1)") })); + const input = calls[0]!; + expect(input.code).toBe("print(1)"); + expect(input.argv).toEqual(["-"]); + expect(input.stdin).toBe(""); // stdin was consumed as the program + }); +}); + +describe("pyodide command — drain into the filesystem", () => { + it("drains a created file back into ctx.fs", async () => { + const fs = await freshFs(); + const { sandbox } = fakeSandbox(() => makeResponse({ created: [fileEntry("/home/user/out.txt", "result")] })); + const [python3] = createPyodideCommands(sandbox); + const res = await run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs)); + expect(res.exitCode).toBe(0); + expect(await fs.readFile("/home/user/out.txt")).toBe("result"); + }); + + it("creates dirs before files and applies deletions", async () => { + const fs = await freshFs(); + await fs.writeFile("/home/user/old.txt", "gone soon"); + const { sandbox } = fakeSandbox(() => + makeResponse({ + created: [ + { path: "/home/user/sub", kind: "dir", mode: 0o755, data: "" }, + fileEntry("/home/user/sub/inner.txt", "nested"), + ], + deleted: ["/home/user/old.txt"], + }), + ); + const [python3] = createPyodideCommands(sandbox); + await run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs)); + expect((await fs.stat("/home/user/sub")).isDirectory).toBe(true); + expect(await fs.readFile("/home/user/sub/inner.txt")).toBe("nested"); + expect(await fs.exists("/home/user/old.txt")).toBe(false); + }); + + it("drains nothing when the signal is aborted after the response, before the drain", async () => { + const fs = await freshFs(); + const ac = new AbortController(); + ac.abort(); + const { sandbox } = fakeSandbox(() => makeResponse({ created: [fileEntry("/home/user/out.txt", "x")] })); + const [python3] = createPyodideCommands(sandbox); + await expect( + run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs, { signal: ac.signal })), + ).rejects.toMatchObject({ + name: "AbortError", + }); + expect(await fs.exists("/home/user/out.txt")).toBe(false); // nothing drained + }); + + it("preserves a non-default mode (exec bit) via chmod", async () => { + const fs = await freshFs(); + const { sandbox } = fakeSandbox(() => + makeResponse({ created: [fileEntry("/home/user/run.sh", "#!/bin/sh\n", 0o755)] }), + ); + const [python3] = createPyodideCommands(sandbox); + await run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs)); + expect((await fs.stat("/home/user/run.sh")).mode & 0o777).toBe(0o755); + }); +}); + +describe("pyodide command — drain rejects escaping / invalid paths (fail closed)", () => { + async function expectRejectedDrain(resp: RunResponse): Promise { + const fs = await freshFs(); + const { sandbox } = fakeSandbox(() => resp); + const [python3] = createPyodideCommands(sandbox); + await expect(run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs))).rejects.toBeInstanceOf(PyodideDrainError); + } + + it("rejects an absolute path outside cwd", async () => { + await expectRejectedDrain(makeResponse({ created: [fileEntry("/etc/passwd", "x")] })); + }); + + it("rejects a path that escapes cwd via ..", async () => { + await expectRejectedDrain(makeResponse({ created: [fileEntry("/home/user/../etc/evil", "x")] })); + }); + + it("rejects a null byte in a created path", async () => { + await expectRejectedDrain(makeResponse({ created: [fileEntry("/home/user/ab", "x")] })); + }); + + it("rejects a deleted path outside cwd", async () => { + await expectRejectedDrain(makeResponse({ deleted: ["/etc/passwd"] })); + }); + + it("does not write any file when a later drain entry is rejected (fail closed)", async () => { + const fs = await freshFs(); + const { sandbox } = fakeSandbox(() => + makeResponse({ created: [fileEntry("/home/user/ok.txt", "ok"), fileEntry("/etc/evil", "bad")] }), + ); + const [python3] = createPyodideCommands(sandbox); + await expect(run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs))).rejects.toBeInstanceOf(PyodideDrainError); + // Validation happens before any write, so the valid sibling is NOT written. + expect(await fs.exists("/home/user/ok.txt")).toBe(false); + }); +}); + +describe("pyodide command — byte caps", () => { + it("rejects staging a file over the per-file cap", async () => { + const fs = await freshFs(); + await fs.writeFile("/home/user/big.bin", "x".repeat(2048)); + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox, { maxFileBytes: 1024, maxTotalBytes: 1_000_000 }); + const res = await run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs)); + expect(res.exitCode).toBe(1); + expect(res.stderr).toContain("per-file stage cap"); + expect(calls).toHaveLength(0); // never reached the sandbox + }); + + it("rejects draining a file over the per-file cap", async () => { + const fs = await freshFs(); + const { sandbox } = fakeSandbox(() => + makeResponse({ created: [fileEntry("/home/user/big.out", "y".repeat(2048))] }), + ); + const [python3] = createPyodideCommands(sandbox, { maxFileBytes: 1024, maxTotalBytes: 1_000_000 }); + await expect(run(python3 as CustomCommand, ["-c", "..."], makeCtx(fs))).rejects.toBeInstanceOf(PyodideDrainError); + expect(await fs.exists("/home/user/big.out")).toBe(false); + }); + + it("enforces the per-file cap on a script resolved outside cwd", async () => { + const fs = await freshFs(); + await fs.mkdir("/outside", { recursive: true }); + await fs.writeFile("/outside/big.py", `x = '${"a".repeat(2048)}'`); + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox, { maxFileBytes: 1024, maxTotalBytes: 1_000_000 }); + const res = await run(python3 as CustomCommand, ["/outside/big.py"], makeCtx(fs)); + expect(res.exitCode).toBe(1); + expect(res.stderr).toContain("per-file stage cap"); + expect(calls).toHaveLength(0); // never reached the sandbox + }); +}); + +describe("pyodide command — script resolved outside cwd (FILE parity)", () => { + it("stages an out-of-cwd script with its real mode, not a hardcoded 0644", async () => { + const fs = await freshFs(); + await fs.mkdir("/outside", { recursive: true }); + await fs.writeFile("/outside/tool.py", "print('hi')"); + await fs.chmod("/outside/tool.py", 0o755); + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + await run(python3 as CustomCommand, ["/outside/tool.py", "arg"], makeCtx(fs)); + const input = calls[0]!; + expect(input.argv).toEqual(["/outside/tool.py", "arg"]); + const staged = input.files.find((f) => f.path === "/outside/tool.py"); + expect(staged?.kind).toBe("file"); + expect(staged?.mode).toBe(0o755); // real mode captured (was hardcoded 0o644 before the fix) + }); +}); diff --git a/src/api/session-manager.ts b/src/api/session-manager.ts index b40c24b..fc69a60 100644 --- a/src/api/session-manager.ts +++ b/src/api/session-manager.ts @@ -14,7 +14,14 @@ import type { Redis } from "ioredis"; import { Bash } from "just-bash"; -import type { BashExecResult, DefenseInDepthConfig, ExecOptions, IFileSystem, SecurityViolation } from "just-bash"; +import type { + BashExecResult, + CustomCommand, + DefenseInDepthConfig, + ExecOptions, + IFileSystem, + SecurityViolation, +} from "just-bash"; import { createPostgresSandboxFs, destroyPostgresSandbox } from "../sql-fs/index.js"; import type { RedisBlobCache } from "../sql-fs/redis-blob-cache.js"; import { type RedisPathSnapshot, versionKey } from "../sql-fs/redis-path-snapshot.js"; @@ -22,6 +29,7 @@ import { SessionScopedFs } from "../sql-fs/session-scoped-fs.js"; import type { ICoherentFs, IReadOnlyScopeFs, IScriptTxFs } from "../sql-fs/sql-fs.js"; import type { PathCacheEntry, PythonRuntime, SandboxListEntry, SandboxMeta } from "../sql-fs/types.js"; import { nodeCommand } from "./commands/node-command.js"; +import { createPyodideCommands } from "./commands/pyodide-command.js"; import { execLockKey, withDistributedLock } from "./distributed-lock.js"; import { type DistributedRWLockOptions, rwLockKeys, withDistributedRWLock } from "./distributed-rw-lock.js"; import { logAudit } from "./lib/audit.js"; @@ -29,6 +37,7 @@ import { logAudit } from "./lib/audit.js"; // It spawned the HOST python3 with full `process.env`, which is a sandbox // escape (RCE + secret/credential exfil — audit C1). The WASM `python3` // command from just-bash (enabled via `python: true` below) is the safe path. +import { PyodideSandbox } from "./pyodide/manager.js"; import { type ReadOnlyContext, readOnlyContext } from "./read-only-context.js"; import { RWLock } from "./rw-lock.js"; import type { TenantConfig } from "./tenants.js"; @@ -152,6 +161,14 @@ export interface Session { readonly runtimeOptions: RuntimeOptions; readonly tenantId: string; readonly scriptTx: SessionScopedFs | undefined; + /** + * Per-session Pyodide subprocess manager — present ONLY when + * `runtimeOptions.pythonRuntime === "pyodide"`. Owned first-class by the + * session: the Deno child spawns lazily on the first `python3`/`python` + * exec and is killed via {@link SessionManager.disposePyodide} on every + * teardown path (destroy / reaper / shutdown / failed-create). + */ + pyodideSandbox?: PyodideSandbox; lastUsed: number; inFlight: number; /** @@ -264,6 +281,13 @@ export interface SessionManagerOptions { * Flip to `false` once logs are clean to enforce the security boundary. */ readonly defenseAuditMode?: boolean; + /** + * Factory for a per-session {@link PyodideSandbox}, invoked in `getOrCreate` + * only when `pythonRuntime === "pyodide"`. Defaults to `() => new + * PyodideSandbox()` (env-driven `PYODIDE_ASSET_DIR`/`DENO_BIN_PATH`). Override + * to inject a fake child for tests, or to pass runtime caps/timeouts. + */ + readonly createPyodideSandbox?: () => PyodideSandbox; } interface SemaphoreWaiter { @@ -315,6 +339,7 @@ export class SessionManager { private readonly jsSem: Semaphore; private readonly defenseInDepth: boolean; private readonly defenseAuditMode: boolean; + private readonly createPyodideSandbox: () => PyodideSandbox; private shuttingDown = false; constructor({ @@ -335,6 +360,7 @@ export class SessionManager { listSandboxesFn, defenseInDepth, defenseAuditMode, + createPyodideSandbox, }: SessionManagerOptions) { this.tenantConfig = tenantConfig; this.createFsOverride = createFs; @@ -375,6 +401,7 @@ export class SessionManager { }; this.defenseInDepth = defenseInDepth ?? process.env.JUST_BASH_DEFENSE_IN_DEPTH === "true"; this.defenseAuditMode = defenseAuditMode ?? process.env.JUST_BASH_DEFENSE_AUDIT_MODE !== "false"; + this.createPyodideSandbox = createPyodideSandbox ?? ((): PyodideSandbox => new PyodideSandbox()); } private sessionKey(tenantId: string, sandboxId: string): string { @@ -462,6 +489,7 @@ export class SessionManager { const creationPromise = (async (): Promise => { let createdFs: IFileSystem | undefined; + let createdSandbox: PyodideSandbox | undefined; try { const { fs, resolvedOwner, createdAt: fsCreatedAt } = await this.buildFs(tenantId, sandboxId, owner); createdFs = fs; @@ -476,7 +504,7 @@ export class SessionManager { // NOTE: the `py-exec` warm-host-Python custom command is deliberately // not registered (audit C1 — host sandbox escape). Python sandboxes // run via just-bash's WASM `python3` (`python: true`), which is isolated. - const customCommands = [ + const customCommands: CustomCommand[] = [ // Override just-bash's built-in nodeStubCommand with a smarter // version that translates `node -e CODE` → `js-exec -c CODE` and // `node FILE` → `js-exec FILE` instead of dumping a help wall. @@ -485,6 +513,14 @@ export class SessionManager { ...(resolvedRuntime.javascript ? [nodeCommand] : []), ]; + // "pyodide" runtime: own a per-session PyodideSandbox and register the + // custom python3/python commands that route through it. The Deno child + // spawns lazily on first exec; disposePyodide kills it on teardown. + if (resolvedRuntime.pythonRuntime === "pyodide") { + createdSandbox = this.createPyodideSandbox(); + customCommands.push(...createPyodideCommands(createdSandbox)); + } + const bash = new Bash({ fs, // "stdlib" → just-bash's WASM python3; "pyodide" leaves python @@ -538,9 +574,11 @@ export class SessionManager { lastSeenVersion: initialVersion, publishPending: false, cwd: bash.getCwd(), + pyodideSandbox: createdSandbox, }; this.sessions.set(key, session); createdFs = undefined; // ownership transferred to session + createdSandbox = undefined; // ownership transferred to session return session; } catch (err) { // Disconnect any FS we built before the failure so the dialect @@ -553,6 +591,14 @@ export class SessionManager { // best-effort } } + // Likewise kill any partially-built Pyodide child before rethrow. + if (createdSandbox !== undefined) { + try { + await createdSandbox.dispose(); + } catch { + // best-effort + } + } throw err; } finally { this.pending.delete(key); @@ -1040,6 +1086,7 @@ export class SessionManager { } } } finally { + await this.disposePyodide(session); await this.disconnectFs(session.fs); } if (cleanupError !== undefined) throw cleanupError; @@ -1114,6 +1161,7 @@ export class SessionManager { } this.sessions.delete(key); try { + await this.disposePyodide(session); await this.disconnectFs(session.fs); } catch { // best-effort @@ -1153,12 +1201,23 @@ export class SessionManager { }) .catch(() => {}) .finally(() => { + void this.disposePyodide(session); void this.disconnectFs(session.fs); }); } } } + /** Best-effort SIGKILL of a session's Pyodide child on any teardown path. */ + private async disposePyodide(session: Session): Promise { + if (session.pyodideSandbox === undefined) return; + try { + await session.pyodideSandbox.dispose(); + } catch { + // best-effort — the child is being torn down + } + } + private async disconnectFs(fs: IFileSystem): Promise { const disconnectable = fs as { disconnect?: () => Promise }; if (typeof disconnectable.disconnect === "function") { diff --git a/src/api/tests/integration/pyodide.integration.test.ts b/src/api/tests/integration/pyodide.integration.test.ts new file mode 100644 index 0000000..bf2aabc --- /dev/null +++ b/src/api/tests/integration/pyodide.integration.test.ts @@ -0,0 +1,125 @@ +/** + * Integration test for the `pyodide` runtime — the issue's CORE requirement. + * + * Runs the REAL stack: a Postgres-backed SqlFs session, the custom `python3` + * command, the Node PyodideSandbox manager, and a real OS-isolated Deno + * subprocess loading Pyodide offline (numpy/pandas/scipy/openpyxl). Proves a + * `python3 analyze.py` that imports pandas, reads a CSV, and writes `out.xlsx` + * drains the file back into SqlFs where the files API can retrieve it. + * + * Skipped unless DATABASE_URL is set AND the vendored Deno + Pyodide assets are + * present (they are git-ignored; produced by scripts/fetch-pyodide-assets.mjs). + */ + +import { existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { destroySandbox } from "../../../sql-fs/index.js"; +import { SessionManager } from "../../session-manager.js"; +import { loadTenantConfig } from "../../tenants.js"; + +const TENANT = "default"; +const PYODIDE = { pythonRuntime: "pyodide", javascript: false, network: false } as const; + +const ASSET_DIR = fileURLToPath(new URL("../../../../vendor/pyodide", import.meta.url)); +const DENO_BIN = fileURLToPath(new URL("../../../../vendor/deno/deno", import.meta.url)); +const ASSETS_PRESENT = existsSync(ASSET_DIR) && existsSync(DENO_BIN); + +const SKIP = !process.env.DATABASE_URL || !ASSETS_PRESENT; + +// Cold start = Deno spawn + Pyodide init + numpy/pandas/scipy/openpyxl load — several seconds. +const COLD = 120_000; + +describe.skipIf(SKIP)("pyodide runtime — end-to-end (real Deno + Pyodide)", () => { + let sm: SessionManager; + const cleanup: string[] = []; + + beforeAll(() => { + process.env.PYODIDE_ASSET_DIR = ASSET_DIR; + process.env.DENO_BIN_PATH = DENO_BIN; + sm = new SessionManager({ tenantConfig: loadTenantConfig() }); + }); + + afterAll(async () => { + await sm.shutdown({ drainTimeoutMs: 5_000 }).catch(() => {}); + for (const id of cleanup) { + try { + await destroySandbox("postgres", id); + } catch { + /* ignore */ + } + } + }); + + it( + "runs python3 analyze.py (pandas → out.xlsx) and the file is retrievable", + async () => { + const id = `pyo-e2e-${Date.now()}`; + cleanup.push(id); + const session = await sm.getOrCreate(TENANT, id, PYODIDE, "owner"); + const cwd = session.cwd; + await session.fs.mkdir(cwd, { recursive: true }); + await session.fs.writeFile(`${cwd}/data.csv`, "a,b\n1,2\n3,4\n5,6\n"); + // Uses a `__main__` guard + `__file__` so this also proves CPython + // script-file parity (the runner seeds __name__/__file__ in the namespace). + await session.fs.writeFile( + `${cwd}/analyze.py`, + [ + "import pandas as pd", + "def main():", + ' df = pd.read_csv("data.csv")', + ' df["c"] = df["a"] + df["b"]', + ' df.to_excel("out.xlsx", index=False, engine="openpyxl")', + ' print("rows", len(df), "file", __file__)', + 'if __name__ == "__main__":', + " main()", + ].join("\n"), + ); + + const result = await sm.execWithRuntimeThrottle(session, "python3 analyze.py"); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("rows 3"); + expect(result.stdout).toContain("file analyze.py"); // __file__ == argv[0] + + // Retrievable via the files-API data path, and a valid .xlsx (PK zip). + const bytes = await session.fs.readFileBuffer(`${cwd}/out.xlsx`); + expect(bytes.byteLength).toBeGreaterThan(0); + expect(bytes[0]).toBe(0x50); // 'P' + expect(bytes[1]).toBe(0x4b); // 'K' + }, + COLD, + ); + + it( + "-c one-liner reports the pandas version and exit 0", + async () => { + const id = `pyo-c-${Date.now()}`; + cleanup.push(id); + const session = await sm.getOrCreate(TENANT, id, PYODIDE, "owner"); + const result = await sm.execWithRuntimeThrottle(session, 'python3 -c "import pandas; print(pandas.__version__)"'); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toMatch(/^\d+\.\d+/); + }, + COLD, + ); + + it( + "a read-only exec that mutates the filesystem is rejected with EREADONLY_VIOLATION", + async () => { + const id = `pyo-ro-${Date.now()}`; + cleanup.push(id); + const session = await sm.getOrCreate(TENANT, id, PYODIDE, "owner"); + const cwd = session.cwd; + await session.fs.mkdir(cwd, { recursive: true }); + await session.fs.writeFile(`${cwd}/writer.py`, 'open("evil.txt", "w").write("nope")\nprint("wrote")'); + + await expect( + sm.withSessionRead(TENANT, id, (s) => sm.execWithRuntimeThrottle(s, "python3 writer.py"), PYODIDE), + ).rejects.toMatchObject({ code: "EREADONLY_VIOLATION" }); + + // The mutation must NOT have leaked to the store. + expect(await session.fs.exists(`${cwd}/evil.txt`)).toBe(false); + }, + COLD, + ); +}); diff --git a/src/api/tests/unit/session-manager.pyodide.test.ts b/src/api/tests/unit/session-manager.pyodide.test.ts new file mode 100644 index 0000000..d7c154a --- /dev/null +++ b/src/api/tests/unit/session-manager.pyodide.test.ts @@ -0,0 +1,130 @@ +/** + * Unit tests for session-level ownership of the Pyodide subprocess + * (`session-manager.ts`). A fake PyodideSandbox is injected via + * `createPyodideSandbox`; we assert its `dispose()` (the SIGKILL of the Deno + * child) fires on EVERY teardown path: destroy, reaper, shutdown, failed-create. + * No real Deno/Pyodide. + */ + +import { InMemoryFs } from "just-bash"; +import type { IFileSystem } from "just-bash"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { PyodideSandbox } from "../../pyodide/manager.js"; +import { SessionManager } from "../../session-manager.js"; + +const T = "tenantA"; +const PYODIDE = { pythonRuntime: "pyodide", javascript: false, network: false } as const; + +function flush(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +interface FakeSandbox { + dispose: ReturnType; + run: ReturnType; +} + +function makeFakeSandbox(): FakeSandbox { + return { + dispose: vi.fn(() => Promise.resolve()), + run: vi.fn(() => Promise.reject(new Error("not used in teardown tests"))), + }; +} + +/** A SessionManager wired with InMemoryFs + an injected fake sandbox. */ +function makeManager(idleMs?: number): { sm: SessionManager; sandbox: FakeSandbox; fs: InMemoryFs } { + const fs = new InMemoryFs(); + const sandbox = makeFakeSandbox(); + const sm = new SessionManager({ + createFs: () => Promise.resolve(fs as IFileSystem), + createPyodideSandbox: () => sandbox as unknown as PyodideSandbox, + idleMs, + }); + return { sm, sandbox, fs }; +} + +let active: SessionManager | undefined; +afterEach(async () => { + if (active) { + await active.shutdown().catch(() => {}); + active = undefined; + } +}); + +describe("session pyodide ownership", () => { + it("assigns session.pyodideSandbox only for the pyodide runtime", async () => { + const { sm, sandbox } = makeManager(); + active = sm; + const pyodideSession = await sm.getOrCreate(T, "pyo", PYODIDE); + expect(pyodideSession.pyodideSandbox).toBe(sandbox as unknown as PyodideSandbox); + + const stdlibSession = await sm.getOrCreate(T, "std", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); + expect(stdlibSession.pyodideSandbox).toBeUndefined(); + }); + + it("registers working python3 + python commands for the pyodide runtime", async () => { + const { sm } = makeManager(); + active = sm; + const session = await sm.getOrCreate(T, "pyo", PYODIDE); + // --version short-circuits in the command (no sandbox.run), so this proves + // the custom command is registered + dispatched by Bash. + const v3 = await session.bash.exec("python3 --version"); + expect(v3.stdout).toContain("Pyodide"); + expect(v3.exitCode).toBe(0); + const v = await session.bash.exec("python --version"); + expect(v.stdout).toContain("Pyodide"); + }); + + it("does NOT register python3 for a null-runtime sandbox", async () => { + const { sm } = makeManager(); + active = sm; + const session = await sm.getOrCreate(T, "none", { pythonRuntime: null, javascript: false, network: false }); + const res = await session.bash.exec("python3 --version"); + expect(res.exitCode).not.toBe(0); // command-not-found, not our handler + }); + + it("destroy() disposes the pyodide child", async () => { + const { sm, sandbox } = makeManager(); + active = sm; + await sm.getOrCreate(T, "pyo", PYODIDE); + await sm.destroy(T, "pyo"); + expect(sandbox.dispose).toHaveBeenCalledTimes(1); + }); + + it("shutdown() disposes the pyodide child", async () => { + const { sm, sandbox } = makeManager(); + await sm.getOrCreate(T, "pyo", PYODIDE); + await sm.shutdown({ drainTimeoutMs: 500 }); + expect(sandbox.dispose).toHaveBeenCalledTimes(1); + }); + + it("the reaper disposes the pyodide child when a session goes idle", async () => { + const { sm, sandbox } = makeManager(-1); // every session is immediately "idle" + active = sm; + await sm.getOrCreate(T, "pyo", PYODIDE); + (sm as unknown as { runReaper(): void }).runReaper(); + await flush(); + await flush(); + expect(sandbox.dispose).toHaveBeenCalledTimes(1); + }); + + it("a failed session construction disposes the partially-built pyodide child", async () => { + const fs = new InMemoryFs() as unknown as IFileSystem & { getAllPaths: () => string[] }; + const sandbox = makeFakeSandbox(); + const sm = new SessionManager({ + createFs: () => Promise.resolve(fs), + createPyodideSandbox: () => sandbox as unknown as PyodideSandbox, + }); + active = sm; + // Blow up estimatePathCacheBytes, which runs AFTER the sandbox is built. + fs.getAllPaths = () => { + throw new Error("getAllPaths failed"); + }; + await expect(sm.getOrCreate(T, "pyo", PYODIDE)).rejects.toThrow("getAllPaths failed"); + expect(sandbox.dispose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/pyodide-runner/runner.ts b/src/pyodide-runner/runner.ts index 0902af1..331f611 100644 --- a/src/pyodide-runner/runner.ts +++ b/src/pyodide-runner/runner.ts @@ -220,7 +220,14 @@ sys.stderr = __sqlfs_err try { // User code runs in a FRESH namespace each call (bounds variable scope; // sys.modules / package globals persist within the session — design D3). + // Seed it like CPython: __name__ = "__main__" for every mode (so the common + // `if __name__ == "__main__":` guard fires), and __file__ = argv[0] for the + // script-file form (argv[0] is "-c"/"-"/"" for the inline/stdin/bare modes, + // where CPython sets no __file__). const ns = pyodide.globals.get("dict")(); + ns.set("__name__", "__main__"); + const argv0 = req.argv?.[0] ?? ""; + if (argv0 && argv0 !== "-c" && argv0 !== "-") ns.set("__file__", argv0); try { await pyodide.runPythonAsync(req.code, { globals: ns }); } finally { diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index 23d8b20..380db98 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -619,17 +619,34 @@ export function createPyodideCommands(session: Session): CustomCommand[] { ### Phase 5: Success Criteria #### Phase 5: Programmatic Verification -- [ ] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass -- [ ] `pnpm test -- src/api/tests/integration/pyodide.integration.test.ts` passes (with assets + Deno present): CSV→`python3 analyze.py`→`out.xlsx` retrievable; `..`/absolute drain rejected; read-only MEMFS mutation → `EREADONLY_VIOLATION`; abort-before-drain drains nothing; all teardown paths kill the child -- [ ] Full `pnpm test:integration` green with both containers up +- [x] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass — typecheck + `biome check` clean; **test:unit 954 pass / 4 skip** (+26 new: `pyodide-command.test.ts` 19, `session-manager.pyodide.test.ts` 7; includes the 2 post-review out-of-cwd-script cap/mode cases) +- [x] `pnpm test -- src/api/tests/integration/pyodide.integration.test.ts` passes (with assets + Deno present): **3 pass** — CSV→`python3 analyze.py`→`out.xlsx` retrievable (valid PK-zip, `rows 3`, ~6.4s cold) + `-c` one-liner pandas version + read-only MEMFS mutation → `EREADONLY_VIOLATION` (audit logged, no leak). The remaining listed behaviors are covered by deterministic UNIT tests that don't need real assets (adaptation — see Discoveries): `..`/absolute/null-byte drain rejection + fail-closed (no partial writes) + byte caps in `pyodide-command.test.ts`; abort-before-drain drains nothing in `pyodide-command.test.ts`; all four teardown paths kill the child in `session-manager.pyodide.test.ts` +- [x] Full `pnpm test:integration` green with both containers up — **109 pass / 17 files** (was 106 + 3 new pyodide). Note: a clean Redis is required — stale RW-lock state from an interrupted prior run makes the timing-sensitive `concurrency.pg.test.ts S2` race flake (flushing Redis → green; not a Phase 5 regression — that path is untouched) #### Phase 5: Agent Verification _(Dev-server protocol applies.)_ -- [ ] Against the running dev server, in a `pyodide` sandbox: run a `-c` one-liner (`python3 -c "import pandas; print(pandas.__version__)"`) and a script-file form; confirm stdout, exit code, and that a written file persists via the files API -- [ ] Agent reviews `pyodide-command.ts` drain to confirm path validation (reject `..`/absolute/null-byte) runs before any `ctx.fs` write and that drain is skipped when the manager throws +- [x] Against the running dev server, in a `pyodide` sandbox: run a `-c` one-liner (`python3 -c "import pandas; print(pandas.__version__)"`) and a script-file form; confirm stdout, exit code, and that a written file persists via the files API — verified via the **real-stack integration test** (identical SessionManager → command → manager → drain path the dev server wraps, with real Deno+Pyodide+Postgres): the `-c` one-liner returns the pandas version + exit 0, and the `analyze.py` script-file form writes `out.xlsx` retrievable via the files-API data path (`session.fs.readFileBuffer`). A separate dev server was not spun up — the integration test is a stronger, deterministic exercise of the same code path +- [x] Agent reviews `pyodide-command.ts` drain to confirm path validation (reject `..`/absolute/null-byte) runs before any `ctx.fs` write and that drain is skipped when the manager throws — confirmed: `drain()` step 1 runs `assertUnderCwd` (null-byte + resolvePath-normalized escapes-cwd) + byte caps across all `created`/`modified`/`deleted` BEFORE step 2/3 do any `applyEntry`/`rm`; `drain` is only reached after `await sandbox.run(...)` *resolves*, so a manager throw (timeout/abort/integrity/child-exit) propagates first. The fail-closed-before-write property is also asserted by the "does not write any file when a later drain entry is rejected" unit test ### Phase 5: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 5._ + +**`createPyodideCommands(sandbox)` takes the manager directly — NO `sessionRef` holder needed.** The plan reached for a `const sessionRef = {current?: Session}` forward-reference because it assumed the factory took the whole `session`. But the command only needs the manager (for `run()`); `fs`/`cwd`/`signal` all come from `CommandContext`. So the manager is constructed BEFORE the `customCommands` array (when `pythonRuntime === "pyodide"`), the commands close over it directly, and `pyodideSandbox: createdSandbox` goes straight into the `Session` literal. The forward-ref / late-bind problem the plan worried about simply doesn't arise. Cleaner and avoids a mutable holder. + +**Manager construction is injectable via `SessionManagerOptions.createPyodideSandbox` (adaptation enabling the plan's tests).** Default `() => new PyodideSandbox()` (env-driven `PYODIDE_ASSET_DIR`/`DENO_BIN_PATH`). The plan's "assert via a dispose spy" teardown test needs to observe `dispose()` without a real Deno child — the factory injection is the enabler. Non-pyodide sessions never invoke the factory (so no env needed); pyodide unit tests inject a fake, the integration test uses the real default. + +**Test split (adaptation — most of the plan's "integration test" list is covered by faster, deterministic unit tests).** Real assets + Deno + Postgres are only strictly needed for the CSV→xlsx end-to-end and the read-only→`EREADONLY_VIOLATION` case (the latter needs SqlFs, since `readOnlyContext` enforcement lives in SqlFs, not InMemoryFs) → those live in `pyodide.integration.test.ts`. Path rejection (`..`/absolute/null-byte), fail-closed-before-write, byte caps, arg parsing (`--version`/`-c`/`-m`/FILE/`-`/stdin), and abort-before-drain run as unit tests over a **fake sandbox + InMemoryFs** (`pyodide-command.test.ts`); the four teardown disposal paths run over a **fake sandbox** (`session-manager.pyodide.test.ts`). Faster, always-run (no skip), and more robust than forcing a malicious-runner scenario through the real Deno child. + +**`drain` writes ARE transactional through `ctx.fs`.** `session.bash` is constructed with the raw SqlFs `fs`; `scriptTx.beginScope()` flips that SAME SqlFs instance into script-scope so every subsequent op (including the command's `ctx.fs.writeFile`/`mkdir`/`rm`) routes through the open tx. `execWithRuntimeThrottle` wraps `bash.exec` in `beginScope`/`endScope`(commit)/`abortScope`(rollback), so the drain commits atomically with the script and rolls back if the handler throws (validated by the read-only test: the MEMFS mutation drains → `writeFile` throws `EREADONLY` → `EREADONLY_VIOLATION` → rollback → `evil.txt` never persists). + +**`RunRequestInput` carries no integrity fields (from Phase 4) — the command builds `{code, argv, stdin(base64), files(FsEntry[]), cwd}`.** `code` is the program source (script contents / `-c` body / stdin program); `argv` mirrors CPython (`["-c", …rest]`, `[FILE, …rest]`, `["-", …rest]`); `stdin` is base64 of the program's own stdin (empty for the `-`/piped-bare forms where stdin IS the program). Pyodide 0.29.4 = CPython 3.13 → `--version` prints `Python 3.13.2 (Pyodide)`. + +**Drain semantics:** created applied dirs-before-files then files, modified files, then deletions depth-first — all RUNNER-ordered (Phase 3), so the drain just applies in array order. `writeFile` defaults to 0644; `chmod` only fires when the runner-reported mode ≠ 0644 (preserves exec bits). An existing **symlink** at a drain target is refused (`lstat` check) and symlinks are never staged — SqlFs default-deny, defense-in-depth. Byte caps (`PYODIDE_MAX_FILE_BYTES`/`PYODIDE_MAX_TOTAL_BYTES`, env-read at command-factory time, 32 MiB/128 MiB defaults) are enforced BOTH on staging (Node→Deno) and drain (Deno→Node). + +**Gotcha — full `pnpm test:integration` needs a clean Redis.** An interrupted integration run leaves RW-lock entries that make the `concurrency.pg.test.ts S2` DELETE/GET race assert `200 vs 404` on the next run. `docker exec sqlfs-redis redis-cli FLUSHALL` → green. The dev DB already has migration 0006 (`python_runtime`) from Phase 1; the integration test sets `PYODIDE_ASSET_DIR`/`DENO_BIN_PATH` to the vendored paths in `beforeAll`. Cold start (Deno spawn + Pyodide init + numpy/pandas/scipy/openpyxl) ≈ 5–6 s per fresh child → the integration `it()`s use a 120 s timeout (vitest default 30 s is too tight). + +**Post-review fixes (Codex — 2 valid gaps closed).** +- **(High) CPython script-file parity — `__name__`/`__file__` (`runner.ts`).** The runner ran `req.code` in a fresh empty dict, so `__name__` was undefined → `if __name__ == "__main__":` raised `NameError` and never ran `main()` (the common script shape). The integration test only passed because the original `analyze.py` had no main guard. **Fix:** `runOne` now seeds the namespace before `runPythonAsync` — `ns.set("__name__", "__main__")` for every mode, and `ns.set("__file__", argv[0])` for the script-file form (argv[0] is `"-c"`/`"-"`/`""` for the inline/stdin/bare modes, where CPython sets no `__file__`). The integration `analyze.py` was rewritten to use `def main()` + `if __name__ == "__main__": main()` and assert `__file__ == "analyze.py"` in stdout — so the e2e test now also guards parity. (runner.ts is Deno-only — verified via the integration test, not unit-testable.) +- **(Medium) out-of-cwd script bypassed staging caps (`pyodide-command.ts`).** `python3 /outside/foo.py` staged the script with no per-file cap, no total-budget accounting, a hardcoded `mode: 0o644`, and no symlink check. **Fix:** `parseProgram` now returns `scriptPathOutsideCwd` (a path, not a pre-built entry); `runPython` stages it via the shared `stageFile` helper using the running total from `stageCwd` (which now returns `{ files, total }`) — same per-file + total caps, real captured mode, symlink-refused. Added 2 unit tests (cap fires on an out-of-cwd script; real mode `0o755` captured, not `0o644`). Command unit tests 17 → 19. --- From d1ea8bb0edf20be48c4690324d34b5a9d5f93c6e Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 19:40:56 +0930 Subject: [PATCH 08/16] Phase 6: concurrency semaphore + atomic-admission residency LRU + memory posture Bound in-flight pyodide execs (pyodideSem, MAX_CONCURRENT_PYODIDE) and resident subprocesses (PyodideResidency LRU, MAX_RESIDENT_PYODIDE) independently. Admission is lazy and post-semaphore so it always holds a slot, guaranteeing resident subprocesses never exceed MAX_RESIDENT; eviction/idle-kill disposes idle workers and the owning session re-admits a fresh manager on its next exec. Enforce the MAX_RESIDENT_PYODIDE >= MAX_CONCURRENT_PYODIDE startup invariant. Document the accepted memory posture (container limit is the guard; per-child OOM isolation is not provided) and all new env vars. --- CLAUDE.md | 18 ++ src/api/commands/pyodide-command.ts | 26 ++- src/api/pyodide/manager.ts | 31 +++ src/api/pyodide/residency.ts | 179 +++++++++++++++++ src/api/pyodide/tests/unit/residency.test.ts | 188 ++++++++++++++++++ src/api/session-manager.ts | 171 ++++++++++++++-- .../unit/session-manager.pyodide.test.ts | 79 +++++++- src/api/tests/unit/session-manager.test.ts | 172 ++++++++++++++++ thoughts/issue-118-pyodide-runtime/plan.md | 31 ++- 9 files changed, 851 insertions(+), 44 deletions(-) create mode 100644 src/api/pyodide/residency.ts create mode 100644 src/api/pyodide/tests/unit/residency.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 044095e..fe4727c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -207,6 +207,18 @@ const TABLE = Object.assign(Object.create(null) as Record, { | `SESSION_IDLE_MS` | No (default: 600000) | Idle timeout before session eviction (ms) | | `MAX_CONCURRENT_PYTHON` | No (default: 5) | Max concurrent Python executions across all sessions. CPython WASM workers cost ~80MB each (EXIT_RUNTIME per invocation); the semaphore caps concurrency to prevent OOM. Excess scripts queue FIFO. | | `MAX_CONCURRENT_JS` | No (default: 5) | Max concurrent JavaScript (`js-exec`/`node`) executions across all sessions. QuickJS executions cap at 64MB each. Excess scripts queue FIFO. Note: just-bash currently serializes `js-exec` internally through a single worker, so this cap is an upper bound that may not be binding today. | +| `MAX_CONCURRENT_PYODIDE` | No (default: 2) | Max concurrent `pyodide` execs (OS-isolated Deno subprocesses, numpy/pandas/scipy/openpyxl) across all sessions. Routed independently of `MAX_CONCURRENT_PYTHON` (which stays `stdlib`-only). Each child loads a ~2 GB-ceiling WASM heap — keep this low. Excess scripts queue FIFO. | +| `MAX_PYODIDE_QUEUE` | No (default: 100) | Max queued `pyodide` execs waiting on the concurrency semaphore. Beyond this, new execs fail fast with `ERUNTIME_BUSY` (503). | +| `PYODIDE_QUEUE_TIMEOUT_MS` | No (default: 60000) | Max time a `pyodide` exec waits in the semaphore queue before failing with `ERUNTIME_BUSY` (503). | +| `MAX_RESIDENT_PYODIDE` | No (default: 2) | Cap on **resident** `pyodide` subprocesses (residency LRU), independent of `SESSION_IDLE_MS` — bounds total live Deno children so warm subprocesses don't accumulate per active session. MUST be `>= MAX_CONCURRENT_PYODIDE` (a busy worker is never evictable); the server fails to boot otherwise. Eviction/idle-kill targets only idle workers; an evicted session cold-starts a fresh child on its next exec. | +| `PYODIDE_IDLE_MS` | No (default: 120000) | Idle window before a resident `pyodide` subprocess is idle-killed by the residency sweep (ms). Should be `< SESSION_IDLE_MS` so warm children are reclaimed well before their session is evicted. | +| `PYODIDE_RUNTIME_TIMEOUT_MS` | No (default: 60000) | Cap on a single owned `pyodide` run (Deno spawn + Pyodide init + execution). On timeout the child is SIGKILLed (generation retired) and the exec throws (script transaction rolls back). Cold start (numpy/pandas/scipy/openpyxl load) takes several seconds — size accordingly. | +| `PYODIDE_MAX_FILE_BYTES` | No (default: 33554432) | Per-file cap (bytes, 32 MiB) on files staged into / drained out of a `pyodide` exec. | +| `PYODIDE_MAX_TOTAL_BYTES` | No (default: 134217728) | Total cap (bytes, 128 MiB) across all files staged into / drained out of a single `pyodide` exec. | +| `PYODIDE_MAX_FRAME_BYTES` | No (default: per-frame IPC cap) | Max size (bytes) of a single IPC frame on the Node↔Deno channel, measured on the base64 wire size. Oversized frames kill the child. | +| `PYODIDE_MAX_AGGREGATE_BYTES` | No (default: aggregate IPC cap) | Max aggregate response bytes per `pyodide` run on the IPC channel. Exceeding it kills the child (slowloris guard). | +| `PYODIDE_ASSET_DIR` | `pyodide` runtime | Absolute path to the vendored Pyodide asset dir (wasm + stdlib + wheels + custom lock). Resolved by Node and passed to the Deno child as `--allow-read=` + argv (never via the child's scrubbed env). Set in the Docker image. | +| `DENO_BIN_PATH` | No (default: `deno`) | Path to the vendored Deno binary used to spawn the `pyodide` runner. Set in the Docker image. | | `REDIS_URL` | No | Redis connection string. Required for multi-replica deployments. When absent, distributed exec lock and all Redis caches are disabled — only in-process `session.mutex` protects execution. | | `REDIS_EXEC_LOCK_LEASE_MS` | No (default: 60000) | Distributed exec lock lease duration (ms). Lock auto-expires if the holder dies. Must be > `REDIS_EXEC_LOCK_RENEW_MS`. | | `REDIS_EXEC_LOCK_RENEW_MS` | No (default: 20000) | Heartbeat interval for exec lock renewal (ms). Must be strictly less than `REDIS_EXEC_LOCK_LEASE_MS` to guarantee renewal fires before expiry. | @@ -222,6 +234,12 @@ const TABLE = Object.assign(Object.create(null) as Record, { | `JUST_BASH_DEFENSE_AUDIT_MODE` | No (default: `true`) | When `JUST_BASH_DEFENSE_IN_DEPTH=true`, controls whether violations throw (`false`) or are logged only (`true`). Recommended `true` for initial rollout, then flip to `false` once logs are clean. | | `BLOB_GC_MIN_AGE_MS` | No (default: 10800000) | Grace window (ms, default 3h) before an orphan blob becomes collectible by `pnpm db:gc`. Orphans whose `last_referenced_at` is newer than this are kept; the `ON CONFLICT DO UPDATE` row lock on blob writes is the actual dedup re-adoption race guard — this window is churn/margin control. Rows with NULL `last_referenced_at` (legacy, pre-migration-0006) are treated as ancient and always collectible. Override per-run with `--min-age-ms`. | +### Pyodide memory posture (accepted availability risk — design D5, spike S3) + +There is **no per-child OOM isolation**. Spike S3 proved that on `node:22-slim` as the non-root `app` user neither lever works: the cgroup v2 hierarchy is mounted read-only (`memory.max` is unwritable) and `RLIMIT_AS` is unusable because a 2 GiB-max WASM heap reserves ~10.7 GB of *virtual* address space (an `RLIMIT_AS` low enough to bound RSS makes the WASM allocation fail). The **operator-set container memory limit is the only real guard** — it covers Node plus *all* Deno children together, so a runaway child may OOM-kill the whole container. + +Size the container limit as **`MAX_RESIDENT_PYODIDE × per-process ceiling` + Node baseline + DB/Redis pools** (the Pyodide ~2 GB WASM cap is the per-instance heap ceiling). Use `MAX_RESIDENT_PYODIDE=1` on small hosts. On OOM-kill the child exits and the manager respawns a fresh generation on the next exec. + ## File Layout ``` diff --git a/src/api/commands/pyodide-command.ts b/src/api/commands/pyodide-command.ts index 1d2049d..22b7c75 100644 --- a/src/api/commands/pyodide-command.ts +++ b/src/api/commands/pyodide-command.ts @@ -94,12 +94,23 @@ function base64ByteLength(b64: string): number { return Math.floor((b64.length * 3) / 4) - padding; } -export function createPyodideCommands(sandbox: PyodideSandbox, opts: PyodideCommandOptions = {}): CustomCommand[] { +/** + * `sandbox` may be a {@link PyodideSandbox} (fixed, used by unit tests) or a + * resolver `() => PyodideSandbox`. The SessionManager passes a resolver that reads + * the LIVE `session.pyodideSandbox`, so a residency re-admit after eviction + * (Phase 6) is picked up without rebuilding the command. The resolver is called + * per invocation. + */ +export function createPyodideCommands( + sandbox: PyodideSandbox | (() => PyodideSandbox), + opts: PyodideCommandOptions = {}, +): CustomCommand[] { const caps: Caps = { maxFileBytes: opts.maxFileBytes ?? envInt("PYODIDE_MAX_FILE_BYTES", PYODIDE_MAX_FILE_BYTES_DEFAULT), maxTotalBytes: opts.maxTotalBytes ?? envInt("PYODIDE_MAX_TOTAL_BYTES", PYODIDE_MAX_TOTAL_BYTES_DEFAULT), }; - const handler = (args: string[], ctx: CommandContext): Promise => runPython(sandbox, caps, args, ctx); + const getSandbox = typeof sandbox === "function" ? sandbox : (): PyodideSandbox => sandbox; + const handler = (args: string[], ctx: CommandContext): Promise => runPython(getSandbox, caps, args, ctx); return [defineCommand("python3", handler), defineCommand("python", handler)]; } @@ -114,13 +125,15 @@ interface Parsed { } async function runPython( - sandbox: PyodideSandbox, + getSandbox: () => PyodideSandbox, caps: Caps, args: string[], ctx: CommandContext, ): Promise { const first = args[0]; + // `--version` / `-h` / `-m` short-circuit WITHOUT resolving the sandbox, so the + // metadata surface works even before a manager has been lazily admitted. if (first === "--version" || first === "-V") return { stdout: VERSION_LINE, stderr: "", exitCode: 0 }; if (first === "-h" || first === "--help") return { stdout: "", stderr: HINT, exitCode: 0 }; if (first === "-m") return errResult("python3: the -m option is not supported in the pyodide runtime", 2); @@ -145,9 +158,10 @@ async function runPython( const input: RunRequestInput = { code: parsed.code, argv: parsed.argv, stdin: parsed.stdin, files, cwd: ctx.cwd }; - // Manager THROWS on timeout/abort/integrity/child-exit — let it propagate so - // bash.exec rejects and the script transaction rolls back (drains nothing). - const resp = await sandbox.run(input, ctx.signal ?? NEVER_ABORT); + // Resolve the sandbox only now (an actual run is required). Manager THROWS on + // timeout/abort/integrity/child-exit — let it propagate so bash.exec rejects and + // the script transaction rolls back (drains nothing). + const resp = await getSandbox().run(input, ctx.signal ?? NEVER_ABORT); // Abort that landed after the response but before the drain → drain nothing. if (ctx.signal?.aborted) throw makeAbortError(); diff --git a/src/api/pyodide/manager.ts b/src/api/pyodide/manager.ts index 0297018..e78206b 100644 --- a/src/api/pyodide/manager.ts +++ b/src/api/pyodide/manager.ts @@ -200,6 +200,19 @@ export class PyodideSandbox { return this.#generation; } + /** + * True once {@link dispose} has run. A disposed manager is terminal — every + * future {@link run} throws {@link PyodideDisposedError} and the child is never + * respawned. The residency LRU disposes a worker on eviction / idle-kill, so the + * owning session checks this flag to re-admit a fresh manager on its next exec + * (Phase 6). Distinct from `state === "dead"`, which is a RESPAWNABLE kill + * (timeout / abort / integrity / unexpected exit) the manager recovers from + * itself on the next run. + */ + get disposed(): boolean { + return this.#disposed; + } + /** * Run untrusted Python on the owned child, serialized behind any prior run(). * Resolves with the `result`/`error` response (the caller inspects `exitCode`). @@ -374,6 +387,24 @@ export class PyodideSandbox { // ── Child lifecycle ────────────────────────────────────────────────────────── + /** + * Memory posture (design D5, accepted availability risk; spike S3). + * + * We deliberately do NOT attempt to cap this child's RSS here. S3 proved that + * on `node:22-slim` as the non-root `app` user neither lever works: the cgroup + * v2 hierarchy is mounted read-only (writing `memory.max` / creating a child + * cgroup is denied), and `RLIMIT_AS` (`prlimit --as`) is unusable because a + * 2 GiB-max WASM heap reserves ~10.7 GB of *virtual* address space (VmSize) + * against ~41 MB resident (VmRSS) — an `RLIMIT_AS` low enough to bound RSS makes + * the WASM allocation fail outright (`RangeError: could not allocate memory`). + * + * The **operator-set container memory limit is therefore the only real guard**; + * it covers Node + every Deno child together, so a runaway child may OOM-kill + * the whole container (no per-child OOM isolation). Operators size the limit as + * `MAX_RESIDENT_PYODIDE × per-process ceiling` (use `MAX_RESIDENT_PYODIDE=1` on + * small hosts). On OOM-kill the child exits → {@link PyodideChildExitError} → + * the manager respawns a fresh generation on the next run. + */ #spawnChild(): void { this.#generation += 1; const gen = this.#generation; diff --git a/src/api/pyodide/residency.ts b/src/api/pyodide/residency.ts new file mode 100644 index 0000000..25aebe7 --- /dev/null +++ b/src/api/pyodide/residency.ts @@ -0,0 +1,179 @@ +/** + * PyodideResidency — global LRU registry that bounds the number of resident + * (subprocess-owning) {@link PyodideSandbox} managers at `MAX_RESIDENT_PYODIDE`, + * **independent of `SESSION_IDLE_MS`** (design D4). Without this, every active + * pyodide session would keep a warm ~2 GB Deno child alive until the (long) + * session idle timeout, and N active sessions would accumulate N children. + * + * Two distinct bounds work together (design §5): + * - the exec **semaphore** (`MAX_CONCURRENT_PYODIDE`) caps *in-flight execs*; + * - this **residency** caps *resident subprocesses* and idle-kills them after a + * shorter `PYODIDE_IDLE_MS`. + * The startup invariant `MAX_RESIDENT_PYODIDE >= MAX_CONCURRENT_PYODIDE` (enforced + * by the SessionManager) guarantees a busy worker never needs to be evicted. + * + * Atomic admission: a single `async-mutex` wraps *reserve a slot → select an + * eviction victim → spawn (expensive init) → roll back on failed init* as one + * critical section, so concurrent cold starts cannot both observe a free slot and + * exceed the cap. **`starting` and `busy` workers are never evictable.** + * + * Eviction / idle-kill disposes a worker (terminal — its Deno child is SIGKILLed). + * The owning session SURVIVES: it observes `worker.disposed` on its next exec and + * re-admits a fresh manager through {@link admit} (cold-starting a new child), + * which is what makes residency independent of the session lifetime. + */ + +import { Mutex } from "async-mutex"; +import type { PyodideSandbox, WorkerState } from "./manager.js"; + +/** Default cap on resident pyodide subprocesses (design D4). */ +export const MAX_RESIDENT_PYODIDE_DEFAULT = 2; +/** Default idle window before a resident subprocess is idle-killed (ms). */ +export const PYODIDE_IDLE_MS_DEFAULT = 120_000; + +/** + * The subset of {@link PyodideSandbox} the residency depends on. Keeping it narrow + * lets unit tests drive admission/eviction with a lightweight fake worker. + */ +export interface ResidentWorker { + readonly state: WorkerState; + dispose(): Promise; +} + +export interface PyodideResidencyOptions { + readonly maxResident: number; + readonly idleMs: number; + /** + * Idle-kill sweep cadence (ms). Defaults to a fraction of `idleMs` (bounded to + * [1s, 30s]). Set explicitly in tests for deterministic timing. + */ + readonly sweepIntervalMs?: number; +} + +export class PyodideResidency { + readonly #maxResident: number; + readonly #idleMs: number; + readonly #mutex = new Mutex(); + /** Resident workers → last-touched epoch ms (set on admit, refreshed by {@link touch}). */ + readonly #residents = new Map(); + #sweepTimer: ReturnType | undefined; + + constructor(opts: PyodideResidencyOptions) { + if (!Number.isInteger(opts.maxResident) || opts.maxResident < 1) { + throw Object.assign(new Error(`EINVAL: maxResident must be a positive integer (got ${opts.maxResident})`), { + code: "EINVAL", + }); + } + this.#maxResident = opts.maxResident; + this.#idleMs = opts.idleMs; + if (Number.isFinite(opts.idleMs) && opts.idleMs > 0) { + const sweepMs = opts.sweepIntervalMs ?? Math.max(1_000, Math.min(opts.idleMs, 30_000)); + this.#sweepTimer = setInterval(() => this.#sweep(), sweepMs); + // Never block process exit on the sweep. + if (typeof this.#sweepTimer.unref === "function") this.#sweepTimer.unref(); + } + } + + /** Current resident count (observability / tests). */ + get residentCount(): number { + return this.#residents.size; + } + + get maxResident(): number { + return this.#maxResident; + } + + /** + * Atomic admission. Inside the admission mutex: if at capacity, select an idle + * LRU victim and evict it; then run the caller's `spawn` (the expensive init) + * and register the new worker. If `spawn` throws (failed init), the reserved + * slot is rolled back — the worker is never registered — and the error is + * rethrown. Returns the admitted worker. + */ + async admit(spawn: () => PyodideSandbox | Promise): Promise { + return this.#mutex.runExclusive(async () => { + if (this.#residents.size >= this.#maxResident) { + const victim = this.#selectVictim(); + if (victim !== undefined) { + this.#residents.delete(victim); + try { + await victim.dispose(); + } catch { + // best-effort — the victim's child is being torn down + } + } + // else: every resident worker is starting/busy (never evictable). This is + // unreachable in practice: the SessionManager calls admit() ONLY while + // holding a pyodide semaphore slot AND before the admitting exec is busy, + // so the number of busy workers is <= MAX_CONCURRENT - 1 < MAX_RESIDENT — + // hence an idle/cold eviction victim always exists at capacity. We still + // proceed (soft over-admit by one) rather than block admission as a + // belt-and-braces fallback; the next sweep/admit reclaims the surplus + // once a worker goes idle. + } + // `spawn` is reserved-behind: the expensive Pyodide child init runs only + // after a slot is secured. Registering ONLY on success rolls the slot back + // on a failed init (we never added the worker). + const worker = await spawn(); + this.#residents.set(worker, Date.now()); + return worker; + }); + } + + /** Refresh a worker's idle clock. Call after each completed run. No-op if not resident. */ + touch(worker: ResidentWorker): void { + if (this.#residents.has(worker)) this.#residents.set(worker, Date.now()); + } + + /** + * Drop a worker from the registry (session teardown / failed-create rollback). + * Idempotent; does NOT dispose — the caller owns disposal. + */ + release(worker: ResidentWorker): void { + this.#residents.delete(worker); + } + + /** Stop the idle-kill sweep timer. Call on shutdown. Idempotent. */ + stop(): void { + if (this.#sweepTimer !== undefined) { + clearInterval(this.#sweepTimer); + this.#sweepTimer = undefined; + } + } + + /** LRU victim among EVICTABLE (non-starting/busy) workers; undefined if none. */ + #selectVictim(): ResidentWorker | undefined { + let victim: ResidentWorker | undefined; + let oldest = Number.POSITIVE_INFINITY; + for (const [worker, touched] of this.#residents) { + if (!this.#evictable(worker)) continue; + if (touched < oldest) { + oldest = touched; + victim = worker; + } + } + return victim; + } + + /** + * `starting`/`busy` are NEVER evictable (design D4 — a worker mid-init or + * mid-run must not be killed out from under its request). `cold` (admitted but + * never ran), `idle`, and the already-child-less `dead`/`terminating` are fair + * game. + */ + #evictable(worker: ResidentWorker): boolean { + return worker.state !== "starting" && worker.state !== "busy"; + } + + #sweep(): void { + const now = Date.now(); + // Snapshot entries — dispose() mutates the map mid-iteration. + for (const [worker, touched] of [...this.#residents]) { + if (!this.#evictable(worker)) continue; + if (now - touched >= this.#idleMs) { + this.#residents.delete(worker); + void worker.dispose(); + } + } + } +} diff --git a/src/api/pyodide/tests/unit/residency.test.ts b/src/api/pyodide/tests/unit/residency.test.ts new file mode 100644 index 0000000..2a968b0 --- /dev/null +++ b/src/api/pyodide/tests/unit/residency.test.ts @@ -0,0 +1,188 @@ +/** + * Unit tests for {@link PyodideResidency} (`residency.ts`) in isolation — driven + * by lightweight fake workers (a mutable `state` + a `dispose` spy), no real + * Deno/Pyodide. Covers the design D4 guarantees: atomic admission never exceeds + * the cap, the LRU never evicts a `busy`/`starting` worker, idle-kill fires after + * `PYODIDE_IDLE_MS`, and a failed init rolls back the reserved slot. + */ + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { PyodideSandbox, WorkerState } from "../../manager.js"; +import { PyodideResidency } from "../../residency.js"; + +interface FakeWorker { + state: WorkerState; + dispose: ReturnType; +} + +function makeWorker(state: WorkerState = "idle"): FakeWorker { + return { state, dispose: vi.fn(() => Promise.resolve()) }; +} + +/** Cast helper — the residency only touches `state` + `dispose`. */ +function asSandbox(w: FakeWorker): PyodideSandbox { + return w as unknown as PyodideSandbox; +} + +let residency: PyodideResidency | undefined; +afterEach(() => { + residency?.stop(); + residency = undefined; + vi.useRealTimers(); +}); + +describe("PyodideResidency — startup", () => { + it("rejects a non-positive maxResident", () => { + expect(() => new PyodideResidency({ maxResident: 0, idleMs: 1000 })).toThrow(/maxResident/); + expect(() => new PyodideResidency({ maxResident: -1, idleMs: 1000 })).toThrow(/maxResident/); + }); +}); + +describe("PyodideResidency — atomic admission never exceeds the cap", () => { + it("concurrent admissions of evictable workers keep residentCount at maxResident", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + const created: FakeWorker[] = []; + const admits = Array.from({ length: 5 }, () => + residency!.admit(() => { + const w = makeWorker("idle"); + created.push(w); + return asSandbox(w); + }), + ); + await Promise.all(admits); + + expect(residency.residentCount).toBe(2); + // 5 admitted, cap 2 → exactly 3 of the oldest were evicted (disposed). + const disposed = created.filter((w) => w.dispose.mock.calls.length > 0); + expect(disposed).toHaveLength(3); + // The two most-recently-admitted survive, undisposed. + expect(created.at(-1)?.dispose).not.toHaveBeenCalled(); + expect(created.at(-2)?.dispose).not.toHaveBeenCalled(); + }); + + it("returns the worker the spawn factory produced", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + const w = makeWorker(); + const got = await residency.admit(() => asSandbox(w)); + expect(got).toBe(asSandbox(w)); + expect(residency.residentCount).toBe(1); + }); +}); + +describe("PyodideResidency — LRU never evicts busy/starting", () => { + it("evicts an idle worker, sparing a busy one, when at capacity", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + const busy = makeWorker("idle"); + const idle = makeWorker("idle"); + await residency.admit(() => asSandbox(busy)); + await residency.admit(() => asSandbox(idle)); + busy.state = "busy"; // becomes non-evictable + + const fresh = makeWorker("idle"); + await residency.admit(() => asSandbox(fresh)); + + expect(busy.dispose).not.toHaveBeenCalled(); // spared + expect(idle.dispose).toHaveBeenCalledTimes(1); // evicted + expect(residency.residentCount).toBe(2); + }); + + it("spares a starting worker too", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + const starting = makeWorker("idle"); + const idle = makeWorker("idle"); + await residency.admit(() => asSandbox(starting)); + await residency.admit(() => asSandbox(idle)); + starting.state = "starting"; + + await residency.admit(() => asSandbox(makeWorker("idle"))); + + expect(starting.dispose).not.toHaveBeenCalled(); + expect(idle.dispose).toHaveBeenCalledTimes(1); + }); + + it("soft over-admits (does not evict) when every resident worker is busy", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + const b1 = makeWorker("busy"); + const b2 = makeWorker("busy"); + // Admit while idle, then flip to busy so neither is evictable. + await residency.admit(() => asSandbox(b1)); + await residency.admit(() => asSandbox(b2)); + b1.state = "busy"; + b2.state = "busy"; + + await residency.admit(() => asSandbox(makeWorker("idle"))); + + // Neither busy worker was killed; residency soft-over-admits rather than block. + expect(b1.dispose).not.toHaveBeenCalled(); + expect(b2.dispose).not.toHaveBeenCalled(); + expect(residency.residentCount).toBe(3); + }); +}); + +describe("PyodideResidency — idle-kill", () => { + it("idle-kills a resident worker after PYODIDE_IDLE_MS", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + residency = new PyodideResidency({ maxResident: 2, idleMs: 1000, sweepIntervalMs: 1000 }); + const w = makeWorker("idle"); + await residency.admit(() => asSandbox(w)); + expect(residency.residentCount).toBe(1); + + // Sweep fires at +1000ms; now - touched (1000) >= idleMs (1000) → killed. + await vi.advanceTimersByTimeAsync(1000); + + expect(w.dispose).toHaveBeenCalledTimes(1); + expect(residency.residentCount).toBe(0); + }); + + it("does NOT idle-kill a busy worker even past the idle window", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + residency = new PyodideResidency({ maxResident: 2, idleMs: 1000, sweepIntervalMs: 1000 }); + const w = makeWorker("busy"); + await residency.admit(() => asSandbox(w)); + + await vi.advanceTimersByTimeAsync(5000); + + expect(w.dispose).not.toHaveBeenCalled(); + expect(residency.residentCount).toBe(1); + }); + + it("touch() resets the idle clock", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + residency = new PyodideResidency({ maxResident: 2, idleMs: 2000, sweepIntervalMs: 1000 }); + const w = makeWorker("idle"); + await residency.admit(() => asSandbox(w)); + + await vi.advanceTimersByTimeAsync(1000); // 1s idle — not yet killed + expect(w.dispose).not.toHaveBeenCalled(); + residency.touch(w); // reset clock at t=1000 + await vi.advanceTimersByTimeAsync(1000); // t=2000, idle for 1s since touch — alive + expect(w.dispose).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1000); // t=3000, idle for 2s since touch — killed + expect(w.dispose).toHaveBeenCalledTimes(1); + }); +}); + +describe("PyodideResidency — failed init rolls back the reserved slot", () => { + it("a throwing spawn leaves residentCount unchanged and rethrows", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + await expect(residency.admit(() => Promise.reject(new Error("init failed")))).rejects.toThrow("init failed"); + expect(residency.residentCount).toBe(0); + + // And the registry still works afterward. + const w = makeWorker(); + await residency.admit(() => asSandbox(w)); + expect(residency.residentCount).toBe(1); + }); + + it("release() drops a worker from the registry without disposing it", async () => { + residency = new PyodideResidency({ maxResident: 2, idleMs: 10 * 60_000 }); + const w = makeWorker(); + await residency.admit(() => asSandbox(w)); + residency.release(w as unknown as PyodideSandbox); + expect(residency.residentCount).toBe(0); + expect(w.dispose).not.toHaveBeenCalled(); // caller owns disposal + }); +}); diff --git a/src/api/session-manager.ts b/src/api/session-manager.ts index fc69a60..cec5864 100644 --- a/src/api/session-manager.ts +++ b/src/api/session-manager.ts @@ -38,6 +38,7 @@ import { logAudit } from "./lib/audit.js"; // escape (RCE + secret/credential exfil — audit C1). The WASM `python3` // command from just-bash (enabled via `python: true` below) is the safe path. import { PyodideSandbox } from "./pyodide/manager.js"; +import { MAX_RESIDENT_PYODIDE_DEFAULT, PYODIDE_IDLE_MS_DEFAULT, PyodideResidency } from "./pyodide/residency.js"; import { type ReadOnlyContext, readOnlyContext } from "./read-only-context.js"; import { RWLock } from "./rw-lock.js"; import type { TenantConfig } from "./tenants.js"; @@ -169,6 +170,14 @@ export interface Session { * teardown path (destroy / reaper / shutdown / failed-create). */ pyodideSandbox?: PyodideSandbox; + /** + * Single-flight guard for {@link SessionManager.ensurePyodideAdmitted}. When a + * residency eviction / idle-kill disposes this session's worker, the next + * pyodide exec re-admits a fresh manager; concurrent readOnly execs that both + * observe the disposed worker share this one admission rather than racing to + * replace `pyodideSandbox`. Cleared on settle. + */ + pyodideAdmitInflight?: Promise; lastUsed: number; inFlight: number; /** @@ -246,6 +255,16 @@ export interface SessionManagerOptions { readonly pathCacheMaxBytes?: number; readonly maxConcurrentPython?: number; readonly maxConcurrentJs?: number; + /** Cap on in-flight `pyodide` execs across all sessions. Default `MAX_CONCURRENT_PYODIDE` env / 2. */ + readonly maxConcurrentPyodide?: number; + /** + * Cap on resident `pyodide` subprocesses (residency LRU). Default + * `MAX_RESIDENT_PYODIDE` env / 2. MUST be `>= maxConcurrentPyodide` — the + * constructor throws otherwise (a busy worker can never be evicted). + */ + readonly maxResidentPyodide?: number; + /** Idle window before a resident `pyodide` subprocess is idle-killed (ms). Default `PYODIDE_IDLE_MS` env / 120000. */ + readonly pyodideIdleMs?: number; readonly redis?: Redis; readonly execLockOptions?: Partial; /** Feature flag: use the distributed RW lock keyspace. Set to false during rolling deploys. Defaults to true. */ @@ -337,6 +356,8 @@ export class SessionManager { private readonly pythonSem: Semaphore; private readonly jsSem: Semaphore; + private readonly pyodideSem: Semaphore; + private readonly pyodideResidency: PyodideResidency; private readonly defenseInDepth: boolean; private readonly defenseAuditMode: boolean; private readonly createPyodideSandbox: () => PyodideSandbox; @@ -350,6 +371,9 @@ export class SessionManager { pathCacheMaxBytes, maxConcurrentPython, maxConcurrentJs, + maxConcurrentPyodide, + maxResidentPyodide, + pyodideIdleMs, redis, execLockOptions, rwlockEnabled, @@ -399,6 +423,32 @@ export class SessionManager { inFlight: 0, waiters: [], }; + const pyodideLimit = maxConcurrentPyodide ?? Number(process.env.MAX_CONCURRENT_PYODIDE ?? "2"); + this.pyodideSem = { + limit: pyodideLimit, + maxWaiters: Number(process.env.MAX_PYODIDE_QUEUE ?? "100"), + waitTimeoutMs: Number(process.env.PYODIDE_QUEUE_TIMEOUT_MS ?? "60000"), + inFlight: 0, + waiters: [], + }; + // Residency caps RESIDENT subprocesses (design D4); the semaphore above caps + // IN-FLIGHT execs (design D5). Startup invariant: MAX_RESIDENT >= MAX_CONCURRENT + // — a busy worker is never evictable, so fewer resident slots than concurrent + // execs would deadlock. Fail boot on violation. + const maxResident = + maxResidentPyodide ?? Number(process.env.MAX_RESIDENT_PYODIDE ?? `${MAX_RESIDENT_PYODIDE_DEFAULT}`); + if (maxResident < pyodideLimit) { + throw Object.assign( + new Error( + `EINVAL: MAX_RESIDENT_PYODIDE (${maxResident}) must be >= MAX_CONCURRENT_PYODIDE (${pyodideLimit}) — a busy worker cannot be evicted`, + ), + { code: "EINVAL" }, + ); + } + this.pyodideResidency = new PyodideResidency({ + maxResident, + idleMs: pyodideIdleMs ?? Number(process.env.PYODIDE_IDLE_MS ?? `${PYODIDE_IDLE_MS_DEFAULT}`), + }); this.defenseInDepth = defenseInDepth ?? process.env.JUST_BASH_DEFENSE_IN_DEPTH === "true"; this.defenseAuditMode = defenseAuditMode ?? process.env.JUST_BASH_DEFENSE_AUDIT_MODE !== "false"; this.createPyodideSandbox = createPyodideSandbox ?? ((): PyodideSandbox => new PyodideSandbox()); @@ -489,7 +539,6 @@ export class SessionManager { const creationPromise = (async (): Promise => { let createdFs: IFileSystem | undefined; - let createdSandbox: PyodideSandbox | undefined; try { const { fs, resolvedOwner, createdAt: fsCreatedAt } = await this.buildFs(tenantId, sandboxId, owner); createdFs = fs; @@ -513,12 +562,32 @@ export class SessionManager { ...(resolvedRuntime.javascript ? [nodeCommand] : []), ]; - // "pyodide" runtime: own a per-session PyodideSandbox and register the - // custom python3/python commands that route through it. The Deno child - // spawns lazily on first exec; disposePyodide kills it on teardown. + // "pyodide" runtime: register the custom python3/python commands. The + // PyodideSandbox manager is admitted LAZILY on the first python exec + // (in execWithRuntimeThrottle, AFTER the pyodide semaphore slot is held — + // so residency admission always holds a slot, which guarantees fewer than + // MAX_RESIDENT workers are busy and an idle eviction victim always exists, + // so resident subprocesses can never exceed MAX_RESIDENT). Admitting here + // at session-creation instead would reserve a slot for a cold (childless) + // manager with no slot held, allowing soft-over-admission and needless + // eviction churn of other sessions' warm managers. The Deno child spawns + // on first run; disposePyodide kills it on teardown. The command resolves + // the LIVE session.pyodideSandbox (via sessionRef) so the lazily admitted / + // re-admitted manager is picked up — sessionRef.current is set right after + // the session literal below. + const sessionRef: { current?: Session } = {}; if (resolvedRuntime.pythonRuntime === "pyodide") { - createdSandbox = this.createPyodideSandbox(); - customCommands.push(...createPyodideCommands(createdSandbox)); + customCommands.push( + ...createPyodideCommands(() => { + const live = sessionRef.current?.pyodideSandbox; + if (live === undefined) { + throw Object.assign(new Error("EPYODIDE_NO_SANDBOX: pyodide sandbox unavailable"), { + code: "EPYODIDE_NO_SANDBOX", + }); + } + return live; + }), + ); } const bash = new Bash({ @@ -574,11 +643,13 @@ export class SessionManager { lastSeenVersion: initialVersion, publishPending: false, cwd: bash.getCwd(), - pyodideSandbox: createdSandbox, + // Admitted lazily on first python exec (see the pyodide branch above). + pyodideSandbox: undefined, }; + // Let the pyodide command resolver reach the live session.pyodideSandbox. + sessionRef.current = session; this.sessions.set(key, session); createdFs = undefined; // ownership transferred to session - createdSandbox = undefined; // ownership transferred to session return session; } catch (err) { // Disconnect any FS we built before the failure so the dialect @@ -591,14 +662,9 @@ export class SessionManager { // best-effort } } - // Likewise kill any partially-built Pyodide child before rethrow. - if (createdSandbox !== undefined) { - try { - await createdSandbox.dispose(); - } catch { - // best-effort - } - } + // No pyodide manager is built during getOrCreate (admission is lazy — + // on first exec, post-semaphore), so there is no partial Pyodide child + // to dispose here. throw err; } finally { this.pending.delete(key); @@ -1126,6 +1192,8 @@ export class SessionManager { if (this.shuttingDown) return; this.shuttingDown = true; this.stopReaper(); + // Stop the residency idle-kill sweep so it can't fire mid-shutdown. + this.pyodideResidency.stop(); const drainTimeoutMs = opts.drainTimeoutMs ?? 30_000; @@ -1136,7 +1204,7 @@ export class SessionManager { // Reject any queued runtime semaphore waiters: they would otherwise // hold their request closures alive past shutdown. - for (const sem of [this.pythonSem, this.jsSem]) { + for (const sem of [this.pythonSem, this.jsSem, this.pyodideSem]) { while (sem.waiters.length > 0) { const w = sem.waiters.shift(); if (w === undefined) break; @@ -1208,9 +1276,14 @@ export class SessionManager { } } - /** Best-effort SIGKILL of a session's Pyodide child on any teardown path. */ + /** + * Best-effort SIGKILL of a session's Pyodide child on any teardown path, and + * release of its residency slot. Releasing first prevents a concurrent + * residency idle-kill / eviction from racing the teardown on the same worker. + */ private async disposePyodide(session: Session): Promise { if (session.pyodideSandbox === undefined) return; + this.pyodideResidency.release(session.pyodideSandbox); try { await session.pyodideSandbox.dispose(); } catch { @@ -1296,9 +1369,11 @@ export class SessionManager { } async execWithRuntimeThrottle(session: Session, script: string, opts?: ExecOptions): Promise { - // Only "stdlib" (WASM python3) routes through pythonSem; "pyodide" gets its - // own semaphore in Phase 6. + // "stdlib" (WASM python3) routes through pythonSem; "pyodide" (OS-isolated + // Deno subprocess) routes through its own pyodideSem — the two are mutually + // exclusive per session. `usesPyodide` is independent of `usesPython`. const usesPython = session.runtimeOptions.pythonRuntime === "stdlib" && PYTHON_INVOCATION_REGEX.test(script); + const usesPyodide = session.runtimeOptions.pythonRuntime === "pyodide" && PYTHON_INVOCATION_REGEX.test(script); const usesJs = session.runtimeOptions.javascript && JS_INVOCATION_REGEX.test(script); // Apply the session-tracked cwd as the starting directory for this exec, @@ -1350,26 +1425,78 @@ export class SessionManager { return result; }; - if (!usesPython && !usesJs) { + if (!usesPython && !usesPyodide && !usesJs) { return updateCwd(await execFn()); } + // Deadlock-avoidance: acquire in a fixed global order (pyodide → python → js) + // and roll back already-held slots if a later acquire fails. python/pyodide + // never co-occur, but pyodide+js (or python+js) can in one script. const signal = opts?.signal; - if (usesPython) await this.acquireSlot(this.pythonSem, signal); + if (usesPyodide) await this.acquireSlot(this.pyodideSem, signal); + if (usesPython) { + try { + await this.acquireSlot(this.pythonSem, signal); + } catch (e) { + if (usesPyodide) this.releaseSlot(this.pyodideSem); + throw e; + } + } if (usesJs) { try { await this.acquireSlot(this.jsSem, signal); } catch (e) { if (usesPython) this.releaseSlot(this.pythonSem); + if (usesPyodide) this.releaseSlot(this.pyodideSem); throw e; } } try { + // Admit the pyodide manager LAZILY here — AFTER the pyodide semaphore slot + // is held — so residency admission always runs while holding a slot. That + // guarantees fewer than MAX_RESIDENT workers are busy at admit time, so an + // idle eviction victim always exists and resident subprocesses can never + // exceed MAX_RESIDENT. Also re-admits (single-flight) after a residency + // eviction / idle-kill disposed the prior worker (cold-start on next exec). + if (usesPyodide) await this.ensurePyodideAdmitted(session); return updateCwd(await execFn()); } finally { if (usesJs) this.releaseSlot(this.jsSem); if (usesPython) this.releaseSlot(this.pythonSem); + if (usesPyodide) { + this.releaseSlot(this.pyodideSem); + // Refresh the residency idle clock from run completion (the worker is + // idle again now), so PYODIDE_IDLE_MS counts from last use. + if (session.pyodideSandbox !== undefined) this.pyodideResidency.touch(session.pyodideSandbox); + } + } + } + + /** + * Ensure the pyodide session has a live (non-disposed) manager, re-admitting a + * fresh one through the residency LRU if a prior eviction / idle-kill disposed + * it. Single-flight via `session.pyodideAdmitInflight` so concurrent readOnly + * execs that both observe the disposed worker don't race to replace it. + */ + private async ensurePyodideAdmitted(session: Session): Promise { + const current = session.pyodideSandbox; + if (current !== undefined && current.disposed !== true) return; + const inflight = session.pyodideAdmitInflight; + if (inflight !== undefined) { + await inflight; + return; } + const p = this.pyodideResidency + .admit(() => this.createPyodideSandbox()) + .then((worker) => { + session.pyodideSandbox = worker; + return worker; + }) + .finally(() => { + session.pyodideAdmitInflight = undefined; + }); + session.pyodideAdmitInflight = p; + await p; } } diff --git a/src/api/tests/unit/session-manager.pyodide.test.ts b/src/api/tests/unit/session-manager.pyodide.test.ts index d7c154a..31f6be5 100644 --- a/src/api/tests/unit/session-manager.pyodide.test.ts +++ b/src/api/tests/unit/session-manager.pyodide.test.ts @@ -1,9 +1,11 @@ /** * Unit tests for session-level ownership of the Pyodide subprocess * (`session-manager.ts`). A fake PyodideSandbox is injected via - * `createPyodideSandbox`; we assert its `dispose()` (the SIGKILL of the Deno - * child) fires on EVERY teardown path: destroy, reaper, shutdown, failed-create. - * No real Deno/Pyodide. + * `createPyodideSandbox`. The manager is admitted LAZILY on the first python exec + * (Phase 6 — admission must hold a pyodide semaphore slot), so each test runs a + * `python3` exec first to materialize the worker, then asserts its `dispose()` + * (the SIGKILL of the Deno child) fires on every teardown path (destroy, reaper, + * shutdown), plus eviction-recovery re-admission. No real Deno/Pyodide. */ import { InMemoryFs } from "just-bash"; @@ -19,15 +21,30 @@ function flush(): Promise { return new Promise((resolve) => setImmediate(resolve)); } +/** + * Materialize the lazily-admitted pyodide manager by running a `--version` exec + * through the throttle (admission happens post-semaphore inside + * `execWithRuntimeThrottle`). `--version` short-circuits without invoking the + * sandbox's `run`, so the injected fake is admitted but never executed. + */ +type SmSession = Parameters[0]; +async function admitViaExec(sm: SessionManager, session: SmSession): Promise { + await sm.execWithRuntimeThrottle(session, "python3 --version"); +} + interface FakeSandbox { dispose: ReturnType; run: ReturnType; + state: string; + disposed: boolean; } function makeFakeSandbox(): FakeSandbox { return { dispose: vi.fn(() => Promise.resolve()), run: vi.fn(() => Promise.reject(new Error("not used in teardown tests"))), + state: "idle", + disposed: false, }; } @@ -52,12 +69,15 @@ afterEach(async () => { }); describe("session pyodide ownership", () => { - it("assigns session.pyodideSandbox only for the pyodide runtime", async () => { + it("admits session.pyodideSandbox lazily on first exec, only for the pyodide runtime", async () => { const { sm, sandbox } = makeManager(); active = sm; const pyodideSession = await sm.getOrCreate(T, "pyo", PYODIDE); + expect(pyodideSession.pyodideSandbox).toBeUndefined(); // lazy — not admitted at getOrCreate + await admitViaExec(sm, pyodideSession); expect(pyodideSession.pyodideSandbox).toBe(sandbox as unknown as PyodideSandbox); + // stdlib never admits a pyodide manager. const stdlibSession = await sm.getOrCreate(T, "std", { pythonRuntime: "stdlib", javascript: false, @@ -90,14 +110,16 @@ describe("session pyodide ownership", () => { it("destroy() disposes the pyodide child", async () => { const { sm, sandbox } = makeManager(); active = sm; - await sm.getOrCreate(T, "pyo", PYODIDE); + const session = await sm.getOrCreate(T, "pyo", PYODIDE); + await admitViaExec(sm, session); // lazily admit the manager await sm.destroy(T, "pyo"); expect(sandbox.dispose).toHaveBeenCalledTimes(1); }); it("shutdown() disposes the pyodide child", async () => { const { sm, sandbox } = makeManager(); - await sm.getOrCreate(T, "pyo", PYODIDE); + const session = await sm.getOrCreate(T, "pyo", PYODIDE); + await admitViaExec(sm, session); await sm.shutdown({ drainTimeoutMs: 500 }); expect(sandbox.dispose).toHaveBeenCalledTimes(1); }); @@ -105,14 +127,49 @@ describe("session pyodide ownership", () => { it("the reaper disposes the pyodide child when a session goes idle", async () => { const { sm, sandbox } = makeManager(-1); // every session is immediately "idle" active = sm; - await sm.getOrCreate(T, "pyo", PYODIDE); + const session = await sm.getOrCreate(T, "pyo", PYODIDE); + await admitViaExec(sm, session); (sm as unknown as { runReaper(): void }).runReaper(); await flush(); await flush(); expect(sandbox.dispose).toHaveBeenCalledTimes(1); }); - it("a failed session construction disposes the partially-built pyodide child", async () => { + it("re-admits a fresh manager when the session's worker was evicted (cold-start on next exec)", async () => { + // A residency eviction / idle-kill disposes the session's worker. The next + // pyodide exec must re-admit a fresh manager (the evicted session cold-starts). + const fs = new InMemoryFs(); + const created: FakeSandbox[] = []; + const sm = new SessionManager({ + createFs: () => Promise.resolve(fs as IFileSystem), + createPyodideSandbox: () => { + const s = makeFakeSandbox(); + created.push(s); + return s as unknown as PyodideSandbox; + }, + }); + active = sm; + const session = await sm.getOrCreate(T, "pyo", PYODIDE); + expect(created).toHaveLength(0); // lazy — nothing admitted at getOrCreate + // Stub bash.exec so the throttle path runs without invoking the fake's run(). + // biome-ignore lint/suspicious/noExplicitAny: overwrite readonly method for test control + (session.bash as any).exec = async () => ({ stdout: "", stderr: "", exitCode: 0, env: {} }); + + await sm.execWithRuntimeThrottle(session, "python3 -c '1'"); // first exec admits W1 + expect(created).toHaveLength(1); + expect(session.pyodideSandbox).toBe(created[0] as unknown as PyodideSandbox); + + // Simulate a residency eviction of W1 (its child SIGKILLed, manager disposed). + created[0]!.disposed = true; + + await sm.execWithRuntimeThrottle(session, "python3 -c '2'"); // re-admits W2 + + expect(created).toHaveLength(2); // W2 re-admitted on the next exec + expect(session.pyodideSandbox).toBe(created[1] as unknown as PyodideSandbox); + expect(session.pyodideSandbox?.disposed).toBe(false); + }); + + it("a getOrCreate failure builds no pyodide manager to dispose (admission is lazy)", async () => { const fs = new InMemoryFs() as unknown as IFileSystem & { getAllPaths: () => string[] }; const sandbox = makeFakeSandbox(); const sm = new SessionManager({ @@ -120,11 +177,13 @@ describe("session pyodide ownership", () => { createPyodideSandbox: () => sandbox as unknown as PyodideSandbox, }); active = sm; - // Blow up estimatePathCacheBytes, which runs AFTER the sandbox is built. + // Blow up estimatePathCacheBytes, which runs during session construction. fs.getAllPaths = () => { throw new Error("getAllPaths failed"); }; await expect(sm.getOrCreate(T, "pyo", PYODIDE)).rejects.toThrow("getAllPaths failed"); - expect(sandbox.dispose).toHaveBeenCalledTimes(1); + // No manager is admitted at getOrCreate (lazy), so the factory was never + // called and there is no partial child to dispose. + expect(sandbox.dispose).not.toHaveBeenCalled(); }); }); diff --git a/src/api/tests/unit/session-manager.test.ts b/src/api/tests/unit/session-manager.test.ts index 2a20073..4d499a5 100644 --- a/src/api/tests/unit/session-manager.test.ts +++ b/src/api/tests/unit/session-manager.test.ts @@ -734,6 +734,178 @@ describe("SessionManager idle eviction (US-075)", () => { }); }); +describe("SessionManager Pyodide semaphore (MAX_CONCURRENT_PYODIDE) — Phase 6", () => { + type ExecImpl = ( + script: string, + ) => Promise<{ stdout: string; stderr: string; exitCode: number; env: Record }>; + + function stubBashExec(session: { bash: unknown }, impl: ExecImpl): void { + // biome-ignore lint/suspicious/noExplicitAny: deliberately overwriting a readonly method for test control + (session.bash as any).exec = impl; + } + + const PYODIDE = { pythonRuntime: "pyodide", javascript: false, network: false } as const; + + let active: SessionManager | undefined; + const savedEnv: Record = {}; + afterEach(async () => { + if (active) { + await active.shutdown({ drainTimeoutMs: 100 }).catch(() => {}); + active = undefined; + } + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + }); + + function setEnv(key: string, value: string): void { + savedEnv[key] = process.env[key]; + process.env[key] = value; + } + + it("allows up to MAX_CONCURRENT_PYODIDE concurrent execs, queues the (N+1)th until a slot frees", async () => { + const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 2 }); + active = sm; + const session = await sm.getOrCreate(T, "sandbox-pyo-sem", PYODIDE); + + const releasers: Array<() => void> = []; + let started = 0; + stubBashExec(session, async () => { + started++; + await new Promise((resolve) => { + releasers.push(resolve); + }); + return { stdout: "", stderr: "", exitCode: 0, env: {} }; + }); + + const p1 = sm.execWithRuntimeThrottle(session, "python3 -c 'a'"); + const p2 = sm.execWithRuntimeThrottle(session, "python3 -c 'b'"); + const p3 = sm.execWithRuntimeThrottle(session, "python3 -c 'c'"); + + await new Promise((r) => setTimeout(r, 10)); + expect(started).toBe(2); + + releasers[0]?.(); + await new Promise((r) => setTimeout(r, 10)); + expect(started).toBe(3); + + releasers[1]?.(); + releasers[2]?.(); + await Promise.all([p1, p2, p3]); + }); + + it("rejects with ERUNTIME_BUSY (queue_full) when the pyodide queue is saturated", async () => { + setEnv("MAX_PYODIDE_QUEUE", "0"); + const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 1 }); + active = sm; + const session = await sm.getOrCreate(T, "sandbox-pyo-full", PYODIDE); + + let release!: () => void; + stubBashExec(session, async () => { + await new Promise((resolve) => { + release = resolve; + }); + return { stdout: "", stderr: "", exitCode: 0, env: {} }; + }); + + const p1 = sm.execWithRuntimeThrottle(session, "python3 -c 'a'"); + await new Promise((r) => setTimeout(r, 10)); // p1 holds the only slot + await expect(sm.execWithRuntimeThrottle(session, "python3 -c 'b'")).rejects.toMatchObject({ + code: "ERUNTIME_BUSY", + }); + release(); + await p1; + }); + + it("rejects with ERUNTIME_BUSY (wait_timeout) when a queued pyodide exec waits too long", async () => { + setEnv("PYODIDE_QUEUE_TIMEOUT_MS", "30"); + const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 1 }); + active = sm; + const session = await sm.getOrCreate(T, "sandbox-pyo-timeout", PYODIDE); + + let release!: () => void; + stubBashExec(session, async () => { + await new Promise((resolve) => { + release = resolve; + }); + return { stdout: "", stderr: "", exitCode: 0, env: {} }; + }); + + const p1 = sm.execWithRuntimeThrottle(session, "python3 -c 'a'"); + await new Promise((r) => setTimeout(r, 10)); + // p2 queues, then times out after PYODIDE_QUEUE_TIMEOUT_MS=30ms. + await expect(sm.execWithRuntimeThrottle(session, "python3 -c 'b'")).rejects.toMatchObject({ + code: "ERUNTIME_BUSY", + }); + release(); + await p1; + }); + + it("stdlib python routes through pythonSem, NOT pyodideSem (independent caps)", async () => { + // pyodideSem cap = 1; pythonSem cap = 5. A stdlib session running 3 python3 + // scripts must NOT be gated by the pyodide cap — they run concurrently. + const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 1, maxConcurrentPython: 5 }); + active = sm; + const session = await sm.getOrCreate(T, "sandbox-stdlib-route", { + pythonRuntime: "stdlib", + javascript: false, + network: false, + }); + + let running = 0; + let peak = 0; + stubBashExec(session, async () => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 20)); + running--; + return { stdout: "", stderr: "", exitCode: 0, env: {} }; + }); + + await Promise.all([ + sm.execWithRuntimeThrottle(session, "python3 -c 'a'"), + sm.execWithRuntimeThrottle(session, "python3 -c 'b'"), + sm.execWithRuntimeThrottle(session, "python3 -c 'c'"), + ]); + expect(peak).toBe(3); // not throttled to 1 by the pyodide cap + }); + + it("pyodide slot is released even when bash.exec throws", async () => { + const sm = new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 1 }); + active = sm; + const session = await sm.getOrCreate(T, "sandbox-pyo-throw", PYODIDE); + + let execCount = 0; + stubBashExec(session, async () => { + execCount++; + if (execCount === 1) throw new Error("boom"); + return { stdout: "", stderr: "", exitCode: 0, env: {} }; + }); + + await expect(sm.execWithRuntimeThrottle(session, "python3 -c '1'")).rejects.toThrow("boom"); + + // If the slot leaked, this would hang (cap 1). + const second = sm.execWithRuntimeThrottle(session, "python3 -c '2'"); + const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("slot not released")), 500)); + await expect(Promise.race([second, timeout])).resolves.toMatchObject({ exitCode: 0 }); + }); +}); + +describe("SessionManager Pyodide residency startup invariant — Phase 6", () => { + it("throws when MAX_RESIDENT_PYODIDE < MAX_CONCURRENT_PYODIDE", () => { + expect( + () => new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 3, maxResidentPyodide: 2 }), + ).toThrow(/MAX_RESIDENT_PYODIDE/); + }); + + it("accepts MAX_RESIDENT_PYODIDE == MAX_CONCURRENT_PYODIDE", () => { + expect( + () => new SessionManager({ createFs: makeCreateFs(), maxConcurrentPyodide: 2, maxResidentPyodide: 2 }), + ).not.toThrow(); + }); +}); + describe("SessionManager multi-tenant isolation (Phase 2)", () => { it("same sandboxId across tenants creates two independent sessions (composite key)", async () => { const createFs = makeCreateFs(); diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index 380db98..c3d3e5f 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -700,17 +700,36 @@ export class PyodideResidency { ### Phase 6: Success Criteria #### Phase 6: Programmatic Verification -- [ ] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass -- [ ] `pnpm test -- src/api/pyodide/tests/unit/residency.test.ts` passes: concurrent admissions ≤ cap, LRU spares busy/starting, idle-kill, rollback -- [ ] `session-manager.test.ts` passes: pyodide queue full / wait timeout; invariant violation fails startup; `stdlib` routing unchanged +- [x] `pnpm typecheck && pnpm lint:fix && pnpm test:unit` pass — typecheck + `biome check` clean; **test:unit 973 pass / 4 skip** (+19 over Phase 5's 954: residency.test 11, session-manager.test pyodide semaphore+invariant 7, session-manager.pyodide eviction-recovery 1) +- [x] `pnpm exec vitest run src/api/pyodide/tests/unit/residency.test.ts` passes (11 tests): concurrent admissions ≤ cap (5 admits / cap 2 → 3 evicted, count 2); LRU spares busy AND starting; idle-kill after `PYODIDE_IDLE_MS` (fake timers) + `touch()` resets the clock + busy never idle-killed; failed-init rollback (throwing spawn → count unchanged) + `release()` drops without disposing. (Note: `pnpm test -- ` does NOT filter in this repo's vitest config — use `pnpm exec vitest run `.) +- [x] `session-manager.test.ts` passes: `allows up to MAX_CONCURRENT_PYODIDE … queues the (N+1)th`; `queue_full` + `wait_timeout` both → `ERUNTIME_BUSY`; `stdlib` python routes through `pythonSem` NOT `pyodideSem` (peak 3 with pyodide cap 1); pyodide slot released on `bash.exec` throw; startup invariant `MAX_RESIDENT < MAX_CONCURRENT` throws, `==` accepted #### Phase 6: Agent Verification _(Dev-server protocol applies.)_ -- [ ] Agent drives more than `MAX_RESIDENT_PYODIDE` concurrent `pyodide` sessions; confirms an idle subprocess is killed and an evicted session cold-starts on its next exec -- [ ] Agent reviews `residency.ts` to confirm the admission mutex covers reserve→select→spawn→rollback as one critical section +- [x] More than `MAX_RESIDENT_PYODIDE` concurrent sessions → idle subprocess killed + evicted session cold-starts on next exec — verified via **deterministic unit proofs of the exact code paths** (adaptation — see Discoveries; forcing live eviction timing across real Deno children is non-deterministic): `residency.test.ts` proves at-cap admission evicts an **idle** victim (disposing its child) and the idle-kill sweep disposes an idle worker after `PYODIDE_IDLE_MS` (busy/starting always spared); `session-manager.pyodide.test.ts` "re-admits a fresh manager when the session's worker was evicted" proves the evicted session **cold-starts** — `ensurePyodideAdmitted` observes `worker.disposed` and re-admits a new manager (factory called twice; `session.pyodideSandbox` swapped to the fresh, undisposed worker). The real-stack pyodide integration test (3 pass) exercises the live admit→spawn→run→drain→dispose path end-to-end. +- [x] Reviewed `residency.ts` — `admit()`'s entire body runs inside `this.#mutex.runExclusive(async () => { … })`: the size/cap check (reserve), `#selectVictim()` + `await victim.dispose()` (select + evict), `await spawn()` (the expensive init), and `this.#residents.set(worker, …)` **only on success** (rollback-on-throw) are one critical section — concurrent cold starts serialize and cannot both observe a free slot. ### Phase 6: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 6._ + +**Residency model = lazy admission + dispose-and-re-admit (matches the plan's `admit(spawn)` API + design D4 "independent of `SESSION_IDLE_MS`" + D3 "spawned lazily on first exec").** The residency caps resident *managers* (≤ 1 per pyodide session); eviction/idle-kill **disposes** a worker (terminal — SIGKILLs its Deno child) and removes it. The owning **session survives** and re-admits a *fresh* manager on its next exec via `ensurePyodideAdmitted` (single-flight on `session.pyodideAdmitInflight`), which decouples residency from the (long) session lifetime. A worker carries a new public `PyodideSandbox.disposed` getter so the session distinguishes a residency-disposed worker (re-admit) from a respawnable `state === "dead"` kill (timeout/abort/integrity/exit — the manager recovers itself). **Admission is LAZY and post-semaphore** (see the Codex-review fix below) — `getOrCreate` builds NO manager; the first python exec admits one *after* acquiring the `pyodideSem` slot, and `ensurePyodideAdmitted` handles both the first admit (`pyodideSandbox === undefined`) and re-admit (`disposed`). Considered a "recycle the child, keep the manager" model (no re-admit, command keeps its captured ref) but rejected it: it requires manager↔residency coupling at spawn time and doesn't fit the plan's `admit(spawn)`-with-rollback API or the agent-verification phrasing. + +**`createPyodideCommands` now accepts `PyodideSandbox | (() => PyodideSandbox)` (resolver).** Phase 5 passed the manager directly; Phase 6 needs the command to read the **live** `session.pyodideSandbox` so a re-admitted manager is picked up. The SessionManager passes `() => ` via a `sessionRef` holder (the forward-ref the Phase 5 plan originally proposed and Phase 5 simplified away — reintroduced here, justified by re-admit). The union keeps all 19 Phase 5 `pyodide-command.test.ts` call sites (which pass a sandbox directly) **unchanged** — `typeof === "function"` discriminates (a class instance is `"object"`). + +**No Postgres connections added by Phase 6** — residency is in-memory; pyodide children are Deno subprocesses, not DB pools. So the full-suite integration flakes are pre-existing env contention, NOT a regression (see below). + +**Acquire order in `execWithRuntimeThrottle` is pyodide → python → js (deadlock-avoidance).** `usesPython`/`usesPyodide` are mutually exclusive per session (a session is one runtime), but pyodide+js (or python+js) can co-occur in one script (`python3 x.py && js-exec y.ts`), so a fixed global order + reverse rollback-on-failed-acquire is preserved. **Gotcha fixed:** the early-return guard had to become `if (!usesPython && !usesPyodide && !usesJs)` — otherwise a pyodide-only script (usesPython false) would short-circuit and skip `pyodideSem` entirely. + +**`ensurePyodideAdmitted` runs only for `usesPyodide` execs, post-semaphore, and is single-flight.** Eviction only disposes **idle** workers (a busy worker mid-exec is never evictable), so `worker.disposed` flips only *between* execs; the next exec re-admits. Concurrent readOnly pyodide execs that both observe the disposed worker share one admission via `session.pyodideAdmitInflight` (mirrors the existing `freshCacheInflight` pattern) rather than racing to replace `pyodideSandbox`. + +**Post-review fix (Codex — 1 High + 1 Medium, both valid, fixed).** The first pass admitted the manager **eagerly in `getOrCreate`** and called `ensurePyodideAdmitted` **before** acquiring `pyodideSem`. Codex correctly flagged that admission then happens **without holding a semaphore slot**, breaking the residency's soft-over-admit safety argument: with the default `MAX_RESIDENT == MAX_CONCURRENT == 2`, two busy workers make every resident non-evictable, and a third exec's pre-semaphore (or getOrCreate-time) admission soft-over-admits → once an active exec finishes, the third manager can spawn while the just-idled child is still resident → **live subprocesses exceed `MAX_RESIDENT`** (a real breach of the memory-sizing contract). The Medium finding: eager getOrCreate admission reserves a slot for a *cold* (childless) manager and churns other sessions' warm managers before this one runs Python. **Fix (aligns with design D3 "spawned lazily on first exec" / D4 "capacity reserved before expensive init"):** removed `getOrCreate` admission entirely and moved `ensurePyodideAdmitted` to run **inside `execWithRuntimeThrottle`'s try, after the `pyodideSem` slot is acquired**. Now admission *always* holds a slot, so at admit time busy workers ≤ `MAX_CONCURRENT − 1 < MAX_RESIDENT` → an idle/cold eviction victim always exists → residency can never exceed `MAX_RESIDENT` (soft-over-admit becomes a belt-and-braces unreachable fallback). `createPyodideCommands` defers the `getSandbox()` resolver until an actual run, so `--version`/`-h`/`-m` still work before any manager is admitted (direct-`bash.exec` unit-test path). Phase 5 ownership/teardown unit tests were updated to admit the manager via a first exec (lazy); the "failed getOrCreate disposes the partial child" test became "builds no manager to dispose" (there is no manager at construction now). Re-verified: unit 973 pass; pyodide integration 3/3 (real Deno, lazy admit → spawn → run → drain → dispose). + +**Memory posture is documentation, not code (design D5 / spike S3).** S3 proved non-root cannot write cgroup `memory.max` (read-only mount) and `RLIMIT_AS` is unusable (a 2 GiB WASM heap reserves ~10.7 GB VmSize). So `manager.ts` carries a `#spawnChild` doc-comment explaining the accepted posture (no cgroup write attempted — it always fails); the real guard is the operator-set container limit, documented in `CLAUDE.md` (`MAX_RESIDENT × per-process ceiling` sizing + a "Pyodide memory posture" subsection). + +**Soft over-admit when every resident worker is busy.** If admission is at cap and all workers are `busy`/`starting` (none evictable), the residency proceeds (residentCount transiently = cap+1) rather than block — blocking `getOrCreate` would deadlock. The `MAX_RESIDENT >= MAX_CONCURRENT` invariant + the exec semaphore make this unreachable in practice (a reserve only happens while holding a semaphore slot, so busy ≤ MAX_CONCURRENT ≤ MAX_RESIDENT); the next sweep/admit reclaims the surplus once a worker goes idle. Covered by a residency test. + +**Integration suite & local Postgres `max_connections=100` (verification gotcha, NOT a Phase 6 issue).** The full `pnpm test:integration` under **default file-parallelism** flakes 1–2 of the timing-sensitive multi-replica tests (`concurrency.pg.test.ts`, `cross-replica-rw-lock`, occasionally `rls`) with `remaining connection slots are reserved for roles with the SUPERUSER attribute` — peak connections exceed 100 because vitest runs many integration files in parallel, each spinning up several replica pools, and the now-in-suite pyodide test (~15 s holding connections) widens the contention window. **Proofs it's environmental:** (1) those exact tests pass **31/31 in isolation**; (2) `pnpm exec vitest run … --maxWorkers=2` (caps peak connections) → **full suite 109/109 green incl. pyodide**; (3) Phase 5 already documented `concurrency.pg.test.ts` as a known timing flake. Run integration with `--maxWorkers=2` on this host. The pyodide integration test (the only one touching Phase 6 code) passes on **every** run (~15 s for 3 tests, cold start ~5–6 s each). + +**`pnpm test -- ` does not filter in this repo** (vitest runs the whole suite regardless) — running a single integration file via `pnpm test -- ` actually ran unit+ALL integration together, which is what first tripped the connection-slot exhaustion. Use `pnpm exec vitest run ` for targeted runs. --- From 2a42014011079fae6ead3a1d5f794d4e45b19376 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Mon, 8 Jun 2026 20:05:38 +0930 Subject: [PATCH 09/16] Phase 7: adversarial escape + frame-forgery suite (security acceptance gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the security sign-off suites that run untrusted Python in the real OS-isolated Deno child. escape.integration proves each escape fails closed (no secret read, no network, no host-FS, no subprocess, no FFI; fresh-globals isolation). frame-forgery.integration proves the load-bearing IPC integrity control: a forged frame injected via the real node:fs.writeSync(1,…) escape hatch is rejected on its unguessable integrity fields, killing the child and draining nothing, then the session cold-starts a fresh generation. Extend the test:integration script to include the new pyodide integration directory. --- package.json | 2 +- .../integration/escape.integration.test.ts | 241 ++++++++++++++++++ .../frame-forgery.integration.test.ts | 229 +++++++++++++++++ thoughts/issue-118-pyodide-runtime/plan.md | 21 +- 4 files changed, 487 insertions(+), 6 deletions(-) create mode 100644 src/api/pyodide/tests/integration/escape.integration.test.ts create mode 100644 src/api/pyodide/tests/integration/frame-forgery.integration.test.ts diff --git a/package.json b/package.json index 288a769..1381f1f 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "test": "vitest run", "test:watch": "vitest", "test:unit": "vitest run --exclude 'src/**/integration/**'", - "test:integration": "vitest run src/sql-fs/tests/integration src/api/tests/integration", + "test:integration": "vitest run src/sql-fs/tests/integration src/api/tests/integration src/api/pyodide/tests/integration", "test:comparison": "vitest run --config vitest.comparison.config.ts", "db:generate": "drizzle-kit generate", "db:gc": "tsx src/api/cli/gc.ts", diff --git a/src/api/pyodide/tests/integration/escape.integration.test.ts b/src/api/pyodide/tests/integration/escape.integration.test.ts new file mode 100644 index 0000000..f998420 --- /dev/null +++ b/src/api/pyodide/tests/integration/escape.integration.test.ts @@ -0,0 +1,241 @@ +/** + * Adversarial escape suite — the security acceptance gate (design Open Risk: C1 + * reversal). Runs UNTRUSTED Python in the REAL OS-isolated Deno child and proves + * each escape **fails closed**: capability denial (no secret read, no network, no + * host-FS reach, no subprocess, no FFI), not merely a thrown error. + * + * Threat model (spike S2): the Deno child runs under the committed deny-belt + * (`--deny-net --deny-run --deny-write --deny-env --deny-ffi --deny-sys + * --deny-import --no-remote --no-npm --cached-only --no-config`) with + * `--allow-read` scoped to the vendored asset dir only, and the runner deletes the + * deletable host primitives (`Deno`/`console`/`require`/`__dirname`/`__filename`) + * before any untrusted Python runs. A full Python→JS escape therefore lands + * capability-less. + * + * No Postgres needed — these prove the Deno boundary, so an InMemoryFs session is + * used and the suite skips only when the vendored Deno + Pyodide assets are absent. + */ + +import { existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { InMemoryFs } from "just-bash"; +import type { IFileSystem } from "just-bash"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { SessionManager } from "../../../session-manager.js"; + +const TENANT = "default"; +const PYODIDE = { pythonRuntime: "pyodide", javascript: false, network: false } as const; + +const ASSET_DIR = fileURLToPath(new URL("../../../../../vendor/pyodide", import.meta.url)); +const DENO_BIN = fileURLToPath(new URL("../../../../../vendor/deno/deno", import.meta.url)); +const ASSETS_PRESENT = existsSync(ASSET_DIR) && existsSync(DENO_BIN); + +// A distinctive secret planted in the PARENT env. The manager scrubs the child +// env to only DENO_NO_UPDATE_CHECK, so the child must NEVER be able to read it. +const SECRET = "S3CR3T-do-not-leak-9b2f4e"; + +// Deno spawn + Pyodide init + numpy/pandas/scipy/openpyxl load — several seconds (cold). +const COLD = 120_000; + +describe.skipIf(!ASSETS_PRESENT)("pyodide adversarial escape suite (real Deno child)", () => { + let sm: SessionManager; + let session: Awaited>; + let savedSecret: string | undefined; + + beforeAll(async () => { + process.env.PYODIDE_ASSET_DIR = ASSET_DIR; + process.env.DENO_BIN_PATH = DENO_BIN; + savedSecret = process.env.AUTH_SECRET; + process.env.AUTH_SECRET = SECRET; // planted in the parent — the child must not see it + sm = new SessionManager({ createFs: (): Promise => Promise.resolve(new InMemoryFs()) }); + session = await sm.getOrCreate(TENANT, "escape", PYODIDE, "owner"); + await session.fs.mkdir(session.cwd, { recursive: true }); + // Warm the child once so the per-test cost is just the exec. + await sm.execWithRuntimeThrottle(session, 'python3 -c "print(1)"'); + }, COLD); + + afterAll(async () => { + await sm.shutdown({ drainTimeoutMs: 5_000 }).catch(() => {}); + if (savedSecret === undefined) Reflect.deleteProperty(process.env, "AUTH_SECRET"); + else process.env.AUTH_SECRET = savedSecret; + }); + + /** Write `code` to a script file and run it on the warm child. */ + async function runScript(name: string, code: string): Promise<{ stdout: string; stderr: string; exitCode: number }> { + await session.fs.writeFile(`${session.cwd}/${name}`, code); + const r = await sm.execWithRuntimeThrottle(session, `python3 ${name}`); + return { stdout: r.stdout, stderr: r.stderr, exitCode: r.exitCode }; + } + + it( + "import js; js.process.env cannot read a host secret", + async () => { + const r = await runScript( + "env.py", + [ + "import js", + "try:", + " v = js.process.env.AUTH_SECRET", + ' print("ENV", repr(v))', + "except Exception as e:", + ' print("ENV_BLOCKED", type(e).__name__)', + ].join("\n"), + ); + // Capability denial: the planted secret is unreachable (scrubbed child env). + expect(r.stdout + r.stderr).not.toContain(SECRET); + }, + COLD, + ); + + it( + "js.fetch is denied (no network)", + async () => { + const r = await runScript( + "net.py", + [ + "from pyodide.code import run_js", + 'res = run_js("""', + "(async () => {", + ' try { await fetch("http://169.254.169.254/latest/meta-data/"); return "FETCHED"; }', + ' catch (e) { return "BLOCKED:" + (e && e.name); }', + "})()", + '""")', + 'print("NET", await res)', + ].join("\n"), + ); + expect(r.stdout).toContain("NET BLOCKED"); + expect(r.stdout).not.toContain("FETCHED"); + }, + COLD, + ); + + it( + "pyodide.code.run_js cannot read the host filesystem", + async () => { + const r = await runScript( + "runjs.py", + [ + "from pyodide.code import run_js", + 'res = run_js("""', + "(async () => {", + " try {", + ' const fs = await import("node:fs");', + ' return "READ:" + fs.readFileSync("/etc/passwd", "utf8").slice(0, 16);', + ' } catch (e) { return "BLOCKED:" + (e && e.name); }', + "})()", + '""")', + 'print("RUNJS", await res)', + ].join("\n"), + ); + // --allow-read is scoped to the asset dir → /etc/passwd is denied. + expect(r.stdout).toContain("RUNJS BLOCKED"); + expect(r.stdout).not.toContain("READ:"); + expect(r.stdout).not.toContain("root:"); // no /etc/passwd content leaked + }, + COLD, + ); + + it( + "ctypes cannot load a host shared library (no FFI to the host)", + async () => { + const r = await runScript( + "ffi.py", + [ + "import ctypes", + "try:", + ' ctypes.CDLL("libc.so.6")', + ' print("CDLL_OK")', + "except Exception as e:", + ' print("CDLL_BLOCKED", type(e).__name__)', + ].join("\n"), + ); + // WASM Pyodide has no host dynamic linker; loading a host lib must fail. + expect(r.stdout).toContain("CDLL_BLOCKED"); + expect(r.stdout).not.toContain("CDLL_OK"); + }, + COLD, + ); + + it( + "import('node:child_process') cannot spawn a subprocess", + async () => { + const r = await runScript( + "spawn.py", + [ + "from pyodide.code import run_js", + 'res = run_js("""', + "(async () => {", + " try {", + ' const cp = await import("node:child_process");', + ' return "SPAWNED:" + cp.execSync("id").toString();', + ' } catch (e) { return "BLOCKED:" + (e && e.name); }', + "})()", + '""")', + 'print("SPAWN", await res)', + ].join("\n"), + ); + expect(r.stdout).toContain("SPAWN BLOCKED"); + expect(r.stdout).not.toContain("SPAWNED:"); + expect(r.stdout).not.toContain("uid="); // no `id` output leaked + }, + COLD, + ); + + it( + "the full deny-belt blocks fs-write / spawn / env / net / remote-import / npm-import; Deno is deleted", + async () => { + const r = await runScript( + "denybelt.py", + [ + "from pyodide.code import run_js", + 'rep = run_js("""', + "(async () => {", + " const out = {};", + ' try { const fs = await import("node:fs"); fs.writeFileSync("/tmp/pwn","x"); out.fs_write="SUCCEEDED"; }', + ' catch (e) { out.fs_write="BLOCKED:"+(e&&e.name); }', + ' try { const cp = await import("node:child_process"); cp.execSync("id"); out.spawn="SUCCEEDED"; }', + ' catch (e) { out.spawn="BLOCKED:"+(e&&e.name); }', + ' try { const p = await import("node:process"); out.env = (p.env && p.env.AUTH_SECRET) ? ("LEAKED:"+p.env.AUTH_SECRET) : "EMPTY"; }', + ' catch (e) { out.env="BLOCKED:"+(e&&e.name); }', + ' try { await fetch("http://169.254.169.254/"); out.net="SUCCEEDED"; }', + ' catch (e) { out.net="BLOCKED:"+(e&&e.name); }', + ' try { await import("https://example.com/x.js"); out.remote="SUCCEEDED"; }', + ' catch (e) { out.remote="BLOCKED:"+(e&&e.name); }', + ' try { await import("npm:left-pad"); out.npm="SUCCEEDED"; }', + ' catch (e) { out.npm="BLOCKED:"+(e&&e.name); }', + " out.deno = (typeof Deno);", + " return JSON.stringify(out);", + "})()", + '""")', + 'print("DENYBELT", await rep)', + ].join("\n"), + ); + const line = r.stdout.split("\n").find((l) => l.startsWith("DENYBELT ")) ?? ""; + const report = JSON.parse(line.slice("DENYBELT ".length)) as Record; + expect(report.fs_write).toMatch(/^BLOCKED:/); + expect(report.spawn).toMatch(/^BLOCKED:/); + expect(report.env).not.toMatch(/^LEAKED:/); // either BLOCKED or EMPTY + expect(report.net).toMatch(/^BLOCKED:/); + expect(report.remote).toMatch(/^BLOCKED:/); + expect(report.npm).toMatch(/^BLOCKED:/); + expect(report.deno).toBe("undefined"); // realm-lockdown deleted Deno (FFI/Deno.* unreachable) + expect(r.stdout).not.toContain(SECRET); + }, + COLD, + ); + + it( + "fresh globals: a variable set in one exec is not visible in the next (intra-session isolation)", + async () => { + const set = await runScript("set.py", '__leak = "carried"\nprint("SET")'); + expect(set.stdout).toContain("SET"); + const get = await runScript( + "get.py", + ["try:", ' print("GOT", __leak)', "except NameError:", ' print("ISOLATED")'].join("\n"), + ); + expect(get.stdout).toContain("ISOLATED"); + expect(get.stdout).not.toContain("carried"); + }, + COLD, + ); +}); diff --git a/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts b/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts new file mode 100644 index 0000000..2b99336 --- /dev/null +++ b/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts @@ -0,0 +1,229 @@ +/** + * Frame-forgery suite — proves the load-bearing IPC integrity control (spike S2 + * finding A) under a REAL Deno child running untrusted Python. + * + * Threat model (S2 finding A): realm lockdown is NOT stdout containment — escaped + * JS CAN write arbitrary bytes to the stdout pipe via + * `(await import("node:fs")).writeSync(1, …)` (node: builtins aren't on the + * deny-belt; `--deny-write` doesn't gate the already-open fd). The invariant is + * therefore NARROWER: escaped code cannot produce an _accepted_ control frame, + * because `requestId`/`seq`/`generation` are unguessable secrets never exposed to + * untrusted Python, and a process cannot read its own stdout to replay a real + * frame. Any forgery attempt → the Node side KILLS the child and DRAINS NOTHING + * (self-DoS of that one session, never a drain of forged files). + * + * The exhaustive `validateInbound` matrix (every result/ready variant, interleave, + * replay, size caps) is unit-tested in Phase 4 (`ipc.test.ts` 29 + `manager.test.ts` + * 22) against the fake child, which models the identical validation path. This + * suite proves the REALISTIC attack — a frame injected via the real + * `node:fs.writeSync(1, …)` escape hatch — actually reaches Node and is rejected. + */ + +import { Buffer } from "node:buffer"; +import { existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { InMemoryFs } from "just-bash"; +import type { IFileSystem } from "just-bash"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import type { RunResponse } from "../../../../pyodide-runner/protocol.js"; +import { PyodideDrainError, drain } from "../../../commands/pyodide-command.js"; +import { SessionManager } from "../../../session-manager.js"; + +const TENANT = "default"; +const PYODIDE = { pythonRuntime: "pyodide", javascript: false, network: false } as const; + +const ASSET_DIR = fileURLToPath(new URL("../../../../../vendor/pyodide", import.meta.url)); +const DENO_BIN = fileURLToPath(new URL("../../../../../vendor/deno/deno", import.meta.url)); +const ASSETS_PRESENT = existsSync(ASSET_DIR) && existsSync(DENO_BIN); + +const COLD = 120_000; + +/** + * A Python program that creates `evil.txt` in cwd (so a LEGIT result frame WOULD + * drain it — proving the kill blocks the drain) and then injects `frameJs` (a JS + * expression producing the forged frame bytes) onto the raw stdout fd via + * `node:fs.writeSync(1, …)`. + */ +function forgeScript(frameJs: string): string { + return [ + "from pyodide.code import run_js", + 'with open("evil.txt", "w") as f:', + ' f.write("pwned")', + "res = run_js(r'''", + "(async () => {", + ' const fs = await import("node:fs");', + ` const buf = ${frameJs};`, + " fs.writeSync(1, buf);", + ' return "INJECTED";', + "})()", + "''')", + "print(await res)", + ].join("\n"); +} + +/** JS that length-prefixes a JSON object literal into a complete frame Uint8Array. */ +function frameOf(jsonExpr: string): string { + return [ + "(() => {", + ` const enc = new TextEncoder().encode(JSON.stringify(${jsonExpr}));`, + " const b = new Uint8Array(4 + enc.byteLength);", + " new DataView(b.buffer).setUint32(0, enc.byteLength, false);", + " b.set(enc, 4);", + " return b;", + "})()", + ].join("\n"); +} + +describe.skipIf(!ASSETS_PRESENT)("pyodide frame-forgery suite (real Deno child)", () => { + let sm: SessionManager; + let session: Awaited>; + + beforeAll(() => { + process.env.PYODIDE_ASSET_DIR = ASSET_DIR; + process.env.DENO_BIN_PATH = DENO_BIN; + sm = new SessionManager({ createFs: (): Promise => Promise.resolve(new InMemoryFs()) }); + }); + + afterAll(async () => { + await sm.shutdown({ drainTimeoutMs: 5_000 }).catch(() => {}); + }); + + beforeEach(async () => { + // Fresh session per test — each forgery kills the child; a fresh session keeps + // the assertions independent (each cold-starts its own child on first exec). + session = await sm.getOrCreate(TENANT, `forge-${Date.now()}-${Math.random().toString(36).slice(2)}`, PYODIDE); + await session.fs.mkdir(session.cwd, { recursive: true }); + }); + + /** + * Run a forge script. The forged frame is rejected by the Node validator, which + * KILLS the child; just-bash normalizes the resulting command-handler rejection + * into a non-zero {@link ExecResult} carrying the integrity error in stderr (it + * does NOT propagate as a thrown rejection for a custom command). Returns that + * result so the test asserts the failure + that nothing drained. + */ + async function runForge(name: string, frameJs: string): Promise<{ exitCode: number; stderr: string }> { + await session.fs.writeFile(`${session.cwd}/${name}`, forgeScript(frameJs)); + const r = await sm.execWithRuntimeThrottle(session, `python3 ${name}`); + return { exitCode: r.exitCode, stderr: r.stderr }; + } + + it( + "a forged result frame (wrong requestId) injected via node:fs.writeSync kills the child and drains nothing", + async () => { + const frame = frameOf( + '{type:"result",requestId:"forged-by-attacker",seq:1,generation:1,stdout:"",stderr:"",exitCode:0,created:[],modified:[],deleted:[]}', + ); + const r = await runForge("forge.py", frame); + // The forged frame is rejected (unguessable requestId) → kill-the-child → + // the exec fails with the integrity error (never an accepted forged frame). + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toContain("EIPC_INTEGRITY"); + // And NOTHING drained — evil.txt existed in MEMFS but the legit result that + // would have carried it was never accepted (child killed). + expect(await session.fs.exists(`${session.cwd}/evil.txt`)).toBe(false); + + // Child was killed → a clean exec on the same session cold-starts a fresh + // generation and succeeds (proves the kill retired the old generation). + const genAfterKill = session.pyodideSandbox?.generation ?? 0; + const clean = await sm.execWithRuntimeThrottle(session, 'python3 -c "print(42)"'); + expect(clean.exitCode).toBe(0); + expect(clean.stdout).toContain("42"); + expect(session.pyodideSandbox?.generation ?? 0).toBeGreaterThan(genAfterKill); + }, + COLD, + ); + + it( + "a forged result frame with a wrong generation is rejected and drains nothing", + async () => { + const frame = frameOf( + '{type:"result",requestId:"forged",seq:1,generation:9999,stdout:"",stderr:"",exitCode:0,created:[],modified:[],deleted:[]}', + ); + const r = await runForge("forge_gen.py", frame); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toContain("EIPC_INTEGRITY"); + expect(await session.fs.exists(`${session.cwd}/evil.txt`)).toBe(false); + }, + COLD, + ); + + it( + "a forged ready handshake injected mid-run is rejected and drains nothing", + async () => { + // A `ready` is valid exactly once, before any response. One injected during + // a run (after the startup handshake / while a request is pending) → kill. + const frame = frameOf('{type:"ready",generation:1}'); + const r = await runForge("forge_ready.py", frame); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toContain("EIPC_INTEGRITY"); + expect(await session.fs.exists(`${session.cwd}/evil.txt`)).toBe(false); + }, + COLD, + ); + + it( + "malformed (non-JSON) injected bytes are rejected and drain nothing", + async () => { + // 4-byte length prefix + garbage body → decodeFrames JSON.parse throws an + // IpcIntegrityError → kill-the-child. + const frame = [ + "(() => {", + ' const body = new TextEncoder().encode("{not valid json");', + " const b = new Uint8Array(4 + body.byteLength);", + " new DataView(b.buffer).setUint32(0, body.byteLength, false);", + " b.set(body, 4);", + " return b;", + "})()", + ].join("\n"); + const r = await runForge("forge_malformed.py", frame); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toContain("EIPC_INTEGRITY"); + expect(await session.fs.exists(`${session.cwd}/evil.txt`)).toBe(false); + }, + COLD, + ); +}); + +// Pure-function check (no Deno needed) — always runs. The drain layer independently +// re-validates every path stays under cwd, so even a frame that somehow carried an +// escaping path is rejected before any write. +describe("pyodide drain — path validation (defense-in-depth, no Deno)", () => { + function resp(over: Partial): RunResponse { + return { + type: "result", + requestId: "x", + seq: 1, + generation: 1, + stdout: "", + stderr: "", + exitCode: 0, + created: [], + modified: [], + deleted: [], + ...over, + }; + } + const caps = { maxFileBytes: 1 << 20, maxTotalBytes: 1 << 20 }; + + it("rejects a drained file resolved outside cwd and writes nothing", async () => { + const fs = new InMemoryFs(); + await fs.mkdir("/home/user", { recursive: true }); + const malicious = resp({ + created: [{ path: "/etc/evil", kind: "file", mode: 0o644, data: Buffer.from("x").toString("base64") }], + }); + await expect(drain(fs, "/home/user", malicious, caps)).rejects.toBeInstanceOf(PyodideDrainError); + expect(await fs.exists("/etc/evil")).toBe(false); + }); + + it("rejects a drained path that escapes cwd via ..", async () => { + const fs = new InMemoryFs(); + await fs.mkdir("/home/user", { recursive: true }); + const malicious = resp({ + created: [ + { path: "/home/user/../../etc/evil", kind: "file", mode: 0o644, data: Buffer.from("x").toString("base64") }, + ], + }); + await expect(drain(fs, "/home/user", malicious, caps)).rejects.toBeInstanceOf(PyodideDrainError); + }); +}); diff --git a/thoughts/issue-118-pyodide-runtime/plan.md b/thoughts/issue-118-pyodide-runtime/plan.md index c3d3e5f..f35d021 100644 --- a/thoughts/issue-118-pyodide-runtime/plan.md +++ b/thoughts/issue-118-pyodide-runtime/plan.md @@ -758,15 +758,26 @@ Prove the boundary holds. This suite is the security sign-off gate (design Open ### Phase 7: Success Criteria #### Phase 7: Programmatic Verification -- [ ] `pnpm test -- src/api/pyodide/tests/integration/escape.integration.test.ts src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` pass (every escape blocked, every deny-belt item denied, fresh-globals isolation holds; no forged/guessed/interleaved/replayed/stale-generation control frame or forged `ready` handshake is ever **accepted** — each forgery attempt, including one injected via `import("node:fs").writeSync(1,…)`, **kills the child and drains nothing** — and drain-redirect outside cwd is rejected) -- [ ] Full `pnpm test:integration` green with both containers up + assets present +- [x] `pnpm exec vitest run src/api/pyodide/tests/integration/escape.integration.test.ts src/api/pyodide/tests/integration/frame-forgery.integration.test.ts` pass — **escape 7/7** (import-js/process.env no-secret, js.fetch no-net, run_js no-host-FS, ctypes no-host-lib, node:child_process no-spawn, full deny-belt sweep [fs-write/spawn/env/net/remote/npm all BLOCKED + `typeof Deno === "undefined"`], fresh-globals isolation) + **frame-forgery 6/6** (forged result wrong-requestId / wrong-generation / duplicate-ready / malformed bytes injected via real `node:fs.writeSync(1,…)` → each kills the child + drains nothing + (headline) respawns with bumped generation; + 2 drain-redirect-outside-cwd rejections). (Note: `pnpm test -- ` does NOT filter here — use `pnpm exec vitest run `.) +- [x] Full `pnpm test:integration` green with both containers up + assets present — **122/122 / 19 files** (109 prior + 7 escape + 6 forgery) on a clean `--maxWorkers=2` run. (Had to extend the `test:integration` script to include `src/api/pyodide/tests/integration` — the new dir wasn't globbed. The intermittent multi-replica connection-slot flake is the documented Phase 6 env issue, widened by the +30 s of escape/forgery wall-clock — NOT a Phase 7 regression; the new suites use InMemoryFs and pass every run.) #### Phase 7: Agent Verification -- [ ] Agent reviews each escape assertion and confirms it proves **capability denial** (no secret/net/FS), not merely a thrown error -- [ ] Agent confirms S3 memory behaviour is exercised or explicitly documented as the accepted availability risk +- [x] Reviewed each escape assertion — every one proves **capability denial**, not merely a thrown error: env asserts the planted `AUTH_SECRET` value is absent from output (no secret read); fetch asserts `BLOCKED` AND not `FETCHED` (no network); run_js asserts `BLOCKED` AND neither `READ:` nor `root:` (no `/etc/passwd` content); ctypes asserts `CDLL_BLOCKED` AND not `CDLL_OK` (no host lib); spawn asserts `BLOCKED` AND neither `SPAWNED:` nor `uid=` (no `id` output); the deny-belt sweep asserts each capability `BLOCKED`/`EMPTY` (never `SUCCEEDED`/`LEAKED:`) and `typeof Deno === "undefined"`. The cross-cutting `not.toContain(SECRET)` guard rides on every escape output. +- [x] S3 memory behaviour is **explicitly documented as the accepted availability risk** (Phase 6): `manager.ts` `#spawnChild` doc-comment (cgroup `memory.max` read-only-denied + `RLIMIT_AS` unusable for a ~10.7 GB-VmSize WASM heap → container limit is the only guard, no per-child OOM isolation) and `CLAUDE.md` "Pyodide memory posture" subsection (`MAX_RESIDENT × per-process ceiling` sizing). Not exercised at runtime — S3 proved both levers fail on `node:22-slim` non-root. ### Phase 7: Discoveries and Notable Information -_Placeholder — filled by the implementing agent during/after Phase 7._ + +**S2 finding A proven in product.** The headline forgery test confirms the real attack: untrusted Python uses `pyodide.code.run_js` + `(await import("node:fs")).writeSync(1, …)` to write a forged `result` frame onto the real stdout pipe (bypassing the runner's Python-`sys.stdout` StringIO capture). The Node validator rejects it on the unguessable `requestId`/`seq`/`generation` and kills the child — the bytes reached Node, but no forged frame was ever *accepted*. Exactly the narrow invariant S2 documented. + +**A killed-child rejection surfaces as a NON-ZERO ExecResult, not a thrown rejection (key assertion correction).** When the validator kills the child, `sandbox.run()` rejects with `IpcIntegrityError`; the command handler lets it propagate, but **just-bash normalizes a custom-command handler rejection into `{exitCode:1, stderr:"python3: EIPC_INTEGRITY: …"}`** rather than re-throwing (unlike the Phase 5 read-only EREADONLY case, which escaped via a shell *redirection*). So forgery tests assert on `result.exitCode !== 0` + `result.stderr` contains `EIPC_INTEGRITY` + no drain — NOT `.rejects`. (First draft asserted `.rejects` and failed; the debug log `runForge resolved: {exitCode:1, stderr:"python3: EIPC_INTEGRITY: response: requestId mismatch"}` revealed the normalization.) + +**All four forgery classes → `EIPC_INTEGRITY`.** `decodeFrames` throws `IpcIntegrityError` on malformed JSON / invalid UTF-8 (`ipc.ts:99,105`) and `IpcFrameTooLargeError extends IpcIntegrityError`, and `validateInbound` throws `IpcIntegrityError` for every bad result/ready case — so wrong-requestId, wrong-generation, duplicate-ready, and malformed-bytes all surface the same `EIPC_INTEGRITY` code, simplifying the assertions. + +**Suites use InMemoryFs + `skipIf(!ASSETS_PRESENT)` — NO Postgres.** Both suites prove the *Deno boundary*, not the FS backend, so they inject `createFs: () => new InMemoryFs()` and skip only on missing vendored Deno+Pyodide assets (no `DATABASE_URL` needed). Benefit: they add zero Postgres connection pressure, so they never themselves flake on the `max_connections=100` contention. `session.fs.exists("evil.txt")` against the InMemoryFs is the "nothing drained" proof. **The drain-redirect-outside-cwd check is a pure-function `drain()` call** in a non-skipped `describe` (runs even without assets), satisfying that gate item without needing a forged-but-accepted frame (impossible by construction — `validateInbound` kills before `drain` is ever reached). + +**`test:integration` script needed the new dir.** Phase 7 introduces the first `src/api/pyodide/tests/integration/` dir; `test:integration` globbed only `src/sql-fs/tests/integration` + `src/api/tests/integration`, so the new suites were invisible to the gate until the script was extended. `test:unit`'s `--exclude 'src/**/integration/**'` already correctly excludes them. + +**Gotchas:** relative asset paths are **5** `../` from `src/api/pyodide/tests/integration/` (vs 4 from `src/api/tests/integration/`). biome's `noDelete` rejects `delete process.env.X` in the env-restore — use `Reflect.deleteProperty(process.env, "X")` (and never `= undefined`, which pollutes with the string `"undefined"`). Escape execs reuse one warm child (~6 s cold, then fast); forgery execs each kill+respawn (~3 s each) so the suite uses a fresh session per test. Run the full integration suite with `--maxWorkers=2` to keep peak Postgres connections under 100. --- From 53d9392ae952bbbd53a62a5986db1d70762bdfc8 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Tue, 9 Jun 2026 11:27:58 +0930 Subject: [PATCH 10/16] Harden pyodide runtime per code review Addresses the maintainer + Codex review findings on the pyodide runtime: - Runner: explicit ordered init sequence with a pre-`ready` adversarial self-test and an init try/catch that fails the spawn on error; snapshot and restore the FULL os.environ per run so env is strictly per-execution. - Commands: read script files through one capped + symlink-refusing helper before decoding; reject read-only execs that report any mutation before touching the FS; stage out-of-cwd scripts at a reserved non-drainable path; validate the full drain manifest (uniqueness, reserved-prefix, write/delete ancestor conflicts, file-as-directory) before any write. - Timeout: surface the internal EPYODIDE_TIMEOUT as a fatal throw that aborts the script transaction, mapped to 408 / SSE / batch timeout. - Manager: resolve a bare Deno binary to an absolute path (the child env omits PATH); inject exported bash env into Python os.environ per run. - Residency: clarify that admission is lazy/post-semaphore and the spawn thunk is cheap (no expensive init under the mutex). Adds unit + integration coverage and updates the design record. --- src/api/commands/pyodide-command.ts | 179 +++++++++++++++--- src/api/lib/batch-exec.ts | 11 +- src/api/pyodide-runtime-context.ts | 30 +++ src/api/pyodide/manager.ts | 31 ++- src/api/pyodide/residency.ts | 23 ++- .../tests/unit/pyodide-command.test.ts | 143 +++++++++++++- src/api/routes/exec.ts | 8 +- src/api/session-manager.ts | 22 ++- .../integration/pyodide.integration.test.ts | 28 +++ .../unit/session-manager.pyodide.test.ts | 15 ++ src/pyodide-runner/protocol.ts | 16 ++ src/pyodide-runner/runner.ts | 156 +++++++++++---- thoughts/issue-118-pyodide-runtime/design.md | 54 ++++-- 13 files changed, 615 insertions(+), 101 deletions(-) create mode 100644 src/api/pyodide-runtime-context.ts diff --git a/src/api/commands/pyodide-command.ts b/src/api/commands/pyodide-command.ts index 22b7c75..cbd8d95 100644 --- a/src/api/commands/pyodide-command.ts +++ b/src/api/commands/pyodide-command.ts @@ -30,8 +30,12 @@ import { latin1FromBytes, } from "just-bash"; import type { IFileSystem } from "just-bash"; +import { PYODIDE_EXT_STAGING_DIR } from "../../pyodide-runner/protocol.js"; import type { FsEntry, RunResponse } from "../../pyodide-runner/protocol.js"; +import { pyodideRuntimeContext } from "../pyodide-runtime-context.js"; +import { PyodideTimeoutError } from "../pyodide/manager.js"; import type { PyodideSandbox, RunRequestInput } from "../pyodide/manager.js"; +import { readOnlyContext } from "../read-only-context.js"; /** Default per-file cap on staged-in / drained-out file bytes (32 MiB). */ export const PYODIDE_MAX_FILE_BYTES_DEFAULT = 32 * 1024 * 1024; @@ -119,9 +123,16 @@ interface Parsed { readonly argv: string[]; /** base64 of the program's own stdin ("" when stdin was consumed as the program). */ readonly stdin: string; - /** Absolute path of a FILE script resolved OUTSIDE cwd that must also be staged - * (python3 FILE parity). Staged in `runPython` under the same caps as the cwd walk. */ - readonly scriptPathOutsideCwd?: string; + /** When the FILE script resolved OUTSIDE cwd (python3 FILE parity): stage its + * bytes at a reserved, non-drainable MEMFS path (collision-free, excluded from + * the cwd-scoped diff). `argv[0]` / `__file__` point at `stagePath`. Carries the + * ALREADY-read+capped bytes + mode so staging never re-reads the file. */ + readonly extScript?: { + readonly srcPath: string; + readonly stagePath: string; + readonly bytes: Uint8Array; + readonly mode: number; + }; } async function runPython( @@ -138,7 +149,7 @@ async function runPython( if (first === "-h" || first === "--help") return { stdout: "", stderr: HINT, exitCode: 0 }; if (first === "-m") return errResult("python3: the -m option is not supported in the pyodide runtime", 2); - const parsed = await parseProgram(args, ctx); + const parsed = await parseProgram(args, ctx, caps); if ("error" in parsed) return parsed.error; // Stage the cwd subtree (+ a script file outside cwd) into the request, under @@ -147,25 +158,68 @@ async function runPython( try { const staged = await stageCwd(ctx.fs, ctx.cwd, caps); files = staged.files; - if (parsed.scriptPathOutsideCwd !== undefined) { - const { entry } = await stageFile(ctx.fs, parsed.scriptPathOutsideCwd, caps, staged.total); - files.push(entry); + if (parsed.extScript !== undefined) { + // The out-of-cwd script was already read + symlink-refused + per-file-capped + // in parseProgram; here we only enforce the shared TOTAL budget and stage the + // already-read bytes at the reserved path (no second read). + enforceStageCaps(parsed.extScript.bytes.byteLength, parsed.extScript.srcPath, caps, staged.total); + files.push({ + path: parsed.extScript.stagePath, + kind: "file", + mode: parsed.extScript.mode, + data: Buffer.from(parsed.extScript.bytes).toString("base64"), + }); } } catch (err) { if (err instanceof PyodideDrainError) return errResult(err.message, 1); throw err; } - const input: RunRequestInput = { code: parsed.code, argv: parsed.argv, stdin: parsed.stdin, files, cwd: ctx.cwd }; + const input: RunRequestInput = { + code: parsed.code, + argv: parsed.argv, + stdin: parsed.stdin, + files, + cwd: ctx.cwd, + // Exported bash env vars → Python os.environ for THIS run only (subprocess + // inherit semantics). The Deno/host env is separately scrubbed; this is the + // sandbox's own bash environment, safe to surface to the script. + env: ctx.exportedEnv, + }; // Resolve the sandbox only now (an actual run is required). Manager THROWS on - // timeout/abort/integrity/child-exit — let it propagate so bash.exec rejects and - // the script transaction rolls back (drains nothing). - const resp = await getSandbox().run(input, ctx.signal ?? NEVER_ABORT); + // timeout/abort/integrity/child-exit; we let those propagate so bash.exec fails + // and nothing drains. For the INTERNAL runtime timeout we ALSO tag the per-exec + // context so execWithRuntimeThrottle can re-raise it as a fatal timeout (mapped + // to a consistent HTTP timeout) instead of just-bash flattening it to a generic + // non-zero exit. + let resp: RunResponse; + try { + resp = await getSandbox().run(input, ctx.signal ?? NEVER_ABORT); + } catch (err) { + if (err instanceof PyodideTimeoutError) { + const store = pyodideRuntimeContext.getStore(); + if (store !== undefined) store.timeoutError = err; + } + throw err; + } // Abort that landed after the response but before the drain → drain nothing. if (ctx.signal?.aborted) throw makeAbortError(); + // Explicit read-only enforcement (Decision 6): compare the run's reported MEMFS + // manifest against the staged snapshot and reject BEFORE any ctx.fs mutation if + // the run produced ANY persistent change. Created-then-deleted temp files are + // not in the final diff and are intentionally allowed (no persistent mutation). + // This fails closed one step earlier than relying on SqlFs to throw mid-drain; + // marking the shared read-only context lets the session layer surface a uniform + // EREADONLY_VIOLATION. + const roStore = readOnlyContext.getStore(); + if (roStore !== undefined && (resp.created.length > 0 || resp.modified.length > 0 || resp.deleted.length > 0)) { + roStore.violated = true; + return errResult("python3: readOnly script attempted to mutate the filesystem", 1); + } + await drain(ctx.fs, ctx.cwd, resp, caps); return { @@ -175,7 +229,7 @@ async function runPython( }; } -async function parseProgram(args: string[], ctx: CommandContext): Promise { +async function parseProgram(args: string[], ctx: CommandContext, caps: Caps): Promise { const first = args[0]; if (first === "-c") { @@ -200,20 +254,64 @@ async function parseProgram(args: string[], ctx: CommandContext): Promise { + const st = await fs.lstat(path); + if (st.isSymbolicLink) throw new PyodideDrainError(`refusing to run a symlink: ${path}`); + const bytes = await fs.readFileBuffer(path); + if (bytes.byteLength > caps.maxFileBytes) { + throw new PyodideDrainError(`'${path}' (${bytes.byteLength} bytes) exceeds the per-file stage cap`); + } + return { code: Buffer.from(bytes).toString("utf8"), bytes, mode: st.mode & 0o777 }; +} + +/** Final path segment (basename) of an absolute MEMFS/SqlFs path. */ +function baseName(path: string): string { + const i = path.lastIndexOf("/"); + return i >= 0 ? path.slice(i + 1) : path; +} + function stdinBase64(ctx: CommandContext): string { return Buffer.from(latin1FromBytes(ctx.stdin), "latin1").toString("base64"); } @@ -298,25 +396,62 @@ async function stageCwd(fs: IFileSystem, cwd: string, caps: Caps): Promise<{ fil * which rolls the transaction back. */ export async function drain(fs: IFileSystem, cwd: string, resp: RunResponse, caps: Caps): Promise { - const assertUnderCwd = (path: string, label: string): string => { + const assertSafePath = (path: string, label: string): string => { if (path.includes("\0")) throw new PyodideDrainError(`${label} path contains a null byte`); const resolved = fs.resolvePath(cwd, path); if (!isUnderCwd(resolved, cwd)) throw new PyodideDrainError(`${label} path escapes cwd: ${path}`); + // Defense-in-depth: never drain into the reserved out-of-cwd staging area. + // (Already excluded by the cwd check since it lives outside cwd — asserted + // explicitly so a forged frame can't target it even if cwd ever overlapped.) + if (resolved === PYODIDE_EXT_STAGING_DIR || resolved.startsWith(`${PYODIDE_EXT_STAGING_DIR}/`)) { + throw new PyodideDrainError(`${label} path targets the reserved staging area: ${path}`); + } return resolved; }; - // 1. Validate paths + caps for everything before writing anything. + // 1. Validate paths + caps + manifest consistency before writing anything. let total = 0; + const writes = new Set(); // resolved created+modified paths (uniqueness) + const writeFiles: string[] = []; // resolved written-FILE paths (ancestor check) for (const e of [...resp.created, ...resp.modified]) { - assertUnderCwd(e.path, "drain"); + const resolved = assertSafePath(e.path, "drain"); + if (writes.has(resolved)) throw new PyodideDrainError(`duplicate drain path: ${e.path}`); + writes.add(resolved); if (e.kind === "file") { + writeFiles.push(resolved); const n = base64ByteLength(e.data); if (n > caps.maxFileBytes) throw new PyodideDrainError(`'${e.path}' (${n} bytes) exceeds the per-file drain cap`); total += n; } } if (total > caps.maxTotalBytes) throw new PyodideDrainError("drained files exceed the total byte cap"); - for (const p of resp.deleted) assertUnderCwd(p, "deleted"); + + // Deletes: reject duplicates (a plain Set would silently collapse them). + const deletes = new Set(); + for (const p of resp.deleted) { + const resolved = assertSafePath(p, "deleted"); + if (deletes.has(resolved)) throw new PyodideDrainError(`duplicate deleted path: ${p}`); + deletes.add(resolved); + } + + // No write may equal, contain, or be contained by a delete: deleting an ancestor + // directory would silently remove a "written" path (and vice versa). The real + // diff never produces such a manifest — reject it before any mutation. + for (const w of writes) { + for (const d of deletes) { + if (w === d || w.startsWith(`${d}/`) || d.startsWith(`${w}/`)) { + throw new PyodideDrainError(`drain path conflicts with a deletion: '${w}' vs '${d}'`); + } + } + } + // A written FILE cannot also be a directory ancestor of another written path + // (would require treating a file as a directory). + for (const f of writeFiles) { + const prefix = `${f}/`; + for (const w of writes) { + if (w.startsWith(prefix)) throw new PyodideDrainError(`drain path uses a file as a directory: ${f}`); + } + } // 2. Apply created (dirs shallow→deep, then files), then modified files. for (const e of resp.created) await applyEntry(fs, cwd, e); diff --git a/src/api/lib/batch-exec.ts b/src/api/lib/batch-exec.ts index 45dfc75..1c166f2 100644 --- a/src/api/lib/batch-exec.ts +++ b/src/api/lib/batch-exec.ts @@ -93,10 +93,12 @@ async function runSequential( durationMs: Date.now() - scriptStart, }); } - } catch { + } catch (err) { clearTimeout(timer); outerSignal?.removeEventListener("abort", abortFromOuter); - if (timedOut) { + // The script's own timer (timedOut) OR the manager's internal pyodide + // runtime timeout both surface as a per-script timeout result. + if (timedOut || (err as Error & { code?: string }).code === "EPYODIDE_TIMEOUT") { results.push({ id: entry.id, stdout: "", @@ -199,7 +201,8 @@ async function runParallel( exitCode: execResult.exitCode, durationMs: Date.now() - scriptStart, }; - } catch { + } catch (err) { + const isPyodideTimeout = (err as Error & { code?: string }).code === "EPYODIDE_TIMEOUT"; results[idx] = { id: entry.id, stdout: "", @@ -207,7 +210,7 @@ async function runParallel( exitCode: -1, durationMs: Date.now() - scriptStart, error: - timedOut || perScriptTimedOut + timedOut || perScriptTimedOut || isPyodideTimeout ? "timeout" : sharedController.signal.aborted ? "aborted" diff --git a/src/api/pyodide-runtime-context.ts b/src/api/pyodide-runtime-context.ts new file mode 100644 index 0000000..4209e1c --- /dev/null +++ b/src/api/pyodide-runtime-context.ts @@ -0,0 +1,30 @@ +/** + * Per-exec attribution for the pyodide runtime's INTERNAL timeout. + * + * just-bash normalizes a custom-command handler rejection into a non-zero + * `ExecResult` (the error lands in `stderr`, the exec RESOLVES) rather than + * propagating it as a throw — only certain FS/builtin errors escape `bash.exec` + * (see `withSessionReadEntry`'s `EREADONLY` remap). So when the manager's own + * `PYODIDE_RUNTIME_TIMEOUT_MS` fires and `PyodideSandbox.run()` rejects with + * {@link PyodideTimeoutError}, that typed error would otherwise be flattened into + * a generic non-zero exit and the route would return 200 instead of a timeout. + * + * We bridge it the same way the read-only path bridges its violation: an + * `AsyncLocalStorage` context that follows the async stack down into the pyodide + * command. The command records the typed error on the context; `execWithRuntimeThrottle` + * reads it back AFTER `bash.exec` resolves and re-throws it as a fatal error, so + * the script transaction is ABORTED (a timed-out run never commits) and the route + * layer maps `EPYODIDE_TIMEOUT` to a consistent timeout response. + * + * Per-exec (not per-session) because concurrent readOnly pyodide execs share one + * session; a session-level flag would mis-attribute a timeout across readers. + */ + +import { AsyncLocalStorage } from "node:async_hooks"; + +export interface PyodideRuntimeContext { + /** Set by the pyodide command when the manager's internal runtime timeout fired. */ + timeoutError?: Error; +} + +export const pyodideRuntimeContext = new AsyncLocalStorage(); diff --git a/src/api/pyodide/manager.ts b/src/api/pyodide/manager.ts index e78206b..1570f24 100644 --- a/src/api/pyodide/manager.ts +++ b/src/api/pyodide/manager.ts @@ -28,6 +28,7 @@ import { Buffer } from "node:buffer"; import { type ChildProcess, type SpawnOptions, spawn as nodeSpawn } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { accessSync, constants as fsConstants } from "node:fs"; import { fileURLToPath } from "node:url"; import type { Frame, RunRequest, RunResponse } from "../../pyodide-runner/protocol.js"; import { @@ -155,6 +156,33 @@ interface OwnedOp { const DEFAULT_RUNNER_PATH = fileURLToPath(new URL("../../pyodide-runner/runner.ts", import.meta.url)); +/** + * Resolve the Deno binary to an ABSOLUTE path (design D1 / review #6). + * + * The child is spawned with a minimal allowlisted env (`{ DENO_NO_UPDATE_CHECK }`) + * that intentionally omits `PATH`. Node's `spawn` resolves a *bare* command name + * (e.g. `"deno"`) against the CHILD env's `PATH` — which is absent — so a bare name + * would fail `ENOENT`. Production sets `DENO_BIN_PATH` to the vendored absolute + * path; for dev/test convenience we also resolve a bare name against the PARENT + * `PATH` here so what we hand to `spawn` is always absolute. A path that already + * contains a separator (absolute or relative) is used verbatim. If a bare name + * can't be found, it is returned unchanged so the spawn surfaces a clear error. + */ +function resolveDenoBin(bin: string): string { + if (bin.includes("/")) return bin; // already absolute or relative — not PATH-resolved + for (const dir of (process.env.PATH ?? "").split(":")) { + if (dir.length === 0) continue; + const candidate = `${dir}/${bin}`; + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // not here / not executable — keep scanning + } + } + return bin; // not found on PATH; let spawn surface ENOENT with the bare name +} + export class PyodideSandbox { #state: WorkerState = "cold"; #generation = 0; @@ -182,7 +210,7 @@ export class PyodideSandbox { constructor(opts: PyodideSandboxOptions = {}) { this.#assetDir = opts.assetDir ?? process.env.PYODIDE_ASSET_DIR ?? ""; - this.#denoBin = opts.denoBin ?? process.env.DENO_BIN_PATH ?? "deno"; + this.#denoBin = resolveDenoBin(opts.denoBin ?? process.env.DENO_BIN_PATH ?? "deno"); this.#runnerPath = opts.runnerPath ?? DEFAULT_RUNNER_PATH; this.#runtimeTimeoutMs = opts.runtimeTimeoutMs ?? PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT; this.#maxFrameBytes = opts.maxFrameBytes ?? PYODIDE_MAX_FRAME_BYTES_DEFAULT; @@ -348,6 +376,7 @@ export class PyodideSandbox { stdin: op.input.stdin, files: op.input.files, cwd: op.input.cwd, + env: op.input.env, }; op.requestId = frame.requestId; op.seq = frame.seq; diff --git a/src/api/pyodide/residency.ts b/src/api/pyodide/residency.ts index 25aebe7..ab92fe4 100644 --- a/src/api/pyodide/residency.ts +++ b/src/api/pyodide/residency.ts @@ -12,10 +12,14 @@ * The startup invariant `MAX_RESIDENT_PYODIDE >= MAX_CONCURRENT_PYODIDE` (enforced * by the SessionManager) guarantees a busy worker never needs to be evicted. * - * Atomic admission: a single `async-mutex` wraps *reserve a slot → select an - * eviction victim → spawn (expensive init) → roll back on failed init* as one - * critical section, so concurrent cold starts cannot both observe a free slot and - * exceed the cap. **`starting` and `busy` workers are never evictable.** + * Atomic admission: a single `async-mutex` wraps *select an eviction victim → + * construct the manager → register* as one critical section, so concurrent + * admissions cannot both observe a free slot and exceed the cap. The `spawn` + * thunk is CHEAP — it only constructs a `PyodideSandbox` (no Deno child, no + * Pyodide load). The multi-second Deno spawn + package init happens LAZILY in + * `PyodideSandbox.run()`, OUTSIDE this mutex (lazy post-semaphore admission), so + * the critical section never serializes cold starts. **`starting` and `busy` + * workers are never evictable.** * * Eviction / idle-kill disposes a worker (terminal — its Deno child is SIGKILLed). * The owning session SURVIVES: it observes `worker.disposed` on its next exec and @@ -85,8 +89,9 @@ export class PyodideResidency { /** * Atomic admission. Inside the admission mutex: if at capacity, select an idle - * LRU victim and evict it; then run the caller's `spawn` (the expensive init) - * and register the new worker. If `spawn` throws (failed init), the reserved + * LRU victim and evict it; then run the caller's `spawn` (a CHEAP `PyodideSandbox` + * construction — NOT the Deno child / Pyodide load, which happen lazily in + * `run()` outside this mutex) and register the new worker. If `spawn` throws, the * slot is rolled back — the worker is never registered — and the error is * rethrown. Returns the admitted worker. */ @@ -111,9 +116,9 @@ export class PyodideResidency { // belt-and-braces fallback; the next sweep/admit reclaims the surplus // once a worker goes idle. } - // `spawn` is reserved-behind: the expensive Pyodide child init runs only - // after a slot is secured. Registering ONLY on success rolls the slot back - // on a failed init (we never added the worker). + // `spawn` only CONSTRUCTS the manager (cheap); the Deno child + Pyodide load + // happen lazily in run(), outside this mutex. Registering ONLY on success + // rolls the slot back if construction throws (the worker is never added). const worker = await spawn(); this.#residents.set(worker, Date.now()); return worker; diff --git a/src/api/pyodide/tests/unit/pyodide-command.test.ts b/src/api/pyodide/tests/unit/pyodide-command.test.ts index b8deb3e..2cfc491 100644 --- a/src/api/pyodide/tests/unit/pyodide-command.test.ts +++ b/src/api/pyodide/tests/unit/pyodide-command.test.ts @@ -10,7 +10,8 @@ import { type ByteString, EMPTY_BYTES, InMemoryFs, encodeUtf8ToBytes } from "jus import type { CommandContext, CustomCommand, ExecResult, IFileSystem } from "just-bash"; import { describe, expect, it } from "vitest"; import type { FsEntry, RunResponse } from "../../../../pyodide-runner/protocol.js"; -import { PyodideDrainError, createPyodideCommands } from "../../../commands/pyodide-command.js"; +import { PyodideDrainError, createPyodideCommands, drain } from "../../../commands/pyodide-command.js"; +import { type ReadOnlyContext, readOnlyContext } from "../../../read-only-context.js"; import type { PyodideSandbox, RunRequestInput } from "../../manager.js"; function makeResponse(over: Partial = {}): RunResponse { @@ -50,12 +51,13 @@ function fakeSandbox(respond: (input: RunRequestInput) => RunResponse): { function makeCtx( fs: IFileSystem, - opts: { cwd?: string; stdin?: ByteString; signal?: AbortSignal } = {}, + opts: { cwd?: string; stdin?: ByteString; signal?: AbortSignal; exportedEnv?: Record } = {}, ): CommandContext { return { fs, cwd: opts.cwd ?? "/home/user", env: new Map(), + exportedEnv: opts.exportedEnv, stdin: opts.stdin ?? EMPTY_BYTES, signal: opts.signal, } as CommandContext; @@ -261,7 +263,7 @@ describe("pyodide command — byte caps", () => { }); describe("pyodide command — script resolved outside cwd (FILE parity)", () => { - it("stages an out-of-cwd script with its real mode, not a hardcoded 0644", async () => { + it("stages an out-of-cwd script at a reserved non-drainable path with its real mode", async () => { const fs = await freshFs(); await fs.mkdir("/outside", { recursive: true }); await fs.writeFile("/outside/tool.py", "print('hi')"); @@ -270,9 +272,140 @@ describe("pyodide command — script resolved outside cwd (FILE parity)", () => const [python3] = createPyodideCommands(sandbox); await run(python3 as CustomCommand, ["/outside/tool.py", "arg"], makeCtx(fs)); const input = calls[0]!; - expect(input.argv).toEqual(["/outside/tool.py", "arg"]); - const staged = input.files.find((f) => f.path === "/outside/tool.py"); + // The out-of-cwd script is re-homed at the reserved staging path and argv[0] + // (→ __file__) points there, so it can't collide with Pyodide MEMFS internals + // and is excluded from the cwd-scoped drain. + expect(input.argv).toEqual(["/__sqlfs_ext__/tool.py", "arg"]); + const staged = input.files.find((f) => f.path === "/__sqlfs_ext__/tool.py"); expect(staged?.kind).toBe("file"); expect(staged?.mode).toBe(0o755); // real mode captured (was hardcoded 0o644 before the fix) + // The original absolute path is NOT staged (no MEMFS collision risk). + expect(input.files.find((f) => f.path === "/outside/tool.py")).toBeUndefined(); + }); + + it("refuses to run a symlinked script (symlink-refused on the capped script read, review #1)", async () => { + const fs = await freshFs(); + await fs.writeFile("/home/user/real.py", "print('hi')"); + await fs.symlink("/home/user/real.py", "/home/user/link.py"); + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + const res = await run(python3 as CustomCommand, ["link.py"], makeCtx(fs)); + expect(res.exitCode).toBe(1); + expect(res.stderr).toContain("symlink"); + expect(calls).toHaveLength(0); // refused before reaching the sandbox + }); +}); + +describe("pyodide command — exec env forwarding (review #6)", () => { + it("forwards exported bash env vars to the run request (→ os.environ)", async () => { + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + const ctx = makeCtx(await freshFs(), { exportedEnv: { FOO: "bar", API_BASE: "http://x" } }); + await run(python3 as CustomCommand, ["-c", "print(1)"], ctx); + expect(calls[0]?.env).toEqual({ FOO: "bar", API_BASE: "http://x" }); + }); + + it("omits env when nothing is exported", async () => { + const { sandbox, calls } = fakeSandbox(() => makeResponse()); + const [python3] = createPyodideCommands(sandbox); + await run(python3 as CustomCommand, ["-c", "print(1)"], makeCtx(await freshFs())); + expect(calls[0]?.env).toBeUndefined(); + }); +}); + +describe("pyodide command — explicit read-only enforcement (review #3)", () => { + it("rejects a readOnly run that reported any mutation, before touching ctx.fs", async () => { + const fs = await freshFs(); + // The run reports a created file → a persistent mutation. + const { sandbox, calls } = fakeSandbox(() => makeResponse({ created: [fileEntry("/home/user/out.txt", "x")] })); + const [python3] = createPyodideCommands(sandbox); + const roCtx: ReadOnlyContext = { violated: false }; + const res = await readOnlyContext.run(roCtx, () => + run(python3 as CustomCommand, ["-c", "open('out.txt','w').write('x')"], makeCtx(fs)), + ); + expect(res.exitCode).toBe(1); + expect(roCtx.violated).toBe(true); // the session layer maps this to EREADONLY_VIOLATION + expect(calls).toHaveLength(1); // the run happened… + expect(await fs.exists("/home/user/out.txt")).toBe(false); // …but nothing drained + }); + + it("allows a readOnly run that reported no mutation", async () => { + const fs = await freshFs(); + const { sandbox } = fakeSandbox(() => makeResponse({ stdout: Buffer.from("ok").toString("base64") })); + const [python3] = createPyodideCommands(sandbox); + const roCtx: ReadOnlyContext = { violated: false }; + const res = await readOnlyContext.run(roCtx, () => + run(python3 as CustomCommand, ["-c", "print('ok')"], makeCtx(fs)), + ); + expect(res.exitCode).toBe(0); + expect(roCtx.violated).toBe(false); + }); +}); + +describe("pyodide drain — manifest validation (review #7)", () => { + const caps = { maxFileBytes: 1 << 20, maxTotalBytes: 1 << 20 }; + async function fsAt(cwd: string): Promise { + const fs = new InMemoryFs(); + await fs.mkdir(cwd, { recursive: true }); + return fs; + } + + it("rejects a duplicate drain path", async () => { + const fs = await fsAt("/home/user"); + const resp = makeResponse({ + created: [fileEntry("/home/user/a.txt", "1"), fileEntry("/home/user/a.txt", "2")], + }); + await expect(drain(fs, "/home/user", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + }); + + it("rejects a path that is both written and deleted", async () => { + const fs = await fsAt("/home/user"); + const resp = makeResponse({ + created: [fileEntry("/home/user/a.txt", "1")], + deleted: ["/home/user/a.txt"], + }); + await expect(drain(fs, "/home/user", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + }); + + it("rejects a delete that is an ancestor of a written path (would erase the write)", async () => { + const fs = await fsAt("/home/user"); + const resp = makeResponse({ + created: [fileEntry("/home/user/d/x.txt", "child")], + deleted: ["/home/user/d"], + }); + await expect(drain(fs, "/home/user", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + expect(await fs.exists("/home/user/d/x.txt")).toBe(false); // nothing applied + }); + + it("rejects duplicate deleted paths (not silently collapsed)", async () => { + const fs = await fsAt("/home/user"); + await fs.writeFile("/home/user/gone.txt", "x"); + const resp = makeResponse({ deleted: ["/home/user/gone.txt", "/home/user/gone.txt"] }); + await expect(drain(fs, "/home/user", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + expect(await fs.exists("/home/user/gone.txt")).toBe(true); // not deleted (rejected pre-mutation) + }); + + it("rejects a written file used as a directory ancestor", async () => { + const fs = await fsAt("/home/user"); + const resp = makeResponse({ + created: [fileEntry("/home/user/a", "file"), fileEntry("/home/user/a/b.txt", "child")], + }); + await expect(drain(fs, "/home/user", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + }); + + it("rejects a drain path targeting the reserved staging area (cwd = root)", async () => { + const fs = new InMemoryFs(); + const resp = makeResponse({ created: [fileEntry("/__sqlfs_ext__/evil.py", "x")] }); + await expect(drain(fs, "/", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + expect(await fs.exists("/__sqlfs_ext__/evil.py")).toBe(false); + }); + + it("applies a valid (dirs-before-files) manifest", async () => { + const fs = await fsAt("/home/user"); + const resp = makeResponse({ + created: [{ path: "/home/user/d", kind: "dir", mode: 0o755, data: "" }, fileEntry("/home/user/d/x.txt", "hi")], + }); + await drain(fs, "/home/user", resp, caps); + expect(await fs.readFile("/home/user/d/x.txt", "utf8")).toBe("hi"); }); }); diff --git a/src/api/routes/exec.ts b/src/api/routes/exec.ts index d2ac08f..1f90431 100644 --- a/src/api/routes/exec.ts +++ b/src/api/routes/exec.ts @@ -195,7 +195,9 @@ export function execRoutes(sessionManager: SessionManager): Hono<{ Variables: Au }; } catch (e) { clearTimeout(timer); - if (timedOut) { + // Route-level abort (timedOut) OR the manager's internal pyodide runtime + // timeout both map to the same 408 EXEC_TIMEOUT response. + if (timedOut || (e as Error & { code?: string }).code === "EPYODIDE_TIMEOUT") { return { kind: "timeout", durationMs: Date.now() - startMs }; } throw e; @@ -314,7 +316,9 @@ export function execRoutes(sessionManager: SessionManager): Hono<{ Variables: Au }); } catch (e) { clearTimeout(timer); - if (timedOut) { + // Route-level abort (timedOut) OR the manager's internal pyodide runtime + // timeout both emit the same terminal timeout exit event. + if (timedOut || (e as Error & { code?: string }).code === "EPYODIDE_TIMEOUT") { await stream.writeSSE({ event: "exit", data: JSON.stringify({ t: "exit", exitCode: -1, durationMs: Date.now() - startMs, error: "timeout" }), diff --git a/src/api/session-manager.ts b/src/api/session-manager.ts index cec5864..146175b 100644 --- a/src/api/session-manager.ts +++ b/src/api/session-manager.ts @@ -33,6 +33,7 @@ import { createPyodideCommands } from "./commands/pyodide-command.js"; import { execLockKey, withDistributedLock } from "./distributed-lock.js"; import { type DistributedRWLockOptions, rwLockKeys, withDistributedRWLock } from "./distributed-rw-lock.js"; import { logAudit } from "./lib/audit.js"; +import { type PyodideRuntimeContext, pyodideRuntimeContext } from "./pyodide-runtime-context.js"; // NOTE: `py-exec` (warm host Python) is intentionally NOT imported/wired here. // It spawned the HOST python3 with full `process.env`, which is a sandbox // escape (RCE + secret/credential exfil — audit C1). The WASM `python3` @@ -1392,11 +1393,20 @@ export class SessionManager { // nothing to commit, and beginScope/endScope on the shared SessionScopedFs // would race across concurrent parallel readers. const inReadOnlyScope = readOnlyContext.getStore() !== undefined; + // A pyodide INTERNAL runtime timeout is flattened by just-bash into a non-zero + // ExecResult; the pyodide command records the typed error on this per-exec + // context. Recover it here and treat it as FATAL so the script tx is aborted + // (a timed-out run never commits) and the typed timeout escapes to the route. + const assertNoPyodideTimeout = (): void => { + const pyTimeout = pyodideRuntimeContext.getStore()?.timeoutError; + if (pyTimeout !== undefined) throw pyTimeout; + }; const execFn = async (): Promise => { if (!inReadOnlyScope && session.scriptTx !== undefined) { session.scriptTx.beginScope(); try { const result = await session.bash.exec(script, resolvedOpts); + assertNoPyodideTimeout(); await session.scriptTx.endScope(); return result; } catch (err) { @@ -1404,7 +1414,9 @@ export class SessionManager { throw err; } } - return session.bash.exec(script, resolvedOpts); + const result = await session.bash.exec(script, resolvedOpts); + assertNoPyodideTimeout(); + return result; }; const updateCwd = (result: BashExecResult): BashExecResult => { @@ -1459,7 +1471,13 @@ export class SessionManager { // idle eviction victim always exists and resident subprocesses can never // exceed MAX_RESIDENT. Also re-admits (single-flight) after a residency // eviction / idle-kill disposed the prior worker (cold-start on next exec). - if (usesPyodide) await this.ensurePyodideAdmitted(session); + if (usesPyodide) { + await this.ensurePyodideAdmitted(session); + // Run inside the per-exec pyodide context so the command can record an + // internal runtime timeout that `execFn` re-raises as a fatal timeout. + const pyCtx: PyodideRuntimeContext = {}; + return updateCwd(await pyodideRuntimeContext.run(pyCtx, execFn)); + } return updateCwd(await execFn()); } finally { if (usesJs) this.releaseSlot(this.jsSem); diff --git a/src/api/tests/integration/pyodide.integration.test.ts b/src/api/tests/integration/pyodide.integration.test.ts index bf2aabc..96257d9 100644 --- a/src/api/tests/integration/pyodide.integration.test.ts +++ b/src/api/tests/integration/pyodide.integration.test.ts @@ -122,4 +122,32 @@ describe.skipIf(SKIP)("pyodide runtime — end-to-end (real Deno + Pyodide)", () }, COLD, ); + + it( + "injects exec env into os.environ and does not leak it across runs (reviews #6/#3)", + async () => { + const id = `pyo-env-${Date.now()}`; + cleanup.push(id); + const session = await sm.getOrCreate(TENANT, id, PYODIDE, "owner"); + // #6: a bash-exported var is visible in Python's os.environ for the run; the + // script also sets a NEW os.environ key to probe cross-run leakage. + const r1 = await sm.execWithRuntimeThrottle( + session, + `export FOO=barvalue; python3 -c "import os; print('FOO=' + os.environ.get('FOO','none')); os.environ['LEAK']='run1'"`, + ); + expect(r1.exitCode).toBe(0); + expect(r1.stdout).toContain("FOO=barvalue"); + + // #3: on the SAME warm child, neither the injected FOO nor the script-set + // LEAK may survive into the next run (env is strictly per-execution). + const r2 = await sm.execWithRuntimeThrottle( + session, + `python3 -c "import os; print('LEAK=' + os.environ.get('LEAK','clean') + ' FOO=' + os.environ.get('FOO','clean'))"`, + ); + expect(r2.exitCode).toBe(0); + expect(r2.stdout).toContain("LEAK=clean"); + expect(r2.stdout).toContain("FOO=clean"); + }, + COLD, + ); }); diff --git a/src/api/tests/unit/session-manager.pyodide.test.ts b/src/api/tests/unit/session-manager.pyodide.test.ts index 31f6be5..9b1d408 100644 --- a/src/api/tests/unit/session-manager.pyodide.test.ts +++ b/src/api/tests/unit/session-manager.pyodide.test.ts @@ -11,6 +11,7 @@ import { InMemoryFs } from "just-bash"; import type { IFileSystem } from "just-bash"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { PyodideTimeoutError } from "../../pyodide/manager.js"; import type { PyodideSandbox } from "../../pyodide/manager.js"; import { SessionManager } from "../../session-manager.js"; @@ -169,6 +170,20 @@ describe("session pyodide ownership", () => { expect(session.pyodideSandbox?.disposed).toBe(false); }); + it("surfaces an internal runtime timeout as a fatal EPYODIDE_TIMEOUT throw (review #4)", async () => { + const { sm, sandbox } = makeManager(); + active = sm; + const session = await sm.getOrCreate(T, "pyo", PYODIDE); + // The manager's internal runtime timeout: run() rejects with PyodideTimeoutError. + // just-bash flattens that into a non-zero ExecResult; the command tags the + // per-exec context and execWithRuntimeThrottle re-raises it as a fatal throw so + // the route layer can map it to a consistent timeout response. + sandbox.run = vi.fn(() => Promise.reject(new PyodideTimeoutError(25))); + await expect(sm.execWithRuntimeThrottle(session, "python3 -c '1'")).rejects.toMatchObject({ + code: "EPYODIDE_TIMEOUT", + }); + }); + it("a getOrCreate failure builds no pyodide manager to dispose (admission is lazy)", async () => { const fs = new InMemoryFs() as unknown as IFileSystem & { getAllPaths: () => string[] }; const sandbox = makeFakeSandbox(); diff --git a/src/pyodide-runner/protocol.ts b/src/pyodide-runner/protocol.ts index 9e73915..9ae2ca9 100644 --- a/src/pyodide-runner/protocol.ts +++ b/src/pyodide-runner/protocol.ts @@ -20,6 +20,18 @@ export const PROTOCOL_VERSION = 1; +/** + * Reserved MEMFS directory for staging a script file that resolves OUTSIDE the + * exec cwd (the `python3 /abs/elsewhere.py` parity case). Staging it here instead + * of at its original absolute path makes non-drainability STRUCTURAL — it lives + * outside the runner's cwd-scoped diff so it can never drain back — and avoids + * colliding with Pyodide's own MEMFS layout (`/lib`, `/usr`, `/home/pyodide`, …) + * if the sandbox path happened to overlap. The runner writes the script here, runs + * it with `argv[0]`/`__file__` pointing here, and wipes the directory after each + * run. The drain layer independently rejects any path under this prefix. + */ +export const PYODIDE_EXT_STAGING_DIR = "/__sqlfs_ext__"; + export type FrameType = "run" | "result" | "error" | "ready"; /** @@ -46,6 +58,10 @@ export interface RunRequest { readonly stdin: string; // base64 readonly files: readonly FsEntry[]; // cwd subtree staged into MEMFS (files + dirs) readonly cwd: string; + // Exported bash env vars injected into Python's `os.environ` for THIS run only + // (subprocess-inherit semantics), then restored — never the host/Deno env, which + // is separately scrubbed. Optional for backward compatibility with old callers. + readonly env?: Record; } /** child → Node: the single response to a `run`. */ diff --git a/src/pyodide-runner/runner.ts b/src/pyodide-runner/runner.ts index 331f611..a5770db 100644 --- a/src/pyodide-runner/runner.ts +++ b/src/pyodide-runner/runner.ts @@ -25,6 +25,7 @@ import { createRequire } from "node:module"; import { type Frame, type FsEntry, + PYODIDE_EXT_STAGING_DIR, type ReadyFrame, type RunRequest, type RunResponse, @@ -36,6 +37,7 @@ import { // deno-lint-ignore no-explicit-any const denoRef = (globalThis as any).Deno; const stdoutWriteSync: (b: Uint8Array) => number = denoRef.stdout.writeSync.bind(denoRef.stdout); +const stderrWriteSync: (b: Uint8Array) => number = denoRef.stderr.writeSync.bind(denoRef.stderr); const stdinReadable: ReadableStream = denoRef.stdin.readable; const denoExit: (code: number) => never = denoRef.exit.bind(denoRef); const denoArgs: string[] = denoRef.args; @@ -72,48 +74,90 @@ function emit(frame: Frame): void { } } -// ── Load Pyodide + packages (spike S1 proven, fully offline) ──────────────── -// deno-lint-ignore no-explicit-any -const pyodideModule = await import(`file://${indexURL}pyodide.mjs`); -const loadPyodide = pyodideModule.loadPyodide; - -const pyodide = await loadPyodide({ - indexURL, - lockFileURL: `${indexURL}pyodide-lock.json`, - // Discard Pyodide's own load-time banner/print; per-run capture is wired below. - stdout: () => {}, - stderr: () => {}, -}); - -// numpy/pandas/scipy ship in the distribution → load by name. -await pyodide.loadPackage(["numpy", "pandas", "scipy"]); -// openpyxl + et_xmlfile are NOT in the distribution; load the vendored pure-python -// wheels by local file:// URL (discovered in the asset dir). loadPackage reads -// them via node:fs under --allow-read — no network. (Phase 0 Discoveries: the -// stock lock has no openpyxl, so loadPackage-by-name would throw; file:// wheels -// are the S1-proven offline path. The supplementary custom lock from -// build-pyodide-lock.mjs is not required by this runner.) -const wheelNames: string[] = []; -for (const entry of denoRef.readDirSync(assetRoot)) { - if (entry.isFile && (/^openpyxl-.*\.whl$/.test(entry.name) || /^et_xmlfile-.*\.whl$/.test(entry.name))) { - wheelNames.push(entry.name); +// ── Pyodide instance + MEMFS handle (assigned by the init sequence below) ──── +let pyodide; +let FS; + +/** + * Adversarial self-test (spike S2 hard gate). Runs AFTER realm lockdown and + * BEFORE `ready`: proves the deletable host primitives are actually gone and that + * Pyodide's `js` proxy retains no reachable reference to them. Deleted globals + + * closure privacy do NOT by themselves prove no usable writer survived, so we + * verify it directly here. A failure throws → the init sequence kills the child + * and the run that triggered the spawn fails (admission fails). + */ +function selfTest() { + const gg = globalThis as any; + for (const name of ["Deno", "console", "require", "__dirname", "__filename"]) { + if (gg[name] !== undefined) throw new Error(`realm lockdown self-test failed: globalThis.${name} survived`); + } + // Pyodide's `js` foreign module proxies globalThis; confirm the deleted host + // primitives are unreachable through it. + const probe = pyodide.runPython("import js\n[hasattr(js, _n) for _n in ('Deno', 'console', 'require')]"); + let reachable; + try { + reachable = probe && typeof probe.toJs === "function" ? probe.toJs() : probe; + } finally { + if (probe && typeof probe.destroy === "function") probe.destroy(); } + if (Array.isArray(reachable) && reachable.some(Boolean)) { + throw new Error("realm lockdown self-test failed: js proxy still exposes a deleted host primitive"); + } +} + +// ── INIT SEQUENCE (spike S1 + S2) ─────────────────────────────────────────── +// Load offline Pyodide + packages, lock down the realm, run the adversarial +// self-test — ALL before announcing `ready`. Any failure here is fatal: write a +// diagnostic to stderr and exit non-zero so the Node manager retires the child +// and fails the run that triggered the spawn. +try { + // deno-lint-ignore no-explicit-any + const pyodideModule = await import(`file://${indexURL}pyodide.mjs`); + pyodide = await pyodideModule.loadPyodide({ + indexURL, + lockFileURL: `${indexURL}pyodide-lock.json`, + // Discard Pyodide's own load-time banner/print; per-run capture is wired below. + stdout: () => {}, + stderr: () => {}, + }); + + // numpy/pandas/scipy ship in the distribution → load by name. + await pyodide.loadPackage(["numpy", "pandas", "scipy"]); + // openpyxl + et_xmlfile are NOT in the distribution; load the vendored pure-python + // wheels by local file:// URL (discovered in the asset dir). loadPackage reads + // them via node:fs under --allow-read — no network. (Phase 0 Discoveries: the + // stock lock has no openpyxl, so loadPackage-by-name would throw; file:// wheels + // are the S1-proven offline path. The supplementary custom lock from + // build-pyodide-lock.mjs is not required by this runner.) + const wheelNames: string[] = []; + for (const entry of denoRef.readDirSync(assetRoot)) { + if (entry.isFile && (/^openpyxl-.*\.whl$/.test(entry.name) || /^et_xmlfile-.*\.whl$/.test(entry.name))) { + wheelNames.push(entry.name); + } + } + // et_xmlfile before openpyxl (dependency order). + wheelNames.sort((a, b) => (a.startsWith("et_xmlfile") ? -1 : b.startsWith("et_xmlfile") ? 1 : 0)); + await pyodide.loadPackage(wheelNames.map((w) => `file://${indexURL}${w}`)); + + FS = pyodide.FS; + + // ── REALM LOCKDOWN — before any untrusted runPythonAsync (spike S2) ───────── + // Delete every deletable host / Node-compat write primitive. import("node:fs") + // cannot be deleted (it is syntax), so this is hardening, not containment — the + // Node-side validator is the real guarantee (see file header). + delete g.Deno; + delete g.console; + delete g.require; + delete g.__dirname; + delete g.__filename; + + // ── ADVERSARIAL SELF-TEST — must pass before any untrusted code runs ───────── + selfTest(); +} catch (err) { + const msg = err instanceof Error ? (err.stack ?? err.message) : String(err); + stderrWriteSync(new TextEncoder().encode(`RUNNER FATAL: pyodide init / lockdown self-test failed: ${msg}\n`)); + denoExit(3); } -// et_xmlfile before openpyxl (dependency order). -wheelNames.sort((a, b) => (a.startsWith("et_xmlfile") ? -1 : b.startsWith("et_xmlfile") ? 1 : 0)); -await pyodide.loadPackage(wheelNames.map((w) => `file://${indexURL}${w}`)); - -const FS = pyodide.FS; - -// ── REALM LOCKDOWN — before any untrusted runPythonAsync (spike S2) ───────── -// Delete every deletable host / Node-compat write primitive. import("node:fs") -// cannot be deleted (it is syntax), so this is hardening, not containment — the -// Node-side validator is the real guarantee (see file header). -delete g.Deno; -delete g.console; -delete g.require; -delete g.__dirname; -delete g.__filename; // ── MEMFS helpers ─────────────────────────────────────────────────────────── function mkdirTree(dir: string): void { @@ -204,11 +248,18 @@ async function runOne(req: RunRequest): Promise { pyodide.globals.set("__sqlfs_argv", JSON.stringify(req.argv ?? [])); pyodide.globals.set("__sqlfs_stdin", req.stdin ? Buffer.from(req.stdin, "base64").toString("utf-8") : ""); pyodide.globals.set("__sqlfs_cwd", cwd); + pyodide.globals.set("__sqlfs_env", JSON.stringify(req.env ?? {})); await pyodide.runPythonAsync(` import sys, os, io, json as __json sys.argv = __json.loads(__sqlfs_argv) or [""] os.chdir(__sqlfs_cwd) sys.stdin = io.StringIO(__sqlfs_stdin) +# Snapshot the FULL os.environ, then apply the exec's exported env. The warm child +# persists os.environ across runs, so we snapshot/restore the WHOLE mapping (not just +# injected keys) to keep env strictly per-execution — a script that sets a NEW +# os.environ key cannot leak it into the next run. +__sqlfs_env_saved = dict(os.environ) +os.environ.update(__json.loads(__sqlfs_env)) __sqlfs_out = io.StringIO() __sqlfs_err = io.StringIO() sys.stdout = __sqlfs_out @@ -245,7 +296,14 @@ sys.stderr = __sqlfs_err let capturedErr = (errProxy.getvalue() as string) ?? ""; outProxy.destroy(); errProxy.destroy(); - await pyodide.runPythonAsync("sys.stdout = sys.__stdout__\nsys.stderr = sys.__stderr__"); + // Restore the real streams AND the prior os.environ (undo this run's env injection). + await pyodide.runPythonAsync(` +sys.stdout = sys.__stdout__ +sys.stderr = sys.__stderr__ +os.environ.clear() +os.environ.update(__sqlfs_env_saved) +__sqlfs_env_saved.clear() +`); if (jsError) capturedErr = capturedErr && !capturedErr.endsWith("\n") ? `${capturedErr}\n${jsError}` : capturedErr + jsError; // ── Diff the cwd subtree (dirs + files) against the staged baseline ─────── @@ -284,6 +342,22 @@ sys.stderr = __sqlfs_err } } + // Also wipe the reserved out-of-cwd staging dir (a `python3 FILE` resolved + // outside cwd was staged there) so it never leaks across execs in the warm child. + for (const node of walkTree(PYODIDE_EXT_STAGING_DIR).sort((a, b) => depth(b.path) - depth(a.path))) { + try { + if (node.kind === "file") FS.unlink(node.path); + else FS.rmdir(node.path); + } catch { + /* already gone */ + } + } + try { + FS.rmdir(PYODIDE_EXT_STAGING_DIR); + } catch { + /* not created this run */ + } + return { type: exitCode === 0 ? "result" : "error", requestId: req.requestId, diff --git a/thoughts/issue-118-pyodide-runtime/design.md b/thoughts/issue-118-pyodide-runtime/design.md index f25b7f2..4df0071 100644 --- a/thoughts/issue-118-pyodide-runtime/design.md +++ b/thoughts/issue-118-pyodide-runtime/design.md @@ -110,17 +110,25 @@ not merely a thrown error. ## Design Decisions 1. **Isolation = explicit single-layer Deno subprocess (headline; findings 1, 2, 3).** The per-session - `PyodideSandbox` spawns a **Deno subprocess** with a **scrubbed env** (`env:{}` plus `DENO_NO_UPDATE_CHECK=1`). - Flags (spike-validated against the pinned Deno version, S1): `--no-prompt` + the deny belt `--deny-net - --deny-run --deny-write --deny-env --deny-ffi --deny-sys --deny-import`, **plus the module-loading air-gap - flags `--no-remote --no-npm --cached-only --no-config`** (finding 2 — `--deny-net` alone does not gate the - module graph; remote registries load by default), with `--allow-read` scoped to the vendored asset dir only. + `PyodideSandbox` spawns a **Deno subprocess** with a **minimal allowlisted env** — Node's `child_process` + `env` option set to `{ DENO_NO_UPDATE_CHECK: "1" }` only (no parent env, no `AUTH_SECRET`/`DATABASE_URL`); this + is a Node-side env replacement, *not* Deno's `clearEnv`. Because the child env omits `PATH`, Node must spawn + Deno by an **absolute** path (the vendored `DENO_BIN_PATH`); a bare `deno` is resolved against the parent + `PATH` to an absolute path before spawn. Flags (spike-validated against the pinned Deno version, S1): + `--no-prompt` + the deny belt `--deny-net --deny-run --deny-write --deny-env --deny-ffi --deny-sys + --deny-import`, **plus the module-loading air-gap flags `--no-remote --no-npm --cached-only --no-config`** + (finding 2 — `--deny-net` alone does not gate the module graph; remote registries load by default), with + `--allow-read` scoped to the vendored asset dir only. Note `--deny-import` blocks *remote* imports only — **local** dynamic imports under `--allow-read` remain possible, contained because the read scope is read-only Pyodide assets. Deno gates the *JS layer itself*, so a full Python→JS escape lands capability-less. - **IPC is committed, not an alternative (finding 3).** Transport = **Node↔Deno over the child's stdin/stdout**, - with **realm-lockdown**: at startup the harness captures the writer, then deletes `Deno`/`console`/other - write primitives from `globalThis` **before** any untrusted Python runs. Every frame (both directions) is + with **realm-lockdown** in an explicit, ordered init sequence (review #2): (1) capture the writer into a + closure; (2) load Pyodide + preload packages; (3) delete `Deno`/`console`/`require`/`__dirname`/`__filename` + from `globalThis`; (4) run an **adversarial self-test** asserting the deletions took and that Pyodide's `js` + proxy exposes none of them; (5) only then emit `ready`. An init failure **or** a failed self-test writes a + diagnostic and exits non-zero → the manager retires the child and the spawning run fails. All of this runs + **before** any untrusted Python. Every frame (both directions) is **length-prefixed JSON** carrying **mandatory integrity fields**: random **requestId**, monotonic **sequence number**, exact **message type**, **child-generation id**; with **max per-frame and aggregate size caps** and **exactly one response per request**. The Node side **kills the child immediately** on any @@ -143,14 +151,22 @@ not merely a thrown error. multi-second cold start for the iterative LibreChat loop), serialized by the per-subprocess mutex. Between execs: fresh Python `globals` + wipe staged MEMFS paths (bounds variable scope + staged files only; `sys.modules`/package globals persist within a session — same trust boundary). Cross-session isolation comes - from per-session subprocesses. Timeout/abort kills the subprocess; next exec re-inits (new generation). + from per-session subprocesses. Timeout/abort kills the subprocess; next exec re-inits (new generation). The + manager's **internal** runtime timeout throws a typed `EPYODIDE_TIMEOUT` (review #4): just-bash flattens a + custom-command rejection into a non-zero `ExecResult`, so the command tags a per-exec context that + `execWithRuntimeThrottle` re-raises as a fatal timeout (aborting the script tx — a timed-out run never commits) + and the routes map it consistently: **408 `EXEC_TIMEOUT`** (sync), **terminal timeout exit event** (SSE), + **per-script `error:"timeout"`** (batch). 4. **Residency LRU — explicit state machine + atomic admission (findings 7, 8).** A **global registry** caps resident subprocesses at `MAX_RESIDENT_PYODIDE` (small default, e.g. 2), independent of `SESSION_IDLE_MS` (else warm subprocesses accumulate per active session); a shorter `PYODIDE_IDLE_MS` idle-kills them. Each worker has an explicit state: `cold → starting → idle → busy → terminating → dead`. **`starting` and `busy` - are never evictable.** A **registry mutex** makes admission atomic — it covers *reserve a slot → select an - eviction victim → spawn → roll back on failed init* as one critical section, so concurrent cold starts cannot - both observe a free slot and exceed the cap. **Capacity is reserved before** expensive Pyodide init. + are never evictable.** A **registry mutex** makes admission atomic — it covers *select an eviction victim → + construct the manager → register* as one critical section, so concurrent admissions cannot both observe a free + slot and exceed the cap. Admission is **lazy and post-semaphore** (review #5): it runs only while the admitting + exec holds a `pyodide` semaphore slot, and the `spawn` thunk is **cheap** (it constructs a `PyodideSandbox`, + nothing more). The **expensive Deno spawn + Pyodide package load happens later, inside `run()`, OUTSIDE the + registry mutex** — so the critical section never serializes multi-second cold starts. 5. **Concurrency + memory — dedicated semaphore; OOM isolation NOT guaranteed (Q4 + finding 4).** New `MAX_CONCURRENT_PYODIDE` (low default, **2**) + queue/wait-timeout env vars mirroring the python set; routed by `python_runtime==="pyodide"` so `stdlib` keeps `MAX_CONCURRENT_PYTHON=5`. The semaphore caps *in-flight execs*, @@ -163,10 +179,18 @@ not merely a thrown error. (`MAX_RESIDENT=1` on small hosts). The Pyodide ~2 GB WASM cap is only a per-instance heap ceiling. The manager reports an error + respawns (new generation) on child exit. 6. **File staging — cwd subtree + script over IPC, diff-and-drain with explicit semantics (Q2 + finding 5/6).** - Before run: ship the cwd subtree **plus the resolved script path** (even if outside cwd, for `python3 FILE` - parity) to Deno, written into MEMFS. After a **successful** run only: Deno reports the created/modified/deleted - set; Node applies it to `ctx.fs` **inside the existing script transaction** (atomic rollback on failure, - `research.md` Q6). **Never drain from a timed-out / aborted / protocol-invalid run** (finding 6). **Semantics:** + Before run: ship the cwd subtree **plus the resolved script path** to Deno, written into MEMFS; a script that + resolves **outside cwd** is staged at a **reserved, non-drainable MEMFS path** (`/__sqlfs_ext__/…`, review #7) + with `argv[0]`/`__file__` pointing there — collision-free with Pyodide's own MEMFS and structurally excluded + from the cwd-scoped diff. After a **successful** run only: Deno reports the created/modified/deleted set; Node + applies it to `ctx.fs` **inside the existing script transaction** (atomic rollback on failure, `research.md` Q6). + **Before any `ctx.fs` mutation** the drain validates the whole manifest (review #7): every path under cwd + (reject `..`/absolute/null-byte/reserved-prefix), **unique**, **no write↔delete conflict**, **no + file-as-directory-ancestor**, per-file + aggregate decoded caps. **read-only execs (review #3):** compare the + reported manifest against the staged snapshot and reject with **`EREADONLY_VIOLATION` before any `ctx.fs` + mutation** if the run produced any persistent change (created-then-deleted temp files are not in the diff and + are intentionally allowed — no persistent mutation). **Never drain from a timed-out / aborted / protocol-invalid + run** (finding 6). **Semantics:** reject **symlinks** (SqlFs default-deny); files written 0644 default (`sql-fs.ts:738`), exec-bit via `chmod` only if needed; dirs-before-files, delete depth-first; hardlinks drained as independent copies; **cwd-scoped only** (keep script + inputs under cwd). Per-file + total byte caps on both directions. From 0bf4499c0f2e156654e841c4b65350e266bb69e1 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Tue, 9 Jun 2026 13:43:25 +0930 Subject: [PATCH 11/16] Address Codex + CodeRabbit PR review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - runner: set/restore sys.path[0] for file scripts (sibling-import parity); track entry KIND in the diff baseline so file↔dir replacements drain. - drain: validate file payloads are base64 (ipc); reject symlinked ancestors and handle wrong-kind targets; stop swallowing readdir() staging errors. - residency: refresh the idle clock per Pyodide command (not just per bash.exec); catch dispose() rejections in the idle sweep. - manager: handle async child.stdin stream errors (broken pipe). - supply chain: pin the Deno binary sha256 for all four supported targets; fetch hard-fails on an unpinned or mismatched binary. - tooling: add typecheck:runner (deno check) gate for the tsc-excluded runner; TS SDK validates python_runtime; parseNameVersion guards wheels. - tests: per-execution env isolation + sys.path + file↔dir coverage; restore PYODIDE_ASSET_DIR/DENO_BIN_PATH and clean up sandboxes per test. --- clients/typescript/src/models.ts | 9 ++- package.json | 1 + scripts/build-pyodide-lock.mjs | 3 + scripts/fetch-pyodide-assets.mjs | 40 +++++++++-- src/api/commands/pyodide-command.ts | 52 ++++++++++++-- src/api/pyodide/ipc.ts | 9 +++ src/api/pyodide/manager.ts | 7 ++ src/api/pyodide/residency.ts | 4 +- .../integration/escape.integration.test.ts | 11 +-- .../frame-forgery.integration.test.ts | 18 ++++- src/api/pyodide/tests/unit/ipc.test.ts | 10 +++ .../tests/unit/pyodide-command.test.ts | 26 +++++++ src/api/pyodide/tests/unit/residency.test.ts | 13 ++++ src/api/session-manager.ts | 30 +++++--- .../integration/pyodide.integration.test.ts | 72 ++++++++++++++++--- src/api/tests/unit/mcp-tools.test.ts | 17 ++++- src/pyodide-runner/runner.ts | 57 +++++++++++---- .../spikes/S3-findings.md | 2 +- .../spikes/s1-pyodide-deno.sh | 2 +- 19 files changed, 333 insertions(+), 50 deletions(-) diff --git a/clients/typescript/src/models.ts b/clients/typescript/src/models.ts index 42e5634..3c9dede 100644 --- a/clients/typescript/src/models.ts +++ b/clients/typescript/src/models.ts @@ -82,13 +82,20 @@ export class ReadResult { type ApiObject = Record; +/** Validate the server's python_runtime instead of blindly asserting the type. */ +function toPythonRuntime(value: unknown): PythonRuntime { + if (value == null) return null; + if (value === "stdlib" || value === "pyodide") return value; + throw new Error(`unexpected python_runtime from server: ${JSON.stringify(value)}`); +} + export function sandboxRecordFromApi(payload: ApiObject): SandboxRecord { return { id: String(payload.id), name: payload.name == null ? null : String(payload.name), owner: String(payload.owner), createdAt: String(payload.createdAt), - python_runtime: (payload.python_runtime ?? null) as PythonRuntime, + python_runtime: toPythonRuntime(payload.python_runtime), javascript: Boolean(payload.javascript), network: Boolean(payload.network), }; diff --git a/package.json b/package.json index 1381f1f..ef77096 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "start": "node dist/api/server.js", "bench:remote-bash": "python3 scripts/benchmark_remote_bash.py", "typecheck": "tsc --noEmit", + "typecheck:runner": "${DENO_BIN_PATH:-vendor/deno/deno} check src/pyodide-runner/runner.ts", "lint": "biome check .", "lint:fix": "biome check --write .", "test": "vitest run", diff --git a/scripts/build-pyodide-lock.mjs b/scripts/build-pyodide-lock.mjs index e6f74a0..5b116f9 100644 --- a/scripts/build-pyodide-lock.mjs +++ b/scripts/build-pyodide-lock.mjs @@ -53,6 +53,9 @@ function findWheel(re) { function parseNameVersion(fileName) { // PEP 427 wheel: {name}-{version}-{pytag}-{abitag}-{platformtag}.whl const [name, version] = fileName.split("-"); + if (!name || !version) { + throw new Error(`malformed wheel filename (expected {name}-{version}-…): ${fileName}`); + } return { name, version }; } diff --git a/scripts/fetch-pyodide-assets.mjs b/scripts/fetch-pyodide-assets.mjs index 449cb07..29f4f14 100644 --- a/scripts/fetch-pyodide-assets.mjs +++ b/scripts/fetch-pyodide-assets.mjs @@ -12,9 +12,10 @@ * * INTEGRITY. Platform-independent runtime bytes are SHA-256-pinned to the exact * artifacts spike S1 validated (pyodide.mjs / pyodide.asm.wasm / python_stdlib.zip - * and the two pure-python wheels). The Deno binary is pinned by version + the - * official dl.deno.land URL only — its bytes are platform-specific, so a single - * cross-arch checksum is impossible; we verify it extracted and is executable. + * and the two pure-python wheels). The Deno binary is platform-specific, so it is + * pinned PER TARGET in `DENO_SHA256` and verified after extraction: an unpinned + * target OR a checksum mismatch is a HARD FAILURE — we never run an unverified Deno + * binary (all four supported targets are pinned). * * Requires `curl`, `unzip`, and `tar` (with bzip2) on PATH — present on macOS and * installed in the Docker builder stage. Mirrors the proven spike @@ -53,6 +54,19 @@ const SHA256 = { [PINS.etXmlfileWheel]: "7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", }; +// SHA-256 of the EXTRACTED Deno binary, per target, for PINS.deno. The binary is +// platform-specific, so it's keyed by target rather than a single hash. After +// download/extraction the binary is verified against this map: an UNPINNED target +// or a MISMATCH is a HARD FAILURE (supply-chain / tamper guard) — we never run an +// unverified Deno binary. To bump PINS.deno: download deno-.zip for every +// target below, and record sha256 of the extracted `deno`. +const DENO_SHA256 = { + "aarch64-apple-darwin": "9d25a1a5a67579eb607ed27a73141548b163e29df38735bc5556b7d887992435", + "x86_64-apple-darwin": "a06e411d2da878b9240ecab047ea4ad3f2d1297dfff6bae9de7059baf34733dd", + "x86_64-unknown-linux-gnu": "30761b46413a814d5f83081bf6011e0c900a5b4154f64b03a065e97511079fa0", + "aarch64-unknown-linux-gnu": "f7dc66b53f77133b4ca9a24c77d1fb48e49cd8c26a4043e49f6b0b8195f09d80", +}; + const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); const VENDOR = join(ROOT, "vendor"); const DENO_DIR = join(VENDOR, "deno"); @@ -95,12 +109,29 @@ function denoTarget() { return target; } +function verifyDeno(target) { + const expected = DENO_SHA256[target]; + const actual = sha256(DENO_BIN); + if (!expected) { + // HARD FAIL — never run an unverified Deno binary. Add the target's hash to + // DENO_SHA256 (download deno-.zip for PINS.deno and sha256 the binary). + throw new Error( + `Deno binary for ${target} is not pinned in DENO_SHA256 (its sha256 is ${actual}) — add it before building`, + ); + } + if (actual !== expected) { + throw new Error(`Deno binary checksum mismatch for ${target}: expected ${expected}, got ${actual}`); + } + log(`verified Deno binary for ${target} (sha256 ok)`); +} + function fetchDeno() { + const target = denoTarget(); if (existsSync(DENO_BIN)) { + verifyDeno(target); // verify even when already present (catches a swapped binary) log(`Deno ${PINS.deno} already present`); return; } - const target = denoTarget(); mkdirSync(DENO_DIR, { recursive: true }); mkdirSync(CACHE, { recursive: true }); const zip = join(CACHE, `deno-${PINS.deno}-${target}.zip`); @@ -117,6 +148,7 @@ function fetchDeno() { /* not macOS / no xattr */ } if (!existsSync(DENO_BIN)) throw new Error("Deno binary not present after extraction"); + verifyDeno(target); log(`Deno ${PINS.deno} ready at ${DENO_BIN}`); } diff --git a/src/api/commands/pyodide-command.ts b/src/api/commands/pyodide-command.ts index cbd8d95..354c483 100644 --- a/src/api/commands/pyodide-command.ts +++ b/src/api/commands/pyodide-command.ts @@ -65,6 +65,14 @@ interface Caps { export interface PyodideCommandOptions { readonly maxFileBytes?: number; readonly maxTotalBytes?: number; + /** + * Called after each completed `run()` (the worker is idle again). The + * SessionManager wires this to refresh the residency idle clock PER COMMAND, so a + * long earlier command in a multi-command bash script can't leave the worker + * idle-and-stale long enough for the residency sweep to dispose it before the next + * command (which would otherwise fail with EPYODIDE_DISPOSED). + */ + readonly onRunComplete?: () => void; } /** A drain / staging policy violation. Surfaces as a non-zero exec, rolling back the script tx. */ @@ -114,7 +122,9 @@ export function createPyodideCommands( maxTotalBytes: opts.maxTotalBytes ?? envInt("PYODIDE_MAX_TOTAL_BYTES", PYODIDE_MAX_TOTAL_BYTES_DEFAULT), }; const getSandbox = typeof sandbox === "function" ? sandbox : (): PyodideSandbox => sandbox; - const handler = (args: string[], ctx: CommandContext): Promise => runPython(getSandbox, caps, args, ctx); + const onRunComplete = opts.onRunComplete; + const handler = (args: string[], ctx: CommandContext): Promise => + runPython(getSandbox, caps, args, ctx, onRunComplete); return [defineCommand("python3", handler), defineCommand("python", handler)]; } @@ -140,6 +150,7 @@ async function runPython( caps: Caps, args: string[], ctx: CommandContext, + onRunComplete?: () => void, ): Promise { const first = args[0]; @@ -203,6 +214,9 @@ async function runPython( } throw err; } + // The worker is idle again — refresh its residency clock NOW (per command), not + // only after the whole bash.exec, so the sweep can't dispose it between commands. + onRunComplete?.(); // Abort that landed after the response but before the drain → drain nothing. if (ctx.signal?.aborted) throw makeAbortError(); @@ -364,8 +378,10 @@ async function stageCwd(fs: IFileSystem, cwd: string, caps: Caps): Promise<{ fil let names: string[]; try { names = await fs.readdir(dir); - } catch { - return; + } catch (err) { + // Do NOT swallow: a readdir failure would silently stage a PARTIAL cwd and + // run the script against it with no signal. Surface it as a staging error. + throw new PyodideDrainError(`cannot read directory while staging: ${dir} (${(err as Error).message})`); } for (const name of names) { if (name === "." || name === "..") continue; @@ -471,10 +487,21 @@ export async function drain(fs: IFileSystem, cwd: string, resp: RunResponse, cap async function applyEntry(fs: IFileSystem, cwd: string, entry: FsEntry): Promise { const resolved = fs.resolvePath(cwd, entry.path); - // Default-deny: never write through / over an existing symlink at the target. + // Default-deny: never write through a symlinked ANCESTOR directory (a symlinked + // ancestor could redirect the write outside cwd — `resolvePath` is lexical and + // doesn't follow links). SqlFs default-denies symlink creation, so this is + // defense-in-depth, but it closes the symlink-traversal gap the final-target + // check alone left open. + await assertNoSymlinkAncestor(fs, cwd, resolved); + + // Default-deny over an existing symlink at the target; and if a target of the + // WRONG KIND already exists (file↔dir replacement), remove it first so the new + // kind can be materialized in place. if (await fs.exists(resolved)) { const st = await fs.lstat(resolved); if (st.isSymbolicLink) throw new PyodideDrainError(`refusing to drain over a symlink: ${entry.path}`); + const existingKind = st.isDirectory ? "dir" : "file"; + if (existingKind !== entry.kind) await fs.rm(resolved, { recursive: true, force: true }); } if (entry.kind === "dir") { @@ -488,3 +515,20 @@ async function applyEntry(fs: IFileSystem, cwd: string, entry: FsEntry): Promise // a non-default mode (e.g. an executable bit). if ((entry.mode & 0o777) !== 0o644) await fs.chmod(resolved, entry.mode & 0o777); } + +/** + * Reject any symlinked directory on the path from cwd down to `resolved`'s parent. + * Walks the ancestor chain (strictly between cwd and the target) and throws on the + * first symlink. Defense-in-depth against symlink traversal during drain. + */ +async function assertNoSymlinkAncestor(fs: IFileSystem, cwd: string, resolved: string): Promise { + const cwdNoSlash = cwd.endsWith("/") ? cwd.slice(0, -1) : cwd; + let parent = resolved.slice(0, resolved.lastIndexOf("/")); + while (parent.length > cwdNoSlash.length && parent.startsWith(`${cwdNoSlash}/`)) { + if (await fs.exists(parent)) { + const st = await fs.lstat(parent); + if (st.isSymbolicLink) throw new PyodideDrainError(`refusing to drain through a symlinked directory: ${parent}`); + } + parent = parent.slice(0, parent.lastIndexOf("/")); + } +} diff --git a/src/api/pyodide/ipc.ts b/src/api/pyodide/ipc.ts index 5933029..3993243 100644 --- a/src/api/pyodide/ipc.ts +++ b/src/api/pyodide/ipc.ts @@ -145,6 +145,15 @@ function assertFsEntry(entry: unknown, label: string): void { if (typeof entry.data !== "string") throw new IpcIntegrityError(`${label}: missing/invalid data`); if (entry.kind === "dir" && entry.data !== "") throw new IpcIntegrityError(`${label}: dir entry must have empty data`); + // `data` MUST be valid base64 for a file (it is decoded straight into persisted + // bytes by the drain). Reject malformed payloads here rather than silently + // decoding garbage. (`""` is a legal empty file.) + if (entry.kind === "file" && !isBase64(entry.data)) throw new IpcIntegrityError(`${label}: data is not valid base64`); +} + +/** Strict canonical-base64 check: length a multiple of 4, only the base64 alphabet + padding. */ +function isBase64(s: string): boolean { + return s.length % 4 === 0 && /^[A-Za-z0-9+/]*={0,2}$/.test(s); } /** diff --git a/src/api/pyodide/manager.ts b/src/api/pyodide/manager.ts index 1570f24..daaab35 100644 --- a/src/api/pyodide/manager.ts +++ b/src/api/pyodide/manager.ts @@ -472,6 +472,13 @@ export class PyodideSandbox { if (this.#child !== child) return; this.#failOwned(err, true); }); + // The synchronous try/catch around `stdin.write` only catches sync throws; a + // torn-down child can also emit an ASYNC stream error (EPIPE / destroyed) on + // stdin. Route it through #failOwned so a broken pipe can't escape unhandled. + child.stdin?.on("error", (err: Error) => { + if (this.#child !== child) return; + this.#failOwned(err, true); + }); } #killChild(): void { diff --git a/src/api/pyodide/residency.ts b/src/api/pyodide/residency.ts index ab92fe4..e8e9f1a 100644 --- a/src/api/pyodide/residency.ts +++ b/src/api/pyodide/residency.ts @@ -177,7 +177,9 @@ export class PyodideResidency { if (!this.#evictable(worker)) continue; if (now - touched >= this.#idleMs) { this.#residents.delete(worker); - void worker.dispose(); + // Swallow a dispose() rejection — the child is being torn down best-effort + // and an unhandled rejection from the timer must not crash the process. + worker.dispose().catch(() => {}); } } } diff --git a/src/api/pyodide/tests/integration/escape.integration.test.ts b/src/api/pyodide/tests/integration/escape.integration.test.ts index f998420..08c4ff7 100644 --- a/src/api/pyodide/tests/integration/escape.integration.test.ts +++ b/src/api/pyodide/tests/integration/escape.integration.test.ts @@ -40,12 +40,12 @@ const COLD = 120_000; describe.skipIf(!ASSETS_PRESENT)("pyodide adversarial escape suite (real Deno child)", () => { let sm: SessionManager; let session: Awaited>; - let savedSecret: string | undefined; + const savedEnv = new Map(); beforeAll(async () => { + for (const k of ["PYODIDE_ASSET_DIR", "DENO_BIN_PATH", "AUTH_SECRET"]) savedEnv.set(k, process.env[k]); process.env.PYODIDE_ASSET_DIR = ASSET_DIR; process.env.DENO_BIN_PATH = DENO_BIN; - savedSecret = process.env.AUTH_SECRET; process.env.AUTH_SECRET = SECRET; // planted in the parent — the child must not see it sm = new SessionManager({ createFs: (): Promise => Promise.resolve(new InMemoryFs()) }); session = await sm.getOrCreate(TENANT, "escape", PYODIDE, "owner"); @@ -56,8 +56,11 @@ describe.skipIf(!ASSETS_PRESENT)("pyodide adversarial escape suite (real Deno ch afterAll(async () => { await sm.shutdown({ drainTimeoutMs: 5_000 }).catch(() => {}); - if (savedSecret === undefined) Reflect.deleteProperty(process.env, "AUTH_SECRET"); - else process.env.AUTH_SECRET = savedSecret; + // Restore every env var we set so this suite doesn't pollute later test files. + for (const [k, v] of savedEnv) { + if (v === undefined) Reflect.deleteProperty(process.env, k); + else process.env[k] = v; + } }); /** Write `code` to a script file and run it on the warm child. */ diff --git a/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts b/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts index 2b99336..50ccdb1 100644 --- a/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts +++ b/src/api/pyodide/tests/integration/frame-forgery.integration.test.ts @@ -24,7 +24,7 @@ import { existsSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { InMemoryFs } from "just-bash"; import type { IFileSystem } from "just-bash"; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import type { RunResponse } from "../../../../pyodide-runner/protocol.js"; import { PyodideDrainError, drain } from "../../../commands/pyodide-command.js"; import { SessionManager } from "../../../session-manager.js"; @@ -77,8 +77,11 @@ function frameOf(jsonExpr: string): string { describe.skipIf(!ASSETS_PRESENT)("pyodide frame-forgery suite (real Deno child)", () => { let sm: SessionManager; let session: Awaited>; + let sandboxId: string; + const savedEnv = new Map(); beforeAll(() => { + for (const k of ["PYODIDE_ASSET_DIR", "DENO_BIN_PATH"]) savedEnv.set(k, process.env[k]); process.env.PYODIDE_ASSET_DIR = ASSET_DIR; process.env.DENO_BIN_PATH = DENO_BIN; sm = new SessionManager({ createFs: (): Promise => Promise.resolve(new InMemoryFs()) }); @@ -86,15 +89,26 @@ describe.skipIf(!ASSETS_PRESENT)("pyodide frame-forgery suite (real Deno child)" afterAll(async () => { await sm.shutdown({ drainTimeoutMs: 5_000 }).catch(() => {}); + for (const [k, v] of savedEnv) { + if (v === undefined) Reflect.deleteProperty(process.env, k); + else process.env[k] = v; + } }); beforeEach(async () => { // Fresh session per test — each forgery kills the child; a fresh session keeps // the assertions independent (each cold-starts its own child on first exec). - session = await sm.getOrCreate(TENANT, `forge-${Date.now()}-${Math.random().toString(36).slice(2)}`, PYODIDE); + sandboxId = `forge-${Date.now()}-${Math.random().toString(36).slice(2)}`; + session = await sm.getOrCreate(TENANT, sandboxId, PYODIDE); await session.fs.mkdir(session.cwd, { recursive: true }); }); + afterEach(async () => { + // Clean up the per-test session (its Deno child) rather than letting them + // accumulate until the final shutdown(). + await sm.destroy(TENANT, sandboxId).catch(() => {}); + }); + /** * Run a forge script. The forged frame is rejected by the Node validator, which * KILLS the child; just-bash normalizes the resulting command-handler rejection diff --git a/src/api/pyodide/tests/unit/ipc.test.ts b/src/api/pyodide/tests/unit/ipc.test.ts index 622fb15..412b0f2 100644 --- a/src/api/pyodide/tests/unit/ipc.test.ts +++ b/src/api/pyodide/tests/unit/ipc.test.ts @@ -205,6 +205,16 @@ describe("validateInbound — drain entry schema (FsEntry[] / string[])", () => const frame = responseFrame({ deleted: [42] as never }); expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); }); + + it("rejects a file entry whose data is not valid base64 (review #4)", () => { + const frame = responseFrame({ created: [{ path: "/cwd/x", kind: "file", mode: 0o644, data: "not base64!!" }] }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); + + it("rejects a file entry whose base64 length is not a multiple of 4 (review #4)", () => { + const frame = responseFrame({ created: [{ path: "/cwd/x", kind: "file", mode: 0o644, data: "QQQ" }] }); + expect(() => validateInbound(frame, RESPONSE_CTX)).toThrow(IpcIntegrityError); + }); }); describe("validateInbound — direction / shape", () => { diff --git a/src/api/pyodide/tests/unit/pyodide-command.test.ts b/src/api/pyodide/tests/unit/pyodide-command.test.ts index 2cfc491..38d49fb 100644 --- a/src/api/pyodide/tests/unit/pyodide-command.test.ts +++ b/src/api/pyodide/tests/unit/pyodide-command.test.ts @@ -408,4 +408,30 @@ describe("pyodide drain — manifest validation (review #7)", () => { await drain(fs, "/home/user", resp, caps); expect(await fs.readFile("/home/user/d/x.txt", "utf8")).toBe("hi"); }); + + it("replaces an existing file with a directory at the same path (file→dir, review #2)", async () => { + const fs = await fsAt("/home/user"); + await fs.writeFile("/home/user/x", "i was a file"); + const resp = makeResponse({ created: [{ path: "/home/user/x", kind: "dir", mode: 0o755, data: "" }] }); + await drain(fs, "/home/user", resp, caps); + expect((await fs.stat("/home/user/x")).isDirectory).toBe(true); + }); + + it("replaces an existing directory with a file at the same path (dir→file, review #2)", async () => { + const fs = await fsAt("/home/user"); + await fs.mkdir("/home/user/x", { recursive: true }); + const resp = makeResponse({ created: [fileEntry("/home/user/x", "now a file")] }); + await drain(fs, "/home/user", resp, caps); + expect((await fs.stat("/home/user/x")).isFile).toBe(true); + expect(await fs.readFile("/home/user/x", "utf8")).toBe("now a file"); + }); + + it("rejects a drain through a symlinked ancestor directory (review #7)", async () => { + const fs = await fsAt("/home/user"); + await fs.mkdir("/home/user/real", { recursive: true }); + await fs.symlink("/home/user/real", "/home/user/link"); + const resp = makeResponse({ created: [fileEntry("/home/user/link/evil.txt", "x")] }); + await expect(drain(fs, "/home/user", resp, caps)).rejects.toBeInstanceOf(PyodideDrainError); + expect(await fs.exists("/home/user/real/evil.txt")).toBe(false); + }); }); diff --git a/src/api/pyodide/tests/unit/residency.test.ts b/src/api/pyodide/tests/unit/residency.test.ts index 2a968b0..c5fc653 100644 --- a/src/api/pyodide/tests/unit/residency.test.ts +++ b/src/api/pyodide/tests/unit/residency.test.ts @@ -135,6 +135,19 @@ describe("PyodideResidency — idle-kill", () => { expect(residency.residentCount).toBe(0); }); + it("swallows a dispose() rejection in the idle-kill sweep (review #8)", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + residency = new PyodideResidency({ maxResident: 2, idleMs: 1000, sweepIntervalMs: 1000 }); + const w: FakeWorker = { state: "idle", dispose: vi.fn(() => Promise.reject(new Error("dispose boom"))) }; + await residency.admit(() => asSandbox(w)); + // The sweep must not surface an unhandled rejection when dispose rejects (an + // unhandled rejection here would fail this test). + await vi.advanceTimersByTimeAsync(1000); + expect(w.dispose).toHaveBeenCalledTimes(1); + expect(residency.residentCount).toBe(0); // still removed from the registry + }); + it("does NOT idle-kill a busy worker even past the idle window", async () => { vi.useFakeTimers(); vi.setSystemTime(0); diff --git a/src/api/session-manager.ts b/src/api/session-manager.ts index 146175b..10f4f3f 100644 --- a/src/api/session-manager.ts +++ b/src/api/session-manager.ts @@ -579,15 +579,27 @@ export class SessionManager { const sessionRef: { current?: Session } = {}; if (resolvedRuntime.pythonRuntime === "pyodide") { customCommands.push( - ...createPyodideCommands(() => { - const live = sessionRef.current?.pyodideSandbox; - if (live === undefined) { - throw Object.assign(new Error("EPYODIDE_NO_SANDBOX: pyodide sandbox unavailable"), { - code: "EPYODIDE_NO_SANDBOX", - }); - } - return live; - }), + ...createPyodideCommands( + () => { + const live = sessionRef.current?.pyodideSandbox; + if (live === undefined) { + throw Object.assign(new Error("EPYODIDE_NO_SANDBOX: pyodide sandbox unavailable"), { + code: "EPYODIDE_NO_SANDBOX", + }); + } + return live; + }, + { + // Refresh the residency idle clock after EACH python command (not only + // after the whole bash.exec), so a long earlier command in a + // multi-command script can't leave the worker idle-and-stale for the + // sweep to dispose before the next command. + onRunComplete: () => { + const live = sessionRef.current?.pyodideSandbox; + if (live !== undefined) this.pyodideResidency.touch(live); + }, + }, + ), ); } diff --git a/src/api/tests/integration/pyodide.integration.test.ts b/src/api/tests/integration/pyodide.integration.test.ts index 96257d9..052f4cb 100644 --- a/src/api/tests/integration/pyodide.integration.test.ts +++ b/src/api/tests/integration/pyodide.integration.test.ts @@ -13,7 +13,7 @@ import { existsSync } from "node:fs"; import { fileURLToPath } from "node:url"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { destroySandbox } from "../../../sql-fs/index.js"; import { SessionManager } from "../../session-manager.js"; import { loadTenantConfig } from "../../tenants.js"; @@ -32,22 +32,31 @@ const COLD = 120_000; describe.skipIf(SKIP)("pyodide runtime — end-to-end (real Deno + Pyodide)", () => { let sm: SessionManager; - const cleanup: string[] = []; + let cleanup: string[] = []; + const savedEnv = new Map(); beforeAll(() => { + for (const k of ["PYODIDE_ASSET_DIR", "DENO_BIN_PATH"]) savedEnv.set(k, process.env[k]); process.env.PYODIDE_ASSET_DIR = ASSET_DIR; process.env.DENO_BIN_PATH = DENO_BIN; sm = new SessionManager({ tenantConfig: loadTenantConfig() }); }); + // Clean up each test's sandbox (session + Deno child + DB rows) per test, rather + // than letting them accumulate until the final shutdown(). + afterEach(async () => { + for (const id of cleanup) { + await sm.destroy(TENANT, id).catch(() => {}); + await destroySandbox("postgres", id).catch(() => {}); + } + cleanup = []; + }); + afterAll(async () => { await sm.shutdown({ drainTimeoutMs: 5_000 }).catch(() => {}); - for (const id of cleanup) { - try { - await destroySandbox("postgres", id); - } catch { - /* ignore */ - } + for (const [k, v] of savedEnv) { + if (v === undefined) Reflect.deleteProperty(process.env, k); + else process.env[k] = v; } }); @@ -150,4 +159,51 @@ describe.skipIf(SKIP)("pyodide runtime — end-to-end (real Deno + Pyodide)", () }, COLD, ); + + it( + "a file script can import a sibling module from its own directory (sys.path, review #1)", + async () => { + const id = `pyo-syspath-${Date.now()}`; + cleanup.push(id); + const session = await sm.getOrCreate(TENANT, id, PYODIDE, "owner"); + const cwd = session.cwd; + await session.fs.mkdir(`${cwd}/pkg`, { recursive: true }); + await session.fs.writeFile(`${cwd}/pkg/helper.py`, "VALUE = 42\n"); + await session.fs.writeFile(`${cwd}/pkg/main.py`, "import helper\nprint('val', helper.VALUE)\n"); + // Sibling import only resolves if the script's dir (pkg/) is on sys.path[0]. + const r = await sm.execWithRuntimeThrottle(session, "python3 pkg/main.py"); + expect(r.exitCode).toBe(0); + expect(r.stdout).toContain("val 42"); + }, + COLD, + ); + + it( + "replaces a file with a directory and drains the replacement (file↔dir baseline, review #2)", + async () => { + const id = `pyo-replace-${Date.now()}`; + cleanup.push(id); + const session = await sm.getOrCreate(TENANT, id, PYODIDE, "owner"); + const cwd = session.cwd; + await session.fs.mkdir(cwd, { recursive: true }); + await session.fs.writeFile(`${cwd}/x`, "i am a file"); + await session.fs.writeFile( + `${cwd}/replace.py`, + [ + "import os", + "os.remove('x')", + "os.mkdir('x')", + "open('x/inside.txt','w').write('hi')", + "print('replaced')", + ].join("\n"), + ); + const r = await sm.execWithRuntimeThrottle(session, "python3 replace.py"); + expect(r.exitCode).toBe(0); + expect(r.stdout).toContain("replaced"); + // The file→dir replacement must persist to SqlFs (was invisible before #2). + expect((await session.fs.stat(`${cwd}/x`)).isDirectory).toBe(true); + expect(await session.fs.readFile(`${cwd}/x/inside.txt`, "utf8")).toBe("hi"); + }, + COLD, + ); }); diff --git a/src/api/tests/unit/mcp-tools.test.ts b/src/api/tests/unit/mcp-tools.test.ts index a5c8b38..c117742 100644 --- a/src/api/tests/unit/mcp-tools.test.ts +++ b/src/api/tests/unit/mcp-tools.test.ts @@ -124,7 +124,11 @@ describe("MCP tools — python_runtime echo", () => { const result = (await getHandler()({ python_runtime: "pyodide" }, {})) as { content: { text: string }[] }; const body = JSON.parse(result.content[0]!.text); - expect(body.python_runtime).toBe("pyodide"); + // Assert the capability echo present in the MCP record (python_runtime + + // javascript; `network` is intentionally not surfaced in MCP records) plus the + // dynamic id. + expect(body).toMatchObject({ python_runtime: "pyodide", javascript: false }); + expect(typeof body.id).toBe("string"); }); it("sandbox_list echoes python_runtime per sandbox", async () => { @@ -147,6 +151,15 @@ describe("MCP tools — python_runtime echo", () => { const result = (await getHandler()({}, {})) as { content: { text: string }[] }; const body = JSON.parse(result.content[0]!.text); - expect(body.sandboxes[0].python_runtime).toBe("stdlib"); + // Assert the whole per-sandbox MCP record (not just python_runtime); `network` + // is intentionally not surfaced in MCP records. + expect(body.sandboxes).toHaveLength(1); + expect(body.sandboxes[0]).toMatchObject({ + id: "sb-1", + name: null, + owner: "test-owner", + python_runtime: "stdlib", + javascript: false, + }); }); }); diff --git a/src/pyodide-runner/runner.ts b/src/pyodide-runner/runner.ts index a5770db..7dcea73 100644 --- a/src/pyodide-runner/runner.ts +++ b/src/pyodide-runner/runner.ts @@ -75,8 +75,10 @@ function emit(frame: Frame): void { } // ── Pyodide instance + MEMFS handle (assigned by the init sequence below) ──── -let pyodide; -let FS; +// deno-lint-ignore no-explicit-any +let pyodide: any; +// deno-lint-ignore no-explicit-any +let FS: any; /** * Adversarial self-test (spike S2 hard gate). Runs AFTER realm lockdown and @@ -233,9 +235,11 @@ async function runOne(req: RunRequest): Promise { // the diff baseline (excludes staging infrastructure dirs, which pre-exist // from the caller's SqlFs tree). const baseFiles = new Map(); - const basePaths = new Set(); + // Track KIND per path (not just presence) so a file↔dir replacement at the same + // path is detectable — otherwise `os.remove('x'); os.mkdir('x')` is invisible. + const baseKind = new Map(); for (const node of walkTree(cwd)) { - basePaths.add(node.path); + baseKind.set(node.path, node.kind); if (node.kind === "file" && node.bytes) baseFiles.set(node.path, node.bytes); } @@ -254,6 +258,13 @@ import sys, os, io, json as __json sys.argv = __json.loads(__sqlfs_argv) or [""] os.chdir(__sqlfs_cwd) sys.stdin = io.StringIO(__sqlfs_stdin) +# CPython parity: a FILE script runs with its OWN directory at sys.path[0] so sibling +# imports resolve (argv[0] is "-c"/"-"/"" for the inline/stdin/bare modes, where cwd +# applies). Snapshot sys.path so the insert is undone per run (the warm child persists it). +__sqlfs_syspath_saved = list(sys.path) +__sqlfs_argv0 = sys.argv[0] if sys.argv else "" +if __sqlfs_argv0 and __sqlfs_argv0 not in ("-c", "-"): + sys.path.insert(0, os.path.dirname(os.path.abspath(__sqlfs_argv0))) # Snapshot the FULL os.environ, then apply the exec's exported env. The warm child # persists os.environ across runs, so we snapshot/restore the WHOLE mapping (not just # injected keys) to keep env strictly per-execution — a script that sets a NEW @@ -300,6 +311,7 @@ sys.stderr = __sqlfs_err await pyodide.runPythonAsync(` sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ +sys.path[:] = __sqlfs_syspath_saved os.environ.clear() os.environ.update(__sqlfs_env_saved) __sqlfs_env_saved.clear() @@ -308,27 +320,44 @@ __sqlfs_env_saved.clear() // ── Diff the cwd subtree (dirs + files) against the staged baseline ─────── const after = walkTree(cwd); + const afterKind = new Map(); + for (const node of after) afterKind.set(node.path, node.kind); + const createdDirs: FsEntry[] = []; const createdFiles: FsEntry[] = []; const modified: FsEntry[] = []; - const afterPaths = new Set(); for (const node of after) { - afterPaths.add(node.path); if (node.kind === "dir") { - if (!basePaths.has(node.path)) createdDirs.push({ path: node.path, kind: "dir", mode: node.mode, data: "" }); + // New, OR a path that was a FILE before (file→dir replacement) — both are a + // "created dir" the drain materializes (replacing the old file in place). + if (baseKind.get(node.path) !== "dir") createdDirs.push({ path: node.path, kind: "dir", mode: node.mode, data: "" }); continue; } const bytes = node.bytes ?? new Uint8Array(0); - const before = baseFiles.get(node.path); const entry: FsEntry = { path: node.path, kind: "file", mode: node.mode, data: Buffer.from(bytes).toString("base64") }; - if (before === undefined) createdFiles.push(entry); - else if (!sameBytes(before, bytes)) modified.push(entry); + if (baseKind.get(node.path) !== "file") createdFiles.push(entry); // new, or dir→file replacement + else if (!sameBytes(baseFiles.get(node.path) ?? new Uint8Array(0), bytes)) modified.push(entry); } // dirs-before-files, dirs shallow→deep, so the drain can apply created in order. createdDirs.sort((a, b) => depth(a.path) - depth(b.path)); const created: FsEntry[] = [...createdDirs, ...createdFiles]; - // deleted: any baseline path gone, deepest-first (children before parents). - const deleted = [...basePaths].filter((p) => !afterPaths.has(p)).sort((a, b) => depth(b) - depth(a)); + + // deleted: a baseline path gone from the after-tree, EXCEPT those shadowed by an + // ancestor that is now a FILE (a dir→file replacement implicitly drops the old + // children; emitting them as separate deletes would also collide with the created + // file under drain validation). Deepest-first (children before parents). + const cwdNoSlash = cwd.endsWith("/") ? cwd.slice(0, -1) : cwd; + const shadowedByFile = (p: string): boolean => { + let parent = p.slice(0, p.lastIndexOf("/")); + while (parent.length > cwdNoSlash.length && parent.startsWith(`${cwdNoSlash}/`)) { + if (afterKind.get(parent) === "file") return true; + parent = parent.slice(0, parent.lastIndexOf("/")); + } + return false; + }; + const deleted = [...baseKind.keys()] + .filter((p) => !afterKind.has(p) && !shadowedByFile(p)) + .sort((a, b) => depth(b) - depth(a)); // Wipe the ENTIRE cwd subtree (files + dirs, deepest-first) so the next exec // in this warm child starts from a clean cwd — no leftover dirs leak across @@ -377,7 +406,9 @@ __sqlfs_env_saved.clear() const ready: ReadyFrame = { type: "ready", generation }; emit(ready); -let buf = new Uint8Array(0); +// `decodeFrames` returns `rest` as a `Uint8Array`; widen the +// accumulator's type so the reassignment type-checks under `deno check`. +let buf: Uint8Array = new Uint8Array(0); const reader = stdinReadable.getReader(); for (;;) { const { value, done } = await reader.read(); diff --git a/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md b/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md index c5202bf..c390210 100644 --- a/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md +++ b/thoughts/issue-118-pyodide-runtime/spikes/S3-findings.md @@ -7,7 +7,7 @@ Decision 5: as the non-root `node` user in `node:22-slim`, **per-child memory limiting is unavailable/unusable**, so the **operator-set container memory limit + accepted availability risk is the guard**. -``` +```text cgroup_write_denied=1 rlimit_as_unusable=1 (vaddr_decoupled=1 rlimit_breaks_wasm=1) S3 PASS: non-root cannot set cgroup memory.max; prlimit --as is unusable for RSS (V8/WASM vaddr >> RSS). Container memory limit is the real guard. diff --git a/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh b/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh index 254ac19..3782923 100755 --- a/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh +++ b/thoughts/issue-118-pyodide-runtime/spikes/s1-pyodide-deno.sh @@ -68,7 +68,7 @@ if [[ ! -f "${PYODIDE_DIR}/pyodide.mjs" ]]; then else echo "[s1] Pyodide ${PYODIDE_VERSION} already extracted" >&2 fi -echo "[s1] pyodide assets: $(ls "${PYODIDE_DIR}" | wc -l | tr -d ' ') files in ${PYODIDE_DIR}" >&2 +echo "[s1] pyodide assets: $(find "${PYODIDE_DIR}" -maxdepth 1 -type f | wc -l | tr -d ' ') files in ${PYODIDE_DIR}" >&2 # --- 2b. Vendor openpyxl + et_xmlfile wheels (NOT in the pyodide dist) ------- # FINDING (gates Phase 3): openpyxl + et_xmlfile are absent from pyodide From 8606d104bb998b466be102c178e21e1aea60f7b7 Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Tue, 9 Jun 2026 14:05:55 +0930 Subject: [PATCH 12/16] clients/ts: give thrown response-mapping errors a code property Per the error-handling guideline, sandboxRecordFromApi and streamEventFromApi now throw Error instances carrying a code (EINVALID_PYTHON_RUNTIME / EUNKNOWN_SSE_EVENT) instead of bare Errors. --- clients/typescript/src/models.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clients/typescript/src/models.ts b/clients/typescript/src/models.ts index 3c9dede..fdeb629 100644 --- a/clients/typescript/src/models.ts +++ b/clients/typescript/src/models.ts @@ -86,7 +86,9 @@ type ApiObject = Record; function toPythonRuntime(value: unknown): PythonRuntime { if (value == null) return null; if (value === "stdlib" || value === "pyodide") return value; - throw new Error(`unexpected python_runtime from server: ${JSON.stringify(value)}`); + throw Object.assign(new Error(`unexpected python_runtime from server: ${JSON.stringify(value)}`), { + code: "EINVALID_PYTHON_RUNTIME", + }); } export function sandboxRecordFromApi(payload: ApiObject): SandboxRecord { @@ -174,5 +176,5 @@ export function streamEventFromSse(eventName: string, payload: ApiObject): Strea t: typeof payload.t === "number" ? payload.t : undefined, }; } - throw new Error(`unknown SSE event: ${eventName}`); + throw Object.assign(new Error(`unknown SSE event: ${eventName}`), { code: "EUNKNOWN_SSE_EVENT" }); } From baecea786826c3a306574028d701a3ad47b87f5a Mon Sep 17 00:00:00 2001 From: Neil Mazumdar Date: Tue, 9 Jun 2026 14:16:51 +0930 Subject: [PATCH 13/16] migrations: renumber python_runtime 0006 -> 0007 Upstream added 0006_blob_last_referenced_at.sql; renumber the python_runtime migration to avoid the duplicate 0006 prefix. Content unchanged (idempotent; runs after the blob-gc migration). --- .../postgres/{0006_python_runtime.sql => 0007_python_runtime.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/sql-fs/migrations/postgres/{0006_python_runtime.sql => 0007_python_runtime.sql} (100%) diff --git a/src/sql-fs/migrations/postgres/0006_python_runtime.sql b/src/sql-fs/migrations/postgres/0007_python_runtime.sql similarity index 100% rename from src/sql-fs/migrations/postgres/0006_python_runtime.sql rename to src/sql-fs/migrations/postgres/0007_python_runtime.sql From 54dd35a43148d1b03013cd6d627905bc25710800 Mon Sep 17 00:00:00 2001 From: QuangNguyen2609 Date: Tue, 9 Jun 2026 23:46:33 +0930 Subject: [PATCH 14/16] feat(pyodide): preload matplotlib, fix asset pin, document runtime - runner: preload numpy/pandas/matplotlib (swap scipy out) so the pyodide runtime covers CSV/Excel analysis + plotting; loaded by name from the stock lock with automatic dependency resolution. - fetch-pyodide-assets: re-pin pyodide.mjs to the actual 0.29.4 GitHub release hash. The previous pin was stale (only the JS loader re-rolled upstream; pyodide.asm.wasm + python_stdlib.zip are byte-identical), which hard-failed the offline checksum guard during provisioning. - README: document the python -> python_runtime breaking change, the stdlib vs pyodide runtimes, the preloaded package set, and memory/cap sizing (16 MiB file cap + MAX_CONCURRENT_PYODIDE=1 on a 2 GB host). --- README.md | 58 +++++++++++++++++++++++++++++++- scripts/fetch-pyodide-assets.mjs | 2 +- src/pyodide-runner/runner.ts | 4 +-- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 44ea7a0..8f7e6a3 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,56 @@ await sandbox.delete(); See [clients/typescript/README.md](clients/typescript/README.md) for the full TypeScript API. +## Language runtimes + +Sandboxes are bash-only by default. Opt into a Python and/or JavaScript runtime at **creation** (it cannot be changed later). + +> ### ⚠️ Breaking change: `python` → `python_runtime` +> +> The boolean `python` create option has been **replaced** by the `python_runtime` enum. +> There are now **two** Python runtimes: +> +> | Old (removed) | New | +> |---|---| +> | `python: true` | `python_runtime: "stdlib"` | +> | `python: false` / omitted | omit `python_runtime` (no Python) | +> +> `SandboxRecord.python` (boolean) is likewise replaced by `SandboxRecord.python_runtime` +> (`"stdlib" \| "pyodide" \| null`). Update SDK calls accordingly. + +```python +# stdlib — lightweight CPython, stdlib only +sb = client.sandboxes.create(python_runtime="stdlib") + +# pyodide — heavy, with the scientific stack +sb = client.sandboxes.create(python_runtime="pyodide") + +# JavaScript (QuickJS) — unchanged +sb = client.sandboxes.create(javascript=True) +``` + +### The two Python runtimes + +| | `stdlib` | `pyodide` | +|---|---|---| +| Engine | CPython → WASM (in-process worker) | Pyodide → WASM in an **OS-isolated Deno subprocess** | +| Packages | Python **stdlib only** (no pip) | stdlib **+ numpy, pandas, matplotlib, openpyxl** (preloaded) | +| Cold start | ~1.4 s | ~3–5 s | +| Per-worker memory | ~80 MB | ~0.6 GB floor; ~1 GB+ with a real DataFrame | +| File reads | **8 MiB cap** per `open()` (IPC bridge) — chunk larger files | no 8 MiB cap; bounded by `PYODIDE_MAX_FILE_BYTES` (32 MiB default) | +| Use it for | quick scripts, glue, capability probes | CSV/Excel data analysis, plotting (charts drain back to the FS and are retrievable via `GET /files/...`) | + +The full Pyodide distribution (~380 packages) ships on disk, but **only the preloaded set is importable** — user scripts cannot `loadPackage` others at runtime. To add a package (e.g. `scipy`, `scikit-learn`), add it to the preload list in `src/pyodide-runner/runner.ts` (which raises the memory floor). + +### Sizing the Pyodide runtime + +Pyodide workers are the dominant memory cost (bash and the `stdlib` runtime are comparatively free). WASM memory does **not** shrink mid-run, so size for the worst-case single file: + +- **2 GB container, `MAX_CONCURRENT_PYODIDE=1`** → set `PYODIDE_MAX_FILE_BYTES=16777216` (**16 MiB**). A ~16 MB CSV peaks ~1 GB/worker (~1.6 GB combined) — fits with headroom. The shipped 32 MiB default is borderline for 2 GB. +- **`cap=2` doubles the memory** → use ~8 MiB files or a ~4 GB container. +- A ~50 MB CSV needs the stage cap raised **and** ~3–4 GB; `chunksize`/dtype tricks do **not** help (WASM high-water + the fully-staged file dominate). Reduce columns/rows *before* upload instead. +- The Pyodide runtime requires a vendored Deno binary + Pyodide assets (`scripts/fetch-pyodide-assets.mjs`, baked into the Docker image via `DENO_BIN_PATH` / `PYODIDE_ASSET_DIR`). + ## How It Works > **Postgres is always the source of truth. Everything else is a cache or a lock.** @@ -164,8 +214,14 @@ Key design choices: | `AUTH_SECRET` | Yes | — | Secret for Bearer token validation | | `PORT` | No | `8080` | HTTP server port | | `SESSION_IDLE_MS` | No | `600000` | Evict idle Bash instances after this many ms | -| `MAX_CONCURRENT_PYTHON` | No | `5` | Cap on concurrent CPython WASM workers (~80 MB each) | +| `MAX_CONCURRENT_PYTHON` | No | `5` | Cap on concurrent `stdlib` CPython WASM workers (~80 MB each) | | `MAX_CONCURRENT_JS` | No | `5` | Cap on concurrent QuickJS workers (~64 MB each) | +| `MAX_CONCURRENT_PYODIDE` | No | `2` | Cap on concurrent `pyodide` workers (OS-isolated Deno; ~0.6–1 GB+ each). Keep low; **use `1` on a 2 GB host**. Routed separately from `MAX_CONCURRENT_PYTHON`. | +| `MAX_RESIDENT_PYODIDE` | No | `2` | Resident `pyodide` workers (idle LRU). Must be `>=` `MAX_CONCURRENT_PYODIDE` or the server refuses to boot. | +| `PYODIDE_MAX_FILE_BYTES` | No | `33554432` | Per-file cap (32 MiB) on files staged into a `pyodide` exec. **Lower to `16777216` (16 MiB) on a 2 GB host.** | +| `PYODIDE_MAX_TOTAL_BYTES` | No | `134217728` | Total staged bytes across one `pyodide` exec (128 MiB). | +| `PYODIDE_ASSET_DIR` | `pyodide` runtime | — | Absolute path to vendored Pyodide assets (wasm + stdlib + wheels). Set in the Docker image. | +| `DENO_BIN_PATH` | No | `deno` | Path to the vendored Deno binary that runs the `pyodide` runner. Set in the Docker image. | | `MAX_REQUEST_BODY_BYTES` | No | `268435456` | Hard cap on any HTTP request body (256 MB) — file write, bulk write, ingest. Applied before auth/handlers. Since base64 inflates content ~33%, this is usually the binding limit on ingest: ~190 MB of raw file bytes per call. | | `MAX_INGEST_BYTES` | No | `536870912` | Max total decoded bytes across one `ingest-files` manifest (512 MB). The request-body cap above normally trips first. | | `MAX_INGEST_FILES` | No | `10000` | Max number of entries (files + paths) in one `ingest-files` manifest. | diff --git a/scripts/fetch-pyodide-assets.mjs b/scripts/fetch-pyodide-assets.mjs index 29f4f14..0948169 100644 --- a/scripts/fetch-pyodide-assets.mjs +++ b/scripts/fetch-pyodide-assets.mjs @@ -47,7 +47,7 @@ const WHEEL_URLS = { // SHA-256 of the platform-independent artifacts, pinned to the exact bytes // spike S1 validated. Mismatch ⇒ hard failure (supply-chain / corruption guard). const SHA256 = { - "pyodide.mjs": "c8dffeefeb6f9c4bf635baf0cdb51f4da06df0e3aab4fe1a99b8ad3570065461", + "pyodide.mjs": "8fdfed5eaf81bde14bcdeaeea11f2672675b2362248f8537446b6fda5e4a4751", "pyodide.asm.wasm": "10090fe41e019ae669d512e1f747021a8db2aaab0f6dd6f85fa9368c55d681e3", "python_stdlib.zip": "92cb24faa546818f3ef4050fd5bd2b6487bd2042efed2113af141d035f30efb4", [PINS.openpyxlWheel]: "5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", diff --git a/src/pyodide-runner/runner.ts b/src/pyodide-runner/runner.ts index 7dcea73..35c87cb 100644 --- a/src/pyodide-runner/runner.ts +++ b/src/pyodide-runner/runner.ts @@ -123,8 +123,8 @@ try { stderr: () => {}, }); - // numpy/pandas/scipy ship in the distribution → load by name. - await pyodide.loadPackage(["numpy", "pandas", "scipy"]); + // numpy/pandas/matplotlib loaded by name from the stock lock. + await pyodide.loadPackage(["numpy", "pandas", "matplotlib"]); // openpyxl + et_xmlfile are NOT in the distribution; load the vendored pure-python // wheels by local file:// URL (discovered in the asset dir). loadPackage reads // them via node:fs under --allow-read — no network. (Phase 0 Discoveries: the From 10c8ef303bc3f66b3af61157e3d0a35171fdcf04 Mon Sep 17 00:00:00 2001 From: QuangNguyen2609 Date: Wed, 10 Jun 2026 08:36:37 +0930 Subject: [PATCH 15/16] perf(pyodide): cut runtime memory ~1.6GB to ~1.1GB Profiling an 18.8 MB-CSV workload showed the spike was IPC/staging machinery, not pandas. Three measured fixes: - O(n) chunk-list framing in the Deno runner stdin loop and the Node manager #onStdoutData (was O(n^2): a full-buffer realloc+copy per pipe chunk for a ~25 MB base64 frame). Steady state ~1316MB -> ~804MB child. - Streamed SHA-256 diff baseline in runOne: store size+hash per staged file instead of holding every file's bytes for the whole run and re-reading the tree afterwards for a byte compare. - Lazy offline package loading: only PYODIDE_PRELOAD_PACKAGES (default numpy,pandas) is resident at init; other distribution packages load on first import from the local lock. matplotlib defaults to the headless Agg backend so savefig() works in the child (webagg needs the DOM). New env vars PYODIDE_PRELOAD_PACKAGES and PYODIDE_MAX_CHILD_RSS_BYTES (optional RSS-based child retirement after a high-water run). Adds the memory-analysis and streaming/staging follow-up plan under thoughts/. US-118: pyodide runtime memory optimization --- .changeset/python-runtime-enum.md | 7 + .env.example | 4 + CLAUDE.md | 2 + README.md | 8 +- src/api/pyodide/manager.ts | 146 +++++-- src/api/pyodide/tests/unit/manager.test.ts | 84 +++- src/pyodide-runner/runner.ts | 189 +++++++-- .../memory-analysis.md | 364 ++++++++++++++++++ .../streaming-staging-plan.md | 330 ++++++++++++++++ 9 files changed, 1064 insertions(+), 70 deletions(-) create mode 100644 thoughts/issue-118-pyodide-runtime/memory-analysis.md create mode 100644 thoughts/issue-118-pyodide-runtime/streaming-staging-plan.md diff --git a/.changeset/python-runtime-enum.md b/.changeset/python-runtime-enum.md index dc3ad56..77c9f7b 100644 --- a/.changeset/python-runtime-enum.md +++ b/.changeset/python-runtime-enum.md @@ -11,3 +11,10 @@ `python_runtime: "stdlib"` is the air-gapped CPython-WASM runtime (the previous `python: true`). `python_runtime: "pyodide"` adds a numpy/pandas/scipy/openpyxl runtime in an OS-isolated Deno subprocess. The DB layer migrates rolling-deploy-safe (migration 0006 dual-writes/back-reads the legacy column). Clients must migrate `python: true` → `python_runtime: "stdlib"` and `python: false` → omit (or `null`). + +**Pyodide runtime memory & throughput tuning** (part of the same unreleased feature): + +- Cut combined runtime memory ~1.6 GB → ~1.1 GB on an 18.8 MB-CSV workload: O(n) chunk-list IPC framing (replaces an O(n²) per-chunk reallocation on both the Deno runner and the Node manager), and a streamed SHA-256 diff baseline (replaces holding every staged file's bytes for the whole run). +- Lazy, offline package loading: only `PYODIDE_PRELOAD_PACKAGES` (default `numpy,pandas`) is resident at init; other distribution packages load on first import from the local lock. Lowers the idle floor and lets operators trade latency against RSS. +- `matplotlib` now defaults to the headless `Agg` backend so `savefig()` works in the Deno child (previously failed resolving the DOM `webagg` backend). +- New env vars `PYODIDE_PRELOAD_PACKAGES` and `PYODIDE_MAX_CHILD_RSS_BYTES` (optional RSS-based child retirement). diff --git a/.env.example b/.env.example index 43d0123..40da16e 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,10 @@ PORT=8080 SESSION_IDLE_MS=600000 # evict idle sessions after 10 min MAX_CONCURRENT_PYTHON=5 # cap CPython WASM workers (~80 MB each) MAX_CONCURRENT_JS=5 # cap QuickJS workers (~64 MB each) +# MAX_CONCURRENT_PYODIDE=2 # cap concurrent Deno/Pyodide workers +# MAX_RESIDENT_PYODIDE=2 # cap warm resident workers; must be >= concurrent +# PYODIDE_PRELOAD_PACKAGES=numpy,pandas +# PYODIDE_MAX_CHILD_RSS_BYTES=0 # retire after a run above this RSS; 0 disables # ── Optional: Redis (required for multi-replica deployments) ────────────────── diff --git a/CLAUDE.md b/CLAUDE.md index fe4727c..e07df5b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,6 +213,8 @@ const TABLE = Object.assign(Object.create(null) as Record, { | `MAX_RESIDENT_PYODIDE` | No (default: 2) | Cap on **resident** `pyodide` subprocesses (residency LRU), independent of `SESSION_IDLE_MS` — bounds total live Deno children so warm subprocesses don't accumulate per active session. MUST be `>= MAX_CONCURRENT_PYODIDE` (a busy worker is never evictable); the server fails to boot otherwise. Eviction/idle-kill targets only idle workers; an evicted session cold-starts a fresh child on its next exec. | | `PYODIDE_IDLE_MS` | No (default: 120000) | Idle window before a resident `pyodide` subprocess is idle-killed by the residency sweep (ms). Should be `< SESSION_IDLE_MS` so warm children are reclaimed well before their session is evicted. | | `PYODIDE_RUNTIME_TIMEOUT_MS` | No (default: 60000) | Cap on a single owned `pyodide` run (Deno spawn + Pyodide init + execution). On timeout the child is SIGKILLed (generation retired) and the exec throws (script transaction rolls back). Cold start (numpy/pandas/scipy/openpyxl load) takes several seconds — size accordingly. | +| `PYODIDE_PRELOAD_PACKAGES` | No (default: `numpy,pandas`) | Comma-separated packages loaded at child init, trading cold-start latency against resident RSS. Packages NOT preloaded are loaded on demand from the same offline lock on first import (`loadPackagesFromImports`), so the resident floor only pays for what's preloaded. Set empty for no stock-package preload (smallest baseline). | +| `PYODIDE_MAX_CHILD_RSS_BYTES` | No (default: 0 = disabled) | Retire a `pyodide` child after a completed run when its RSS exceeds this many bytes (Linux `/proc//status`, else `ps`); the next exec cold-starts a fresh generation. Bounds the grow-only WASM/V8 heap's idle retention — it does NOT prevent the in-flight peak. The container memory limit remains the hard guard. | | `PYODIDE_MAX_FILE_BYTES` | No (default: 33554432) | Per-file cap (bytes, 32 MiB) on files staged into / drained out of a `pyodide` exec. | | `PYODIDE_MAX_TOTAL_BYTES` | No (default: 134217728) | Total cap (bytes, 128 MiB) across all files staged into / drained out of a single `pyodide` exec. | | `PYODIDE_MAX_FRAME_BYTES` | No (default: per-frame IPC cap) | Max size (bytes) of a single IPC frame on the Node↔Deno channel, measured on the base64 wire size. Oversized frames kill the child. | diff --git a/README.md b/README.md index 8f7e6a3..e96f79d 100644 --- a/README.md +++ b/README.md @@ -154,19 +154,19 @@ sb = client.sandboxes.create(javascript=True) | | `stdlib` | `pyodide` | |---|---|---| | Engine | CPython → WASM (in-process worker) | Pyodide → WASM in an **OS-isolated Deno subprocess** | -| Packages | Python **stdlib only** (no pip) | stdlib **+ numpy, pandas, matplotlib, openpyxl** (preloaded) | +| Packages | Python **stdlib only** (no pip) | stdlib **+ offline Pyodide packages**; numpy/pandas preloaded, others loaded on first import | | Cold start | ~1.4 s | ~3–5 s | | Per-worker memory | ~80 MB | ~0.6 GB floor; ~1 GB+ with a real DataFrame | | File reads | **8 MiB cap** per `open()` (IPC bridge) — chunk larger files | no 8 MiB cap; bounded by `PYODIDE_MAX_FILE_BYTES` (32 MiB default) | | Use it for | quick scripts, glue, capability probes | CSV/Excel data analysis, plotting (charts drain back to the FS and are retrievable via `GET /files/...`) | -The full Pyodide distribution (~380 packages) ships on disk, but **only the preloaded set is importable** — user scripts cannot `loadPackage` others at runtime. To add a package (e.g. `scipy`, `scikit-learn`), add it to the preload list in `src/pyodide-runner/runner.ts` (which raises the memory floor). +The full Pyodide distribution (~380 packages) ships on disk. Imports are resolved from that offline lock on demand; `PYODIDE_PRELOAD_PACKAGES` controls which packages pay their memory and startup cost at child initialization. Matplotlib defaults to the headless `Agg` backend so `savefig()` works in the Deno child. ### Sizing the Pyodide runtime Pyodide workers are the dominant memory cost (bash and the `stdlib` runtime are comparatively free). WASM memory does **not** shrink mid-run, so size for the worst-case single file: -- **2 GB container, `MAX_CONCURRENT_PYODIDE=1`** → set `PYODIDE_MAX_FILE_BYTES=16777216` (**16 MiB**). A ~16 MB CSV peaks ~1 GB/worker (~1.6 GB combined) — fits with headroom. The shipped 32 MiB default is borderline for 2 GB. +- **2 GB container, `MAX_CONCURRENT_PYODIDE=1`** → set `PYODIDE_MAX_FILE_BYTES=16777216` (**16 MiB**) for conservative headroom. The optimized transport measured ~0.84 GB child RSS and ~1.1 GB combined on an 18.8 MB CSV; host and workload shape still matter. - **`cap=2` doubles the memory** → use ~8 MiB files or a ~4 GB container. - A ~50 MB CSV needs the stage cap raised **and** ~3–4 GB; `chunksize`/dtype tricks do **not** help (WASM high-water + the fully-staged file dominate). Reduce columns/rows *before* upload instead. - The Pyodide runtime requires a vendored Deno binary + Pyodide assets (`scripts/fetch-pyodide-assets.mjs`, baked into the Docker image via `DENO_BIN_PATH` / `PYODIDE_ASSET_DIR`). @@ -218,6 +218,8 @@ Key design choices: | `MAX_CONCURRENT_JS` | No | `5` | Cap on concurrent QuickJS workers (~64 MB each) | | `MAX_CONCURRENT_PYODIDE` | No | `2` | Cap on concurrent `pyodide` workers (OS-isolated Deno; ~0.6–1 GB+ each). Keep low; **use `1` on a 2 GB host**. Routed separately from `MAX_CONCURRENT_PYTHON`. | | `MAX_RESIDENT_PYODIDE` | No | `2` | Resident `pyodide` workers (idle LRU). Must be `>=` `MAX_CONCURRENT_PYODIDE` or the server refuses to boot. | +| `PYODIDE_PRELOAD_PACKAGES` | No | `numpy,pandas` | Comma-separated packages loaded at child initialization. Other packages from the vendored lock load on first import. Use an empty value for no stock-package preload. | +| `PYODIDE_MAX_CHILD_RSS_BYTES` | No | `0` | Retire a child after a completed run when RSS exceeds this many bytes. `0` disables retirement; this bounds idle retention, not the in-flight peak. | | `PYODIDE_MAX_FILE_BYTES` | No | `33554432` | Per-file cap (32 MiB) on files staged into a `pyodide` exec. **Lower to `16777216` (16 MiB) on a 2 GB host.** | | `PYODIDE_MAX_TOTAL_BYTES` | No | `134217728` | Total staged bytes across one `pyodide` exec (128 MiB). | | `PYODIDE_ASSET_DIR` | `pyodide` runtime | — | Absolute path to vendored Pyodide assets (wasm + stdlib + wheels). Set in the Docker image. | diff --git a/src/api/pyodide/manager.ts b/src/api/pyodide/manager.ts index daaab35..07ae9dd 100644 --- a/src/api/pyodide/manager.ts +++ b/src/api/pyodide/manager.ts @@ -26,13 +26,15 @@ */ import { Buffer } from "node:buffer"; -import { type ChildProcess, type SpawnOptions, spawn as nodeSpawn } from "node:child_process"; +import { type ChildProcess, type SpawnOptions, execFile, spawn as nodeSpawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { accessSync, constants as fsConstants } from "node:fs"; +import { readFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import type { Frame, RunRequest, RunResponse } from "../../pyodide-runner/protocol.js"; import { type InboundContext, + IpcFrameTooLargeError, IpcIntegrityError, PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT, PYODIDE_MAX_FRAME_BYTES_DEFAULT, @@ -74,6 +76,7 @@ export const COMMITTED_FLAGS: readonly string[] = [ /** Default cap on a single owned run (init/preload + execution). */ export const PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT = 60_000; +export const PYODIDE_PRELOAD_PACKAGES_DEFAULT = ["numpy", "pandas"] as const; /** Injectable spawn signature (defaults to `child_process.spawn`). */ export type SpawnFn = (command: string, args: readonly string[], options: SpawnOptions) => ChildProcess; @@ -91,10 +94,16 @@ export interface PyodideSandboxOptions { readonly maxFrameBytes?: number; /** Aggregate per-response wire cap (bytes). Default {@link PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT}. */ readonly maxAggregateBytes?: number; + /** Packages loaded at child init. Defaults to PYODIDE_PRELOAD_PACKAGES or numpy,pandas. */ + readonly preloadPackages?: readonly string[]; + /** Retire a child after a run when its RSS exceeds this threshold. Disabled by default. */ + readonly maxChildRssBytes?: number; /** Injected spawn (tests). Defaults to `child_process.spawn`. */ readonly spawnFn?: SpawnFn; /** Injected requestId generator (tests). Defaults to `crypto.randomUUID`. */ readonly randomRequestId?: () => string; + /** Injected RSS sampler (tests). */ + readonly readChildRssBytes?: (pid: number) => Promise; } /** Thrown when an owned run exceeds {@link PyodideSandboxOptions.runtimeTimeoutMs}. */ @@ -183,11 +192,43 @@ function resolveDenoBin(bin: string): string { return bin; // not found on PATH; let spawn surface ENOENT with the bare name } +function packagesFromEnv(value: string | undefined): string[] { + if (value === undefined) return [...PYODIDE_PRELOAD_PACKAGES_DEFAULT]; + return value + .split(",") + .map((name) => name.trim()) + .filter((name) => name.length > 0); +} + +async function readProcessRssBytes(pid: number): Promise { + if (process.platform === "linux") { + try { + const status = await readFile(`/proc/${pid}/status`, "utf8"); + const match = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status); + if (match?.[1]) return Number(match[1]) * 1024; + } catch { + // Fall through to ps for non-procfs Linux environments. + } + } + return await new Promise((resolve) => { + execFile("ps", ["-o", "rss=", "-p", String(pid)], (err, stdout) => { + if (err) { + resolve(null); + return; + } + const rssKiB = Number(stdout.trim()); + resolve(Number.isFinite(rssKiB) && rssKiB >= 0 ? rssKiB * 1024 : null); + }); + }); +} + export class PyodideSandbox { #state: WorkerState = "cold"; #generation = 0; #child: ChildProcess | null = null; - #readBuf: Buffer = Buffer.alloc(0); + #readChunks: Buffer[] = []; + #readBytes = 0; + #expectedFrameBytes = -1; #aggregateBytes = 0; #readyReceived = false; #seqCounter = 0; @@ -205,8 +246,11 @@ export class PyodideSandbox { readonly #runtimeTimeoutMs: number; readonly #maxFrameBytes: number; readonly #maxAggregateBytes: number; + readonly #preloadPackages: readonly string[]; + readonly #maxChildRssBytes: number | undefined; readonly #spawnFn: SpawnFn; readonly #randomRequestId: () => string; + readonly #readChildRssBytes: (pid: number) => Promise; constructor(opts: PyodideSandboxOptions = {}) { this.#assetDir = opts.assetDir ?? process.env.PYODIDE_ASSET_DIR ?? ""; @@ -215,8 +259,12 @@ export class PyodideSandbox { this.#runtimeTimeoutMs = opts.runtimeTimeoutMs ?? PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT; this.#maxFrameBytes = opts.maxFrameBytes ?? PYODIDE_MAX_FRAME_BYTES_DEFAULT; this.#maxAggregateBytes = opts.maxAggregateBytes ?? PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT; + this.#preloadPackages = opts.preloadPackages ?? packagesFromEnv(process.env.PYODIDE_PRELOAD_PACKAGES); + const configuredRss = opts.maxChildRssBytes ?? Number(process.env.PYODIDE_MAX_CHILD_RSS_BYTES ?? "0"); + this.#maxChildRssBytes = Number.isFinite(configuredRss) && configuredRss > 0 ? configuredRss : undefined; this.#spawnFn = opts.spawnFn ?? (nodeSpawn as SpawnFn); this.#randomRequestId = opts.randomRequestId ?? randomUUID; + this.#readChildRssBytes = opts.readChildRssBytes ?? readProcessRssBytes; } get state(): WorkerState { @@ -427,12 +475,11 @@ export class PyodideSandbox { * against ~41 MB resident (VmRSS) — an `RLIMIT_AS` low enough to bound RSS makes * the WASM allocation fail outright (`RangeError: could not allocate memory`). * - * The **operator-set container memory limit is therefore the only real guard**; - * it covers Node + every Deno child together, so a runaway child may OOM-kill - * the whole container (no per-child OOM isolation). Operators size the limit as - * `MAX_RESIDENT_PYODIDE × per-process ceiling` (use `MAX_RESIDENT_PYODIDE=1` on - * small hosts). On OOM-kill the child exits → {@link PyodideChildExitError} → - * the manager respawns a fresh generation on the next run. + * The operator-set container limit remains the hard guard while + * `PYODIDE_MAX_CHILD_RSS_BYTES` optionally retires a child after a completed + * high-water run. The latter bounds idle retention but cannot prevent the + * in-flight spike itself. Operators still size the container as + * `MAX_RESIDENT_PYODIDE × per-process ceiling`. */ #spawnChild(): void { this.#generation += 1; @@ -443,6 +490,7 @@ export class PyodideSandbox { `--allow-read=${this.#assetDir}`, this.#runnerPath, this.#assetDir, + JSON.stringify(this.#preloadPackages), String(gen), ]; const child = this.#spawnFn(this.#denoBin, args, { @@ -454,7 +502,7 @@ export class PyodideSandbox { this.#child = child; this.#readyReceived = false; this.#pending = null; - this.#readBuf = Buffer.alloc(0); + this.#resetReadBuffer(); this.#aggregateBytes = 0; this.#state = "starting"; @@ -486,7 +534,7 @@ export class PyodideSandbox { this.#child = null; this.#readyReceived = false; this.#pending = null; - this.#readBuf = Buffer.alloc(0); + this.#resetReadBuffer(); this.#aggregateBytes = 0; if (this.#state !== "terminating") this.#state = "dead"; if (child) { @@ -517,37 +565,63 @@ export class PyodideSandbox { this.#failOwned(new IpcIntegrityError("aggregate response bytes exceeded cap"), true); return; } - this.#readBuf = this.#readBuf.byteLength === 0 ? Buffer.from(chunk) : Buffer.concat([this.#readBuf, chunk]); - - let decoded: { frames: ReturnType["frames"]; rest: Buffer }; - try { - decoded = decodeFrames(this.#readBuf, this.#maxFrameBytes); - } catch (err) { - this.#failOwned(err instanceof Error ? err : new Error(String(err)), true); - return; - } - this.#readBuf = decoded.rest; + this.#readChunks.push(chunk); + this.#readBytes += chunk.byteLength; - for (const frame of decoded.frames) { - const ctx: InboundContext = { - generation: this.#generation, - ready: this.#readyReceived, - pending: this.#pending ? { requestId: this.#pending.requestId, seq: this.#pending.seq } : null, - }; + for (;;) { try { - validateInbound(frame, ctx); + if (this.#expectedFrameBytes < 0) { + if (this.#readBytes < 4) return; + this.#expectedFrameBytes = this.#peekFrameLength(); + if (this.#expectedFrameBytes > this.#maxFrameBytes) { + throw new IpcFrameTooLargeError(this.#expectedFrameBytes, this.#maxFrameBytes); + } + } + if (this.#readBytes < 4 + this.#expectedFrameBytes) return; + + const decoded = decodeFrames(Buffer.concat(this.#readChunks, this.#readBytes), this.#maxFrameBytes); + this.#readChunks = decoded.rest.byteLength > 0 ? [decoded.rest] : []; + this.#readBytes = decoded.rest.byteLength; + this.#expectedFrameBytes = -1; + + for (const frame of decoded.frames) { + const ctx: InboundContext = { + generation: this.#generation, + ready: this.#readyReceived, + pending: this.#pending ? { requestId: this.#pending.requestId, seq: this.#pending.seq } : null, + }; + validateInbound(frame, ctx); + this.#dispatchFrame(frame); + if (this.#isTerminal()) return; + } } catch (err) { this.#failOwned(err instanceof Error ? err : new Error(String(err)), true); return; } - this.#dispatchFrame(frame); - if (this.#isTerminal()) return; } } + #peekFrameLength(): number { + const header = Buffer.allocUnsafe(4); + let offset = 0; + for (const chunk of this.#readChunks) { + const take = Math.min(chunk.byteLength, 4 - offset); + chunk.copy(header, offset, 0, take); + offset += take; + if (offset === 4) break; + } + return header.readUInt32BE(0); + } + + #resetReadBuffer(): void { + this.#readChunks = []; + this.#readBytes = 0; + this.#expectedFrameBytes = -1; + } + #dispatchFrame(frame: Frame): void { // Reset the aggregate window on each accepted complete frame. - this.#aggregateBytes = this.#readBuf.byteLength; + this.#aggregateBytes = this.#readBytes; if (frame.type === "ready") { this.#readyReceived = true; @@ -569,7 +643,17 @@ export class PyodideSandbox { if (op.timer !== undefined) clearTimeout(op.timer); op.signal.removeEventListener("abort", op.onAbort); this.#current = null; - op.resolve(asRunResponse(frame)); + void this.#resolveResponse(op, asRunResponse(frame), this.#child); + } + } + + async #resolveResponse(op: OwnedOp, response: RunResponse, responseChild: ChildProcess | null): Promise { + const threshold = this.#maxChildRssBytes; + const pid = responseChild?.pid; + if (threshold !== undefined && pid !== undefined) { + const rss = await this.#readChildRssBytes(pid).catch(() => null); + if (rss !== null && rss > threshold && this.#child === responseChild && !this.#disposed) this.#killChild(); } + op.resolve(response); } } diff --git a/src/api/pyodide/tests/unit/manager.test.ts b/src/api/pyodide/tests/unit/manager.test.ts index 319f237..8e4a683 100644 --- a/src/api/pyodide/tests/unit/manager.test.ts +++ b/src/api/pyodide/tests/unit/manager.test.ts @@ -10,7 +10,7 @@ import { Buffer } from "node:buffer"; import type { SpawnOptions } from "node:child_process"; import { afterEach, describe, expect, it } from "vitest"; -import { IpcFrameTooLargeError, IpcIntegrityError } from "../../ipc.js"; +import { IpcFrameTooLargeError, IpcIntegrityError, encodeFrame } from "../../ipc.js"; import type { RunRequestInput } from "../../manager.js"; import { COMMITTED_FLAGS, @@ -26,7 +26,13 @@ const INPUT: RunRequestInput = { code: "print(1)", argv: ["x.py"], stdin: "", fi let seq = 0; function makeManager( harness: Harness, - opts: { runtimeTimeoutMs?: number; maxFrameBytes?: number; maxAggregateBytes?: number } = {}, + opts: { + runtimeTimeoutMs?: number; + maxFrameBytes?: number; + maxAggregateBytes?: number; + maxChildRssBytes?: number; + readChildRssBytes?: (pid: number) => Promise; + } = {}, ): PyodideSandbox { return new PyodideSandbox({ assetDir: "/vendor/pyodide", @@ -38,6 +44,9 @@ function makeManager( runtimeTimeoutMs: opts.runtimeTimeoutMs ?? 5_000, maxFrameBytes: opts.maxFrameBytes, maxAggregateBytes: opts.maxAggregateBytes, + maxChildRssBytes: opts.maxChildRssBytes, + readChildRssBytes: opts.readChildRssBytes, + preloadPackages: ["numpy", "pandas"], }); } @@ -123,6 +132,34 @@ describe("PyodideSandbox — happy path & serialization", () => { expect(order).toEqual([1, 2]); expect(harness.children).toHaveLength(1); }); + + it("reassembles ready and result frames delivered one byte at a time", async () => { + const harness = makeHarness(); + const manager = track(makeManager(harness)); + const p = manager.run(INPUT, new AbortController().signal); + const child = await harness.nextChild(); + + for (const byte of encodeFrame({ type: "ready", generation: child.generation })) { + child.stdout.write(Buffer.from([byte])); + } + const run = await child.nextRun(); + const result = encodeFrame({ + type: "result", + requestId: run.requestId, + seq: run.seq, + generation: run.generation, + stdout: "", + stderr: "", + exitCode: 0, + created: [], + modified: [], + deleted: [], + }); + for (const byte of result) child.stdout.write(Buffer.from([byte])); + + await expect(p).resolves.toMatchObject({ exitCode: 0 }); + expect(child.killed).toBe(false); + }); }); describe("PyodideSandbox — cancellation", () => { @@ -302,6 +339,15 @@ describe("PyodideSandbox — frame integrity (each violation kills the child)", expect(child.killed).toBe(true); }); + it("rejects an oversized declared length as soon as the fragmented header completes", async () => { + const { child, p } = await inFlight({ maxFrameBytes: 200 }); + const header = Buffer.alloc(4); + header.writeUInt32BE(201, 0); + for (const byte of header) child.stdout.write(Buffer.from([byte])); + expect(await p).toBeInstanceOf(IpcFrameTooLargeError); + expect(child.killed).toBe(true); + }); + it("a duplicate / replayed response (none in-flight) → kill", async () => { const harness = makeHarness(); const manager = track(makeManager(harness)); @@ -383,6 +429,7 @@ describe("PyodideSandbox — spawn posture", () => { runnerPath: "/dist/pyodide-runner/runner.ts", spawnFn: recordingSpawn, randomRequestId: () => "req-x", + preloadPackages: ["numpy", "pandas"], }), ); @@ -402,6 +449,7 @@ describe("PyodideSandbox — spawn posture", () => { "--allow-read=/vendor/pyodide", "/dist/pyodide-runner/runner.ts", "/vendor/pyodide", + '["numpy","pandas"]', "1", ]); // Scrubbed env: ONLY the update-check suppressor — no AUTH_SECRET/DATABASE_URL. @@ -410,6 +458,38 @@ describe("PyodideSandbox — spawn posture", () => { }); }); +describe("PyodideSandbox — RSS retirement", () => { + it("returns the completed response, then retires an over-limit child and respawns on the next run", async () => { + const harness = makeHarness(); + const sampledPids: number[] = []; + const manager = track( + makeManager(harness, { + maxChildRssBytes: 500, + readChildRssBytes: async (pid) => { + sampledPids.push(pid); + return 501; + }, + }), + ); + + const p1 = manager.run(INPUT, new AbortController().signal); + const child1 = await harness.nextChild(); + child1.sendReady(); + child1.sendResult(await child1.nextRun()); + await expect(p1).resolves.toMatchObject({ exitCode: 0 }); + expect(sampledPids).toEqual([child1.pid]); + expect(child1.killed).toBe(true); + expect(manager.state).toBe("dead"); + + const p2 = manager.run(INPUT, new AbortController().signal); + const child2 = await harness.nextChild(); + expect(child2.generation).toBe(2); + child2.sendReady(); + child2.sendResult(await child2.nextRun()); + await expect(p2).resolves.toMatchObject({ exitCode: 0 }); + }); +}); + describe("PyodideSandbox — dispose", () => { it("dispose kills the child and rejects an in-flight run; further runs reject", async () => { const harness = makeHarness(); diff --git a/src/pyodide-runner/runner.ts b/src/pyodide-runner/runner.ts index 35c87cb..a0c4a0f 100644 --- a/src/pyodide-runner/runner.ts +++ b/src/pyodide-runner/runner.ts @@ -24,7 +24,9 @@ import { Buffer } from "node:buffer"; import { createRequire } from "node:module"; import { type Frame, + FrameTooLargeError, type FsEntry, + MAX_FRAME_BYTES, PYODIDE_EXT_STAGING_DIR, type ReadyFrame, type RunRequest, @@ -36,14 +38,17 @@ import { // ── Capture host primitives BEFORE lockdown (closure-held, never on globalThis) ─ // deno-lint-ignore no-explicit-any const denoRef = (globalThis as any).Deno; +const consoleRef = (globalThis as any).console; const stdoutWriteSync: (b: Uint8Array) => number = denoRef.stdout.writeSync.bind(denoRef.stdout); const stderrWriteSync: (b: Uint8Array) => number = denoRef.stderr.writeSync.bind(denoRef.stderr); const stdinReadable: ReadableStream = denoRef.stdin.readable; const denoExit: (code: number) => never = denoRef.exit.bind(denoRef); const denoArgs: string[] = denoRef.args; +const subtleDigest = crypto.subtle.digest.bind(crypto.subtle); const assetDir: string | undefined = denoArgs[0]; -const generation = Number(denoArgs[1] ?? "0"); +const preloadPackages = JSON.parse(denoArgs[1] ?? "[]") as string[]; +const generation = Number(denoArgs[2] ?? "0"); if (!assetDir) { stdoutWriteSync(new TextEncoder().encode("RUNNER FATAL: asset dir not provided as argv[0]\n")); denoExit(2); @@ -79,6 +84,7 @@ function emit(frame: Frame): void { let pyodide: any; // deno-lint-ignore no-explicit-any let FS: any; +const importToPackage = new Map(); /** * Adversarial self-test (spike S2 hard gate). Runs AFTER realm lockdown and @@ -123,8 +129,21 @@ try { stderr: () => {}, }); - // numpy/pandas/matplotlib loaded by name from the stock lock. - await pyodide.loadPackage(["numpy", "pandas", "matplotlib"]); + // Retain only the small import-name → package-name index from the local lock. + // `loadPackagesFromImports` is attempted per run, with this index providing a + // deterministic offline fallback in the locked-down Deno realm. + const lock = JSON.parse(denoRef.readTextFileSync(`${assetRoot}/pyodide-lock.json`)) as { + packages: Record; + }; + for (const [packageName, metadata] of Object.entries(lock.packages)) { + for (const importName of metadata.imports ?? []) { + if (!importToPackage.has(importName)) importToPackage.set(importName, packageName); + } + } + + // Operators can trade cold-start latency against resident RSS. Packages not + // preloaded here are loaded on demand from the same offline lock per run. + if (preloadPackages.length > 0) await pyodide.loadPackage(preloadPackages); // openpyxl + et_xmlfile are NOT in the distribution; load the vendored pure-python // wheels by local file:// URL (discovered in the asset dir). loadPackage reads // them via node:fs under --allow-read — no network. (Phase 0 Discoveries: the @@ -170,11 +189,10 @@ interface TreeNode { path: string; kind: "file" | "dir"; mode: number; - bytes?: Uint8Array; // present only for files + size: number; } -/** Walk the subtree under `root` (excluding `root` itself), returning every dir - * and file with its mode (and bytes for files). */ +/** Walk the subtree under `root` (excluding `root` itself), returning metadata only. */ function walkTree(root: string): TreeNode[] { const out: TreeNode[] = []; const walk = (dir: string): void => { @@ -187,12 +205,12 @@ function walkTree(root: string): TreeNode[] { for (const name of names) { if (name === "." || name === "..") continue; const full = dir === "/" ? `/${name}` : `${dir}/${name}`; - const { mode } = FS.stat(full); + const { mode, size } = FS.stat(full); if (FS.isDir(mode)) { - out.push({ path: full, kind: "dir", mode: mode & 0o777 }); + out.push({ path: full, kind: "dir", mode: mode & 0o777, size: 0 }); walk(full); } else if (FS.isFile(mode)) { - out.push({ path: full, kind: "file", mode: mode & 0o777, bytes: readFileBytes(full) }); + out.push({ path: full, kind: "file", mode: mode & 0o777, size }); } } }; @@ -200,18 +218,69 @@ function walkTree(root: string): TreeNode[] { return out; } -function readFileBytes(path: string): Uint8Array { - return FS.readFile(path, { encoding: "binary" }) as Uint8Array; +function readFileBytes(path: string): Uint8Array { + return FS.readFile(path, { encoding: "binary" }) as Uint8Array; } function depth(path: string): number { return path.split("/").length; } -function sameBytes(a: Uint8Array, b: Uint8Array): boolean { - if (a.byteLength !== b.byteLength) return false; - for (let i = 0; i < a.byteLength; i++) if (a[i] !== b[i]) return false; - return true; +async function sha256(bytes: Uint8Array): Promise { + const hash = new Uint8Array(await subtleDigest("SHA-256", bytes)); + let hex = ""; + for (const byte of hash) hex += byte.toString(16).padStart(2, "0"); + return hex; +} + +async function loadImportedPackages(code: string): Promise { + // Parse imports structurally with Python's AST. In this Deno/Node-compat realm, + // loadPackagesFromImports may return without installing a known package after + // host globals are locked down, so explicitly load the locally-mapped packages. + pyodide.globals.set("__sqlfs_import_scan_code", code); + const importsProxy = pyodide.runPython(` +import ast as __sqlfs_ast +__sqlfs_import_tree = __sqlfs_ast.parse(__sqlfs_import_scan_code) +sorted({ + name + for node in __sqlfs_ast.walk(__sqlfs_import_tree) + for name in ( + [alias.name.split(".")[0] for alias in node.names] + if isinstance(node, __sqlfs_ast.Import) + else [node.module.split(".")[0]] + if isinstance(node, __sqlfs_ast.ImportFrom) and node.module + else [] + ) +}) +`); + let importNames: string[]; + try { + importNames = importsProxy.toJs() as string[]; + } finally { + importsProxy.destroy(); + pyodide.globals.delete("__sqlfs_import_scan_code"); + } + const packages = [...new Set(importNames.map((name) => importToPackage.get(name)).filter((name): name is string => !!name))]; + + // The package loader's Node-compat path needs host globals that realm lockdown + // removes from Python's `js` proxy. Restore them only around trusted vendored + // package installation, then prove they are unreachable before user code runs. + g.Deno = denoRef; + g.console = consoleRef; + g.require = createRequire(import.meta.url); + g.__dirname = assetRoot; + g.__filename = `${assetRoot}/pyodide.asm.js`; + try { + await pyodide.loadPackagesFromImports(code); + if (packages.length > 0) await pyodide.loadPackage(packages); + } finally { + delete g.Deno; + delete g.console; + delete g.require; + delete g.__dirname; + delete g.__filename; + } + selfTest(); } // ── Run one request ───────────────────────────────────────────────────────── @@ -234,15 +303,22 @@ async function runOne(req: RunRequest): Promise { // Snapshot the cwd subtree AFTER staging, BEFORE running user code — this is // the diff baseline (excludes staging infrastructure dirs, which pre-exist // from the caller's SqlFs tree). - const baseFiles = new Map(); + const baseFiles = new Map(); // Track KIND per path (not just presence) so a file↔dir replacement at the same // path is detectable — otherwise `os.remove('x'); os.mkdir('x')` is invisible. const baseKind = new Map(); for (const node of walkTree(cwd)) { baseKind.set(node.path, node.kind); - if (node.kind === "file" && node.bytes) baseFiles.set(node.path, node.bytes); + if (node.kind === "file") { + const bytes = readFileBytes(node.path); + baseFiles.set(node.path, { size: node.size, sha256: await sha256(bytes) }); + } } + // Resolve imports against the vendored lock before user code executes. This + // keeps the child fully offline while avoiding resident cost for unused packages. + await loadImportedPackages(req.code); + // Prelude: argv + cwd + stdin, plus redirect sys.stdout/sys.stderr to StringIO // buffers. We read those buffers' getvalue() after the run — this captures ALL // Python output regardless of trailing newlines / flushing (Pyodide's batched @@ -271,6 +347,7 @@ if __sqlfs_argv0 and __sqlfs_argv0 not in ("-c", "-"): # os.environ key cannot leak it into the next run. __sqlfs_env_saved = dict(os.environ) os.environ.update(__json.loads(__sqlfs_env)) +os.environ.setdefault("MPLBACKEND", "Agg") __sqlfs_out = io.StringIO() __sqlfs_err = io.StringIO() sys.stdout = __sqlfs_out @@ -333,10 +410,25 @@ __sqlfs_env_saved.clear() if (baseKind.get(node.path) !== "dir") createdDirs.push({ path: node.path, kind: "dir", mode: node.mode, data: "" }); continue; } - const bytes = node.bytes ?? new Uint8Array(0); - const entry: FsEntry = { path: node.path, kind: "file", mode: node.mode, data: Buffer.from(bytes).toString("base64") }; - if (baseKind.get(node.path) !== "file") createdFiles.push(entry); // new, or dir→file replacement - else if (!sameBytes(baseFiles.get(node.path) ?? new Uint8Array(0), bytes)) modified.push(entry); + const bytes = readFileBytes(node.path); + if (baseKind.get(node.path) !== "file") { + createdFiles.push({ + path: node.path, + kind: "file", + mode: node.mode, + data: Buffer.from(bytes).toString("base64"), + }); // new, or dir→file replacement + continue; + } + const baseline = baseFiles.get(node.path); + if (!baseline || baseline.size !== node.size || baseline.sha256 !== (await sha256(bytes))) { + modified.push({ + path: node.path, + kind: "file", + mode: node.mode, + data: Buffer.from(bytes).toString("base64"), + }); + } } // dirs-before-files, dirs shallow→deep, so the drain can apply created in order. createdDirs.sort((a, b) => depth(a.path) - depth(b.path)); @@ -406,24 +498,53 @@ __sqlfs_env_saved.clear() const ready: ReadyFrame = { type: "ready", generation }; emit(ready); -// `decodeFrames` returns `rest` as a `Uint8Array`; widen the -// accumulator's type so the reassignment type-checks under `deno check`. -let buf: Uint8Array = new Uint8Array(0); +let chunks: Uint8Array[] = []; +let totalBytes = 0; +let expectedBodyBytes = -1; + +function peekFrameLength(): number { + const header = new Uint8Array(4); + let offset = 0; + for (const chunk of chunks) { + const take = Math.min(chunk.byteLength, 4 - offset); + header.set(chunk.subarray(0, take), offset); + offset += take; + if (offset === 4) break; + } + return new DataView(header.buffer).getUint32(0, false); +} + const reader = stdinReadable.getReader(); for (;;) { const { value, done } = await reader.read(); if (done) break; - const merged = new Uint8Array(buf.byteLength + value.byteLength); - merged.set(buf, 0); - merged.set(value, buf.byteLength); - const { frames, rest } = decodeFrames(merged); - buf = rest; - for (const frame of frames) { - if (frame.type === "run") { - const resp = await runOne(frame); - emit(resp); + chunks.push(value); + totalBytes += value.byteLength; + for (;;) { + if (expectedBodyBytes < 0) { + if (totalBytes < 4) break; + expectedBodyBytes = peekFrameLength(); + if (expectedBodyBytes > MAX_FRAME_BYTES) throw new FrameTooLargeError(expectedBodyBytes); + } + if (totalBytes < 4 + expectedBodyBytes) break; + + const merged = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + const { frames, rest } = decodeFrames(merged); + chunks = rest.byteLength > 0 ? [rest] : []; + totalBytes = rest.byteLength; + expectedBodyBytes = -1; + for (const frame of frames) { + if (frame.type === "run") { + const resp = await runOne(frame); + emit(resp); + } + // Non-run inbound frames are ignored — Node only sends `run`. } - // Non-run inbound frames are ignored — Node only sends `run`. } } diff --git a/thoughts/issue-118-pyodide-runtime/memory-analysis.md b/thoughts/issue-118-pyodide-runtime/memory-analysis.md new file mode 100644 index 0000000..0d6f7f9 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/memory-analysis.md @@ -0,0 +1,364 @@ +# Memory analysis — pyodide runtime RAM spike (issue #118 follow-up, 2026-06-10) + +**Symptom:** processing a CSV with `MAX_CONCURRENT_PYODIDE=1` spiked the container at ~1.6 GB RSS. + +**Verdict:** the spike is **not** pandas and **not** the Pyodide WASM heap ceiling. ~80 % of the +per-run growth is the **IPC/staging/diff machinery**, dominated by an O(n²) stdin frame +accumulator in `src/pyodide-runner/runner.ts`. A contained chunk-list fix was prototyped and +measured: **~500 MB less steady-state RSS** on the same workload. A second, unrelated bug was +found on the way: the preloaded matplotlib **cannot plot** (wrong default backend). + +--- + +## 1. Environment & methodology + +All numbers measured on darwin/arm64 with the repo's vendored runtime, driving the **real** +`PyodideSandbox` manager (`src/api/pyodide/manager.ts`) — same code path production uses: + +- Deno 2.8.2 (`vendor/deno/deno`), pyodide 0.29.4 (`vendor/pyodide`, 465 MB asset dir) +- Test payload: **18.8 MB CSV**, 400 000 rows × 5 cols (`k,a,b,c,d`; one 20-char string col) +- Workload: `pd.read_csv("data.csv")` → `groupby("k").sum()` → `to_csv("out.csv")` +- Child RSS sampled via `ps -o rss= -p ` every 150 ms during runs +- Node RSS via `process.memoryUsage().rss` + +The complete measurement harness is in **§6** — it is the same script to re-run for +before/after verification. + +## 2. Where the 1.6 GB goes + +### 2.1 Child baseline (cold start, before any user code) + +| Preload set | Idle child RSS | Cold start | +|---|---|---| +| numpy + pandas + matplotlib (current `runner.ts`) | **418 MB** | ~1.4 s | +| numpy + pandas | 361 MB | ~1.1 s | +| none (Pyodide core + openpyxl wheels) | 283 MB | ~0.9 s | + +So: Pyodide core ≈ 283 MB, +numpy/pandas ≈ +78 MB, +matplotlib ≈ **+57 MB**. + +### 2.2 One CSV run (current code, full preload) + +| Stage | Child RSS | +|---|---| +| Idle after cold start | 418 MB | +| During / after the 18.8 MB CSV run | **peak 1114 MB → settles 1116 MB** | +| After a later trivial run (`print`) | 1116 MB — **the heap never shrinks** | + +Node-side RSS reached ~340 MB during staging/drain. Child ~1.1 GB + Node ~0.3–0.4 GB +≈ the reported **1.6 GB container**. + +### 2.3 Decomposition — transport vs. Python (the smoking gun) + +Same CSV staged, on the numpy+pandas runner (baseline 374 MB idle): + +| Run | Child RSS after | +|---|---| +| Stage CSV, **`code = "pass"`** (zero Python work) | **1089 MB** | +| Stage CSV + the full pandas pipeline | 1262 MB | + +- The DataFrame itself: `df.memory_usage(deep=True).sum()` = **39 MB** +- `del df, out; gc.collect()` inside the run: **changes nothing** (grow-only heap) + +**⇒ ~715 MB of the ~890 MB growth is staging/IPC/diff machinery; pandas adds ~100–170 MB.** + +## 3. Root causes (ranked) + +### 3.1 O(n²) stdin frame accumulation in `runner.ts` — measured, fix validated + +`src/pyodide-runner/runner.ts` (IPC loop, ~line 411): + +```ts +const merged = new Uint8Array(buf.byteLength + value.byteLength); // per stdin chunk! +merged.set(buf, 0); +merged.set(value, buf.byteLength); +``` + +A 18.8 MB CSV becomes a ~25 MB base64 JSON frame, arriving in small pipe chunks → hundreds of +full-buffer realloc+copies → **multi-GB transient allocation churn** → V8 grows the child heap +aggressively and Deno never returns it to the OS. + +**Validated fix** (prototyped, measured): accumulate chunks in a list, peek the 4-byte length +prefix, and concat **once per complete frame**: + +```ts +let chunks: Uint8Array[] = []; +let total = 0; +let expected = -1; // declared body length of the in-flight frame; -1 = header unread +const reader = stdinReadable.getReader(); +for (;;) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(value); + total += value.byteLength; + for (;;) { + if (expected < 0) { + if (total < 4) break; + const head = new Uint8Array(4); + let o = 0; + for (const c of chunks) { + for (let i = 0; i < c.byteLength && o < 4; i++) head[o++] = c[i] as number; + if (o >= 4) break; + } + expected = new DataView(head.buffer).getUint32(0, false); + } + if (total < 4 + expected) break; + const merged = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + merged.set(c, off); + off += c.byteLength; + } + const { frames, rest } = decodeFrames(merged); + chunks = rest.byteLength > 0 ? [rest] : []; + total = rest.byteLength; + expected = -1; + for (const frame of frames) { + if (frame.type === "run") { + const resp = await runOne(frame); + emit(resp); + } + // Non-run inbound frames are ignored — Node only sends `run`. + } + } +} +``` + +Measured impact (same CSV, numpy+pandas preload): + +| | current (per-chunk realloc) | patched (chunk list) | +|---|---|---| +| stage-only run (`pass`) | 374 → **1089 MB** | 337 → **559 MB** | +| steady state, repeated stage+pandas runs | **~1316 MB** | **~804 MB** | + +Notes for productionizing: +- The frame-size cap must still be enforced **before** buffering grows: check `expected` + against `MAX_FRAME_BYTES` as soon as the header is peeked (the patched loop can throw/exit + there instead of waiting for `decodeFrames`). +- **Same pattern on the Node side**: `#onStdoutData` in `src/api/pyodide/manager.ts` (~line 520) + does `Buffer.concat([this.#readBuf, chunk])` per chunk. Harmless for small responses; same + O(n²) blow-up when a run drains large outputs. Apply the same chunk-list approach (the + aggregate cap check already runs per chunk and stays where it is). + +### 3.2 matplotlib preload: +57 MB resident — and plotting is broken today + +`loadPackage(["numpy","pandas","matplotlib"])` at init costs +57 MB, and `plt.plot()` +**crashes**: the default backend resolves to `webagg` → +`ImportError: cannot import name 'document' from 'js'` (needs a DOM; the Deno child has none). + +Verified working fix: select Agg before pyplot is imported — +`matplotlib.use("Agg")` → plot → `savefig("plot.png")` succeeded and the PNG **drained +correctly** into the sandbox FS. Options: + +- **Keep preload + make it work**: set `MPLBACKEND=Agg` in the per-run env prelude (or + `matplotlib.use("Agg")` equivalent at init). One line. +- **Lazy-load instead**: drop matplotlib (optionally everything) from the preload and add + `await pyodide.loadPackagesFromImports(req.code)` in `runOne` — still fully offline (same + `loadPackage` path against the local lock); sessions that never plot never pay the 57 MB. + A `PYODIDE_PRELOAD_PACKAGES` env var lets operators trade first-use latency vs baseline RSS. + +### 3.3 Full-byte diff baseline held across the run + +`runOne` snapshots **every staged file's full bytes** into `baseFiles` and holds them for the +whole execution, then post-run walks the tree again, materializing **all bytes a second time** +for `sameBytes`. Two extra full copies of the staged tree, one pinned across the run. + +Fix (no protocol change): store **size + SHA-256** per file (`crypto.subtle.digest`, bytes +released immediately after hashing); post-run, hash each file and compare — only materialize +bytes for files whose hash changed (those must be shipped anyway). Read-only-violation +semantics (`EREADONLY_VIOLATION`) are unchanged — the manifest comparison is the same, just +hash-keyed. + +### 3.4 base64-in-JSON transport ⇒ ~4–5 coexisting copies per direction + +Each staged byte exists as: SqlFs bytes → base64 string (×1.33) → inside the JSON frame string +→ encoded frame Buffer → (child) accumulated buffer → JSON.parse'd string → decoded bytes → +MEMFS write. Mirror set on the drain path. Structural options, increasing effort: + +1. **Per-file frames** — stage N frames + a final run frame (response likewise): peak ≈ largest + file instead of the whole tree. Keeps JSON+base64. +2. **Binary payload sections** — JSON header (paths/modes/sizes + requestId/seq/generation) with + raw payload bytes appended in the same length-prefixed frame: kills base64 (−25 % wire, + −several copies). Caps then measure raw bytes; `assertFsEntry`'s base64 check becomes length + bookkeeping. The integrity model is untouched (secrets stay in the JSON header). + +### 3.5 Whole-cwd re-staging every exec (warm child wipes cwd) + +Every exec ships the **entire cwd subtree** even if the script reads one file, and the post-run +wipe guarantees the next exec re-pays full transport. For the iterative agent loop this is the +dominant recurring cost. Optimization (bigger change): **incremental staging** — Node keeps the +manifest (path → hash) staged into the live generation; the child keeps cwd between execs; Node +ships only changed/new files + deletions, falling back to full staging on a new generation. +Same trust boundary (per-session child, same session's data). + +### 3.6 Grow-only heap ⇒ idle child squats at peak RSS + +WASM memory cannot shrink and V8 retains churn-driven heap: after one heavy run the warm child +holds peak RSS for up to `PYODIDE_IDLE_MS` (120 s default). Mitigations: + +- **RSS-based retirement**: after each completed run, sample child RSS (`/proc//statm` on + Linux; `ps -o rss=` fallback). Above `PYODIDE_MAX_CHILD_RSS_BYTES`, dispose the generation so + the next exec cold-starts (~1.4 s locally; "several seconds" on prod hosts). Converts + unbounded retention into a bounded posture without cgroups (spike S3 proved those unusable). +- Partial guard: spawn Deno with `--v8-flags=--max-old-space-size=` to cap the child's **JS** + heap (where the churn from §3.1/§3.4 lives). Does not bound WASM memory, but turns a runaway + JS-side allocation into a respawnable child crash instead of a container OOM. + +## 4. Suggested order of attack + +| # | Change | Effort | Expected effect | +|---|---|---|---| +| 1 | Chunk-list accumulator: `runner.ts` IPC loop + Node `#onStdoutData` | small, no protocol change | **−500 MB** steady state (measured) | +| 2 | `MPLBACKEND=Agg` (bugfix) + preload policy / lazy `loadPackagesFromImports` | small | plotting works; −57…135 MB baseline | +| 3 | Hash-based diff baseline in `runOne` | medium, no protocol change | removes 2 full copies of staged bytes | +| 4 | RSS-based child retirement (`PYODIDE_MAX_CHILD_RSS_BYTES`) | medium | bounds the idle tail at a chosen ceiling | +| 5 | Binary payload framing / incremental staging | large (protocol) | structural; do if file sizes keep growing toward the 32/128 MiB caps | + +Projection with #1–#3: child ~550–650 MB steady state + Node ~300 MB ≈ **~0.9–1.0 GB container** +for the same workload, vs 1.6 GB today. #4 caps the idle tail wherever the operator sets it. + +## 5. Verification protocol — how to confirm memory after the fix + +### 5.1 Pass criteria (same 18.8 MB CSV workload, full preload runner) + +| Metric | Before (measured) | After fix #1 (target) | After #1–#3 (target) | +|---|---|---|---| +| Idle child after cold start | 418 MB | ~420 MB (unchanged) | ≤ 365 MB (if matplotlib lazy) | +| Child after stage-only run (`pass`) | ~1089 MB | **≤ 600 MB** | ≤ 500 MB | +| Child steady state, repeated CSV runs | ~1316 MB | **≤ 850 MB** | ≤ 700 MB | +| Container total during CSV run | ~1.6 GB | ~1.1 GB | ~0.9–1.0 GB | +| `plt.plot()` + `savefig` | crashes (webagg) | — | exit 0, PNG drains | + +(Absolute numbers are host-dependent; the **deltas and ratios** are the signal. Re-measure the +"before" column on the target host first if it differs from this machine.) + +### 5.2 Local measurement harness (re-runnable) + +Save as `scripts/measure-pyodide-memory.mjs` (or `/tmp/measure.mjs`) and run with +`./node_modules/.bin/tsx `. Adjust the two consts to the repo's absolute paths. + +```js +// Memory anatomy measurement for the pyodide runtime. Drives the REAL +// PyodideSandbox through: cold start -> stage-only run -> CSV run -> repeat, +// sampling the Deno child's RSS. Run before AND after the fix; compare. +import { execSync } from "node:child_process"; +import { Buffer } from "node:buffer"; +import { PyodideSandbox } from "../src/api/pyodide/manager.ts"; // adjust path + +const ASSET_DIR = new URL("../vendor/pyodide", import.meta.url).pathname; +const DENO_BIN = new URL("../vendor/deno/deno", import.meta.url).pathname; +const never = new AbortController().signal; + +const childRssMB = () => { + try { + const pid = execSync(`pgrep -f "pyodide-runner/runner.ts" | head -1`).toString().trim(); + return pid ? Math.round(Number(execSync(`ps -o rss= -p ${pid}`).toString().trim()) / 1024) : null; + } catch { return null; } +}; +const nodeRssMB = () => Math.round(process.memoryUsage().rss / 1024 / 1024); + +// ~19 MB synthetic CSV: 400k rows, one string column (object dtype — realistic worst case). +const rows = ["k,a,b,c,d"]; +for (let i = 0; i < 400_000; i++) + rows.push(`g${i % 50},${i},${i * 2},${(i * 0.123).toFixed(4)},${"x".repeat(20)}`); +const csvB64 = Buffer.from(rows.join("\n"), "utf8").toString("base64"); +const files = [{ path: "/work/data.csv", kind: "file", mode: 0o644, data: csvB64 }]; +const pandasCode = `import pandas as pd +df = pd.read_csv("data.csv") +out = df.groupby("k").sum(numeric_only=True) +out.to_csv("out.csv") +print(len(df), len(out))`; + +const sb = new PyodideSandbox({ + assetDir: ASSET_DIR, denoBin: DENO_BIN, runtimeTimeoutMs: 300_000, + maxFrameBytes: 256e6, maxAggregateBytes: 256e6, +}); + +let peak = 0, t = setInterval(() => { const r = childRssMB(); if (r > peak) peak = r; }, 150); + +const t0 = Date.now(); +await sb.run({ code: "print(1)", argv: ["-c"], stdin: "", files: [], cwd: "/work" }, never); +console.log(`[1] cold start ${Date.now() - t0}ms — idle child: ${childRssMB()} MB`); + +await sb.run({ code: "pass", argv: ["-c"], stdin: "", files, cwd: "/work" }, never); +console.log(`[2] stage-only run (no pandas): child ${childRssMB()} MB`); + +for (let i = 1; i <= 3; i++) { + const r = await sb.run({ code: pandasCode, argv: ["-c"], stdin: "", files, cwd: "/work" }, never); + console.log(`[3.${i}] stage+pandas run exit=${r.exitCode}: child ${childRssMB()} MB, node ${nodeRssMB()} MB`); +} +console.log(`peak child RSS observed: ${peak} MB`); + +// matplotlib backend regression check (must exit 0 and drain plot.png). +const mp = await sb.run({ + code: `import matplotlib.pyplot as plt\nplt.plot([1,2,3],[1,4,9])\nplt.savefig("plot.png")\nprint("ok")`, + argv: ["-c"], stdin: "", files: [], cwd: "/work", +}, never); +console.log(`[4] matplotlib: exit=${mp.exitCode}, created=${mp.created.map((e) => e.path)}`); +if (mp.exitCode !== 0) console.log(Buffer.from(mp.stderr, "base64").toString()); + +clearInterval(t); +await sb.dispose(); +process.exit(0); +``` + +Procedure: +1. Run on the **unfixed** branch → record the table (this is your host's "before"). +2. Apply the fix(es); `pnpm typecheck && pnpm lint:fix && pnpm test:unit` (the IPC integrity + suite must stay green — kill-on-malformed/oversized/forged-frame behavior is load-bearing). +3. Re-run the harness → compare against §5.1 targets. The key lines are `[2]` (stage-only — + isolates the transport fix from Python noise) and the `[3.x]` steady state. +4. `[4]` must print `exit=0` with `plot.png` in `created` once the Agg fix lands. + +### 5.3 Isolating a single variable (optional, what this analysis did) + +- **Preload cost**: copy `runner.ts`+`protocol.ts` to a temp dir, edit the `loadPackage([...])` + list, pass `runnerPath:` to `PyodideSandbox` — measure `[1]` per variant. +- **Transport vs Python**: compare `[2]` (stage-only) against `[3.1]` − `[2]`. +- **Heap retention**: any trivial run after a heavy one — RSS must not drop (expected; + documents why RSS-based retirement (#4) matters). + +### 5.4 Container-level verification (staging/prod) + +1. Build the image with the fix; deploy with `MAX_CONCURRENT_PYODIDE=1`, + `MAX_RESIDENT_PYODIDE=1` (the issue's repro config). +2. Watch `docker stats ` (or the k8s `container_memory_working_set_bytes` metric) + while running the same CSV workload through the exec API: + upload an ~19 MB CSV → `python3 analyze.py` (read_csv → groupby → to_csv) ×3. +3. Expect peak working set ≈ **1.0–1.1 GB** (vs ~1.6 GB before) with fix #1 alone; the level + after the runs stay flat (no growth per repeat) — repeat-run growth would indicate a leak, + not heap retention. +4. Idle tail: after `PYODIDE_IDLE_MS` (120 s) the child is reaped and container RSS falls back + to Node baseline. With RSS-based retirement (#4), the fall-back happens right after the run. + +## 6. Raw measurement log (this machine, 2026-06-10) + +``` +CSV size: 18.8 MB +[1] cold start + trivial: 1413ms, child RSS now=418MB peak=416MB (numpy+pandas+matplotlib) +[2] CSV run: 2344ms, exit=0 stdout=400000 50 + child RSS now=1116MB peak-during-run=1114MB + node RSS before=339MB after=277MB +[3] trivial after heavy: child RSS now=1116MB (heap retention check) +[4] matplotlib import+plot: exit=1 — ImportError: cannot import name 'document' from 'js' + +[preload numpy+pandas] cold start 1133ms, idle child RSS = 361MB +[preload none (core+openpyxl)] cold start 870ms, idle child RSS = 283MB +df deep bytes MB: 39 +CSV run on numpy+pandas runner: child RSS 350MB -> 1192MB (after del+gc.collect inside the run) + +UNPATCHED accumulator (numpy+pandas runner): + baseline idle: 374MB + after stage-only run (no pandas): 1089MB + after 2nd stage-only run: 1159MB + after stage+pandas run: 1262MB + after 2nd stage+pandas run: 1316MB + +PATCHED chunk-list accumulator (same runner, same workload): + baseline idle: 337MB + after stage-only run (no pandas): 559MB + after 2nd stage-only run: 612MB + after stage+pandas run: 796MB + after 2nd stage+pandas run: 804MB + +matplotlib with Agg backend: exit 0, plot.png (17017 bytes) drained into /work. +``` diff --git a/thoughts/issue-118-pyodide-runtime/streaming-staging-plan.md b/thoughts/issue-118-pyodide-runtime/streaming-staging-plan.md new file mode 100644 index 0000000..b925da2 --- /dev/null +++ b/thoughts/issue-118-pyodide-runtime/streaming-staging-plan.md @@ -0,0 +1,330 @@ +# Plan — Pyodide IPC streaming responses + incremental staging + +Fixes the two scaling issues left after the memory pass (`memory-analysis.md`): + +1. **Monolithic response-frame ceiling.** The drain response is ONE length-prefixed JSON + frame carrying all `created`/`modified` files as base64. Node's inbound caps are + `PYODIDE_MAX_FRAME_BYTES_DEFAULT = 64 MiB` / aggregate `96 MiB`, but the staging total cap is + `PYODIDE_MAX_TOTAL_BYTES_DEFAULT = 128 MiB` — so a drain above ~64 MiB is **killed before the + documented total is reachable**, and the whole tree is base64-amplified (~4–5 coexisting + copies per direction). Pre-existing, not a regression. +2. **Whole-cwd re-staging every exec.** `stageCwd` ships the entire cwd subtree every exec and + `runOne` wipes cwd afterwards, so the iterative LibreChat loop re-pays the full transport on + every call even when one file changed. + +The work is one protocol revision delivered in **independently shippable phases**. The IPC +channel is internal (Node `manager.ts` ↔ Deno `runner.ts`, both baked into the same image, +versioned by `generation`), so a clean protocol break is safe — there is **no cross-version +compat requirement** (unlike the DB migration). + +## Invariants that MUST hold across all phases + +Carry these forward verbatim — they are the existing security/correctness contract: + +- **Integrity secrets never reach Python.** `requestId` / `seq` / `generation` stay in the Node + manager and the runner's JS closure. Every inbound frame is checked against the pending + request; any anomaly → `IpcIntegrityError` → kill-the-child. A new frame type inherits the same + check. (`ipc.ts` `validateInbound`, `protocol.ts` header.) +- **Drain is transactional.** All `ctx.fs` mutation happens inside the script-scope transaction; + a throw rolls everything back (`abortScriptScope`). Streaming drain relies on this: + validate-before-COMMIT, not validate-before-each-write. +- **cwd-scoped + path-validated.** Every drained / removed / staged path is re-validated on the + Node side under cwd (`..`, absolute, null-byte, reserved `/__sqlfs_ext__` prefix) before any + mutation. Forged frames cannot escape cwd. +- **Never drain a timed-out / aborted / protocol-invalid run.** Unchanged. +- **Realm lockdown unchanged.** No new host globals; the package-load window is untouched by + this work. + +--- + +## Phase 0 — Interim cap re-tune (1-line stopgap, ship today) + +Make the documented 128 MiB total **reachable** as a single frame while the real fix lands. + +- `ipc.ts`: raise `PYODIDE_MAX_FRAME_BYTES_DEFAULT` to `192 * 1024 * 1024` and + `PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT` to `256 * 1024 * 1024` (must stay ≥ frame cap and cover + 128 MiB × ~1.33 base64). +- Wire the existing `PYODIDE_MAX_FRAME_BYTES` / `PYODIDE_MAX_AGGREGATE_BYTES` env vars if not + already read (they are documented but confirm they reach the constructor). +- **Cost:** a 128 MiB drain transiently buffers ~170 MiB in Node — acceptable as a stopgap on a + ≥4 GB host; document it. This does NOT reduce amplification; Phases 1–2 do. +- **Test:** a script that writes ~100 MiB across several files drains successfully (today: killed). + +This is the only change that touches caps without touching the protocol. Everything below +supersedes it. + +--- + +## Phase 1 — Streaming per-file response frames (child → Node) + backpressured incremental drain + +**Goal:** the response stops being one frame. The child emits each created/modified file as its +own frame and releases its bytes; Node drains each into the open transaction and releases it. +Peak buffer ≈ the largest single file (≤ per-file cap), on **both** sides. Fixes the ceiling and +caps memory. Keep JSON+base64 for now (binary is Phase 2). + +### Protocol (`protocol.ts`) + +Add two child→Node frame types, both carrying the integrity triple: + +```ts +interface FilePartFrame { // one created/modified file + type: "file"; + requestId: string; seq: number; generation: number; + path: string; kind: "file"; // dirs go in the terminal frame's createdDirs + mode: number; + data: string; // base64 (Phase 1); raw section (Phase 2) + disposition: "created" | "modified"; +} +interface ResultEndFrame { // terminal frame, replaces RunResponse + type: "result" | "error"; + requestId: string; seq: number; generation: number; + stdout: string; stderr: string; exitCode: number; + createdDirs: FsEntry[]; // dirs only (small; ordered shallow→deep) + deleted: string[]; // depth-first + fileCount: number; // # of FilePartFrames the child emitted (completeness check) +} +``` + +`run`/`ready` unchanged. `RunResponse` (all-files-in-one) is retired on the wire but the command +still assembles the same logical result. + +### Runner (`runner.ts` `runOne`) + +- After the diff, instead of building `created`/`modified` arrays of base64: for each + created/modified FILE, `emit()` a `FilePartFrame` (read bytes → base64 → emit → drop the + reference so it's GC'd before the next file). Created DIRS go into `createdDirs` on the terminal + frame (tiny). Then `emit()` the `ResultEndFrame` with stdout/stderr/exit/deleted/createdDirs + + `fileCount`. +- This removes the single giant `created`/`modified` allocation that the hash-diff pass still + base64s in one go. + +### Manager (`manager.ts`) — the security-critical part + +- **Validation:** extend `validateInbound` to accept `file` frames: same `ready`-gated + + in-flight + secret-match checks as a response, but a `file` frame does **not** clear `#pending` + (the response is not complete until the terminal frame). The terminal `result`/`error` clears + pending exactly as today. A `file` frame outside an in-flight request, or after the terminal + frame, or with a wrong secret → kill-the-child. `fileCount` mismatch at terminal → kill. +- **Per-response aggregate cap:** change `#aggregateBytes` to accumulate across the WHOLE + response (all `file` frames + terminal) and reset only when the terminal frame is accepted. + This enforces `PYODIDE_MAX_TOTAL_BYTES` at the transport layer (a child streaming forever is + killed at the total cap). Per-frame cap still bounds one file. +- **Backpressure + incremental drain via a sink.** Change the manager's run API from + "return one `RunResponse`" to a streaming form: + + ```ts + interface RunSink { + onFile(part: { path: string; mode: number; data: Buffer; disposition: "created"|"modified" }): Promise; + } + run(input, signal, sink): Promise // ResultEnd = terminal metadata (no file bytes) + ``` + + In `#onStdoutData`, when a validated `file` frame is dispatched: `child.stdout.pause()`, `await + sink.onFile(...)`, then `child.stdout.resume()`. Pausing while the SqlFs write is in flight + bounds Node memory to one file and applies natural backpressure to the child (its `stdout.write` + blocks). No deadlock: the child emits the whole response, then loops back to read stdin — it + never waits on Node mid-response. The runtime timeout still bounds the whole thing. + - A `sink.onFile` rejection (drain validation/IO error) → `#failOwned(err)` → kill + reject the + run (transaction rolls back). Same throw-not-return contract. + +### Command (`pyodide-command.ts`) + +- Provide the sink: `onFile` runs the existing per-file path validation + (`assertSafePath` + per-file cap + running total + uniqueness Set + write-path Set for the + ancestor check) and then `applyEntry` into `ctx.fs`. Accumulate the path metadata needed for + the cross-file checks. +- After `run()` resolves with the terminal metadata: run the cross-file consistency checks + (write↔delete conflict, file-as-dir-ancestor) over the accumulated write-path Set + `deleted`, + apply `createdDirs` (before files were already applied — re-order: apply dirs first by + buffering file parts? No — see ordering note), then apply `deleted` depth-first. A failure here + throws → rollback. + - **Ordering note:** today drain applies dirs→files→deletes. Streaming files arrive before the + terminal frame's `createdDirs`. Fix by having the runner emit `createdDirs` FIRST — move + `createdDirs` into a small **leading** `dirs` frame (or send them in the run-ack), OR have + `applyEntry` create missing parent dirs on demand (it already calls `fs.mkdir(..., {recursive:true})` + for dir entries; for file writes ensure parents exist). Simplest: in `onFile`, `mkdir -p` the + file's parent before `writeFile`. Empty created dirs (no children) still come in the terminal + frame and are applied after. This preserves "dirs before files" without buffering. +- **Read-only execs:** the first `file` frame (or any non-empty `deleted`/`createdDirs`) → + set `roStore.violated = true`, drain nothing, reject. Detectable on the first part; no need to + buffer. + +### Tests (Phase 1) + +- A script writing N files totaling > 64 MiB (old ceiling) drains all of them; peak Node RSS + during drain stays ~ one-file-sized (assert via the injectable RSS sampler or a memory probe). +- Forged `file` frame (wrong requestId/seq/generation) injected via `node:fs.writeSync` → child + killed, nothing drained (extend the existing frame-forgery integration suite). +- `fileCount` mismatch (terminal claims more/fewer parts than emitted) → killed. +- `file` frame after the terminal frame, or with no in-flight request → killed. +- Read-only exec that creates a file → `EREADONLY_VIOLATION`, no partial drain. +- Backpressure: a slow `sink.onFile` does not drop frames or reorder (deterministic fake child). +- Per-response aggregate cap: a child streaming past `PYODIDE_MAX_TOTAL_BYTES` → killed. + +--- + +## Phase 2 — Binary payload sections (both directions) + +**Goal:** kill base64 (−25 % wire, removes the base64 string + the JSON-string copies — the bulk +of the amplification measured in `memory-analysis.md`). + +### Wire format (`protocol.ts` `encodeFrame`/`decodeFrames`) + +Replace "4-byte length + JSON body" with a binary-capable frame: + +``` +[4-byte total body length] +[4-byte JSON header length] +[JSON header: UTF-8] # type, requestId, seq, generation, path, kind, mode, payloadLen, … +[raw payload bytes] # payloadLen bytes; file/stdin/stdout/stderr contents — NO base64 +``` + +- Frames with no payload (`ready`, `result`/`error` metadata, the run frame's non-file fields) + set `payloadLen = 0` and are pure JSON header — identical cost to today. +- `FilePartFrame` payload = raw file bytes. `RunRequest` file parts likewise (Phase 3 ships + these incrementally; here, the run frame's staged files become payload sections — see below). +- `stdin`, `stdout`, `stderr` move from base64 JSON strings to payload sections. + +### Integrity model (unchanged in substance) + +- Secrets stay in the **JSON header**, never in the payload, never exposed to Python. The + validator parses the header exactly as today and treats the payload as opaque bytes destined + for MEMFS/SqlFs (already-untrusted content). An attacker writing raw stdout bytes still cannot + forge a header with the right secrets. +- `assertFsEntry`'s base64 check (`isBase64`) is **replaced** by length bookkeeping: the declared + `payloadLen` must match the bytes consumed and stay within the per-frame cap. Caps now measure + **raw** bytes (no ~33 % base64 inflation) — re-document the cap semantics (the comment in + `ipc.ts` about base64 measuring expansion is removed). + +### Multi-file request payloads + +The run frame currently inlines all staged files as base64. With Phase 1's framing in place, +stage them as **request-side `file` frames** too (Node → child), each a binary section, so a big +CSV is one payload section rather than a 25 MiB base64 string inside the run JSON. The run frame +then carries only code/argv/cwd/env + `fileCount`. This also sets up Phase 3. + +### Tests (Phase 2) + +- Round-trip a file with every byte value 0x00–0xFF (binary safety; no UTF-8/base64 corruption). +- An invalid `payloadLen` (declares more/less than the frame carries) → `IpcIntegrityError` → + killed. +- A header that is valid JSON but whose payload would exceed the per-frame cap → killed before + buffering the payload. +- Re-run the full escape + frame-forgery suites unchanged (proves the binary split didn't open a + forge path). +- Byte-for-byte parity: same script, same outputs, vs Phase 1 (golden-file). + +--- + +## Phase 3 — Incremental staging (the recurring-cost win) + +**Goal:** within one child generation, ship only **changed/new files + deletions**; keep the +child's cwd populated between execs; stop wiping cwd post-run. The first exec (or any after a +respawn) full-stages; subsequent execs in the same generation pay only the delta. + +### State: a per-session, generation-keyed manifest cache + +Add `Session.pyodideStaging?: CwdSyncCache` (mirrors `Session.pyodideSandbox`), threaded into +`createPyodideCommands` opts like `onRunComplete`. The cache holds: + +```ts +interface CwdSyncCache { + generation: number; // the sandbox.generation it reflects + entries: Map; +} +``` + +**Validity rule:** usable iff `cache.generation === sandbox.generation`. On any mismatch → +**full stage** (and the child wipes cwd, re-establishing the invariant). The manager already +increments `generation` on every (re)spawn, so respawn/timeout/abort/RSS-retire/eviction all +naturally force a full stage. + +### Delta computation — new `stageCwdDelta(fs, cwd, caps, cache)` + +- Walk SqlFs cwd (authoritative — picks up changes made by other bash commands between execs), + `lstat` + hash each file (Node already reads bytes to stage; now it hashes to decide). +- For each path: if `cache.entries` has a matching `{size, sha256, kind, mode}` → **skip** (already + in the child). Else → include as a `file`/`dir` part with bytes (new or changed; kind change → + the child rm-then-creates). +- Paths in `cache.entries` not seen this walk → add to `removePaths` (the child unlinks them). +- Build the request: `syncMode: "full" | "incremental"`, the delta `files`, `removePaths`, and a + cheap **`expectedChecksum`** = hash over the sorted post-sync `path\0size\0mode\0kind` lines + (metadata only — catches missing/extra/size drift; contents are gated by the per-file hash on + Node's side). Compute the *pending* post-sync manifest locally (don't commit it yet). + +### Runner (`runOne`) + +- `syncMode === "full"` → wipe cwd (today's behavior), stage all parts. +- `syncMode === "incremental"` → **do not wipe**; apply the delta (write changed files, mkdir new + dirs, kind-change rm+create, unlink `removePaths`). After applying, compute the same + `expectedChecksum` over the actual cwd; on mismatch emit an `error` terminal frame with a + distinct `desync` marker. +- **Stop wiping cwd post-run in incremental mode.** The reserved `/__sqlfs_ext__` staging dir is + still wiped per run. cwd persists = baseline + this run's changes (matches Node's pending + manifest). + +### Node post-run cache commit / invalidate + +- **Commit** `cache = pending manifest + (created added, modified updated with the bytes Node + drained, deleted removed)`, `cache.generation = sandbox.generation` — **only** when the run + resolved AND its full diff drained 1:1 into SqlFs. +- **Invalidate** (`cache = undefined` → next exec full-stages) on ANY deviation: generation + change, `desync` marker, read-only violation (child cwd now diverges from rolled-back SqlFs), + drain error, abort, or timeout. Conservative by design — the optimization only persists across + the happy path, which is exactly the iterative loop. + +### Why this is safe + +- Same data kind (file contents Node read from SqlFs), same per-path cwd validation on + `removePaths`, same integrity fields. No new capability, no realm-lockdown change. +- Drift is **detectable** (`expectedChecksum`) and **self-healing** (any deviation → + invalidate → full stage wipes the child). The invariant "cache == child cwd at next exec start" + is maintained by commit-only-on-clean-drain. +- Startup guard: assert `per-file cap < per-frame cap` so a single file always fits one frame + (no intra-file chunking needed); fail boot otherwise. + +### Tests (Phase 3) + +- Stage a 19 MiB CSV, then 4 successive execs touching only a tiny script: assert exec #2–#5 + send a delta whose wire size ≈ the script (not 19 MiB) — instrument the request byte count. +- Change one file between execs → only that file re-ships; unchanged files do not. +- Delete a file in SqlFs between execs → `removePaths` carries it; child cwd no longer has it. +- file→dir and dir→file kind change across execs applies correctly. +- Generation change (force a timeout/respawn) → next exec full-stages and succeeds. +- Read-only violation → cache invalidated → next exec full-stages (no stale child cwd). +- Injected `desync` (fake child reports wrong checksum) → exec fails over to full stage + retry, + never runs against a wrong cwd. +- Cross-exec isolation unchanged: `sys.modules`/globals persist (design D3) but cwd contents are + exactly the synced SqlFs state. + +--- + +## Sequencing, risk, and what to ship + +| Phase | Fixes | Risk | Ship independently? | +|---|---|---|---| +| 0 | ceiling (stopgap) | trivial | yes — today | +| 1 | ceiling + peak buffer | medium (touches the security-critical frame loop + backpressure) | yes | +| 2 | base64 amplification | medium (protocol break; re-run escape/forgery suites) | yes, after 1 | +| 3 | recurring re-staging | high (cache coherence / drift) | yes, after 1 (2 optional) | + +- **Phase 0** unblocks large drains immediately. +- **Phase 1** is the real ceiling fix and the bigger memory win; do it next. It is the riskiest + to get right because it modifies `#onStdoutData` (the load-bearing validator) and adds + backpressure — keep the per-frame validation identical and add the streaming purely around it. +- **Phase 2** is a clean amplification win but a protocol break; gate it behind a full re-run of + the escape + frame-forgery integration suites. +- **Phase 3** has the best payoff for the LibreChat loop but the most correctness surface (cache + drift). The conservative invalidate-on-deviation rule keeps it safe; ship it last. + +Projected effect on the 18.8 MB-CSV workload (child RSS, after the memory pass left it ~0.8 GB): +Phase 1 caps drain peak to one file; Phase 2 removes ~25 % wire + the base64/JSON copies; Phase 3 +makes execs #2+ in a session nearly free on transport. Combined target: keep a single warm child +comfortably under ~0.7 GB with `MAX_CONCURRENT_PYODIDE=1`, and make large drains (up to the +128 MiB total) actually work. + +## Out of scope + +- No change to the residency LRU, semaphore, or RSS-retirement (orthogonal). +- No intra-file chunking (guarded by per-file < per-frame cap instead). +- No change to the DB/capability migration or the `stdlib` runtime. From 2bc9c793368b637b21e75eae63cb6261b58fed07 Mon Sep 17 00:00:00 2001 From: QuangNguyen2609 Date: Wed, 10 Jun 2026 08:38:26 +0930 Subject: [PATCH 16/16] fix(pyodide): raise IPC frame caps so 128 MiB drains succeed (Phase 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drain response is one length-prefixed frame carrying all created/ modified files, but the inbound per-frame cap defaulted to 64 MiB while the staging total cap is 128 MiB — so a drain above ~64 MiB was killed before the documented total was reachable. - Raise PYODIDE_MAX_FRAME_BYTES_DEFAULT 64 -> 192 MiB and PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT 96 -> 256 MiB (clears 128 MiB x ~1.33 base64 expansion; aggregate stays >= frame; both below the protocol 256 MiB ceiling). - Wire PYODIDE_MAX_FRAME_BYTES / PYODIDE_MAX_AGGREGATE_BYTES env overrides in the manager constructor — they were documented but never read. Interim stopgap; the per-file streaming that removes the monolithic- frame coupling is tracked as Phase 1 in the follow-up plan. US-118: pyodide IPC framing --- .changeset/python-runtime-enum.md | 1 + src/api/pyodide/ipc.ts | 15 ++++++--- src/api/pyodide/manager.ts | 14 ++++++-- src/api/pyodide/tests/unit/manager.test.ts | 39 +++++++++++++++++++++- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/.changeset/python-runtime-enum.md b/.changeset/python-runtime-enum.md index 77c9f7b..3b17126 100644 --- a/.changeset/python-runtime-enum.md +++ b/.changeset/python-runtime-enum.md @@ -18,3 +18,4 @@ Clients must migrate `python: true` → `python_runtime: "stdlib"` and `python: - Lazy, offline package loading: only `PYODIDE_PRELOAD_PACKAGES` (default `numpy,pandas`) is resident at init; other distribution packages load on first import from the local lock. Lowers the idle floor and lets operators trade latency against RSS. - `matplotlib` now defaults to the headless `Agg` backend so `savefig()` works in the Deno child (previously failed resolving the DOM `webagg` backend). - New env vars `PYODIDE_PRELOAD_PACKAGES` and `PYODIDE_MAX_CHILD_RSS_BYTES` (optional RSS-based child retirement). +- Raised the default IPC frame/aggregate caps (192 MiB / 256 MiB) so a monolithic drain response carrying the full `PYODIDE_MAX_TOTAL_BYTES` (128 MiB, base64-expanded) is reachable instead of being killed at the old 64 MiB frame cap, and wired the previously-documented-but-unused `PYODIDE_MAX_FRAME_BYTES` / `PYODIDE_MAX_AGGREGATE_BYTES` env overrides. diff --git a/src/api/pyodide/ipc.ts b/src/api/pyodide/ipc.ts index 3993243..e6f04ef 100644 --- a/src/api/pyodide/ipc.ts +++ b/src/api/pyodide/ipc.ts @@ -26,18 +26,23 @@ const HEADER_BYTES = 4; /** * Default per-frame wire cap (the declared JSON-body byte length). Because file * payloads are base64 in the body, this naturally measures the ~33%-expanded - * size. Generous default; Phase 6 wires the `PYODIDE_MAX_FRAME_BYTES` env var. - * Stays well below the protocol-level {@link protocol.MAX_FRAME_BYTES} ceiling. + * size. Set ABOVE the staging total cap (`PYODIDE_MAX_TOTAL_BYTES`, 128 MiB) × + * ~1.33 base64 expansion so a single monolithic drain response carrying the full + * 128 MiB total is reachable rather than killed mid-drain — the response is still + * one frame today (see thoughts/.../streaming-staging-plan.md Phase 1 for the + * per-file streaming that removes this coupling). Overridable via + * `PYODIDE_MAX_FRAME_BYTES`. Stays below the protocol-level + * {@link protocol.MAX_FRAME_BYTES} (256 MiB) ceiling. */ -export const PYODIDE_MAX_FRAME_BYTES_DEFAULT = 64 * 1024 * 1024; +export const PYODIDE_MAX_FRAME_BYTES_DEFAULT = 192 * 1024 * 1024; /** * Default aggregate cap: total bytes the manager will buffer from the child for a * single response (reset on each accepted `ready`/`result`/`error`). Bounds a * slowloris-style stream that never forms a complete/valid frame. Must be ≥ the - * per-frame cap. Phase 6 wires the `PYODIDE_MAX_AGGREGATE_BYTES` env var. + * per-frame cap. Overridable via `PYODIDE_MAX_AGGREGATE_BYTES`. */ -export const PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT = 96 * 1024 * 1024; +export const PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT = 256 * 1024 * 1024; /** A framing / integrity violation. The manager turns this into kill-the-child. */ export class IpcIntegrityError extends Error { diff --git a/src/api/pyodide/manager.ts b/src/api/pyodide/manager.ts index 07ae9dd..7e5bfb5 100644 --- a/src/api/pyodide/manager.ts +++ b/src/api/pyodide/manager.ts @@ -192,6 +192,13 @@ function resolveDenoBin(bin: string): string { return bin; // not found on PATH; let spawn surface ENOENT with the bare name } +/** Parse a positive-integer byte cap from an env var; fall back to `fallback` on absent/invalid. */ +function byteCapFromEnv(value: string | undefined, fallback: number): number { + if (value === undefined) return fallback; + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback; +} + function packagesFromEnv(value: string | undefined): string[] { if (value === undefined) return [...PYODIDE_PRELOAD_PACKAGES_DEFAULT]; return value @@ -257,8 +264,11 @@ export class PyodideSandbox { this.#denoBin = resolveDenoBin(opts.denoBin ?? process.env.DENO_BIN_PATH ?? "deno"); this.#runnerPath = opts.runnerPath ?? DEFAULT_RUNNER_PATH; this.#runtimeTimeoutMs = opts.runtimeTimeoutMs ?? PYODIDE_RUNTIME_TIMEOUT_MS_DEFAULT; - this.#maxFrameBytes = opts.maxFrameBytes ?? PYODIDE_MAX_FRAME_BYTES_DEFAULT; - this.#maxAggregateBytes = opts.maxAggregateBytes ?? PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT; + this.#maxFrameBytes = + opts.maxFrameBytes ?? byteCapFromEnv(process.env.PYODIDE_MAX_FRAME_BYTES, PYODIDE_MAX_FRAME_BYTES_DEFAULT); + this.#maxAggregateBytes = + opts.maxAggregateBytes ?? + byteCapFromEnv(process.env.PYODIDE_MAX_AGGREGATE_BYTES, PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT); this.#preloadPackages = opts.preloadPackages ?? packagesFromEnv(process.env.PYODIDE_PRELOAD_PACKAGES); const configuredRss = opts.maxChildRssBytes ?? Number(process.env.PYODIDE_MAX_CHILD_RSS_BYTES ?? "0"); this.#maxChildRssBytes = Number.isFinite(configuredRss) && configuredRss > 0 ? configuredRss : undefined; diff --git a/src/api/pyodide/tests/unit/manager.test.ts b/src/api/pyodide/tests/unit/manager.test.ts index 8e4a683..73b9478 100644 --- a/src/api/pyodide/tests/unit/manager.test.ts +++ b/src/api/pyodide/tests/unit/manager.test.ts @@ -10,7 +10,13 @@ import { Buffer } from "node:buffer"; import type { SpawnOptions } from "node:child_process"; import { afterEach, describe, expect, it } from "vitest"; -import { IpcFrameTooLargeError, IpcIntegrityError, encodeFrame } from "../../ipc.js"; +import { + IpcFrameTooLargeError, + IpcIntegrityError, + PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT, + PYODIDE_MAX_FRAME_BYTES_DEFAULT, + encodeFrame, +} from "../../ipc.js"; import type { RunRequestInput } from "../../manager.js"; import { COMMITTED_FLAGS, @@ -414,6 +420,37 @@ describe("PyodideSandbox — frame integrity (each violation kills the child)", }); }); +describe("PyodideSandbox — wire caps (defaults + env overrides)", () => { + it("raises the default frame/aggregate caps above the 128 MiB staging total", () => { + // A monolithic drain response carries the full PYODIDE_MAX_TOTAL_BYTES (128 MiB) + // base64-expanded (~1.33x); the frame cap MUST clear that or large drains die. + expect(PYODIDE_MAX_FRAME_BYTES_DEFAULT).toBe(192 * 1024 * 1024); + expect(PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT).toBe(256 * 1024 * 1024); + expect(PYODIDE_MAX_AGGREGATE_BYTES_DEFAULT).toBeGreaterThanOrEqual(PYODIDE_MAX_FRAME_BYTES_DEFAULT); + }); + + it("honors PYODIDE_MAX_FRAME_BYTES when no explicit option is given", async () => { + const saved = process.env.PYODIDE_MAX_FRAME_BYTES; + process.env.PYODIDE_MAX_FRAME_BYTES = "200"; + try { + const harness = makeHarness(); + // makeManager passes maxFrameBytes: undefined → constructor falls back to env. + const manager = track(makeManager(harness)); + const p = manager.run(INPUT, new AbortController().signal).catch((e) => e); + const child = await harness.nextChild(); + child.sendReady(); + const run = await child.nextRun(); + // 1 KiB raw → ~1368 base64 chars, well over the 200-byte env cap. + child.sendResult(run, { stdout: Buffer.alloc(1024, 0x41).toString("base64") }); + expect(await p).toBeInstanceOf(IpcFrameTooLargeError); + expect(child.killed).toBe(true); + } finally { + if (saved === undefined) Reflect.deleteProperty(process.env, "PYODIDE_MAX_FRAME_BYTES"); + else process.env.PYODIDE_MAX_FRAME_BYTES = saved; + } + }); +}); + describe("PyodideSandbox — spawn posture", () => { it("spawns deno with the committed deny-belt, an asset-dir-scoped allow-read, and a scrubbed env", async () => { const calls: { cmd: string; args: readonly string[]; opts: SpawnOptions }[] = [];