feat(offload): stream images and retained geometry on 3DS and PSP

.github/workflows/3ds-runtime.yml

@@ -1,10 +1,10 @@
 name: 3DS runtime contracts
 on:
   pull_request:
-    paths: ['hosts/3ds/**', 'contracts/**', 'tools/3ds*.ts', 'tests/3ds*.test.ts', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml']
+    paths: ['hosts/3ds/**', 'contracts/**', 'framework/src/offload.ts', 'framework/src/resource*.ts', 'framework/src/tile-viewport.ts', 'framework/src/drag-filter.ts', 'framework/src/gesture*.ts', 'tools/offload-*.ts', 'tools/3ds*.ts', 'tests/offload*.test.ts', 'tests/resource*.test.ts', 'tests/tile-viewport.test.ts', 'tests/3ds*.test.ts', 'tests/fixtures/offload*', 'tests/fixtures/offload*/**', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml']
   push:
     branches: [main]
-    paths: ['hosts/3ds/**', 'contracts/**', 'tools/3ds*.ts', 'tests/3ds*.test.ts', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml']
+    paths: ['hosts/3ds/**', 'contracts/**', 'framework/src/offload.ts', 'framework/src/resource*.ts', 'framework/src/tile-viewport.ts', 'framework/src/drag-filter.ts', 'framework/src/gesture*.ts', 'tools/offload-*.ts', 'tools/3ds*.ts', 'tests/offload*.test.ts', 'tests/resource*.test.ts', 'tests/tile-viewport.test.ts', 'tests/3ds*.test.ts', 'tests/fixtures/offload*', 'tests/fixtures/offload*/**', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml']
 permissions:
   contents: read
 jobs:
@@ -18,3 +18,4 @@ jobs:
           bun-version: 1.3.14
       - run: bun install --frozen-lockfile
       - run: bun test tests/3ds-profile.test.ts tests/3ds-runtime-state.test.ts tests/3ds-runtime-wire.test.ts tests/3ds-soc.test.ts
+      - run: bun test --conditions=browser tests/offload.test.ts tests/offload-images.test.ts tests/offload-meshes.test.ts tests/offload-provider.test.ts tests/tile-viewport.test.ts tests/resource-cache.test.ts tests/resource-view.test.ts

.github/workflows/psp-offload.yml

@@ -0,0 +1,46 @@
+name: PSP USB offload
+on:
+  pull_request:
+    paths:
+      - 'hosts/psp/src/**'
+      - 'hosts/psp/build.rs'
+      - 'tools/psp.ts'
+      - 'contracts/spec/platforms.ts'
+      - 'tests/platform-contracts.test.ts'
+      - 'tools/offload-usb-provider.ts'
+      - 'tools/offload-process.ts'
+      - 'tools/offload-provider.ts'
+      - 'tools/offload-wire.ts'
+      - 'tests/offload-usb.test.ts'
+      - 'tests/fixtures/offload-usb/**'
+      - '.github/workflows/psp-offload.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'hosts/psp/src/**'
+      - 'hosts/psp/build.rs'
+      - 'tools/psp.ts'
+      - 'contracts/spec/platforms.ts'
+      - 'tests/platform-contracts.test.ts'
+      - 'tools/offload-usb-provider.ts'
+      - 'tools/offload-process.ts'
+      - 'tools/offload-provider.ts'
+      - 'tools/offload-wire.ts'
+      - 'tests/offload-usb.test.ts'
+      - 'tests/fixtures/offload-usb/**'
+      - '.github/workflows/psp-offload.yml'
+permissions:
+  contents: read
+jobs:
+  contracts:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v7
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+      - run: bun install --frozen-lockfile
+      - run: bun test tests/offload-usb.test.ts tests/platform-contracts.test.ts
+      - run: rustc --test hosts/psp/src/offload_packet.rs -o /tmp/usb-packet-test && /tmp/usb-packet-test
+      - run: rustc --test hosts/psp/src/analog.rs -o /tmp/psp-analog-test && /tmp/psp-analog-test

contracts/spec/offload.ts

@@ -1,8 +1,13 @@
 /** Offload v1. JSON records are length-prefixed UTF-8 on the wire.
  * The UI copies bounded records; only the provider executes capabilities. */
 export const OFFLOAD = Object.freeze({
-  version: 1, recordBytes: 4096, payloadChars: 2500, pending: 8,
-  deliveriesPerFrame: 1, submissionsPerFrame: 2, timeoutFrames: 600,
+  version: 1,
+  recordBytes: 4096,
+  payloadChars: 2500,
+  pending: 8,
+  deliveriesPerFrame: 1,
+  submissionsPerFrame: 2,
+  timeoutFrames: 600,
   port: 8741,
 });
 
@@ -13,11 +18,84 @@ export interface OffloadOps {
   submit(record: string): boolean;
   /** At most one complete record per host frame. Never performs IO. */
   take(): string | undefined;
+  /** Borrow one native image ticket. At most one <=256x256 upload per frame.
+   * Pixels stay outside the JS heap. releaseImage returns staging credit. */
+  uploadImage?(token: number): number;
+  /** Shares image staging and one materialization per frame. */
+  uploadMesh?(token: number): number;
+  releaseMesh?(token: number): void;
+  releaseImage?(token: number): void;
   /** Optional bounded 2-bit coverage upload. At most 512x16, one per frame.
    * Foreground is ABGR; alpha comes from coverage. Optional columns provide one
    * lowercase hex palette index per pixel column; palette is 1..16 RGB hex colors.
    * Coloring uses the same scratch buffer and one upload. Returns a texture handle. */
-  uploadCoverage?(base64: string, width: number, height: number, foreground: number, columns?: string, palette?: string): number;
+  uploadCoverage?(
+    base64: string,
+    width: number,
+    height: number,
+    foreground: number,
+    columns?: string,
+    palette?: string,
+  ): number;
+}
+export interface OffloadRequest {
+  v: 1;
+  id: number;
+  method: string;
+  payload: string;
+  response?: "image" | "mesh";
+}
+export interface OffloadImageTicket {
+  token: number;
+  width: number;
+  height: number;
+}
+export interface OffloadReply {
+  id: number;
+  payload?: string;
+  error?: string;
+  image?: OffloadImageTicket;
+  mesh?: OffloadMeshTicket;
+}
+
+/** Optional image response extension. The record length's high bit selects a
+ * binary image; JSON retains its 4096-byte limit. Header is 16 bytes: PIMG,
+ * u32 request ID, u16 width, u16 height, u32 format (all little endian).
+ * Format 0 = row-major R5G6B5 little endian (PSM_5650), 16..256 power-of-two
+ * dimensions. No codec, base64, palette expansion or pixels in guest JS. */
+export const OFFLOAD_IMAGE = Object.freeze({
+  headerBytes: 16,
+  maxSide: 256,
+  maxBytes: 256 * 256 * 2,
+  slots: 8,
+  flag: 0x80000000,
+});
+export interface OffloadImage {
+  width: number;
+  height: number;
+  pixels: Uint8Array;
+  format: "r5g6b5";
+}
+export interface OffloadProviderReply {
+  id: number;
+  payload?: string;
+  error?: string;
+  image?: OffloadImage;
+  mesh?: OffloadMesh;
+}
+
+/** PMH1: 16-byte header (magic, u16 width/height/vertices/triangles, u32 zero),
+ * u16 x/y in 1/16 logical pixels, then [u16 a,b,c,u32 ABGR] triangles.
+ * Coordinates are inside the declared envelope. No topology/codec in guest JS.
+ * PMSH wire wrapper is magic + u32 request ID + this entry. Shares eight slots. */
+export const OFFLOAD_MESH = Object.freeze({ maxVertices: 4096, maxTriangles: 2048, maxBytes: 36880 });
+export interface OffloadMesh {
+  format: "mesh2d-v1";
+  bytes: Uint8Array;
+}
+export interface OffloadMeshTicket {
+  token: number;
+  width: number;
+  height: number;
+  bytes: number;
 }
-export interface OffloadRequest { v: 1; id: number; method: string; payload: string }
-export interface OffloadReply { id: number; payload?: string; error?: string }

contracts/spec/platforms.ts

@@ -254,6 +254,7 @@ export const POCKET_TARGETS = defineTargetRegistry<PocketCapabilityId, {
       rasterDensity: 1,
     },
     capabilities: [
+      "io.offload",
       "input.analog.left",
       "input.buttons",
       "input.cursor",

contracts/spec/spec.ts

@@ -1444,6 +1444,13 @@ export const FONT_FLAG_BOLD = 1 << 0;
 //                           in shell painter order, so later shell ops remain
 //                           above the child surface.
 
+//   MESH (10 words): op, meshHandle, a,b,c,d,tx,ty (f32 affine), clipXY,clipWH.
+//                           Opt-in retained geometry command. Vertices remain
+//                           in the native mesh resource's logical coordinates;
+//                           the GPU applies the affine and clips. Only hosts
+//                           enabling Ui::set_mesh_commands receive this op.
+//                           Other hosts retain CPU-clipped TRI output. The
+//                           resource revision participates in render invalidation.
 export const DRAW_OP = {
   rect: 1,
   gradRect: 2,
@@ -1455,6 +1462,7 @@ export const DRAW_OP = {
   texTri: 8,
   textRun: 9,
   surfaceQuad: 10,
+  mesh: 11,
 } as const;
 
 // ---------------------------------------------------------------------------

docs/DETERMINISM.md

@@ -77,6 +77,33 @@ Frame-counted APIs (`onFrame` counters, sprite `frameStep`) are per-rate by
 definition — an app that hard-codes "on frame 37" means something different
 at each hz. That is a documented boundary, not a bug; write seconds.
 
+## Sampled input duration
+
+**The seventh frame argument, `inputElapsedUs`, carries elapsed input-sampling
+microseconds as input data.** The 3DS host samples its monotonic counter between
+input frames. Apps can use `inputDeltaSeconds()` from
+`@pocketjs/framework/clock` to integrate controller velocity at a constant
+screen distance per second when presentation misses a vblank:
+
+```ts
+onFrame(() => {
+  camera.step(inputDeltaSeconds(), -analogX() * 320, -analogY() * 320);
+});
+```
+
+**This does not change virtual time, timer deadlines, core ticks or the number
+of resource pumps per frame.** Valid samples are rounded to microseconds and
+bounded to 1–66,666 µs. Missing, zero or invalid samples use `1 / simulationHz()`;
+other hosts and deterministic captures keep that nominal step. A long pause
+therefore advances the camera by at most 66.666 ms on resume, without catch-up
+transactions or a large jump. Frames below 15 Hz cannot preserve real-time
+travel under this bound.
+
+The flight recorder stores the normalized duration in tape v4. Replay uses
+that track and ignores the live host duration; tapes without it use the
+nominal step. Apps still do not read a wall clock: the same recorded input
+and duration sequence produces the same trajectory on another host.
+
 ## The effect shell (`@pocketjs/framework/effects`)
 
 Buttons were already part of `input[n]`. The effect shell makes *everything

docs/DEVTOOLS.md

@@ -235,3 +235,13 @@ pairs as `analog`. **Absent right-stick samples replay as centered**, including
 when live hardware moves during replay. The recorder allocates this track only
 after the first noncenter sample. The sixth frame argument carries the raw right
 stick; the touch, hit and surface arguments keep their existing positions.
+
+## Input duration track (v4)
+
+The optional `inputElapsedUs` RLE track records the seventh frame argument:
+`[microseconds, repeatCount]`, bounded to 1–66,666 µs, with zero meaning the
+nominal simulation step. The recorder allocates this track on the first
+nonzero sample. Replay owns the duration along with buttons, sticks and
+touch: **live host timing cannot alter a recorded trajectory**. Tapes without
+this track retain nominal simulation time. `inputDeltaSeconds()` exposes the
+latched sample to apps; virtual timers and core ticks remain frame-based.

docs/OFFLOAD.md

@@ -12,14 +12,17 @@ transport. Pocket Doc is a separate application using the capability.
 
 | Boundary | Enforced limit |
 | --- | --- |
-| Wire record | 4,096 UTF-8 bytes, 4-byte big-endian length prefix |
+| JSON wire record | 4,096 UTF-8 bytes, 4-byte big-endian length prefix |
 | Request/result payload | 2,500 UTF-16 code units, serialized string |
 | Outstanding guest requests | 8 |
 | Native outgoing/incoming queues | 8 records each |
 | Native submissions | 2 per host frame |
 | Native result copies | 1 per host frame |
 | JS completion callbacks | 1 per service-pump tick, including failures |
 | Coverage resource | At most 512×16 pixels, one upload per host frame |
+| Binary image response | At most 256×256 R5G6B5 pixels; 16-byte header |
+| Native image staging | 8 slots, 131,088 bytes per slot plus metadata |
+| Binary image uploads | 1 per host frame; bilinear filtering |
 | Request deadline | 600 guest frames; provider worker deadline 9 seconds |
 
 `submit` and `take` only copy fixed-size memory slots. **They do not call socket
@@ -92,6 +95,35 @@ versioned requests. An over-budget response or malformed frame closes the
 connection. A stalled worker is terminated instead of retaining its requests
 indefinitely.
 
+### Binary image resources
+
+`requestImage` opts a read into the image response extension. A provider method
+returns `{ width, height, format: "r5g6b5", pixels: Uint8Array }` after decoding
+or rasterizing on its worker. Dimensions are powers of two from 16 through 256.
+Pixels are row-major little-endian words with red in the low five bits, green
+in the next six, and blue in the high five. Alpha is not carried by this format.
+
+The wire length's high bit distinguishes an image from a JSON record. The
+16-byte little-endian header contains `PIMG`, request ID, width, height and
+format zero. **The 3DS network worker receives pixels into fixed native slots;
+the JS heap receives only a token and dimensions.** Socket backpressure applies
+when no staging slot is free. The UI copies no pixel array and runs no image
+codec. It can upload one slot per frame through the core's IMG entry path.
+
+Use `createOffloadImageCollection` from `@pocketjs/framework/resource-offload`
+to connect these tickets to resource demand and ownership. The collection
+reserves staging and old-plus-new texture cost before starting a read. After
+materialization, cancellation, a rejected envelope or late delivery, the
+framework returns staging credit. Eviction frees the uploaded texture. A stale
+token cannot free a newer allocation. A realm reset closes the connection
+before new request IDs become eligible for submission.
+
+JSON-only requests and the coverage path retain their existing bounds. The
+image extension is optional in `OffloadOps`; a host without `uploadImage` and
+`releaseImage` reports an unsupported operation. It does not decode PNG/JPEG
+in guest JavaScript. Other hosts can implement the same bounded staging
+contract without changing application resource definitions.
+
 `@pocketjs/framework/offload/capabilities` exports two provider-side helpers:
 
 - `sqliteQueries(db, queries)`: named, provider-owned SQL with device-supplied
@@ -140,6 +172,50 @@ timeouts, cancellation, stale sessions, no mutation replay, provider grants,
 SQLite result budgets, HTTP redirects and oversized bodies. These checks are
 separate from device performance and interaction acceptance.
 
+`bun test --conditions=browser tests/offload-images.test.ts` exercises binary
+envelopes, staging cleanup and 4,000 concurrent native images under address and
+undefined-behavior sanitizers. A socket test compiles the production 3DS worker
+with POSIX thread shims, connects the real provider, compares every image byte,
+and verifies connection replacement after a realm reset. It does not emulate
+libctru, GPU upload cost or device presentation cadence.
+
 Reusable reads can use the [shared resource scheduler](RESOURCES.md#shared-read-scheduling)
 for admission, priority, caching and bounded materialization. Commands retain
 the direct offload path and application-owned recovery semantics.
+
+## Prepared geometry response extension
+
+An opted-in request uses `response: "mesh"`. The length prefix retains the
+binary high bit; the payload starts with `PMSH` and a little-endian u32 request
+ID, followed by a standalone `PMH1` entry. `encodeOffloadMesh` validates the
+entry before it enters the socket queue. Image requests retain `PIMG` and their
+existing binary representation.
+
+**A mesh entry is at most 36,880 bytes: 4,096 vertices and 2,048 triangles.**
+Its 16-byte header contains `PMH1`, u16 width, height, vertex count, triangle
+count and a zero u32 reserved field. Vertices are u16 x/y in sixteenths of a
+logical pixel. Each triangle contains three u16 indices and a u32 ABGR color.
+All coordinates fit the declared width/height, and each index references a
+vertex in the same entry. Integers inside the binary payload are little-endian.
+
+**Mesh reception uses the existing eight image staging slots.** The worker
+validates size, indices and coordinates before publishing a ticket. The guest
+receives only `{token, width, height, bytes}`. `uploadMesh` shares the one-per-frame
+materialization credit with `uploadImage`; `releaseMesh` returns staging.
+`createOffloadMeshCollection` owns both response cleanup and resident handle
+disposal. Late, cancelled, wrong-kind and failed-consumer responses return their
+tickets. Mesh resources never consume texture handles.
+
+The native core owns generation-tagged geometry. On 3DS, materialization also
+uploads an immutable GPU vertex buffer; opaque Views submit its handle,
+transform and clip rectangle each frame. Other backends receive TRI commands
+after bounded CPU transformation and clipping. PBF parsing, topology,
+style selection and triangulation remain provider responsibilities. See
+[prepared geometry resources](RESOURCES.md#prepared-2d-geometry) for rendering
+and cache ownership.
+
+Providers can use `prepareMesh` from `@pocketjs/framework/offload/provider` to
+pack already tessellated geometry. Vertices use logical coordinates; triangle
+records contain three indices and an unsigned ABGR color. The helper checks
+bounds before integer conversion and quantizes positions to 1/16 pixel, so
+applications do not duplicate the binary header or index encoding.