diff --git a/.github/workflows/3ds-runtime.yml b/.github/workflows/3ds-runtime.yml index 716268c1..9878e381 100644 --- a/.github/workflows/3ds-runtime.yml +++ b/.github/workflows/3ds-runtime.yml @@ -1,10 +1,10 @@ name: 3DS runtime contracts on: pull_request: - paths: ['hosts/3ds/**', 'contracts/**', 'framework/src/offload.ts', 'framework/src/resource*.ts', 'framework/src/tile-viewport.ts', 'framework/src/drag-filter.ts', 'framework/src/gesture*.ts', 'tools/offload-*.ts', 'tools/3ds*.ts', 'tests/offload*.test.ts', 'tests/resource*.test.ts', 'tests/tile-viewport.test.ts', 'tests/3ds*.test.ts', 'tests/fixtures/offload*', 'tests/fixtures/offload*/**', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml'] + paths: ['hosts/3ds/**', 'contracts/**', 'framework/src/offload.ts', 'framework/src/resource*.ts', 'framework/src/tile-viewport.ts', 'framework/src/drag-filter.ts', 'framework/src/gesture*.ts', 'tools/offload-*.ts', 'tools/resource-pack.ts', 'tests/fixtures/resource-pack/**', 'tools/3ds*.ts', 'tests/offload*.test.ts', 'tests/resource*.test.ts', 'tests/tile-viewport.test.ts', 'tests/3ds*.test.ts', 'tests/fixtures/offload*', 'tests/fixtures/offload*/**', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml'] push: branches: [main] - paths: ['hosts/3ds/**', 'contracts/**', 'framework/src/offload.ts', 'framework/src/resource*.ts', 'framework/src/tile-viewport.ts', 'framework/src/drag-filter.ts', 'framework/src/gesture*.ts', 'tools/offload-*.ts', 'tools/3ds*.ts', 'tests/offload*.test.ts', 'tests/resource*.test.ts', 'tests/tile-viewport.test.ts', 'tests/3ds*.test.ts', 'tests/fixtures/offload*', 'tests/fixtures/offload*/**', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml'] + paths: ['hosts/3ds/**', 'contracts/**', 'framework/src/offload.ts', 'framework/src/resource*.ts', 'framework/src/tile-viewport.ts', 'framework/src/drag-filter.ts', 'framework/src/gesture*.ts', 'tools/offload-*.ts', 'tools/resource-pack.ts', 'tests/fixtures/resource-pack/**', 'tools/3ds*.ts', 'tests/offload*.test.ts', 'tests/resource*.test.ts', 'tests/tile-viewport.test.ts', 'tests/3ds*.test.ts', 'tests/fixtures/offload*', 'tests/fixtures/offload*/**', 'tests/fixtures/3ds-*', 'tests/fixtures/3ds-*/**', '.github/workflows/3ds-runtime.yml'] permissions: contents: read jobs: @@ -18,4 +18,4 @@ jobs: bun-version: 1.3.14 - run: bun install --frozen-lockfile - run: bun test tests/3ds-profile.test.ts tests/3ds-runtime-state.test.ts tests/3ds-runtime-wire.test.ts tests/3ds-soc.test.ts - - run: bun test --conditions=browser tests/offload.test.ts tests/offload-images.test.ts tests/offload-meshes.test.ts tests/offload-provider.test.ts tests/tile-viewport.test.ts tests/resource-cache.test.ts tests/resource-view.test.ts + - run: bun test --conditions=browser tests/offload.test.ts tests/offload-images.test.ts tests/offload-meshes.test.ts tests/offload-provider.test.ts tests/tile-viewport.test.ts tests/resource-cache.test.ts tests/resource-view.test.ts tests/resource-pack.test.ts tests/resource-pack-view.test.ts diff --git a/contracts/spec/platforms.ts b/contracts/spec/platforms.ts index ceac353e..6a02c3a2 100644 --- a/contracts/spec/platforms.ts +++ b/contracts/spec/platforms.ts @@ -176,6 +176,7 @@ export const POCKET_CAPABILITIES = defineCapabilityRegistry([ // remain host-owned; the browser dev host, deterministic sim and reference // core exercise the contract without granting network access to every host. "io.offload", + "io.resource-pack", "net.http", // SQLite behind the db module's own namespace (`globalThis.db`, // contracts/spec/db.ts): five synchronous ops, rows as one JSON line per diff --git a/docs/RESOURCE_PACKS.md b/docs/RESOURCE_PACKS.md new file mode 100644 index 00000000..da3566d3 --- /dev/null +++ b/docs/RESOURCE_PACKS.md @@ -0,0 +1,76 @@ +# Prepared resources on 3DS storage + +An app can install immutable images on its SD card and load them through +`@pocketjs/framework/resource-pack`. **File access and zlib decoding run on a +native worker.** Guest JavaScript submits an indexed address and receives a +bounded image ticket. It never opens a file, parses a directory or copies pixels. + +```ts +const tiles = createPackedImageCollection(runtime, { + key: (tile: Tile) => `${tile.version}/${tile.index}`, + pack: tile => ({ name: `atlas-${tile.version}`, entry: tile.index }), + width: 256, height: 256, + maxEntries: 40, maxViews: 2, maxDemandsPerView: 24, + fallback: { client: offload(), method: "map.tile", payload: JSON.stringify }, +}); +const visible = createResourceView(tiles, { demand: visibleTiles }); +``` + +The collection owns one cache regardless of storage. Views merge demand, +priority and pins; withdrawal cancels delivery, completion releases staging, +and eviction frees the native texture. A missing or corrupt local record can +use the optional paired-desktop fallback. A host without `io.resource-pack` +uses that fallback directly. Loaders that report a disconnected client refuse +admission so they do not consume the shared scheduler's active slots. + +Declare `io.resource-pack` in the app manifest's `enhances` list. The 3DS build +includes the worker only when the resolved profile enables this capability. +PSP and other hosts currently have no local pack implementation. + +## Installation and image preparation + +`tools/resource-pack.ts` exports `prepareTiledRGB565` and +`createResourcePack(path, count)`. The desktop converts core RGB565 pixels to +PICA's channel order, vertical origin and 8×8 Morton ordering, then compresses +each entry with zlib. The writer completes a temporary file before renaming it. +Install the resulting file at +`sdmc:/pocketjs/assets//.prp`. The slot is the first 16 hex +characters of SHA-256 of the app ID. Names contain 1–48 lowercase ASCII letters, +digits or hyphens. Upload to a temporary path and verify before activation; +installed names identify immutable contents and must change when contents change. + +**A 256×256 prepared image occupies 128 KiB of GPU texture storage.** The host +copies its tiled RGB565 bytes without the normal RGBA8 expansion or pixel +conversion. The core retains a generation-tagged descriptor with dimensions, +without a second CPU pixel copy. The renderer adopts or retires storage only +after its previous GPU work completes. It performs at most one image/mesh +materialization per frame across the local and desktop paths. + +The worker owns eight 128 KiB result slots, one 128 KiB + 128 byte compressed +scratch buffer, four open files and a 32 KiB thread stack. Zlib uses bounded +per-record working storage. **No whole atlas or whole index is loaded into +RAM.** A request reads one 24-byte index entry and its bounded payload. +Submission and delivery each allow one record per frame. Saturation rejects +admission; it does not wait. Realm reset fences obsolete jobs by generation. +Canceled reads may finish on the worker, but cannot reach the old view. + +## Format and diagnostics + +PRP1 version 1 uses a 64-byte little-endian header and up to 65,536 indexed +24-byte entries. Header fields are magic, version, count and total file length; +remaining bytes are zero. Entry fields are offset, compressed length, raw length, +CRC32, kind, then 16-bit width and height. Files are limited to 2 GiB minus one +byte for the 3DS stdio seek range. Every entry is independently zlib compressed. + +Kind 1 carries at most 2,500 UTF-8 bytes of application metadata, with zero +dimensions. `resourcePacks()?.request("pack.read", "atlas/0", callback)` reads +it through the same bounded async service. The serialized response must fit +the 4,096-byte offload record envelope; excessive escaping produces an error. +Kind 2 carries tiled RGB565 images with power-of-two dimensions from 16 to 256. +CRC32 detects damage; it is not an authentication mechanism. + +`resourcePackStats()` reports completed reads, failures, cumulative IO/decode/ +upload microseconds and maxima. These counters separate SD and codec time from +GPU upload time; they do not measure full visible-tile latency or frame rate. +Compare identical camera routes and cache budgets on hardware before claiming +a latency or frame-rate improvement. diff --git a/engine/core/src/lib.rs b/engine/core/src/lib.rs index 1d87cddf..e63eef90 100644 --- a/engine/core/src/lib.rs +++ b/engine/core/src/lib.rs @@ -77,7 +77,7 @@ pub struct Texture { byte_len: usize, pub w: u32, pub h: u32, - /// spec::psm::* pixel format. + /// spec::psm::* pixel format, or u32::MAX for backend-owned storage. pub psm: u32, /// CLUT (PSM_T8 only): exactly TEX_PALETTE_BYTES bytes (256 x u32 ABGR) /// in a 16-byte-aligned backing like `data`, so the PSP GE can point @@ -641,6 +641,21 @@ impl Ui { self.upload_texture_flags(data, w, h, psm, 0) } + /// Register an image whose pixels are owned by the native GPU backend. + /// The backend must attach storage before drawing this handle and retire + /// it after its final GPU use. No CPU pixel buffer is allocated or sampled. + pub fn register_external_texture(&mut self, w: u32, h: u32) -> i32 { + let valid = |n: u32| n > 0 && n <= spec::TEX_MAX_DIM && n.is_power_of_two(); + if !valid(w) || !valid(h) { return -1; } + let texture = Texture { + data: alloc::vec::Vec::new(), byte_len: 0, w, h, psm: u32::MAX, + palette: None, linear: true, revision: 0, + }; + let handle = tex_alloc(&mut self.textures, &mut self.tex_free, texture); + if handle >= 0 { self.bump_raster_revision(); } + handle + } + /// `upload_texture` honoring spec::img flags: FLAG_RLE marks the pixel /// stream (for PSM_T8: the index bytes AFTER the palette — the palette /// itself is never compressed) as PackBits-RLE, which must decode to diff --git a/engine/core/src/tests.rs b/engine/core/src/tests.rs index 6c78c68e..877bf9e5 100644 --- a/engine/core/src/tests.rs +++ b/engine/core/src/tests.rs @@ -3919,3 +3919,18 @@ fn retiring_texture_invalidates_handle_before_gpu_owner_drops() { assert_eq!(ui.texture(next).unwrap().pixels[0], 9); assert!(ui.take_texture(old).is_none()); } + +#[test] +fn external_texture_has_dimensions_and_generations_without_cpu_pixels() { + let mut ui = Ui::new(); + assert_eq!(ui.register_external_texture(0, 256), -1); + assert_eq!(ui.register_external_texture(255, 256), -1); + let handle = ui.register_external_texture(256, 256); + let view = ui.texture(handle).unwrap(); + assert_eq!((view.w, view.h), (256, 256)); + assert!(view.pixels.is_empty()); + assert_eq!(view.psm, u32::MAX); + ui.free_texture(handle); + assert!(ui.texture(handle).is_none()); + assert_ne!(handle, ui.register_external_texture(256, 256)); +} diff --git a/framework/compiler/subpaths.ts b/framework/compiler/subpaths.ts index 4f24a11c..ec42abfa 100644 --- a/framework/compiler/subpaths.ts +++ b/framework/compiler/subpaths.ts @@ -62,6 +62,7 @@ export const SUBPATHS: Record = { "resource-state": { file: "framework/src/resource-state.ts", aliases: TWINS }, "resource-cache": { file: "framework/src/resource-cache.ts", aliases: TWINS }, "resource-offload": { file: "framework/src/resource-offload.ts", aliases: TWINS }, + "resource-pack": { file: "framework/src/resource-pack.ts", aliases: TWINS }, "resource-view": { file: { solid: "framework/src/resource-view.ts" } }, "tile-viewport": { file: "framework/src/tile-viewport.ts", aliases: TWINS }, resource: { file: { solid: "framework/src/resource.ts" } }, diff --git a/framework/src/resource-offload.ts b/framework/src/resource-offload.ts index c78d54e2..14a04559 100644 --- a/framework/src/resource-offload.ts +++ b/framework/src/resource-offload.ts @@ -7,11 +7,12 @@ import { getOps } from "./host.ts"; /** An opt-in adapter for reproducible read capabilities. Already-serialized * payloads retain offload's wire bound. Mutating methods must use offload directly. */ export function offloadResource( - client: Pick, "request" | "cancel">, + client: Pick, "request" | "cancel"> & { connected?(): boolean }, method: string, payload: (input: I) => string, ): ResourceLoad { return (input, complete) => { + if (client.connected?.() === false) return false; const id = client.request(method, payload(input), (result) => complete(result)); return id ? { cancel: () => client.cancel(id) } : false; }; @@ -22,7 +23,7 @@ export function offloadResource( * owned by the collection. Configure one image materialization per frame. */ export function createOffloadImageCollection( runtime: ReturnType, - client: Pick, "requestImage" | "cancel" | "uploadImage" | "releaseImage">, + client: Pick, "requestImage" | "cancel" | "uploadImage" | "releaseImage"> & { connected?(): boolean }, options: Omit< ResourceCollectionOptions, "load" | "materialize" | "dispose" | "releaseResponse" | "maxResponseBytes" | "cost" | "maxCost" @@ -46,6 +47,7 @@ export function createOffloadImageCollection( cost: () => cost, maxCost: options.maxEntries * cost, load: (input, complete) => { + if (client.connected?.() === false) return false; const id = client.requestImage(options.method, options.payload(input), complete); return id ? { cancel: () => client.cancel(id) } : false; }, @@ -64,7 +66,7 @@ export function createOffloadImageCollection( * One completion per frame covers the bounded native validation/copy. */ export function createOffloadMeshCollection( runtime: ReturnType, - client: Pick, "requestMesh" | "cancel" | "uploadMesh" | "releaseMesh">, + client: Pick, "requestMesh" | "cancel" | "uploadMesh" | "releaseMesh"> & { connected?(): boolean }, options: Omit< ResourceCollectionOptions, "load" | "materialize" | "dispose" | "releaseResponse" | "maxResponseBytes" | "cost" | "maxCost" @@ -82,6 +84,7 @@ export function createOffloadMeshCollection( cost: () => cost, maxCost: options.maxEntries * cost, load: (input, complete) => { + if (client.connected?.() === false) return false; const id = client.requestMesh(options.method, options.payload(input), complete); return id ? { cancel: () => client.cancel(id) } : false; }, diff --git a/framework/src/resource-pack.ts b/framework/src/resource-pack.ts new file mode 100644 index 00000000..690c8df9 --- /dev/null +++ b/framework/src/resource-pack.ts @@ -0,0 +1,193 @@ +import { createOffloadClient, type OffloadResult } from "./offload.ts"; +import { registerServicePump } from "./services.ts"; +import { getOps } from "./host.ts"; +import type { + createResourceRuntime, + ResourceCollectionOptions, +} from "./resource-view.ts"; +import type { TextureResource } from "./resource.ts"; + +interface PackOps { + session(): number; + enqueue(id: number, pack: string, entry: number): boolean; + take(): string | undefined; + uploadImage(token: number): number; + releaseImage(token: number): void; + stats(): string; +} +type ImageClient = Pick< + ReturnType, + "requestImage" | "cancel" | "uploadImage" | "releaseImage" +> & { connected?(): boolean }; +let client: ReturnType | undefined; +const native = () => + (globalThis as unknown as { resourcePacks?: PackOps }).resourcePacks; + +/** A realm service, with native IO on a worker. Missing support is explicit; + * callers can retain the same collection contract through a desktop fallback. */ +export function resourcePacks() { + if (client) return client; + const ops = native(); + if (!ops) return undefined; + client = createOffloadClient({ + session: () => ops.session(), + take: () => ops.take(), + uploadImage: (token) => ops.uploadImage(token), + releaseImage: (token) => ops.releaseImage(token), + submit(record) { + const request = JSON.parse(record), + parts = String(request.payload).split("/"); + if (parts.length !== 2 || !/^[a-z0-9-]{1,48}$/.test(parts[0]!)) + return false; + const entry = Number(parts[1]); + return ( + Number.isInteger(entry) && + entry >= 0 && + entry < 65536 && + ops.enqueue(request.id, parts[0]!, entry) + ); + }, + }); + registerServicePump(() => client!.step()); + return client; +} +export function resourcePackStats() { + return native()?.stats(); +} + +/** One cache owns either a prepared local image or the desktop fallback. + * The application chooses identity and demand. The pack path stays below + * the host's app-specific asset directory; no guest filesystem call exists. */ +export function createPackedImageCollection( + runtime: ReturnType, + options: Omit< + ResourceCollectionOptions, + | "load" + | "materialize" + | "dispose" + | "releaseResponse" + | "maxResponseBytes" + | "cost" + | "maxCost" + > & { + pack(input: I): { name: string; entry: number } | undefined; + width: number; + height: number; + fallback?: { + client: ImageClient; + method: string; + payload(input: I): string; + }; + materialized?(storage: "local" | "desktop"): void; + }, +) { + for (const n of [options.width, options.height]) + if (!Number.isInteger(n) || n < 16 || n > 256 || n & (n - 1)) + throw Error("Invalid image envelope"); + const local = resourcePacks(), + fallback = options.fallback; + // Installed names are immutable. An absent pack stays absent until the + // native realm changes; avoid probing SD on every desktop-only tile miss. + const absent = new Set(); + let generation = local?.session() ?? 0; + const source = (raw: string) => { + const record = JSON.parse(raw) as { local: boolean; ticket: string }; + const owner = record.local ? local : fallback?.client; + if (!owner || typeof record.ticket !== "string") + throw Error("Invalid pack response owner"); + return { owner, record }; + }; + // Reserve the larger remote fallback representation, including transient + // staging and GPU retirement. Local GPU-owned images need no core pixels. + const cost = options.width * options.height * 18 + 512; + return runtime.createCollection({ + ...options, + maxResponseBytes: 512, + cost: () => cost, + maxCost: options.maxEntries * cost, + load(input, complete) { + const pack = options.pack(input); + if ((local?.session() ?? 0) !== generation) { + generation = local?.session() ?? 0; + absent.clear(); + } + let owner: ImageClient | undefined, + id = 0, + cancelled = false; + const finish = (isLocal: boolean, result: OffloadResult) => { + if (cancelled) return; + complete( + result.ok + ? { + ok: true, + value: JSON.stringify({ local: isLocal, ticket: result.value }), + } + : result, + ); + }; + const remote = () => { + if (!fallback || fallback.client.connected?.() === false) return false; + owner = fallback.client; + id = owner.requestImage( + fallback.method, + fallback.payload(input), + (result) => finish(false, result), + ); + return !!id; + }; + if (local?.connected() && pack && !absent.has(pack.name)) { + if ( + !/^[a-z0-9-]{1,48}$/.test(pack.name) || + !Number.isInteger(pack.entry) || + pack.entry < 0 || + pack.entry >= 65536 + ) + throw Error("Invalid resource pack address"); + owner = local; + id = local.requestImage( + "pack.read", + `${pack.name}/${pack.entry}`, + (result) => { + if (cancelled) return; + if (!result.ok && fallback) { + if (result.error === "Resource pack not installed") { + if (absent.size === 4) + absent.delete(absent.values().next().value!); + absent.add(pack.name); + } + if (!remote()) finish(true, result); + } else finish(true, result); + }, + ); + if (!id) return false; + } else if (!remote()) return false; + return { + cancel() { + cancelled = true; + owner?.cancel(id); + }, + }; + }, + materialize(raw) { + const { owner, record } = source(raw), + ticket = JSON.parse(record.ticket); + if (ticket.width > options.width || ticket.height > options.height) + throw Error("Image exceeds collection envelope"); + const value = owner.uploadImage(record.ticket); + try { + options.materialized?.(record.local ? "local" : "desktop"); + } catch (error) { + getOps().freeTexture?.(value.handle); + throw error; + } + return value; + }, + releaseResponse(raw) { + const { owner, record } = source(raw); + owner.releaseImage(record.ticket); + }, + dispose(value) { + getOps().freeTexture?.(value.handle); + }, + }); +} diff --git a/hosts/3ds/Makefile b/hosts/3ds/Makefile index fe606616..ab8a8420 100644 --- a/hosts/3ds/Makefile +++ b/hosts/3ds/Makefile @@ -84,7 +84,11 @@ CFLAGS := -Wall -Wextra -O2 -g -std=gnu11 -mword-relocations -ffunction-sections -DPOCKETJS_AUX_VIEW_H=$(POCKETJS_AUX_LOGICAL_HEIGHT) \ -DPOCKETJS_RASTER_DENSITY=$(POCKETJS_RASTER_DENSITY) \ -I$(INCLUDE_DIR) -I$(BUILD) -I$(POCKETJS_QUICKJS_DIR) \ - -I$(DEVKITPRO)/libctru/include + -I$(DEVKITPRO)/libctru/include -I$(DEVKITPRO)/portlibs/3ds/include + +ifeq ($(POCKETJS_ASSET_PACK),1) +CFLAGS += -DPOCKETJS_ASSET_PACK +endif ifeq ($(POCKETJS_OFFLOAD),1) CFLAGS += -DPOCKETJS_OFFLOAD -DPOCKETJS_OFFLOAD_KEY='"sdmc:/pocketjs/offload/$(POCKETJS_OFFLOAD_SLOT).key"' @@ -99,10 +103,10 @@ CFLAGS += -DPOCKETJS_CAPTURE \ endif LDFLAGS := -specs=3dsx.specs $(ARCH) -Wl,--gc-sections -Wl,-Map,$(BUILD)/pocketjs-3ds.map -LIBPATHS := -L$(DEVKITPRO)/libctru/lib -LIBS := -lcitro3d -lctru -lm +LIBPATHS := -L$(DEVKITPRO)/libctru/lib -L$(DEVKITPRO)/portlibs/3ds/lib +LIBS := -lcitro3d -lctru -lz -lm -OBJECTS := $(BUILD)/main.o $(BUILD)/offload.o $(BUILD)/soc.o $(BUILD)/svcwire.o $(BUILD)/runtime.o $(BUILD)/dev_protocol.o $(BUILD)/devserver.o $(BUILD)/devmenu.o $(BUILD)/gfx.o $(BUILD)/qjs.o $(BUILD)/input.o $(BUILD)/vshader_shbin.o +OBJECTS := $(BUILD)/main.o $(BUILD)/asset_pack.o $(BUILD)/offload.o $(BUILD)/soc.o $(BUILD)/svcwire.o $(BUILD)/runtime.o $(BUILD)/dev_protocol.o $(BUILD)/devserver.o $(BUILD)/devmenu.o $(BUILD)/gfx.o $(BUILD)/qjs.o $(BUILD)/input.o $(BUILD)/vshader_shbin.o ELF := $(BUILD)/pocketjs-3ds.elf SMDH := $(BUILD)/pocketjs-3ds.smdh diff --git a/hosts/3ds/core/src/lib.rs b/hosts/3ds/core/src/lib.rs index ef3f9af2..ba2ae4f4 100644 --- a/hosts/3ds/core/src/lib.rs +++ b/hosts/3ds/core/src/lib.rs @@ -376,6 +376,11 @@ pub extern "C" fn ui_free_texture(handle: i32) { ui().free_texture(handle); } +#[no_mangle] +pub extern "C" fn ui_register_external_texture(width: u32, height: u32) -> i32 { + ui().register_external_texture(width, height) +} + #[no_mangle] pub extern "C" fn ui_set_image(id: i32, texture: i32) { ui().set_image(id, texture); diff --git a/hosts/3ds/include/pocket_core.h b/hosts/3ds/include/pocket_core.h index fdd3fe81..2ffb45dd 100644 --- a/hosts/3ds/include/pocket_core.h +++ b/hosts/3ds/include/pocket_core.h @@ -77,6 +77,7 @@ void ui_set_mesh(int32_t id, int32_t handle); int32_t ui_upload_img_entry(const uint8_t *bytes, size_t length); int32_t ui_upload_tileset_tile(const uint8_t *bytes, size_t length, uint32_t index); void ui_free_texture(int32_t handle); +int32_t ui_register_external_texture(uint32_t width, uint32_t height); void ui_set_image(int32_t id, int32_t texture); void ui_set_sprite( int32_t id, diff --git a/hosts/3ds/src/asset_pack.c b/hosts/3ds/src/asset_pack.c new file mode 100644 index 00000000..e7860ad0 --- /dev/null +++ b/hosts/3ds/src/asset_pack.c @@ -0,0 +1,269 @@ +/* This worker owns every SD open/seek/read/inflate. The UI owns descriptors + * and GPU uploads, and transfers fixed slots with release/acquire atomics. */ +#include "asset_pack.h" +#include "asset_pack_format.h" +#include "gfx.h" +#include "pocket_core.h" +#include <3ds.h> +#include +#include +#include +#include +#ifndef POCKETJS_RUNTIME_SLOT +#define POCKETJS_RUNTIME_SLOT "default" +#endif +enum { FREE, QUEUED, READY, BORROWED }; +typedef struct { + _Atomic unsigned state; + uint32_t generation, id, index, token, kind, width, height, length; + char pack[49], error[80]; + uint8_t bytes[PACK_BYTES]; +} Slot; +static Slot slots[8]; +static _Atomic bool running; +static _Atomic unsigned generation = 1; +static _Atomic unsigned reads, io_us, decode_us, max_io_us, max_decode_us, + failures, upload_us, max_upload_us, uploads; +static Thread worker; +static uint32_t sequence; +static unsigned submitted, taken, uploaded; +static unsigned take_cursor; +static void maximum(_Atomic unsigned *a, unsigned n) { + unsigned old = atomic_load(a); + while (n > old && !atomic_compare_exchange_weak(a, &old, n)) { + }; +} +static unsigned micros(u64 start) { + return (unsigned)((svcGetSystemTick() - start) * 1000000 / SYSCLOCK_ARM11); +} +void asset_pack_frame(void) { submitted = taken = uploaded = 0; } +int asset_pack_session(void) { + return atomic_load(&running) ? (int)atomic_load(&generation) : 0; +} +void asset_pack_stats(char *out, size_t size) { + snprintf(out, size, + "reads=%u ioUs=%u decodeUs=%u maxIoUs=%u maxDecodeUs=%u failures=%u " + "uploads=%u uploadUs=%u maxUploadUs=%u", + atomic_load(&reads), atomic_load(&io_us), atomic_load(&decode_us), + atomic_load(&max_io_us), atomic_load(&max_decode_us), + atomic_load(&failures), atomic_load(&uploads), + atomic_load(&upload_us), atomic_load(&max_upload_us)); +} +bool asset_pack_submit(uint32_t id, const char *name, unsigned length, + uint32_t index) { + if (!asset_pack_session() || !id || !length || length > 48 || + index >= PACK_ENTRIES || submitted >= 1) + return false; + for (unsigned i = 0; i < length; i++) + if (!((name[i] >= 'a' && name[i] <= 'z') || + (name[i] >= '0' && name[i] <= '9') || name[i] == '-')) + return false; + for (unsigned i = 0; i < 8; i++) + if (atomic_load_explicit(&slots[i].state, memory_order_acquire) == FREE) { + Slot *s = &slots[i]; + s->id = id; + s->index = index; + s->generation = atomic_load(&generation); + sequence = (sequence % 0x1fffffff) + 1; + s->token = (sequence << 3) | i; + memcpy(s->pack, name, length); + s->pack[length] = 0; + s->error[0] = 0; + atomic_store_explicit(&s->state, QUEUED, memory_order_release); + submitted++; + return true; + } + return false; +} +bool asset_pack_take(AssetPackResult *out) { + if (taken >= 1) + return false; + for (unsigned n = 0; n < 8; n++) { + unsigned i = (take_cursor + n) & 7; + if (atomic_load_explicit(&slots[i].state, memory_order_acquire) == READY) { + Slot *s = &slots[i]; + if (s->generation != atomic_load(&generation)) { + atomic_store_explicit(&s->state, FREE, memory_order_release); + continue; + } + atomic_store_explicit(&s->state, BORROWED, memory_order_release); + taken++; + take_cursor = (i + 1) & 7; + *out = (AssetPackResult){ + s->id, s->token, s->kind, s->width, + s->height, s->length, s->bytes, s->error[0] ? s->error : NULL}; + return true; + } + } + return false; +} +void asset_pack_release(uint32_t token) { + Slot *s = &slots[token & 7]; + if (atomic_load_explicit(&s->state, memory_order_acquire) == BORROWED && + s->token == token) + atomic_store_explicit(&s->state, FREE, memory_order_release); +} +int32_t asset_pack_upload(uint32_t token) { + Slot *s = &slots[token & 7]; + if (uploaded >= 1 || + atomic_load_explicit(&s->state, memory_order_acquire) != BORROWED || + s->token != token || s->kind != 2 || s->error[0]) + return -1; + uploaded++; + u64 at = svcGetSystemTick(); + int32_t handle = ui_register_external_texture(s->width, s->height); + if (handle >= 0 && !gfx_stage_image(handle, s->bytes, s->width, s->height)) { + ui_free_texture(handle); + return -1; + } + if (handle >= 0) { + unsigned duration = micros(at); + atomic_fetch_add(&uploads, 1); + atomic_fetch_add(&upload_us, duration); + maximum(&max_upload_us, duration); + } + return handle; +} +void asset_pack_reset(void) { + atomic_fetch_add(&generation, 1); + for (unsigned i = 0; i < 8; i++) { + unsigned s = atomic_load_explicit(&slots[i].state, memory_order_acquire); + if (s == READY || s == BORROWED) + atomic_store_explicit(&slots[i].state, FREE, memory_order_release); + } +} +/* Four immutable file descriptors bound open-file memory. Entry indices are + * read directly; neither the full directory nor whole pack enters RAM. */ +typedef struct { + char name[49]; + FILE *file; + PackHeader header; + unsigned used; +} OpenPack; +static void serve(void *unused) { + (void)unused; + OpenPack files[4] = {0}; + unsigned clock = 0; + uint8_t *compressed = malloc(PACK_STORED); + if (!compressed) { + atomic_store(&running, false); + return; + } + while (atomic_load(&running)) { + bool worked = false; + for (unsigned i = 0; i < 8; i++) { + Slot *s = &slots[i]; + if (atomic_load_explicit(&s->state, memory_order_acquire) != QUEUED) + continue; + worked = true; + if (s->generation != atomic_load(&generation)) { + atomic_store_explicit(&s->state, FREE, memory_order_release); + continue; + } + u64 at = svcGetSystemTick(); + OpenPack *p = NULL; + uint8_t header[PACK_HEADER]; + PackEntry e; + for (unsigned j = 0; j < 4; j++) + if (files[j].file && !strcmp(files[j].name, s->pack)) { + p = &files[j]; + break; + } + if (!p) { + p = &files[0]; + for (unsigned j = 0; j < 4; j++) + if (!files[j].file || files[j].used < p->used) + p = &files[j]; + if (p->file) + fclose(p->file); + memset(p, 0, sizeof *p); + char path[192]; + snprintf(path, sizeof path, "sdmc:/pocketjs/assets/%s/%s.prp", + POCKETJS_RUNTIME_SLOT, s->pack); + p->file = fopen(path, "rb"); + if (!p->file) { + snprintf(s->error, sizeof s->error, "Resource pack not installed"); + goto done; + } + if (fseek(p->file, 0, SEEK_END) || ftell(p->file) < 64L) { + snprintf(s->error, sizeof s->error, "Invalid resource pack size"); + goto invalid; + } + long length = ftell(p->file); + if (fseek(p->file, 0, SEEK_SET) || + fread(header, 1, PACK_HEADER, p->file) != PACK_HEADER || + !pack_header(header, (uint32_t)length, &p->header)) { + snprintf(s->error, sizeof s->error, "Invalid resource pack header"); + goto invalid; + } + strcpy(p->name, s->pack); + } + p->used = ++clock; + if (s->index >= p->header.count) { + snprintf(s->error, sizeof s->error, "Resource pack entry out of range"); + goto done; + } + if (fseek(p->file, PACK_HEADER + s->index * PACK_ENTRY, SEEK_SET) || + fread(header, 1, PACK_ENTRY, p->file) != PACK_ENTRY || + !pack_entry(header, &p->header, &e)) { + snprintf(s->error, sizeof s->error, "Invalid resource pack entry"); + goto done; + } + if (fseek(p->file, e.offset, SEEK_SET) || + fread(compressed, 1, e.stored, p->file) != e.stored) { + snprintf(s->error, sizeof s->error, "Resource pack read failed"); + goto invalid; + } + unsigned duration = micros(at); + atomic_fetch_add(&io_us, duration); + maximum(&max_io_us, duration); + at = svcGetSystemTick(); + uLongf size = e.raw; + if (uncompress(s->bytes, &size, compressed, e.stored) != Z_OK || + size != e.raw || crc32(0, s->bytes, size) != e.checksum) { + snprintf(s->error, sizeof s->error, + "Resource pack checksum or codec failure"); + goto done; + } + duration = micros(at); + atomic_fetch_add(&decode_us, duration); + maximum(&max_decode_us, duration); + atomic_fetch_add(&reads, 1); + s->kind = e.kind; + s->width = e.width; + s->height = e.height; + s->length = e.raw; + goto done; + invalid: + if (p && p->file) { + fclose(p->file); + p->file = NULL; + } + done: + if (s->error[0]) + atomic_fetch_add(&failures, 1); + atomic_store_explicit(&s->state, READY, memory_order_release); + } + if (!worked) + svcSleepThread(1000000); + } + for (unsigned j = 0; j < 4; j++) + if (files[j].file) + fclose(files[j].file); + free(compressed); +} +void asset_pack_start(void) { + if (atomic_exchange(&running, true)) + return; + worker = threadCreate(serve, NULL, 32 * 1024, 0x3f, -2, false); + if (!worker) + atomic_store(&running, false); +} +void asset_pack_stop(void) { + atomic_store(&running, false); + if (worker) { + threadJoin(worker, U64_MAX); + threadFree(worker); + worker = NULL; + } +} diff --git a/hosts/3ds/src/asset_pack.h b/hosts/3ds/src/asset_pack.h new file mode 100644 index 00000000..38950d21 --- /dev/null +++ b/hosts/3ds/src/asset_pack.h @@ -0,0 +1,22 @@ +#ifndef POCKET_ASSET_PACK_H +#define POCKET_ASSET_PACK_H +#include +#include +#include +typedef struct { + uint32_t id, token, kind, width, height, length; + const uint8_t *bytes; + const char *error; +} AssetPackResult; +void asset_pack_start(void); +void asset_pack_stop(void); +void asset_pack_reset(void); +void asset_pack_frame(void); +bool asset_pack_submit(uint32_t id, const char *pack, unsigned length, + uint32_t entry); +bool asset_pack_take(AssetPackResult *out); +void asset_pack_release(uint32_t token); +int32_t asset_pack_upload(uint32_t token); +int asset_pack_session(void); +void asset_pack_stats(char *out, size_t size); +#endif diff --git a/hosts/3ds/src/asset_pack_format.h b/hosts/3ds/src/asset_pack_format.h new file mode 100644 index 00000000..010a3ae3 --- /dev/null +++ b/hosts/3ds/src/asset_pack_format.h @@ -0,0 +1,56 @@ +#ifndef POCKET_ASSET_PACK_FORMAT_H +#define POCKET_ASSET_PACK_FORMAT_H +#include +#include +#include +#include +#define PACK_HEADER 64u +#define PACK_ENTRY 24u +#define PACK_ENTRIES 65536u +#define PACK_BYTES 131072u +#define PACK_STORED (PACK_BYTES + 128u) +typedef struct { + uint32_t count, length; +} PackHeader; +typedef struct { + uint32_t offset, stored, raw, checksum, kind, width, height; +} PackEntry; +static inline uint32_t pack_u32(const uint8_t *p) { + return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[3] << 24; +} +static inline bool pack_header(const uint8_t *p, uint32_t length, + PackHeader *h) { + if (length < PACK_HEADER || length > 0x7fffffffu || memcmp(p, "PRP1", 4) || + pack_u32(p + 4) != 1 || pack_u32(p + 12) != length) + return false; + h->count = pack_u32(p + 8); + h->length = length; + if (!h->count || h->count > PACK_ENTRIES || + PACK_HEADER + h->count * PACK_ENTRY > length) + return false; + for (unsigned i = 16; i < PACK_HEADER; i++) + if (p[i]) + return false; + return true; +} +static inline bool pack_entry(const uint8_t *p, const PackHeader *h, + PackEntry *e) { + e->offset = pack_u32(p); + e->stored = pack_u32(p + 4); + e->raw = pack_u32(p + 8); + e->checksum = pack_u32(p + 12); + e->kind = pack_u32(p + 16); + e->width = p[20] | (unsigned)p[21] << 8; + e->height = p[22] | (unsigned)p[23] << 8; + if (e->offset < PACK_HEADER + h->count * PACK_ENTRY || + e->offset > h->length || !e->stored || e->stored > PACK_STORED || + e->stored > h->length - e->offset || !e->raw || e->raw > PACK_BYTES) + return false; + if (e->kind == 1) + return e->raw <= 2500 && !e->width && !e->height; + return e->kind == 2 && e->width >= 16 && e->height >= 16 && e->width <= 256 && + e->height <= 256 && !(e->width & (e->width - 1)) && + !(e->height & (e->height - 1)) && e->raw == e->width * e->height * 2; +} +#endif diff --git a/hosts/3ds/src/gfx.c b/hosts/3ds/src/gfx.c index 3c4c4011..8692f650 100644 --- a/hosts/3ds/src/gfx.c +++ b/hosts/3ds/src/gfx.c @@ -154,6 +154,7 @@ static SurfaceBatch surfaces[MAX_SURFACES]; static C3D_Tex white; static ImageTexture *images; static size_t image_capacity; +static ImageTexture prepared_image; static FontTexture *fonts; static size_t font_capacity; @@ -493,6 +494,31 @@ static void release_image(ImageTexture *entry) { } } +static bool reserve_image_slots(size_t slots) { + if (slots <= image_capacity) return true; + ImageTexture *grown = realloc(images, slots * sizeof *images); + if (grown == NULL) return false; + memset(grown + image_capacity, 0, (slots - image_capacity) * sizeof *grown); + images = grown; + image_capacity = slots; + return true; +} + +bool gfx_stage_image(int32_t handle, const uint8_t *bytes, unsigned width, unsigned height) { + if (prepared_image.live || handle < 0 || !bytes) return false; + if (!reserve_image_slots(ui_texture_slot_count())) return false; + if (!C3D_TexInit(&prepared_image.texture, (u16)width, (u16)height, GPU_RGB565)) return false; + C3D_TexUpload(&prepared_image.texture, bytes); + GSPGPU_FlushDataCache(prepared_image.texture.data, prepared_image.texture.size); + C3D_TexSetFilter(&prepared_image.texture, GPU_LINEAR, GPU_LINEAR); + C3D_TexSetWrap(&prepared_image.texture, GPU_CLAMP_TO_EDGE, GPU_CLAMP_TO_EDGE); + prepared_image.handle = handle; + prepared_image.revision = 0; + prepared_image.u_scale = prepared_image.v_scale = 1.0f; + prepared_image.live = true; + return true; +} + static void release_font(FontTexture *entry) { if (entry->live) { C3D_TexDelete(&entry->texture); @@ -502,16 +528,7 @@ static void release_font(FontTexture *entry) { static void sync_resources(void) { size_t slots = ui_texture_slot_count(); - if (slots > image_capacity) { - ImageTexture *grown = realloc(images, slots * sizeof *images); - if (grown != NULL) { - memset(grown + image_capacity, 0, (slots - image_capacity) * sizeof *grown); - images = grown; - image_capacity = slots; - } else { - slots = image_capacity; - } - } + if (!reserve_image_slots(slots)) slots = image_capacity; for (size_t slot = 0; slot < image_capacity; slot += 1) { ImageTexture *entry = &images[slot]; PocketTexture source; @@ -520,7 +537,12 @@ static void sync_resources(void) { continue; } release_image(entry); - if (upload_image(entry, &source)) { + if (source.pixel_storage == UINT32_MAX) { + if (prepared_image.live && prepared_image.handle == source.handle) { + *entry = prepared_image; + prepared_image.live = false; + } + } else if (upload_image(entry, &source)) { entry->handle = source.handle; entry->revision = source.revision; entry->live = true; @@ -530,6 +552,10 @@ static void sync_resources(void) { } } + /* A staged image freed before its first draw was never submitted to PICA. + * Replaced live storage was retired above, after FrameBegin's GPU fence. */ + release_image(&prepared_image); + size_t font_slots = ui_font_slot_count(); if (font_slots > font_capacity) { FontTexture *grown = realloc(fonts, font_slots * sizeof *fonts); @@ -1099,6 +1125,7 @@ bool gfx_init(uint32_t logical_width, uint32_t logical_height) { void gfx_reset_resources(void) { if (!initialized) return; + release_image(&prepared_image); for (size_t slot = 0; slot < MAX_MESHES; slot += 1) release_mesh(&meshes[slot]); for (size_t slot = 0; slot < retired_mesh_count; slot += 1) release_mesh(&retired_meshes[slot]); retired_mesh_count = 0; diff --git a/hosts/3ds/src/gfx.h b/hosts/3ds/src/gfx.h index 3be3c021..35438097 100644 --- a/hosts/3ds/src/gfx.h +++ b/hosts/3ds/src/gfx.h @@ -18,6 +18,8 @@ bool gfx_init(uint32_t logical_width, uint32_t logical_height); /* Materialize one immutable mesh VBO. Safe before FrameBegin: superseded * buffers remain alive until the next GPU fence. Never allocates while drawing. */ bool gfx_upload_mesh(int32_t handle); +/* Pre-tiled RGB565 storage for a GPU-owned core image descriptor. */ +bool gfx_stage_image(int32_t handle, const uint8_t *bytes, unsigned width, unsigned height); void gfx_begin_frame(void); bool gfx_prepare_surface( uint32_t surface, diff --git a/hosts/3ds/src/main.c b/hosts/3ds/src/main.c index c9040814..681b5926 100644 --- a/hosts/3ds/src/main.c +++ b/hosts/3ds/src/main.c @@ -33,6 +33,7 @@ #include "pocket_core.h" #include "qjs.h" #include "offload.h" +#include "asset_pack.h" #include "devserver.h" #include "devmenu.h" #include "runtime.h" @@ -728,6 +729,9 @@ int main(void) { #endif PocketRuntimeFailureLineage failures = {0}; input_init(); +#ifdef POCKETJS_ASSET_PACK + asset_pack_start(); +#endif #ifdef POCKETJS_OFFLOAD GuestChoice guest = package_choice(embedded, 0, &runtime_state); guest.commit_on_accept = false; @@ -1055,6 +1059,9 @@ int main(void) { #endif ); offload_stop(); +#ifdef POCKETJS_ASSET_PACK + asset_pack_stop(); +#endif input_shutdown(); teardown_guest(); C3D_FrameEnd(0); diff --git a/hosts/3ds/src/qjs.c b/hosts/3ds/src/qjs.c index 2d795466..8d411db2 100644 --- a/hosts/3ds/src/qjs.c +++ b/hosts/3ds/src/qjs.c @@ -21,6 +21,7 @@ #include "qjs.h" #include "gfx.h" #include "offload.h" +#include "asset_pack.h" #include "offload_coverage.h" #include @@ -51,6 +52,7 @@ #define POCKETJS_JS_STACK_SIZE (384 * 1024) typedef enum { + HostPackSession, HostPackSubmit, HostPackTake, HostPackUpload, HostPackRelease, HostPackStats, HostOffloadSession, HostOffloadSubmit, HostOffloadTake, HostOffloadCoverage, HostOffloadImage, HostOffloadReleaseImage, HostOffloadMesh, HostCreateNode, HostDestroyNode, @@ -508,8 +510,93 @@ static JSValue host_operation( } return JS_NewInt32(ctx, handle); } +#ifdef POCKETJS_ASSET_PACK + case HostPackSession: + return JS_NewInt32(ctx, asset_pack_session()); + case HostPackSubmit: { + if (argc < 3 || !JS_IsString(argv[1])) + return JS_FALSE; + JSValue length = JS_GetPropertyStr(ctx, argv[1], "length"); + int32_t chars = 0; + JS_ToInt32(ctx, &chars, length); + JS_FreeValue(ctx, length); + if (chars < 1 || chars > 48) + return JS_FALSE; + text = JS_ToCStringLen2(ctx, &text_length, argv[1], 0); + bool ok = + text && asset_pack_submit((uint32_t)argument_int(ctx, argc, argv, 0), + text, text_length, + (uint32_t)argument_int(ctx, argc, argv, 2)); + if (text) + JS_FreeCString(ctx, text); + return JS_NewBool(ctx, ok); + } + case HostPackTake: { + AssetPackResult result; + if (!asset_pack_take(&result)) + return JS_UNDEFINED; + JSValue reply = JS_NewObject(ctx); + JS_SetPropertyStr(ctx, reply, "id", JS_NewUint32(ctx, result.id)); + if (result.error || (result.kind != 2 && result.length > 2500)) + JS_SetPropertyStr( + ctx, reply, "error", + JS_NewString(ctx, result.error + ? result.error + : "Pack data exceeds response budget")); + else if (result.kind == 2) { + JSValue image = JS_NewObject(ctx); + JS_SetPropertyStr(ctx, image, "token", JS_NewUint32(ctx, result.token)); + JS_SetPropertyStr(ctx, image, "width", JS_NewUint32(ctx, result.width)); + JS_SetPropertyStr(ctx, image, "height", + JS_NewUint32(ctx, result.height)); + JS_SetPropertyStr(ctx, reply, "image", image); + } else + JS_SetPropertyStr( + ctx, reply, "payload", + JS_NewStringLen(ctx, (const char *)result.bytes, result.length)); + JSValue encoded = + JS_JSONStringify(ctx, reply, JS_UNDEFINED, JS_UNDEFINED); + JS_FreeValue(ctx, reply); + if (result.error || result.kind != 2 || JS_IsException(encoded)) + asset_pack_release(result.token); + if (!JS_IsException(encoded)) { + size_t bytes = 0; + const char *json = JS_ToCStringLen2(ctx, &bytes, encoded, 0); + bool oversized = json && bytes > 4096; + if (json) + JS_FreeCString(ctx, json); + if (!json || oversized) { + if (result.kind == 2 && !result.error) + asset_pack_release(result.token); + JS_FreeValue(ctx, encoded); + if (!json) + return JS_EXCEPTION; + char error[128]; + snprintf( + error, sizeof error, + "{\"id\":%u,\"error\":\"Pack data exceeds response budget\"}", + (unsigned)result.id); + return JS_NewString(ctx, error); + } + } + return encoded; + } + case HostPackUpload: + if (image_used) + return JS_NewInt32(ctx, -1); + image_used = true; + return JS_NewInt32( + ctx, asset_pack_upload((uint32_t)argument_int(ctx, argc, argv, 0))); + case HostPackRelease: + asset_pack_release((uint32_t)argument_int(ctx, argc, argv, 0)); + return JS_UNDEFINED; + case HostPackStats: + asset_pack_stats(debug_poll_buffer, 256); + return JS_NewString(ctx, debug_poll_buffer); +#endif case HostOffloadImage: { - if (image_used) return JS_NewInt32(ctx, -1); + if (image_used) + return JS_NewInt32(ctx, -1); unsigned width, height; const uint8_t *pixels = offload_image((uint32_t)argument_int(ctx, argc, argv, 0), &width, &height); if (!pixels) return JS_NewInt32(ctx, -1); @@ -596,6 +683,16 @@ static void set_named_property(JSValueConst object, const uint8_t *name, size_t } static void install_host(void) { +#ifdef POCKETJS_ASSET_PACK + JSValue packs=JS_NewObject(context); + add_operation(packs,"session",0,HostPackSession); + add_operation(packs,"enqueue",3,HostPackSubmit); + add_operation(packs,"take",0,HostPackTake); + add_operation(packs,"uploadImage",1,HostPackUpload); + add_operation(packs,"releaseImage",1,HostPackRelease); + add_operation(packs,"stats",0,HostPackStats); + JS_SetPropertyStr(context,global,"resourcePacks",packs); +#endif #ifdef POCKETJS_OFFLOAD JSValue offload = JS_NewObject(context); add_operation(offload, "uploadCoverage", 6, HostOffloadCoverage); @@ -824,6 +921,9 @@ bool qjs_frame( ) { if (context == NULL) return false; offload_frame(); +#ifdef POCKETJS_ASSET_PACK + asset_pack_frame(); +#endif coverage_used = false; image_used = false; JSValue arguments[7] = { @@ -869,6 +969,9 @@ const char *qjs_last_error(void) { void qjs_shutdown(void) { offload_reset(); +#ifdef POCKETJS_ASSET_PACK + asset_pack_reset(); +#endif if (context != NULL) { JS_FreeValue(context, frame_function); JS_FreeValue(context, global); diff --git a/package.json b/package.json index ec17349e..50dca588 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "./resource-state": "./framework/src/resource-state.ts", "./resource-cache": "./framework/src/resource-cache.ts", "./resource-offload": "./framework/src/resource-offload.ts", + "./resource-pack": "./framework/src/resource-pack.ts", "./resource-view": "./framework/src/resource-view.ts", "./tile-viewport": "./framework/src/tile-viewport.ts", "./resource": "./framework/src/resource.ts", @@ -187,6 +188,7 @@ "./vue-vapor/resource-state": "./framework/src/resource-state.ts", "./vue-vapor/resource-cache": "./framework/src/resource-cache.ts", "./vue-vapor/resource-offload": "./framework/src/resource-offload.ts", + "./vue-vapor/resource-pack": "./framework/src/resource-pack.ts", "./vue-vapor/tile-viewport": "./framework/src/tile-viewport.ts", "./vue-vapor/audio": "./framework/src/audio-api.ts", "./vue-vapor/clock": "./framework/src/clock.ts", @@ -207,6 +209,7 @@ "./octane/resource-state": "./framework/src/resource-state.ts", "./octane/resource-cache": "./framework/src/resource-cache.ts", "./octane/resource-offload": "./framework/src/resource-offload.ts", + "./octane/resource-pack": "./framework/src/resource-pack.ts", "./octane/tile-viewport": "./framework/src/tile-viewport.ts", "./octane/audio": "./framework/src/audio-api.ts", "./octane/clock": "./framework/src/clock.ts", diff --git a/tests/3ds-profile.test.ts b/tests/3ds-profile.test.ts index ffd672b1..b6801670 100644 --- a/tests/3ds-profile.test.ts +++ b/tests/3ds-profile.test.ts @@ -103,6 +103,7 @@ describe("private Nintendo 3DS build profile", () => { }, capabilities: [ "io.offload", + "io.resource-pack", "input.analog.left", "input.analog.right", "input.buttons", diff --git a/tests/fixtures/resource-pack/3ds.h b/tests/fixtures/resource-pack/3ds.h new file mode 100644 index 00000000..0677a62e --- /dev/null +++ b/tests/fixtures/resource-pack/3ds.h @@ -0,0 +1,10 @@ +#ifndef POCKET_PACK_TEST_3DS_H +#define POCKET_PACK_TEST_3DS_H +#include "../offload-native/3ds.h" +#define SYSCLOCK_ARM11 1000000000ULL +static inline u64 svcGetSystemTick(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (u64)t.tv_sec * 1000000000 + t.tv_nsec; +} +#endif diff --git a/tests/fixtures/resource-pack/main.c b/tests/fixtures/resource-pack/main.c new file mode 100644 index 00000000..9c032daf --- /dev/null +++ b/tests/fixtures/resource-pack/main.c @@ -0,0 +1,112 @@ +#include <3ds.h> +#include +#include +#include +#include +static _Atomic int slow; +static size_t slow_read(void *p, size_t size, size_t n, FILE *f) { + if (atomic_load(&slow)) + svcSleepThread(50000000); + return fread(p, size, n, f); +} +#define fread slow_read +#include "../../../hosts/3ds/src/asset_pack.c" +#undef fread +int32_t ui_register_external_texture(uint32_t w, uint32_t h) { + assert(w == 16 && h == 16); + return 17; +} +void ui_free_texture(int32_t handle) { assert(handle == 17); } +bool gfx_stage_image(int32_t handle, const uint8_t *b, unsigned w, unsigned h) { + assert(handle == 17 && w == 16 && h == 16); + for (unsigned i = 0; i < 512; i++) + assert(b[i] == 42); + return true; +} +static AssetPackResult wait_for(unsigned id) { + AssetPackResult r; + u64 end = osGetTime() + 3000; + while (osGetTime() < end) { + asset_pack_frame(); + if (asset_pack_take(&r)) { + assert(r.id == id); + return r; + } + svcSleepThread(1000000); + } + assert(!"pack completion deadline"); + return (AssetPackResult){0}; +} +static void submit(unsigned id, const char *name, unsigned index) { + asset_pack_frame(); + assert(asset_pack_submit(id, name, strlen(name), index)); +} +int main(void) { + asset_pack_start(); + assert(asset_pack_session() > 0); + assert(!asset_pack_submit(1, "../escape", 9, 1)); + submit(1, "valid", 0); + AssetPackResult r = wait_for(1); + assert(!r.error && r.kind == 1 && r.length == 2 && !memcmp(r.bytes, "{}", 2)); + asset_pack_release(r.token); + submit(2, "valid", 1); + r = wait_for(2); + assert(!r.error && r.kind == 2 && r.length == 512); + unsigned old = r.token; + assert(asset_pack_upload(old) == 17); + assert(asset_pack_upload(old) < 0); + asset_pack_release(old); + asset_pack_frame(); + assert(asset_pack_upload(old) < 0); + const char *bad[] = {"missing", "checksum", "offset", "truncated"}; + for (unsigned i = 0; i < 4; i++) { + submit(3 + i, bad[i], 1); + r = wait_for(3 + i); + assert(r.error); + asset_pack_release(r.token); + } + // Eight borrowed completions exhaust storage; stale tokens cannot release + // a reused slot. This is the actual worker and zlib reader, under ASan. + unsigned held[8]; + for (unsigned i = 0; i < 8; i++) { + submit(10 + i, "valid", 1); + r = wait_for(10 + i); + held[i] = r.token; + assert(!r.error); + } + asset_pack_frame(); + assert(!asset_pack_submit(20, "valid", 5, 1)); + asset_pack_release(old); + assert(!asset_pack_submit(20, "valid", 5, 1)); + for (unsigned i = 0; i < 8; i++) + asset_pack_release(held[i]); + submit(40, "noise", 0); + r = wait_for(40); + assert(!r.error && r.kind == 2 && r.length == 131072); + asset_pack_release(r.token); + atomic_store(&slow, 1); + u64 at = osGetTime(); + submit(30, "valid", 1); + assert(osGetTime() - at < 20); + // Reset while IO is in progress: the previous realm never receives a + // completion and the UI can continue ticking without waiting on fread. + svcSleepThread(5000000); + asset_pack_reset(); + unsigned ticks = 0; + at = osGetTime(); + while (osGetTime() - at < 130) { + asset_pack_frame(); + assert(!asset_pack_take(&r)); + ticks++; + svcSleepThread(1000000); + } + assert(ticks > 30); + atomic_store(&slow, 0); + submit(31, "valid", 1); + r = wait_for(31); + assert(!r.error); + asset_pack_release(r.token); + asset_pack_stop(); + puts("pack worker: bounded IO, corruption, ownership, reset and stalled-read " + "UI verified"); +} diff --git a/tests/resource-pack-view.test.ts b/tests/resource-pack-view.test.ts new file mode 100644 index 00000000..c4527b32 --- /dev/null +++ b/tests/resource-pack-view.test.ts @@ -0,0 +1,225 @@ +import { expect, test } from "bun:test"; +import { createRoot } from "solid-js"; +import { createOffloadClient } from "../framework/src/offload.ts"; +import { + createResourceRuntime, + createResourceView, +} from "../framework/src/resource-view.ts"; +import { offloadResource } from "../framework/src/resource-offload.ts"; +import { installHost, type HostOps } from "../framework/src/host.ts"; + +// Separate module realms model separate guest boots, including an unsupported +// host. The real offload client and collection own every ticket in these tests. +let realm = 0; +async function fixture(supported = true) { + const sent: { id: number; name: string; entry: number }[] = []; + const replies: string[] = [], + released: number[] = [], + freed: number[] = []; + let upload = 0; + const ops = { + session: () => 1, + enqueue(id: number, name: string, entry: number) { + sent.push({ id, name, entry }); + return true; + }, + take: () => replies.shift(), + uploadImage: () => ++upload, + releaseImage: (id: number) => released.push(id), + stats: () => "", + }; + const target = globalThis as any; + if (supported) target.resourcePacks = ops; + else delete target.resourcePacks; + installHost({ + kind: "injected", + target: "test", + strict: true, + ops: { freeTexture: (id: number) => freed.push(id) } as unknown as HostOps, + }); + const api: typeof import("../framework/src/resource-pack.ts") = await import( + `../framework/src/resource-pack.ts?realm=${++realm}` + ); + const local = api.resourcePacks(); + const remoteSent: any[] = [], + remoteReplies: string[] = [], + remoteReleased: number[] = []; + let connected = true; + const remote = createOffloadClient({ + session: () => (connected ? 1 : 0), + submit(raw) { + remoteSent.push(JSON.parse(raw)); + return true; + }, + take: () => remoteReplies.shift(), + uploadImage: () => 100, + releaseImage: (id) => remoteReleased.push(id), + }); + return { + api, + local, + remote, + sent, + replies, + released, + freed, + remoteSent, + remoteReplies, + remoteReleased, + disconnect() { + connected = false; + }, + uploaded: () => upload, + reply(id: number, token: number) { + replies.push( + JSON.stringify({ id, image: { token, width: 16, height: 16 } }), + ); + }, + close() { + local?.dispose(); + remote.dispose(); + delete target.resourcePacks; + }, + }; +} +const demand = [{ input: "tile", priority: 0, pin: true }]; +function collection( + f: Awaited>, + runtime: ReturnType, + materialized?: (s: "local" | "desktop") => void, +) { + return f.api.createPackedImageCollection(runtime, { + key: (i: string) => i, + pack: () => ({ name: "atlas", entry: 1 }), + width: 16, + height: 16, + maxEntries: 2, + maxViews: 1, + fallback: { client: f.remote, method: "map.tile", payload: (i) => i }, + materialized, + }); +} +test("prepared local images retain normal view cleanup and need no desktop credit", async () => { + const f = await fixture(); + f.disconnect(); + createRoot((dispose) => { + const runtime = createResourceRuntime({ + maxConcurrent: 1, + maxCollections: 2, + startsPerFrame: 1, + completionsPerFrame: 1, + }); + const query = runtime.createCollection({ + key: (i: string) => i, + maxEntries: 1, + maxViews: 1, + maxCost: 1, + cost: () => 1, + maxResponseBytes: 100, + load: offloadResource(f.remote, "query", (i) => i), + materialize: (raw) => raw, + }); + createResourceView(query, { + demand: () => [{ input: "offline", priority: -10 }], + }); + const sources: string[] = [], + c = collection(f, runtime, (s) => sources.push(s)); + const v = createResourceView(c, { demand: () => demand }); + runtime.step(); + f.local!.step(); + expect(f.sent).toHaveLength(1); + expect(f.remoteSent).toHaveLength(0); + f.reply(f.sent[0]!.id, 8); + f.local!.step(); + runtime.step(); + expect(v.state("tile").status).toBe("ready"); + expect(sources).toEqual(["local"]); + expect(f.released).toEqual([8]); + expect(f.freed).toEqual([]); + dispose(); + expect(f.freed).toEqual([1]); + }); + f.close(); +}); +test("missing packs and unsupported hosts resolve through the same desktop cache", async () => { + for (const supported of [true, false]) { + const f = await fixture(supported); + createRoot((dispose) => { + const runtime = createResourceRuntime({ + maxConcurrent: 1, + maxCollections: 1, + startsPerFrame: 1, + completionsPerFrame: 1, + }); + const sources: string[] = [], + c = collection(f, runtime, (s) => sources.push(s)), + v = createResourceView(c, { demand: () => demand }); + runtime.step(); + if (supported) { + f.local!.step(); + f.replies.push( + JSON.stringify({ + id: f.sent[0]!.id, + error: "Resource pack not installed", + }), + ); + f.local!.step(); + } + f.remote.step(); + expect(f.remoteSent).toHaveLength(1); + f.remoteReplies.push( + JSON.stringify({ + id: f.remoteSent[0].id, + image: { token: 16, width: 16, height: 16 }, + }), + ); + f.remote.step(); + runtime.step(); + expect(v.state("tile").status).toBe("ready"); + expect(sources).toEqual(["desktop"]); + expect(f.remoteReleased).toEqual([16]); + c.clear(); + runtime.step(); + f.local?.step(); + f.remote.step(); + expect(f.sent).toHaveLength(supported ? 1 : 0); + expect(f.remoteSent).toHaveLength(2); + + dispose(); + expect(f.freed).toEqual([100]); + }); + f.close(); + } +}); +test("withdrawn and failed materializations release local staging exactly once", async () => { + for (const failure of [false, true]) { + const f = await fixture(); + createRoot((dispose) => { + const runtime = createResourceRuntime({ + maxConcurrent: 1, + maxCollections: 1, + startsPerFrame: 1, + completionsPerFrame: 1, + }); + const c = collection(f, runtime, () => { + if (failure) throw Error("consumer failed"); + }); + let wanted = true; + const v = createResourceView(c, { demand: () => (wanted ? demand : []) }); + runtime.step(); + f.local!.step(); + f.reply(f.sent[0]!.id, 24); + f.local!.step(); + if (!failure) wanted = false; + runtime.step(); + expect(f.released).toEqual([24]); + if (failure) { + expect(v.state("tile").status).toBe("error"); + expect(f.freed).toEqual([1]); + } else expect(f.uploaded()).toBe(0); + dispose(); + expect(f.released).toEqual([24]); + }); + f.close(); + } +}); diff --git a/tests/resource-pack.test.ts b/tests/resource-pack.test.ts new file mode 100644 index 00000000..00e1c8bb --- /dev/null +++ b/tests/resource-pack.test.ts @@ -0,0 +1,106 @@ +import { test, expect } from "bun:test"; +import { + mkdtempSync, + mkdirSync, + readFileSync, + writeFileSync, + rmSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { inflateSync } from "node:zlib"; +import { + createResourcePack, + crc32, + prepareTiledRGB565, +} from "../tools/resource-pack.ts"; + +test("prepared RGB565 preserves channels, vertical origin and Morton texel locations", () => { + const source = Buffer.alloc(16 * 16 * 2); + for (let y = 0; y < 16; y++) + for (let x = 0; x < 16; x++) + source.writeUInt16LE( + (x & 31) | ((y & 63) << 5) | (7 << 11), + (y * 16 + x) * 2, + ); + const tiled = prepareTiledRGB565(source, 16, 16); + // Independent deinterleaving of the destination Morton index. + for (let block = 0; block < 4; block++) + for (let n = 0; n < 64; n++) { + let x = 0, + y = 0; + for (let b = 0; b < 3; b++) { + x |= ((n >>> (2 * b)) & 1) << b; + y |= ((n >>> (2 * b + 1)) & 1) << b; + } + x += (block % 2) * 8; + y += Math.floor(block / 2) * 8; + expect(tiled.readUInt16LE((block * 64 + n) * 2)).toBe( + (x << 11) | ((15 - y) << 5) | 7, + ); + } + expect(crc32(Buffer.from("123456789"))).toBe(0xcbf43926); +}); + +test("desktop pack bake and actual native worker agree on bounds, CRC and ownership", () => { + const directory = mkdtempSync(join(tmpdir(), "pocket-pack-")); + try { + const root = join(directory, "sdmc:/pocketjs/assets/test"); + mkdirSync(root, { recursive: true }); + const path = join(root, "valid.prp"), + pack = createResourcePack(path, 2); + pack.add(Buffer.from("{}")); + pack.add(Buffer.alloc(512, 42), { width: 16, height: 16 }); + pack.finish(); + const bytes = readFileSync(path), + at = bytes.readUInt32LE(88), + length = bytes.readUInt32LE(92); + expect(inflateSync(bytes.subarray(at, at + length))).toEqual( + Buffer.alloc(512, 42), + ); + const checksum = Buffer.from(bytes); + checksum.writeUInt32LE(0, 100); + writeFileSync(join(root, "checksum.prp"), checksum); + const offset = Buffer.from(bytes); + offset.writeUInt32LE(0xffffffff, 88); + writeFileSync(join(root, "offset.prp"), offset); + writeFileSync(join(root, "truncated.prp"), bytes.subarray(0, 72)); + const noise = Buffer.alloc(131072); + let seed = 1; + for (let n = 0; n < noise.length; n++) { + seed ^= seed << 13; + seed ^= seed >>> 17; + seed ^= seed << 5; + noise[n] = seed & 255; + } + const noisy = createResourcePack(join(root, "noise.prp"), 1); + noisy.add(noise, { width: 256, height: 256 }); + noisy.finish(); + const full = readFileSync(join(root, "noise.prp")); + expect(full.readUInt32LE(68)).toBeGreaterThan(131072); + const binary = join(directory, "worker"); + const compile = Bun.spawnSync( + [ + "cc", + "-std=c11", + "-O1", + "-pthread", + "-fsanitize=address,undefined", + '-DPOCKETJS_RUNTIME_SLOT="test"', + "-I" + resolve("tests/fixtures/resource-pack"), + "-I" + resolve("hosts/3ds/include"), + resolve("tests/fixtures/resource-pack/main.c"), + "-lz", + "-o", + binary, + ], + { timeout: 20000 }, + ); + if (compile.exitCode !== 0) throw Error(compile.stderr.toString()); + const run = Bun.spawnSync([binary], { cwd: directory, timeout: 10000 }); + if (run.exitCode !== 0) throw Error(run.stderr.toString()); + expect(run.stdout.toString()).toContain("stalled-read UI verified"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}, 30000); diff --git a/tools/3ds-profile.ts b/tools/3ds-profile.ts index 33a3527d..b6ac62c2 100644 --- a/tools/3ds-profile.ts +++ b/tools/3ds-profile.ts @@ -43,6 +43,7 @@ export const THREE_DS_DEV_CONTRACTS = definePlatformContractRegistry( }, capabilities: [ "io.offload", + "io.resource-pack", "input.analog.left", "input.analog.right", "input.buttons", diff --git a/tools/3ds.ts b/tools/3ds.ts index 33a5d457..0b50f3aa 100644 --- a/tools/3ds.ts +++ b/tools/3ds.ts @@ -955,6 +955,7 @@ export async function build3ds(argv: readonly string[]): Promise { POCKETJS_BUILD_DIR: containerPathFor(buildDirectory, mounts), POCKETJS_OUT_3DSX: containerPathFor(output, mounts), POCKETJS_OFFLOAD: plan.features["io.offload"] ? "1" : "", + POCKETJS_ASSET_PACK: plan.features["io.resource-pack"] ? "1" : "", POCKETJS_OFFLOAD_SLOT: createHash("sha256").update(plan.app.id).digest("hex").slice(0, 16), POCKETJS_SMDH_TITLE: plan.app.title, POCKETJS_SMDH_AUTHOR: plan.app.id, diff --git a/tools/resource-pack.ts b/tools/resource-pack.ts new file mode 100644 index 00000000..df53b35f --- /dev/null +++ b/tools/resource-pack.ts @@ -0,0 +1,119 @@ +/** Immutable indexed resources. Image decoding and PICA pixel preparation run + * during the desktop bake; the console only inflates a bounded record. */ +import { + openSync, + closeSync, + writeSync, + renameSync, + unlinkSync, +} from "node:fs"; +import { deflateSync } from "node:zlib"; + +export function crc32(bytes: Uint8Array) { + let c = 0xffffffff; + for (const b of bytes) { + c ^= b; + for (let j = 0; j < 8; j++) c = (c >>> 1) ^ (c & 1 ? 0xedb88320 : 0); + } + return (c ^ 0xffffffff) >>> 0; +} +/** Core RGB565 stores red in the low bits; PICA RGB565 stores it high. + * PICA's 8x8 Morton blocks also reverse the vertical texture coordinate. */ +export function prepareTiledRGB565( + bytes: Uint8Array, + width: number, + height: number, +) { + for (const n of [width, height]) + if (!Number.isInteger(n) || n < 16 || n > 256 || n & (n - 1)) + throw Error("Invalid texture size"); + if (bytes.length !== width * height * 2) throw Error("Invalid RGB565 pixels"); + const out = Buffer.alloc(bytes.length); + for (let y = 0; y < height; y++) + for (let x = 0; x < width; x++) { + const sy = height - 1 - y, + source = (sy * width + x) * 2, + v = bytes[source]! | (bytes[source + 1]! << 8); + const mx = x & 7, + my = y & 7, + morton = + (mx & 1) | + ((my & 1) << 1) | + ((mx & 2) << 1) | + ((my & 2) << 2) | + ((mx & 4) << 2) | + ((my & 4) << 3); + const at = (((y >>> 3) * (width >>> 3) + (x >>> 3)) * 64 + morton) * 2; + out.writeUInt16LE(((v & 31) << 11) | (v & 0x7e0) | ((v >>> 11) & 31), at); + } + return out; +} +export function createResourcePack(path: string, count: number) { + if (!Number.isInteger(count) || count < 1 || count > 65536) + throw Error("Invalid pack count"); + const pending = path + ".partial", + fd = openSync(pending, "w"), + header = Buffer.alloc(64 + count * 24); + header.write("PRP1"); + header.writeUInt32LE(1, 4); + header.writeUInt32LE(count, 8); + let offset = header.length, + next = 0, + closed = false; + const write = (b: Uint8Array, at: number) => { + let n = 0; + while (n < b.length) { + const k = writeSync(fd, b, n, b.length - n, at + n); + if (!k) throw Error("Short pack write"); + n += k; + } + }; + write(header, 0); + return { + add(bytes: Uint8Array, image?: { width: number; height: number }) { + if (closed || next >= count || !bytes.length || bytes.length > 131072) + throw Error("Pack entry budget exceeded"); + if (image) { + for (const n of [image.width, image.height]) + if (!Number.isInteger(n) || n < 16 || n > 256 || n & (n - 1)) + throw Error("Invalid image"); + if (bytes.length !== image.width * image.height * 2) + throw Error("Invalid pixels"); + } else if (bytes.length > 2500) + throw Error("Data record exceeds 2500 bytes"); + const encoded = deflateSync(bytes, { level: 3 }); + if (encoded.length > 131200 || offset + encoded.length > 0x7fffffff) + throw Error("Pack exceeds native bounds"); + const at = 64 + next * 24; + for (const [p, v] of [ + [0, offset], + [4, encoded.length], + [8, bytes.length], + [12, crc32(bytes)], + [16, image ? 2 : 1], + ]) + header.writeUInt32LE(v!, at + p!); + header.writeUInt16LE(image?.width ?? 0, at + 20); + header.writeUInt16LE(image?.height ?? 0, at + 22); + write(encoded, offset); + offset += encoded.length; + return next++; + }, + finish() { + if (closed || next !== count) throw Error("Incomplete pack"); + header.writeUInt32LE(offset, 12); + write(header, 0); + closeSync(fd); + closed = true; + renameSync(pending, path); + return { entries: count, bytes: offset }; + }, + abort() { + if (!closed) { + closeSync(fd); + closed = true; + unlinkSync(pending); + } + }, + }; +}