diff --git a/.changeset/encoding-flatbuffers.md b/.changeset/encoding-flatbuffers.md new file mode 100644 index 00000000000..1346ee20b47 --- /dev/null +++ b/.changeset/encoding-flatbuffers.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add experimental `effect/encoding/FlatBuffers` for reading and writing FlatBuffers without a schema compiler. diff --git a/packages/effect/src/encoding/FlatBuffers.ts b/packages/effect/src/encoding/FlatBuffers.ts new file mode 100644 index 00000000000..123adffd215 --- /dev/null +++ b/packages/effect/src/encoding/FlatBuffers.ts @@ -0,0 +1,1033 @@ +/** + * Reads and writes the FlatBuffers binary format: tables, structs, vectors, + * strings and unions, read in place without parsing the whole buffer. + * + * **Mental model** + * + * - A FlatBuffer links tables through offsets. Fields are numbered slots; + * a small vtable records where each present field is, so absent fields cost + * nothing and read as their default. + * - This module knows nothing about schemas. The caller decides what each + * slot holds, as code generated from a `.fbs` schema does: read slot 0 as + * a string, slot 1 as a table, and so on. + * - {@link Builder} writes a buffer back to front, children first: create + * strings, vectors and child tables, then the table that refers to them, + * and call {@link Builder.finish} with the root table. + * - {@link root} gives the root {@link Table} of a buffer; its accessors read + * from the bytes on demand. {@link decode} runs a reading function and + * returns a `Result`. + * + * **Example** (Writing and reading a table) + * + * ```ts import.meta.vitest + * import { FlatBuffers } from "effect/encoding" + * import { Result } from "effect" + * + * // table Monster { name: string; hp: short = 100; inventory: [ubyte]; } + * const builder = FlatBuffers.makeBuilder() + * const name = builder.createString("Orc") + * const inventory = builder.createBytes(Uint8Array.of(1, 2, 3)) + * builder.startTable(3) + * builder.addFieldOffset(0, name) + * builder.addFieldInt16(1, 300, 100) + * builder.addFieldOffset(2, inventory) + * const bytes = builder.finish(builder.endTable(), { identifier: "MONS" }) + * + * const monster = FlatBuffers.decode(bytes, (root) => ({ + * name: root.string(0), + * hp: root.int16(1, 100), + * inventory: root.bytes(2)?.length + * }), { identifier: "MONS" }) + * + * Result.getOrThrow(monster) // => { name: "Orc", hp: 300, inventory: 3 } + * ``` + * + * **Details** + * + * Reads are bounds-checked, not a complete schema verification. Invalid reads + * throw a {@link FlatBuffersError}; {@link decode} catches data errors made + * inside its callback. Table nesting is limited by `maxDepth`. Strings must + * be valid UTF-8, and 64-bit integers are `bigint`s. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Data from "../Data.ts" +import * as Utf8 from "../internal/utf8.ts" +import * as Predicate from "../Predicate.ts" +import * as Result from "../Result.ts" + +const FlatBuffersErrorTypeId = "~effect/encoding/FlatBuffers/FlatBuffersError" + +/** + * Error raised for malformed data, excessive nesting or reader/builder misuse. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `OutOfBounds`: an offset, length or value lies outside the buffer. + * - `InvalidVtable`: a table's vtable has an impossible size. + * - `InvalidUtf8`: a string is not valid UTF-8. + * - `Identifier`: the buffer's file identifier is not the expected one. + * - `Limit`: tables are nested deeper than `maxDepth`. + * - `Usage`: invalid arguments, out-of-order builder calls, format size + * limits, or reads outside the declared struct size or at the wrong vector width. + * {@link decode} rethrows these caller errors. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class FlatBuffersError extends Data.TaggedError("FlatBuffersError")<{ + readonly kind: "OutOfBounds" | "InvalidVtable" | "InvalidUtf8" | "Identifier" | "Limit" | "Usage" + readonly message: string +}> { + /** + * Marks this value as a FlatBuffers error for runtime guards. + * + * @stability experimental + * @since 4.1.5 + */ + readonly [FlatBuffersErrorTypeId] = FlatBuffersErrorTypeId +} + +/** + * Checks whether a value is a {@link FlatBuffersError}. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isFlatBuffersError = (u: unknown): u is FlatBuffersError => + Predicate.hasProperty(u, FlatBuffersErrorTypeId) + +function fail(kind: FlatBuffersError["kind"], message: string): never { + throw new FlatBuffersError({ kind, message }) +} + +const positive = (name: string, n: number): void => { + if (!Number.isInteger(n) || n < 1) fail("Usage", `${name} must be a positive integer, got ${n}`) +} + +// Keeps a leading U+FEFF, which is part of the identifier. +const lenientUtf8 = new TextDecoder("utf-8", { ignoreBOM: true }) +const encoder = new TextEncoder() + +interface Source { + readonly bytes: Uint8Array + readonly view: DataView + readonly maxDepth: number +} + +const check = (view: DataView, at: number, size: number): void => { + if (at < 0 || at + size > view.byteLength) { + fail("OutOfBounds", `${size} bytes at ${at} lie outside the ${view.byteLength}-byte buffer`) + } +} + +/** The absolute position a uoffset at `at` points to. */ +const indirect = (view: DataView, at: number): number => { + check(view, at, 4) + return at + view.getUint32(at, true) +} + +/** Checks a vector (or string) header at `start` and its elements, and returns its length. */ +const lengthAt = (view: DataView, start: number, elementSize: number): number => { + check(view, start, 4) + const length = view.getUint32(start, true) + check(view, start + 4, length * elementSize) + return length +} + +const readString = (source: Source, start: number): string => { + const length = lengthAt(source.view, start, 1) + return Utf8.decode(source.bytes, start + 4, start + 4 + length) ?? + fail("InvalidUtf8", `the string at ${start} is not valid UTF-8`) +} + +// ----------------------------------------------------------------------------- +// Reading +// ----------------------------------------------------------------------------- + +/** Scalar readers shared by structs and vectors; `at` validates and locates each read. */ +abstract class Scalars { + readonly position: number + protected readonly source: Source + + constructor(source: Source, position: number) { + this.source = source + this.position = position + } + + get buffer(): Uint8Array { + return this.source.bytes + } + + protected abstract at(n: number, size: number): number + + bool(n: number): boolean { + return this.source.view.getUint8(this.at(n, 1)) !== 0 + } + int8(n: number): number { + return this.source.view.getInt8(this.at(n, 1)) + } + uint8(n: number): number { + return this.source.view.getUint8(this.at(n, 1)) + } + int16(n: number): number { + return this.source.view.getInt16(this.at(n, 2), true) + } + uint16(n: number): number { + return this.source.view.getUint16(this.at(n, 2), true) + } + int32(n: number): number { + return this.source.view.getInt32(this.at(n, 4), true) + } + uint32(n: number): number { + return this.source.view.getUint32(this.at(n, 4), true) + } + int64(n: number): bigint { + return this.source.view.getBigInt64(this.at(n, 8), true) + } + uint64(n: number): bigint { + return this.source.view.getBigUint64(this.at(n, 8), true) + } + float32(n: number): number { + return this.source.view.getFloat32(this.at(n, 4), true) + } + float64(n: number): number { + return this.source.view.getFloat64(this.at(n, 8), true) + } +} + +/** + * Reads fields at fixed byte offsets: the fields of a struct, stored inline + * in a table or a vector. Reads outside the struct's size throw a `Usage` + * {@link FlatBuffersError}. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export interface Struct { + /** The whole buffer. */ + readonly buffer: Uint8Array + /** Where the struct starts in {@link buffer}. */ + readonly position: number + /** The `bool` at byte `offset` of the struct. */ + bool(offset: number): boolean + /** The `byte` at byte `offset` of the struct. */ + int8(offset: number): number + /** The `ubyte` at byte `offset` of the struct. */ + uint8(offset: number): number + /** The `short` at byte `offset` of the struct. */ + int16(offset: number): number + /** The `ushort` at byte `offset` of the struct. */ + uint16(offset: number): number + /** The `int` at byte `offset` of the struct. */ + int32(offset: number): number + /** The `uint` at byte `offset` of the struct. */ + uint32(offset: number): number + /** The `long` at byte `offset` of the struct. */ + int64(offset: number): bigint + /** The `ulong` at byte `offset` of the struct. */ + uint64(offset: number): bigint + /** The `float` at byte `offset` of the struct. */ + float32(offset: number): number + /** The `double` at byte `offset` of the struct. */ + float64(offset: number): number + /** The struct of `size` bytes nested at byte `offset` of this struct. */ + struct(offset: number, size: number): Struct +} + +class StructImpl extends Scalars implements Struct { + private readonly size: number + + /** The caller has checked that the struct lies inside the buffer. */ + constructor(source: Source, position: number, size: number) { + super(source, position) + this.size = size + } + + protected at(offset: number, size: number): number { + if (!Number.isInteger(offset) || offset < 0 || offset + size > this.size) { + fail("Usage", `${size} bytes at ${offset} lie outside the ${this.size}-byte struct`) + } + return this.position + offset + } + + struct(offset: number, size: number): Struct { + positive("size", size) + return new StructImpl(this.source, this.at(offset, size), size) + } +} + +/** + * The elements of a vector. Accessors take the element index; the caller + * picks the accessor of the vector's element type, whose width must be the + * `elementSize` given to {@link Table.vector} (4 for strings and tables). + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export interface Vector { + /** The whole buffer. */ + readonly buffer: Uint8Array + /** Where the first element starts in {@link buffer}. */ + readonly position: number + /** The number of elements. */ + readonly length: number + /** Element `index` of a `[bool]` vector. */ + bool(index: number): boolean + /** Element `index` of a `[byte]` vector. */ + int8(index: number): number + /** Element `index` of a `[ubyte]` vector. */ + uint8(index: number): number + /** Element `index` of a `[short]` vector. */ + int16(index: number): number + /** Element `index` of a `[ushort]` vector. */ + uint16(index: number): number + /** Element `index` of an `[int]` vector. */ + int32(index: number): number + /** Element `index` of an `[uint]` vector. */ + uint32(index: number): number + /** Element `index` of a `[long]` vector. */ + int64(index: number): bigint + /** Element `index` of a `[ulong]` vector. */ + uint64(index: number): bigint + /** Element `index` of a `[float]` vector. */ + float32(index: number): number + /** Element `index` of a `[double]` vector. */ + float64(index: number): number + /** Element `index` of a `[string]` vector. */ + string(index: number): string + /** Element `index` of a vector of tables. */ + table(index: number): Table + /** Element `index` of a vector of structs. */ + struct(index: number): Struct +} + +class VectorImpl extends Scalars implements Vector { + readonly length: number + private readonly elementSize: number + private readonly depth: number + + /** The caller has checked that all `length` elements lie inside the buffer. */ + constructor(source: Source, position: number, length: number, elementSize: number, depth: number) { + super(source, position) + this.length = length + this.elementSize = elementSize + this.depth = depth + } + + protected at(index: number, size: number): number { + if (size !== this.elementSize) { + fail("Usage", `a ${size}-byte read from a vector of ${this.elementSize}-byte elements`) + } + if (!Number.isInteger(index) || index < 0 || index >= this.length) { + fail("OutOfBounds", `index ${index} is outside the vector of length ${this.length}`) + } + return this.position + index * size + } + + string(index: number): string { + return readString(this.source, indirect(this.source.view, this.at(index, 4))) + } + table(index: number): Table { + return new TableImpl(this.source, indirect(this.source.view, this.at(index, 4)), this.depth + 1) + } + struct(index: number): Struct { + return new StructImpl(this.source, this.at(index, this.elementSize), this.elementSize) + } +} + +/** + * A table: fields by slot number, read on demand. Absent fields read as + * the given default, including slots past the end of the table's vtable + * (fields added to the schema after the buffer was written). + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export interface Table { + /** The whole buffer. */ + readonly buffer: Uint8Array + /** Where the table starts in {@link buffer}. */ + readonly position: number + /** Whether field `slot` is present. */ + has(slot: number): boolean + /** The `bool` in `slot`, or `fallback` when it is absent. */ + bool(slot: number, fallback?: boolean): boolean + /** The `byte` in `slot`, or `fallback` when it is absent. */ + int8(slot: number, fallback?: number): number + /** The `ubyte` in `slot`, or `fallback` when it is absent. */ + uint8(slot: number, fallback?: number): number + /** The `short` in `slot`, or `fallback` when it is absent. */ + int16(slot: number, fallback?: number): number + /** The `ushort` in `slot`, or `fallback` when it is absent. */ + uint16(slot: number, fallback?: number): number + /** The `int` in `slot`, or `fallback` when it is absent. */ + int32(slot: number, fallback?: number): number + /** The `uint` in `slot`, or `fallback` when it is absent. */ + uint32(slot: number, fallback?: number): number + /** The `long` in `slot`, or `fallback` when it is absent. */ + int64(slot: number, fallback?: bigint): bigint + /** The `ulong` in `slot`, or `fallback` when it is absent. */ + uint64(slot: number, fallback?: bigint): bigint + /** The `float` in `slot`, or `fallback` when it is absent. */ + float32(slot: number, fallback?: number): number + /** The `double` in `slot`, or `fallback` when it is absent. */ + float64(slot: number, fallback?: number): number + /** The `string` in `slot`. */ + string(slot: number): string | undefined + /** The table in `slot` (also the value of a union). */ + table(slot: number): Table | undefined + /** The struct of `size` bytes stored inline in `slot`. */ + struct(slot: number, size: number): Struct | undefined + /** The vector in `slot`, whose elements are `elementSize` bytes wide (4 for strings and tables). */ + vector(slot: number, elementSize: number): Vector | undefined + /** The `[ubyte]` (or `[byte]`) vector in `slot`, as a view of the buffer (also a nested FlatBuffer). */ + bytes(slot: number): Uint8Array | undefined +} + +class TableImpl implements Table { + readonly position: number + private readonly source: Source + private readonly depth: number + private readonly vtable: number + private readonly vtableSize: number + + constructor(source: Source, position: number, depth: number) { + if (depth > source.maxDepth) fail("Limit", `tables are nested deeper than maxDepth (${source.maxDepth})`) + const view = source.view + check(view, position, 4) + const vtable = position - view.getInt32(position, true) + check(view, vtable, 4) + const vtableSize = view.getUint16(vtable, true) + if (vtableSize < 4 || (vtableSize & 1) !== 0) fail("InvalidVtable", `vtable at ${vtable} has size ${vtableSize}`) + check(view, vtable, vtableSize) + this.source = source + this.position = position + this.depth = depth + this.vtable = vtable + this.vtableSize = vtableSize + } + + get buffer(): Uint8Array { + return this.source.bytes + } + + /** The position of field `slot`, or -1 when it is absent. */ + private field(slot: number, size: number): number { + if (!Number.isInteger(slot) || slot < 0) fail("Usage", `a slot is a non-negative integer, got ${slot}`) + const entry = 4 + slot * 2 + if (entry >= this.vtableSize) return -1 + const relative = this.source.view.getUint16(this.vtable + entry, true) + if (relative === 0) return -1 + const at = this.position + relative + check(this.source.view, at, size) + return at + } + + has(slot: number): boolean { + return this.field(slot, 0) !== -1 + } + bool(slot: number, fallback = false): boolean { + const at = this.field(slot, 1) + return at === -1 ? fallback : this.source.view.getUint8(at) !== 0 + } + int8(slot: number, fallback = 0): number { + const at = this.field(slot, 1) + return at === -1 ? fallback : this.source.view.getInt8(at) + } + uint8(slot: number, fallback = 0): number { + const at = this.field(slot, 1) + return at === -1 ? fallback : this.source.view.getUint8(at) + } + int16(slot: number, fallback = 0): number { + const at = this.field(slot, 2) + return at === -1 ? fallback : this.source.view.getInt16(at, true) + } + uint16(slot: number, fallback = 0): number { + const at = this.field(slot, 2) + return at === -1 ? fallback : this.source.view.getUint16(at, true) + } + int32(slot: number, fallback = 0): number { + const at = this.field(slot, 4) + return at === -1 ? fallback : this.source.view.getInt32(at, true) + } + uint32(slot: number, fallback = 0): number { + const at = this.field(slot, 4) + return at === -1 ? fallback : this.source.view.getUint32(at, true) + } + int64(slot: number, fallback: bigint = BigInt(0)): bigint { + const at = this.field(slot, 8) + return at === -1 ? fallback : this.source.view.getBigInt64(at, true) + } + uint64(slot: number, fallback: bigint = BigInt(0)): bigint { + const at = this.field(slot, 8) + return at === -1 ? fallback : this.source.view.getBigUint64(at, true) + } + float32(slot: number, fallback = 0): number { + const at = this.field(slot, 4) + return at === -1 ? fallback : this.source.view.getFloat32(at, true) + } + float64(slot: number, fallback = 0): number { + const at = this.field(slot, 8) + return at === -1 ? fallback : this.source.view.getFloat64(at, true) + } + string(slot: number): string | undefined { + const at = this.field(slot, 4) + return at === -1 ? undefined : readString(this.source, indirect(this.source.view, at)) + } + table(slot: number): Table | undefined { + const at = this.field(slot, 4) + return at === -1 ? undefined : new TableImpl(this.source, indirect(this.source.view, at), this.depth + 1) + } + struct(slot: number, size: number): Struct | undefined { + positive("size", size) + const at = this.field(slot, size) + return at === -1 ? undefined : new StructImpl(this.source, at, size) + } + vector(slot: number, elementSize: number): Vector | undefined { + positive("elementSize", elementSize) + const at = this.field(slot, 4) + if (at === -1) return undefined + const start = indirect(this.source.view, at) + const length = lengthAt(this.source.view, start, elementSize) + return new VectorImpl(this.source, start + 4, length, elementSize, this.depth) + } + bytes(slot: number): Uint8Array | undefined { + const v = this.vector(slot, 1) + return v === undefined ? undefined : this.source.bytes.subarray(v.position, v.position + v.length) + } +} + +const identifierBytes = (identifier: string | undefined): Uint8Array | undefined => { + if (identifier === undefined) return undefined + const bytes = encoder.encode(identifier) + return bytes.length === 4 + ? bytes + : fail("Usage", `a file identifier is four bytes, got ${JSON.stringify(identifier)}`) +} + +/** + * Returns the four bytes after the root offset decoded as UTF-8, or + * `undefined` when the buffer is too short. + * + * **Details** + * + * A buffer does not record whether it has an identifier. Without one, these + * bytes are other buffer data. Compare the result with the identifier you + * expect, or pass `identifier` to {@link root} or {@link decode}. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const identifier = ( + bytes: Uint8Array, + options?: { readonly sizePrefixed?: boolean | undefined } +): string | undefined => { + const at = options?.sizePrefixed ? 8 : 4 + return bytes.length < at + 4 ? undefined : lenientUtf8.decode(bytes.subarray(at, at + 4)) +} + +/** + * Returns the root table, throwing a {@link FlatBuffersError} for invalid + * root data or a mismatched file identifier. Fields are checked on access. + * + * **Details** + * + * `identifier` must encode to exactly four UTF-8 bytes. `sizePrefixed` reads + * the leading size and confines reads to that payload. `maxDepth` is a + * positive integer (default 64); the root has depth 1, and exceeding the + * limit throws a `Limit` error. + * + * **Gotchas** + * + * Raising `maxDepth` can expose recursive readers to stack overflow. + * + * @see decode for reading into a `Result` + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const root = (bytes: Uint8Array, options?: { + readonly identifier?: string | undefined + readonly sizePrefixed?: boolean | undefined + readonly maxDepth?: number | undefined +}): Table => { + const maxDepth = options?.maxDepth ?? 64 + positive("maxDepth", maxDepth) + const expected = identifierBytes(options?.identifier) + let data = bytes + if (options?.sizePrefixed) { + if (bytes.length < 4) fail("OutOfBounds", "a size-prefixed buffer needs at least 4 bytes") + const size = new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0, true) + if (size > bytes.length - 4) { + fail("OutOfBounds", `the size prefix ${size} exceeds the ${bytes.length - 4} bytes after it`) + } + data = bytes.subarray(4, 4 + size) + } + if (expected !== undefined) { + const actual = data.subarray(4, 8) + if (actual.length !== 4 || actual.some((b, i) => b !== expected[i])) { + fail("Identifier", `expected file identifier ${JSON.stringify(options?.identifier)}`) + } + } + const view = new DataView(data.buffer, data.byteOffset, data.byteLength) + return new TableImpl({ bytes: data, view, maxDepth }, indirect(view, 0), 1) +} + +/** + * Reads a buffer with `read`, which gets the root table. Malformed input + * becomes a failed `Result` instead of an exception; options are those of + * {@link root}. + * + * **Gotchas** + * + * Reads are lazy, so only reads made inside `read` are covered: a + * {@link Table} or {@link Vector} returned from it can still throw later. + * `Usage` errors and errors other than {@link FlatBuffersError} are rethrown. + * + * Many offsets may point at one string or vector, so what a reader + * materialises is not bounded by the size of the buffer: 1,000 offsets to + * one 100 KB string decode to 100 MB. Cap what you read from untrusted input. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const decode = ( + bytes: Uint8Array, + read: (root: Table) => A, + options?: { + readonly identifier?: string | undefined + readonly sizePrefixed?: boolean | undefined + readonly maxDepth?: number | undefined + } +): Result.Result => { + try { + return Result.succeed(read(root(bytes, options))) + } catch (cause) { + if (isFlatBuffersError(cause) && cause.kind !== "Usage") return Result.fail(cause) + throw cause + } +} + +// ----------------------------------------------------------------------------- +// Writing +// ----------------------------------------------------------------------------- + +/** + * The position of something written by a {@link Builder} (a string, vector + * or table), counted from the end of the buffer. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Offset = number + +const MAX_SIZE = 2 ** 31 +/** The most slots whose vtable (two bytes each, plus four) fits its 16-bit size. */ +const MAX_SLOTS = 32765 + +/** + * Writes FlatBuffers tables, structs, vectors and strings back to front. + * + * **Details** + * + * Children come first: create strings, vectors and child tables, keep their + * {@link Offset}s, then start the table that refers to them. Between + * {@link startTable} and {@link endTable}, add fields with the `addField*` + * methods; a scalar equal to its default is not written (unless + * `forceDefaults`). Identical vtables are written once and shared. + * + * Structs are written in place: write their fields last to first with the + * `add*` methods (and {@link pad}) right before the call that refers to + * them, {@link addFieldStruct} for a table field or inside a vector. + * + * Integers are written modulo their width; 64-bit values are `bigint`s. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export class Builder { + private bytes: Uint8Array + private view: DataView + /** Index of the first written byte: data is `bytes[head..]`. */ + private head: number + private minAlign = 1 + private vtable: Array | undefined = undefined + private objectEnd = 0 + private readonly vtables = new Map() + private vectorLength: number | undefined = undefined + private finished = false + private readonly sharedStrings = new Map() + private readonly forceDefaults: boolean + + constructor(options?: { readonly initialSize?: number | undefined; readonly forceDefaults?: boolean | undefined }) { + const size = options?.initialSize ?? 1024 + if (!Number.isInteger(size) || size < 1 || size > MAX_SIZE) { + fail("Usage", `initialSize must be an integer from 1 to ${MAX_SIZE}, got ${size}`) + } + this.bytes = new Uint8Array(size) + this.view = new DataView(this.bytes.buffer) + this.head = size + this.forceDefaults = options?.forceDefaults ?? false + } + + /** The current position, counted from the end of the buffer. */ + get offset(): Offset { + return this.bytes.length - this.head + } + + private grow(): void { + if (this.bytes.length >= MAX_SIZE) fail("Usage", "a FlatBuffer can't grow beyond 2 GiB") + const size = Math.min(this.bytes.length * 2, MAX_SIZE) + const next = new Uint8Array(size) + next.set(this.bytes, size - this.bytes.length) + this.head += size - this.bytes.length + this.bytes = next + this.view = new DataView(next.buffer) + } + + /** Writes `n` zero bytes. */ + pad(n: number): void { + while (this.head < n) this.grow() + for (let i = 0; i < n; i++) this.bytes[--this.head] = 0 + } + + /** + * Pads so that a value of `size` bytes, written after `additional` more + * bytes, is aligned to `size`; makes room for both. + */ + prep(size: number, additional: number): void { + if (!Number.isInteger(size) || size < 1 || (size & (size - 1)) !== 0) { + fail("Usage", `an alignment must be a power of two, got ${size}`) + } + if (size > this.minAlign) this.minAlign = size + const alignSize = (~(this.bytes.length - this.head + additional) + 1) & (size - 1) + while (this.head < alignSize + size + additional) this.grow() + this.pad(alignSize) + } + + /** Writes a `bool`. */ + addBool(value: boolean): void { + this.prep(1, 0) + this.view.setUint8(--this.head, value ? 1 : 0) + } + /** Writes a `byte`. */ + addInt8(value: number): void { + this.prep(1, 0) + this.view.setInt8(--this.head, value) + } + /** Writes a `ubyte`. */ + addUint8(value: number): void { + this.prep(1, 0) + this.view.setUint8(--this.head, value) + } + /** Writes a `short`. */ + addInt16(value: number): void { + this.prep(2, 0) + this.view.setInt16(this.head -= 2, value, true) + } + /** Writes a `ushort`. */ + addUint16(value: number): void { + this.prep(2, 0) + this.view.setUint16(this.head -= 2, value, true) + } + /** Writes an `int`. */ + addInt32(value: number): void { + this.prep(4, 0) + this.view.setInt32(this.head -= 4, value, true) + } + /** Writes a `uint`. */ + addUint32(value: number): void { + this.prep(4, 0) + this.view.setUint32(this.head -= 4, value, true) + } + /** Writes a `long`. */ + addInt64(value: bigint): void { + this.prep(8, 0) + this.view.setBigInt64(this.head -= 8, value, true) + } + /** Writes a `ulong`. */ + addUint64(value: bigint): void { + this.prep(8, 0) + this.view.setBigUint64(this.head -= 8, value, true) + } + /** Writes a `float`. */ + addFloat32(value: number): void { + this.prep(4, 0) + this.view.setFloat32(this.head -= 4, value, true) + } + /** Writes a `double`. */ + addFloat64(value: number): void { + this.prep(8, 0) + this.view.setFloat64(this.head -= 8, value, true) + } + /** Writes an offset to `target`, something written earlier. */ + addOffset(target: Offset): void { + this.prep(4, 0) + const relative = this.offset - target + 4 + if (target > this.offset) fail("Usage", `offset ${target} points past the current position ${this.offset}`) + this.view.setUint32(this.head -= 4, relative, true) + } + + private notNested(what: string): void { + if (this.vtable !== undefined || this.vectorLength !== undefined) { + fail("Usage", `${what} inside a table or vector that isn't finished`) + } + } + + /** Writes a string (UTF-8, or the given bytes) and returns its offset. */ + createString(value: string | Uint8Array): Offset { + this.notNested("createString") + const bytes = typeof value === "string" ? encoder.encode(value) : value + this.prep(4, bytes.length + 1) + this.bytes[--this.head] = 0 + this.head -= bytes.length + this.bytes.set(bytes, this.head) + this.view.setUint32(this.head -= 4, bytes.length, true) + return this.offset + } + + /** Like {@link createString}, but writes each distinct string once. */ + createSharedString(value: string): Offset { + const existing = this.sharedStrings.get(value) + if (existing !== undefined) return existing + const offset = this.createString(value) + this.sharedStrings.set(value, offset) + return offset + } + + /** Writes a `[ubyte]` vector of the given bytes and returns its offset. */ + createBytes(value: Uint8Array): Offset { + this.notNested("createBytes") + this.prep(4, value.length) + this.head -= value.length + this.bytes.set(value, this.head) + this.view.setUint32(this.head -= 4, value.length, true) + return this.offset + } + + /** + * Starts a vector of `length` elements of `elementSize` bytes, aligned to + * `alignment`. Write the elements last to first, then {@link endVector}. + */ + startVector(elementSize: number, length: number, alignment: number): void { + this.notNested("startVector") + positive("elementSize", elementSize) + if (!Number.isInteger(length) || length < 0 || length > 0xffffffff) { + fail("Usage", `a vector length is a 32-bit unsigned integer, got ${length}`) + } + this.vectorLength = length + this.prep(4, elementSize * length) + this.prep(alignment, elementSize * length) + } + + /** Finishes the vector started with {@link startVector} and returns its offset. */ + endVector(): Offset { + const length = this.vectorLength + if (length === undefined) fail("Usage", "endVector without startVector") + this.vectorLength = undefined + this.view.setUint32(this.head -= 4, length, true) + return this.offset + } + + /** Writes a vector of offsets (strings, tables or vectors) and returns its offset. */ + createOffsetVector(targets: ReadonlyArray): Offset { + this.startVector(4, targets.length, 4) + for (let i = targets.length - 1; i >= 0; i--) this.addOffset(targets[i]) + return this.endVector() + } + + /** Starts a table with `fields` slots. Add fields, then {@link endTable}. */ + startTable(fields: number): void { + this.notNested("startTable") + if (!Number.isInteger(fields) || fields < 0 || fields > MAX_SLOTS) { + fail("Usage", `a table has 0 to ${MAX_SLOTS} slots, got ${fields}`) + } + this.vtable = new Array(fields).fill(0) + this.objectEnd = this.offset + } + + private slot(slot: number): void { + if (this.vtable === undefined) fail("Usage", "a field outside startTable/endTable") + if (!Number.isInteger(slot) || slot < 0 || slot >= this.vtable.length) { + fail("Usage", `slot ${slot} is outside the table's ${this.vtable.length} slots`) + } + this.vtable[slot] = this.offset + } + + /** Adds a `bool` field; skipped when equal to `fallback`. */ + addFieldBool(slot: number, value: boolean, fallback = false): void { + if (value !== fallback || this.forceDefaults) { + this.addBool(value) + this.slot(slot) + } + } + /** Adds a `byte` field; skipped when equal to `fallback`. */ + addFieldInt8(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addInt8(value) + this.slot(slot) + } + } + /** Adds a `ubyte` field; skipped when equal to `fallback`. */ + addFieldUint8(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addUint8(value) + this.slot(slot) + } + } + /** Adds a `short` field; skipped when equal to `fallback`. */ + addFieldInt16(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addInt16(value) + this.slot(slot) + } + } + /** Adds a `ushort` field; skipped when equal to `fallback`. */ + addFieldUint16(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addUint16(value) + this.slot(slot) + } + } + /** Adds an `int` field; skipped when equal to `fallback`. */ + addFieldInt32(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addInt32(value) + this.slot(slot) + } + } + /** Adds a `uint` field; skipped when equal to `fallback`. */ + addFieldUint32(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addUint32(value) + this.slot(slot) + } + } + /** Adds a `long` field; skipped when equal to `fallback`. */ + addFieldInt64(slot: number, value: bigint, fallback: bigint = BigInt(0)): void { + if (value !== fallback || this.forceDefaults) { + this.addInt64(value) + this.slot(slot) + } + } + /** Adds a `ulong` field; skipped when equal to `fallback`. */ + addFieldUint64(slot: number, value: bigint, fallback: bigint = BigInt(0)): void { + if (value !== fallback || this.forceDefaults) { + this.addUint64(value) + this.slot(slot) + } + } + /** Adds a `float` field; skipped when equal to `fallback`. */ + addFieldFloat32(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addFloat32(value) + this.slot(slot) + } + } + /** Adds a `double` field; skipped when equal to `fallback`. */ + addFieldFloat64(slot: number, value: number, fallback = 0): void { + if (value !== fallback || this.forceDefaults) { + this.addFloat64(value) + this.slot(slot) + } + } + /** + * Adds a field pointing to a string, vector or table (a union's value too). + * A `target` of 0 (nothing) leaves the field absent, even with + * `forceDefaults`. + */ + addFieldOffset(slot: number, target: Offset): void { + if (target !== 0) { + this.addOffset(target) + this.slot(slot) + } + } + /** + * Adds a struct field. The struct's fields must have just been written: + * `target` is the current {@link offset}. + */ + addFieldStruct(slot: number, target: Offset): void { + if (target !== 0) { + if (target !== this.offset) fail("Usage", "a struct field must be written right before addFieldStruct") + this.slot(slot) + } + } + + /** Finishes the table started with {@link startTable} and returns its offset. */ + endTable(): Offset { + const vtable = this.vtable + if (vtable === undefined) fail("Usage", "endTable without startTable") + // Placeholder for the offset to the vtable. + this.addInt32(0) + const object = this.offset + const objectSize = object - this.objectEnd + if (objectSize > 0xffff) { + fail("Usage", `a table's inline data is ${objectSize} bytes, over the 65535 a vtable can address`) + } + // Trailing absent fields are left out of the vtable. + while (vtable.length > 0 && vtable[vtable.length - 1] === 0) vtable.pop() + const entries = vtable.map((at) => (at === 0 ? 0 : object - at)) + const key = `${objectSize}:${entries.join(",")}` + let shared = this.vtables.get(key) + if (shared === undefined) { + for (let i = entries.length - 1; i >= 0; i--) this.addInt16(entries[i]) + this.addInt16(objectSize) + this.addInt16((entries.length + 2) * 2) + shared = this.offset + this.vtables.set(key, shared) + } + this.view.setInt32(this.bytes.length - object, shared - object, true) + this.vtable = undefined + return object + } + + /** + * Writes the root offset (after the file identifier and before the size + * prefix, when given) and returns the finished buffer. Call it once. + */ + finish(rootTable: Offset, options?: { + readonly identifier?: string | undefined + readonly sizePrefixed?: boolean | undefined + }): Uint8Array { + this.notNested("finish") + if (this.finished) fail("Usage", "finish called twice") + const identifier = identifierBytes(options?.identifier) + this.prep(this.minAlign, 4 + (identifier === undefined ? 0 : 4) + (options?.sizePrefixed ? 4 : 0)) + if (identifier !== undefined) { + this.prep(4, 4) + for (let i = 3; i >= 0; i--) this.bytes[--this.head] = identifier[i] + } + this.addOffset(rootTable) + if (options?.sizePrefixed) this.addInt32(this.bytes.length - this.head) + this.finished = true + return this.bytes.slice(this.head) + } +} + +/** + * Creates a {@link Builder}. `forceDefaults` writes scalar fields even when + * they equal their default. `initialSize` is the first buffer size in bytes, + * an integer from 1 to 2^31 (default 1024); the buffer grows as needed, and + * other values throw a `Usage` {@link FlatBuffersError}. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export const makeBuilder = (options?: { + readonly initialSize?: number | undefined + readonly forceDefaults?: boolean | undefined +}): Builder => new Builder(options) diff --git a/packages/effect/src/encoding/index.ts b/packages/effect/src/encoding/index.ts index dc7605ca67d..24b5f934af2 100644 --- a/packages/effect/src/encoding/index.ts +++ b/packages/effect/src/encoding/index.ts @@ -29,6 +29,12 @@ export * as Csv from "./Csv.ts" */ export * as EncodingError from "./EncodingError.ts" +/** + * @stability experimental + * @since 4.1.5 + */ +export * as FlatBuffers from "./FlatBuffers.ts" + /** * @stability stable * @since 4.0.0 diff --git a/packages/effect/test/encoding/FlatBuffers.conformance.md b/packages/effect/test/encoding/FlatBuffers.conformance.md new file mode 100644 index 00000000000..e4e6bef063f --- /dev/null +++ b/packages/effect/test/encoding/FlatBuffers.conformance.md @@ -0,0 +1,75 @@ +# FlatBuffers assertion reference + +This reference inventories the public low-level encoding boundary in +`FlatBuffers.test.ts` for maintainers. It maps existing regression assertions before +adding focused format examples; it is not a claim of complete FlatBuffers conformance. + +## Authoritative sources + +The upstream Google FlatBuffers documentation is pinned to commit +`569e6cb461738f0aefccbada0f04afefe1a43a1a`, resolved with the GitHub commits API +for `docs/source/internals.md` and read through the contents API at that SHA. + +- [F: FlatBuffer Internals](https://github.com/google/flatbuffers/blob/569e6cb461738f0aefccbada0f04afefe1a43a1a/docs/source/internals.md): + Format components, Offsets, Structs, Tables, Strings and Vectors, Encoding example. + Only the FlatBuffers portion applies, not the separate FlexBuffers format. +- [S: Schemas](https://github.com/google/flatbuffers/blob/569e6cb461738f0aefccbada0f04afefe1a43a1a/docs/source/schema.md): + UTF-8 strings and four-character file identifiers at bytes 4–7. + +`MONSTER` and `PREFIXED` predate this inventory and record their producer as the +official Python builder, PyPI `flatbuffers` 25.12.19. They are interoperability +fixtures, not normative byte layouts or newly regenerated upstream artifacts. +F explicitly allows different valid layouts for identical values. + +## Assertion-to-test map + +All exact test names below are under `describe("FlatBuffers")` in +`FlatBuffers.test.ts`. IDs are stable and appear beside their tests. +**N** denotes a normative format assertion, **I** a particular interoperability +fixture, and **P** a local API/safety policy (not an upstream requirement). + +| ID | Exact test name | Assertion and source / boundary | +| --------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| FB-I01 | builds the bytes the official builder builds | Exact `MONSTER` and size-prefixed `PREFIXED` byte equality with the recorded Python producer, including its field ordering, padding, sharing and default handling. Not a canonical-layout requirement. | +| FB-N01 / FB-I02 | reads the official builder's bytes | F: reads root-relative offsets, inline float32 structs, absent/default int16, present int16, strings, byte vectors, offset-vector tables, padded signed integer structs, and bool from `MONSTER`. S: reads `MONS` identifier. I02: also reads size-prefixed `ABCD`, its string, and explicitly stored scalar default from `PREFIXED`; prefix framing is interoperability coverage here. | +| FB-N02 | reads absent fields as their defaults | F Tables: slots beyond the trimmed four-byte vtable return caller defaults. Zero-entry defaults are exercised by FB-N01 and FB-N03. Undefined reference results are the local API representation of absence. | +| FB-N03 | reads the specification encoding example | F Encoding example: independently transcribed wire bytes yield inline IEEE-754 little-endian floats `[1, 2, 3]`, absent mana/default 150, hp 50 and string `fred`. No local builder supplies the oracle. | +| FB-N04 | reads a vtable after its table | F Tables: a hand-authored root at 4 with signed vtable displacement -8 resolves the vtable at 12, reads int32 value 42, and returns default 7 for an absent later slot. | +| FB-N05 | reads independently encoded binary64 and unsigned scalars | F Format components: hand-authored little-endian binary64 bytes decode to 1.5, positive/negative zero, positive/negative infinity and NaN; uint32 and uint64 preserve high-bit values and their unsigned maxima. Signed zero is distinguished; NaN payload preservation is not asserted. No local builder supplies the oracle. | +| FB-P01 | decode returns a failure for malformed input | Local `OutOfBounds`, `InvalidVtable`, `InvalidUtf8`, and `Identifier` classification for the listed malformed fixtures. S requires UTF-8, but does not prescribe these errors or lazy validation. | +| FB-P02 | decode rethrows errors that aren't FlatBuffersErrors | Local propagation of an arbitrary callback exception. | +| FB-P03 | only fails with FlatBuffersError on truncated or corrupted buffers | Every truncation before the final string terminator fails the selected traversal; every single-byte XOR corruption completes or reports a FlatBuffersError rather than leaking another error. Does not assert every corruption is invalid. | +| FB-P04 | fails tables nested deeper than maxDepth | Local default depth limit rejects a 65-table chain; configured limit 65 reads it. | +| FB-P05 | throws for reads that don't match the shape asked for, even inside decode | Local Usage errors for mismatched vector width, zero width, out-of-range/negative/NaN struct reads, invalid slots and identifier arguments. | +| FB-P06 | keeps UTF-8 strings and identifiers intact, byte order mark included | Local BOM preservation in strings and a four-UTF-8-byte identifier. This non-ASCII identifier is not evidence of S's four-character identifier requirement. | +| FB-P07 | forceDefaults writes scalar defaults but leaves absent references absent | Local builder option keeps the default-valued scalar slot present while zero reference offsets remain absent; this test checks presence, not the stored scalar value. | +| FB-P08 | Builder rejects misuse | Local Usage errors for nesting/state/finish misuse, non-inline structs, invalid offsets, slots, identifier length, sizes and alignments. Includes rejecting 64-KiB inline data / excessive slot count that exceed F's uint16 vtable representation. | + +## Exclusions and gaps + +- No full-buffer verifier claim: traversal checks only requested fields. The + truncation test deliberately does not require the final string NUL to be read. + F requires NUL-terminated strings; writer termination is covered by fixture + parity, but reader rejection of a missing/nonzero terminator is not asserted. +- Scalar coverage remains selective: FB-N05 covers binary64 special values and + high-bit unsigned values, not every scalar-width edge value or NaN payload. +- Existing fixtures exercise some sharing, padding and vector layouts; they do + not exhaust alternate field orders, all alignments, empty vectors/strings, + repeated-offset aliases or all valid table/vtable sharing arrangements. +- No schema compiler, schema evolution suite, union discriminator validation, + required/key fields, sorted-vector lookup, reflection, 64-bit offsets or + FlexBuffers coverage. These are not inferred from low-level table reads. +- Size-prefixed framing is mapped only to the recorded producer fixture, not + elevated to a requirement from the pinned format document. No broad cross-language + or producer-version compatibility claim is made. + +## Validation + +Run from the repository root: + +```sh +nix develop -c pnpm test --run packages/effect/test/encoding/FlatBuffers.test.ts +``` + +The suite retains all existing regressions and adds only three independent +format-reader examples (FB-N03, FB-N04 and FB-N05). No production behavior is changed. diff --git a/packages/effect/test/encoding/FlatBuffers.test.ts b/packages/effect/test/encoding/FlatBuffers.test.ts new file mode 100644 index 00000000000..d91eb54a8b3 --- /dev/null +++ b/packages/effect/test/encoding/FlatBuffers.test.ts @@ -0,0 +1,389 @@ +import { assert, describe, it } from "@effect/vitest" +import { throws } from "@effect/vitest/utils" +import { Result } from "effect" +import * as FlatBuffers from "effect/encoding/FlatBuffers" +import * as Hex from "effect/encoding/Hex" + +// Assertion IDs and immutable source links: ./FlatBuffers.conformance.md +const fromHex = (hex: string) => Result.getOrThrow(Hex.decode(hex)) + +// The expected bytes were written by the official FlatBuffers Python builder +// (`flatbuffers` 25.12.19 from PyPI) making the same calls as the builders below. +const MONSTER = "1c0000004d4f4e53140028001c0000001a00140010000c00080007001400000000000001200000004400000078000000" + + "8000000000002c010000803f00000040000040400200000001000000020000000300000000000000ff000000feffffff" + + "fdffffffffffffff00000000020000001c00000004000000f4ffffff000005001800000008000c000800060008000000" + + "00000300040000000500000053776f7264000000050000000001020304000000030000004f726300" +const PREFIXED = "4c000000100000004142434408000c000800040008000000070000000400000028000000787878787878787878787878" + + "7878787878787878787878787878787878787878787878787878787800000000" + +// struct { a: byte; b: int; c: long }, as flatc-generated code writes it. +const addPadded = (b: FlatBuffers.Builder, a: number, x: number, y: bigint) => { + b.prep(8, 16) + b.addInt64(y) + b.addInt32(x) + b.pad(3) + b.addInt8(a) + return b.offset +} + +const buildMonster = () => { + const b = FlatBuffers.makeBuilder() + const name = b.createString("Orc") + const inventory = b.createBytes(Uint8Array.of(0, 1, 2, 3, 4)) + const weapons = [3, 5].map((damage) => { + const sword = b.createSharedString("Sword") + b.startTable(2) + b.addFieldOffset(0, sword) + b.addFieldInt16(1, damage) + return b.endTable() + }) + const weaponVector = b.createOffsetVector(weapons) + b.startVector(16, 2, 8) + addPadded(b, -1, -2, BigInt(-3)) + addPadded(b, 1, 2, BigInt(3)) + const path = b.endVector() + b.startTable(8) + b.prep(4, 12) + b.addFloat32(3) + b.addFloat32(2) + b.addFloat32(1) + b.addFieldStruct(0, b.offset) + b.addFieldInt16(1, 100, 100) + b.addFieldInt16(2, 300, 100) + b.addFieldOffset(3, name) + b.addFieldOffset(4, inventory) + b.addFieldOffset(5, weaponVector) + b.addFieldOffset(6, path) + b.addFieldBool(7, true) + return b.finish(b.endTable(), { identifier: "MONS" }) +} + +const buildPrefixed = () => { + const b = FlatBuffers.makeBuilder({ initialSize: 1, forceDefaults: true }) + const s = b.createString("x".repeat(40)) + b.startTable(2) + b.addFieldOffset(0, s) + b.addFieldInt32(1, 7, 7) + return b.finish(b.endTable(), { identifier: "ABCD", sizePrefixed: true }) +} + +const readMonster = (root: FlatBuffers.Table) => { + const pos = root.struct(0, 12) + const inventory = root.bytes(4) + const weapons = root.vector(5, 4) + const path = root.vector(6, 16) + return { + pos: pos && [pos.float32(0), pos.float32(4), pos.float32(8)], + mana: [root.has(1), root.int16(1, 100)], + hp: root.int16(2, 100), + name: root.string(3), + inventory: inventory && Array.from(inventory), + weapons: weapons && Array.from({ length: weapons.length }, (_, i) => { + const weapon = weapons.table(i) + return [weapon.string(0), weapon.int16(1)] + }), + path: path && Array.from({ length: path.length }, (_, i) => { + const s = path.struct(i) + return [s.int8(0), s.int32(4), s.int64(8)] + }), + flag: root.bool(7) + } +} + +const usage = (f: () => unknown) => + throws(f, (error) => { + assert.strictEqual(FlatBuffers.isFlatBuffersError(error) && error.kind, "Usage") + }) + +describe("FlatBuffers", () => { + // FB-N03: internals.md (Encoding example), transcribed independently of our builder. + it("reads the specification encoding example", () => { + const bytes = fromHex( + "1400000010001600040000001400100000000000" + + "100000000000803f00000040000040400800000032000000" + + "040000006672656400000000" + ) + const root = FlatBuffers.root(bytes) + const pos = root.struct(0, 12)! + assert.deepStrictEqual([pos.float32(0), pos.float32(4), pos.float32(8)], [1, 2, 3]) + assert.isFalse(root.has(1)) + assert.strictEqual(root.int16(1, 150), 150) + assert.strictEqual(root.int16(2), 50) + assert.strictEqual(root.string(3), "fred") + }) + + // FB-N04: internals.md (Tables): subtract the signed offset; vtables may follow objects. + it("reads a vtable after its table", () => { + // Root at 4; table contains -8 and value 42; vtable at 12 has size 6, object size 8, field offset 4. + const root = FlatBuffers.root(fromHex("04000000f8ffffff2a000000060008000400")) + assert.isTrue(root.has(0)) + assert.strictEqual(root.int32(0), 42) + assert.strictEqual(root.int32(1, 7), 7) + }) + + // FB-N05: internals.md (Format components): little-endian IEEE-754 and unsigned scalars. + it("reads independently encoded binary64 and unsigned scalars", () => { + // Root at 16; vtable at 4; fields at table offsets 8 (double), 16 (ulong), 4 (uint). + // All wire values are hand-authored, not produced by the local builder. + const header = "100000000a00180008001000040000000c000000" + const cases = [ + { + double: "000000000000f83f", + expected: 1.5, + uint: "00000080", + ulong: "0000000000000080", + u32: 2147483648, + u64: 9223372036854775808n + }, + { + double: "0000000000000000", + expected: 0, + uint: "ffffffff", + ulong: "ffffffffffffffff", + u32: 4294967295, + u64: 18446744073709551615n + }, + { + double: "0000000000000080", + expected: -0, + uint: "01000080", + ulong: "0100000000000080", + u32: 2147483649, + u64: 9223372036854775809n + }, + { + double: "000000000000f07f", + expected: Infinity, + uint: "00000080", + ulong: "0000000000000080", + u32: 2147483648, + u64: 9223372036854775808n + }, + { + double: "000000000000f0ff", + expected: -Infinity, + uint: "ffffffff", + ulong: "ffffffffffffffff", + u32: 4294967295, + u64: 18446744073709551615n + }, + { + double: "000000000000f87f", + expected: NaN, + uint: "01000080", + ulong: "0100000000000080", + u32: 2147483649, + u64: 9223372036854775809n + } + ] + for (const row of cases) { + const root = FlatBuffers.root(fromHex(header + row.uint + row.double + row.ulong)) + // Object.is distinguishes signed zero and compares NaN without asserting a NaN payload. + assert.isTrue(Object.is(root.float64(0), row.expected), row.double) + assert.strictEqual(root.uint64(1), row.u64) + assert.strictEqual(root.uint32(2), row.u32) + } + }) + + // FB-I01: Python-builder byte parity; not a canonical wire layout. + it("builds the bytes the official builder builds", () => { + assert.strictEqual(Hex.encode(buildMonster()), MONSTER) + assert.strictEqual(Hex.encode(buildPrefixed()), PREFIXED) + }) + + // FB-N01 / FB-I02: internals.md (Offsets, Structs, Tables, Strings and Vectors); schema.md (identifiers). + it("reads the official builder's bytes", () => { + const monster = fromHex(MONSTER) + assert.strictEqual(FlatBuffers.identifier(monster), "MONS") + assert.deepStrictEqual(readMonster(FlatBuffers.root(monster, { identifier: "MONS" })), { + pos: [1, 2, 3], + mana: [false, 100], + hp: 300, + name: "Orc", + inventory: [0, 1, 2, 3, 4], + weapons: [["Sword", 3], ["Sword", 5]], + path: [[1, 2, BigInt(3)], [-1, -2, BigInt(-3)]], + flag: true + }) + + const prefixed = fromHex(PREFIXED) + assert.strictEqual(FlatBuffers.identifier(prefixed, { sizePrefixed: true }), "ABCD") + const root = FlatBuffers.root(prefixed, { identifier: "ABCD", sizePrefixed: true }) + assert.strictEqual(root.string(0), "x".repeat(40)) + assert.isTrue(root.has(1)) + assert.strictEqual(root.int32(1), 7) + }) + + // FB-N02: internals.md (Tables): slots beyond the trimmed vtable mean absent. + it("reads absent fields as their defaults", () => { + const b = FlatBuffers.makeBuilder() + b.startTable(1) + const root = FlatBuffers.root(b.finish(b.endTable())) + assert.strictEqual(root.int32(0, 7), 7) + assert.strictEqual(root.string(0), undefined) + assert.strictEqual(root.table(0), undefined) + assert.strictEqual(root.vector(0, 1), undefined) + // Slots past the vtable, as written by an older schema, are absent too. + assert.strictEqual(root.float64(100, 1.5), 1.5) + }) + + // FB-P01: local error classification; schema.md requires UTF-8, not this error API. + it("decode returns a failure for malformed input", () => { + const failure = (bytes: Uint8Array, options?: Parameters[2]) => { + const result = FlatBuffers.decode(bytes, (root) => root.string(0), options) + return Result.isFailure(result) ? result.failure.kind : undefined + } + assert.strictEqual(failure(new Uint8Array(2)), "OutOfBounds") + assert.strictEqual(failure(Uint8Array.of(0xff, 0xff, 0xff, 0x7f)), "OutOfBounds") + // A vtable of size 1. + assert.strictEqual(failure(Uint8Array.of(8, 0, 0, 0, 1, 0, 4, 0, 4, 0, 0, 0)), "InvalidVtable") + // A string field with an invalid UTF-8 byte. + const b = FlatBuffers.makeBuilder() + const s = b.createString(Uint8Array.of(0xc3, 0x28)) + b.startTable(1) + b.addFieldOffset(0, s) + assert.strictEqual(failure(b.finish(b.endTable())), "InvalidUtf8") + assert.strictEqual(failure(fromHex(MONSTER), { identifier: "NOPE" }), "Identifier") + assert.strictEqual(failure(Uint8Array.of(0xff, 0, 0, 0, 0, 0, 0, 0), { sizePrefixed: true }), "OutOfBounds") + }) + + // FB-P02: local callback error propagation. + it("decode rethrows errors that aren't FlatBuffersErrors", () => { + assert.throws(() => + FlatBuffers.decode(fromHex(MONSTER), () => { + throw new Error("boom") + }), "boom") + }) + + // FB-P03: local bounded-read safety, not full-buffer verification. + it("only fails with FlatBuffersError on truncated or corrupted buffers", () => { + const monster = fromHex(MONSTER) + for (let i = 0; i < monster.length; i++) { + // Only the last byte, the NUL after "Orc", is not needed to read the buffer. + if (i < monster.length - 1) { + assert.isTrue(Result.isFailure(FlatBuffers.decode(monster.subarray(0, i), readMonster)), `truncated at ${i}`) + } + const corrupted = monster.slice() + corrupted[i] ^= 0xff + // decode rethrows anything that isn't a FlatBuffersError. + FlatBuffers.decode(corrupted, readMonster) + } + }) + + // FB-P04: local configurable traversal limit. + it("fails tables nested deeper than maxDepth", () => { + const b = FlatBuffers.makeBuilder() + let table = 0 + for (let i = 0; i < 65; i++) { + b.startTable(1) + b.addFieldOffset(0, table) + table = b.endTable() + } + const bytes = b.finish(table) + const depth = (t: FlatBuffers.Table): number => { + const child = t.table(0) + return child === undefined ? 1 : 1 + depth(child) + } + const result = FlatBuffers.decode(bytes, depth) + assert.isTrue(Result.isFailure(result) && result.failure.kind === "Limit") + assert.deepStrictEqual(FlatBuffers.decode(bytes, depth, { maxDepth: 65 }), Result.succeed(65)) + }) + + // FB-P05: local shape/argument validation. + it("throws for reads that don't match the shape asked for, even inside decode", () => { + const monster = fromHex(MONSTER) + const root = FlatBuffers.root(monster) + usage(() => FlatBuffers.decode(monster, (root) => root.vector(4, 1)!.int32(0))) + usage(() => root.vector(4, 0)) + usage(() => root.struct(0, 12)!.float32(12)) + usage(() => root.struct(0, 12)!.float32(-4)) + usage(() => root.struct(0, 12)!.float32(NaN)) + usage(() => root.int32(-1)) + usage(() => root.has(0.5)) + usage(() => FlatBuffers.decode(monster, (root) => root.has(0), { identifier: "TOOLONG" })) + }) + + // FB-P06: local BOM preservation, including a four-byte non-ASCII identifier. + it("keeps UTF-8 strings and identifiers intact, byte order mark included", () => { + const b = FlatBuffers.makeBuilder() + const s = b.createString("\ufeffhello") + b.startTable(1) + b.addFieldOffset(0, s) + const bytes = b.finish(b.endTable(), { identifier: "\ufeffA" }) + assert.strictEqual(FlatBuffers.root(bytes).string(0), "\ufeffhello") + assert.strictEqual(FlatBuffers.identifier(bytes), "\ufeffA") + }) + + // FB-P07: local forceDefaults option; absent-reference behavior. + it("forceDefaults writes scalar defaults but leaves absent references absent", () => { + const b = FlatBuffers.makeBuilder({ forceDefaults: true }) + b.startTable(2) + b.addFieldOffset(0, 0) + b.addFieldInt32(1, 0) + const root = FlatBuffers.root(b.finish(b.endTable())) + assert.isFalse(root.has(0)) + assert.strictEqual(root.string(0), undefined) + assert.isTrue(root.has(1)) + }) + + // FB-P08: local builder state/argument validation (includes 16-bit vtable capacity). + it("Builder rejects misuse", () => { + const builder = (f: (b: FlatBuffers.Builder) => void) => usage(() => f(FlatBuffers.makeBuilder())) + builder((b) => { + b.startTable(1) + b.createString("x") + }) + builder((b) => { + b.startTable(1) + b.startTable(1) + }) + builder((b) => { + b.startVector(1, 1, 1) + b.startTable(1) + }) + builder((b) => b.endTable()) + builder((b) => b.endVector()) + builder((b) => { + b.startTable(1) + b.addFieldInt32(1, 5) + }) + builder((b) => b.addFieldInt32(0, 5)) + builder((b) => { + b.addInt32(1) + const s = b.offset + b.addInt32(2) + b.startTable(1) + b.addFieldStruct(0, s) + }) + builder((b) => { + b.startTable(0) + b.finish(b.endTable(), { identifier: "TOOLONG" }) + }) + builder((b) => b.addOffset(100)) + builder((b) => b.startTable(32766)) + builder((b) => { + // 64 KiB of inline data is more than a vtable entry can address. + b.startTable(1) + b.pad(0x10000) + b.addFieldInt32(0, 1) + b.endTable() + }) + builder((b) => { + b.startTable(0) + const table = b.endTable() + b.finish(table) + b.finish(table) + }) + builder((b) => b.startVector(1, 1, 3)) + builder((b) => b.startVector(1, 1, 1.5)) + builder((b) => b.startVector(1, -1, 1)) + builder((b) => b.startVector(1, 0.5, 1)) + builder((b) => { + b.startTable(1) + b.addFieldInt32(0.5, 111) + }) + for (const initialSize of [0, 1.5, NaN, Infinity, 2 ** 31 + 1]) { + usage(() => FlatBuffers.makeBuilder({ initialSize })) + } + }) +})