diff --git a/.changeset/encoding-thrift.md b/.changeset/encoding-thrift.md new file mode 100644 index 00000000000..f5e20a9605a --- /dev/null +++ b/.changeset/encoding-thrift.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add the experimental `effect/encoding/Thrift` module, which reads and writes the Apache Thrift compact protocol without an IDL. diff --git a/dprint.json b/dprint.json index 0622d2e2642..d8e26de0455 100644 --- a/dprint.json +++ b/dprint.json @@ -22,6 +22,7 @@ "**/CHANGELOG.md", "packages/effect/src/StandardSchema.ts", "packages/tools/graphql-generator/test/generated/**/*.graphql.ts", + "packages/effect/test/fixtures/thrift/gold.json", "!scratchpad/**/*", "**/.*", "!.agents", diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts new file mode 100644 index 00000000000..f424e5cf906 --- /dev/null +++ b/packages/effect/src/encoding/Thrift.ts @@ -0,0 +1,1125 @@ +/** + * Reads and writes the Apache Thrift compact protocol: structs, lists, sets, + * maps and scalars, without an IDL. + * + * **Mental model** + * + * - A Thrift struct is a sequence of fields, each a numbered value of some + * {@link Type}, ended by a stop marker. Lists, sets and maps hold values of + * one element type. Nothing on the wire names a field: the IDL that says + * field 3 is `num_rows` lives with the caller. + * - This module works at the protocol level, as generated Thrift code does. + * A {@link Reader} is a cursor over the bytes: read a field header, look at + * its id and type, read the value or {@link Reader.skip} it. A + * {@link Writer} mirrors those calls. + * - The method names and their meaning follow Thrift's `TProtocol`, so code + * written against another Thrift implementation reads the same: a struct + * is `readStructBegin`, then `readFieldBegin` until it returns + * `Type.Stop`, then `readStructEnd`. + * - {@link Reader.readValue} and {@link Writer.writeValue} read and write any + * value as a {@link Value} tree, for debugging and tests. + * + * **Example** (Writing and reading a struct) + * + * ```ts import.meta.vitest + * import { Thrift } from "effect/encoding" + * import { Result } from "effect" + * + * // struct Person { 1: string name; 2: i64 age; 3: list tags } + * const w = Thrift.makeWriter() + * w.writeStructBegin() + * w.writeFieldBegin(Thrift.Type.Binary, 1) + * w.writeString("Ada") + * w.writeFieldBegin(Thrift.Type.I64, 2) + * w.writeI64Number(36) + * w.writeFieldBegin(Thrift.Type.List, 3) + * w.writeListBegin(Thrift.Type.Binary, 1) + * w.writeString("math") + * w.writeFieldStop() + * w.writeStructEnd() + * const bytes = w.finish() + * + * const person = Thrift.decode(bytes, (r) => { + * let name = "" + * let age = 0 + * const tags: Array = [] + * r.readStructBegin() + * for (let type = r.readFieldBegin(); type !== Thrift.Type.Stop; type = r.readFieldBegin()) { + * const id = r.fieldId + * if (id === 1 && type === Thrift.Type.Binary) name = r.readString() + * else if (id === 2 && type === Thrift.Type.I64) age = r.readI64Number() + * else if (id === 3 && type === Thrift.Type.List) { + * for (let n = r.readListBegin(); n > 0; n--) tags.push(r.readString()) + * } else r.skip(type) // unknown fields, or fields of an unexpected type + * } + * r.readStructEnd() + * return { name, age, tags } + * }) + * + * Result.getOrThrow(person) // => { name: "Ada", age: 36, tags: ["math"] } + * ``` + * + * **Details** + * + * Every read is bounds-checked: malformed or hostile input throws a + * {@link ThriftError} (or makes {@link decode} fail) instead of reading past + * the bytes. A length or container size larger than the bytes that remain + * fails before anything is allocated, varints are capped at the width of + * their type, and nesting is limited by {@link ReaderOptions.maxDepth}. + * Strings must be valid UTF-8. + * + * Binary and `uuid` values, also inside a {@link Value}, are views into the + * input, not copies: they change when the input does and keep its buffer + * alive. Copy one with `slice()` to keep it on its own. + * + * `i64` values read as `bigint` ({@link Reader.readI64}) or, when they are + * known to be safe integers such as offsets and counts, as `number` + * ({@link Reader.readI64Number}). + * + * The binary and JSON protocols and the RPC message envelope are not + * supported. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Data from "../Data.ts" +import * as Utf8 from "../internal/utf8.ts" +import * as Predicate from "../Predicate.ts" +import * as Result from "../Result.ts" + +const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" + +/** + * Error raised for malformed input or misuse of a {@link Reader} or + * {@link Writer}. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `Truncated`: the input ends inside a value, or a length or container + * size is larger than the bytes that remain. + * - `Malformed`: an unknown type code, a varint wider than its type (16 + * bits for `i16` and field ids, 32 for `i32` and sizes, 64 for `i64`), a + * field id over 32767, or a non-empty container of `Stop`. + * - `InvalidUtf8`: a string is not valid UTF-8. + * - `Range`: an `i64` read with {@link Reader.readI64Number} is not a safe + * integer. + * - `Limit`: the input nests deeper than {@link ReaderOptions.maxDepth}, or a + * length or size exceeds {@link ReaderOptions.maxBinaryLength} or + * {@link ReaderOptions.maxContainerSize}. + * - `Usage`: the reader or writer was called out of order or with an + * invalid argument or option, or a written value is out of range for its + * type. {@link decode} rethrows these caller errors. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class ThriftError extends Data.TaggedError("ThriftError")<{ + readonly kind: "Truncated" | "Malformed" | "InvalidUtf8" | "Range" | "Limit" | "Usage" + readonly message: string +}> { + /** + * Marks this value as a Thrift error for runtime guards. + * + * @stability experimental + * @since 4.1.5 + */ + readonly [ThriftErrorTypeId] = ThriftErrorTypeId +} + +/** + * Checks whether a value is a {@link ThriftError}. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isThriftError = (u: unknown): u is ThriftError => Predicate.hasProperty(u, ThriftErrorTypeId) + +const fail = (kind: ThriftError["kind"], message: string): never => { + throw new ThriftError({ kind, message }) +} + +const checkInteger = (what: string, value: number, min: number, max: number): number => { + if (!Number.isInteger(value) || value < min || value > max) { + fail("Usage", `${what} ${value} is not an integer from ${min} to ${max}`) + } + return value +} + +const integerOption = (name: string, value: number | undefined, min: number, max: number, fallback: number): number => + value === undefined ? fallback : checkInteger(name, value, min, max) + +/** + * The type of a field or container element. The codes are Thrift's `TType` + * codes, the same in every protocol; `Stop` ends a struct. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export const Type = { + Stop: 0, + Bool: 2, + I8: 3, + Double: 4, + I16: 6, + I32: 8, + I64: 10, + Binary: 11, + Struct: 12, + Map: 13, + Set: 14, + List: 15, + Uuid: 16 +} as const + +/** + * A {@link Type} code. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Type = typeof Type[keyof typeof Type] + +/** + * The name of a {@link Type}, as used by {@link Value}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TypeName = keyof typeof Type + +/** Compact type nibble → TType. Nibbles 1 and 2 are a bool field's true and false; 14 and 15 are unassigned. */ +const fromCompact: ReadonlyArray = [0, 2, 2, 3, 6, 8, 10, 4, 11, 15, 14, 13, 12, 16] + +/** TType → compact type nibble, with `Bool` as 1. */ +const toCompact: Array = [] +fromCompact.forEach((type, nibble) => { + if (nibble !== 2) toCompact[type] = nibble +}) + +const typeNames: Array = [] +for (const name of Object.keys(Type) as Array) typeNames[Type[name]] = name + +const describe = (type: number): string => typeNames[type] ?? String(type) + +/** + * Any Thrift value, tagged with the name of its {@link Type}: what + * {@link Reader.readValue} returns and {@link Writer.writeValue} writes. + * + * **Details** + * + * The wire does not tell strings from binary, so both read as `Binary`. An + * empty map carries no key and value types on the wire; they read as + * `"Stop"`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Value = + | { readonly _tag: "Bool"; readonly value: boolean } + | { readonly _tag: "I8" | "I16" | "I32" | "Double"; readonly value: number } + | { readonly _tag: "I64"; readonly value: bigint } + | { readonly _tag: "Binary" | "Uuid"; readonly value: Uint8Array } + | { readonly _tag: "Struct"; readonly fields: ReadonlyArray } + | { readonly _tag: "List" | "Set"; readonly elementType: TypeName; readonly elements: ReadonlyArray } + | { + readonly _tag: "Map" + readonly keyType: TypeName + readonly valueType: TypeName + readonly entries: ReadonlyArray + } + +/** + * A field of a `Struct` {@link Value}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Field { + readonly id: number + readonly value: Value +} + +/** + * Limits for reading untrusted input. A value that isn't an integer in range + * throws a {@link ThriftError} of kind `Usage`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ReaderOptions { + /** + * How deep structs and containers may nest, counting each struct and each + * container entered by {@link Reader.readStructBegin}, + * {@link Reader.skip} or {@link Reader.readValue}. An integer from 1 to + * 1000, which keeps their recursion within the stack; defaults to 64. + */ + readonly maxDepth?: number | undefined + /** The longest binary or string value, in bytes: a non-negative integer. No limit by default. */ + readonly maxBinaryLength?: number | undefined + /** The largest list, set or map: a non-negative integer. No limit by default. */ + readonly maxContainerSize?: number | undefined +} + +const MAX_SAFE = Number.MAX_SAFE_INTEGER +const BIGINT_1 = BigInt(1) +const BIGINT_7 = BigInt(7) +const BIGINT_63 = BigInt(63) +const BIGINT_0x7f = BigInt(0x7f) + +const zigzag = (n: number): number => ((n << 1) ^ (n >> 31)) >>> 0 +const unzigzag = (n: number): number => (n >>> 1) ^ -(n & 1) + +/** + * A cursor over compact-protocol bytes. Each `read*` method reads one value + * at {@link position} and moves past it, or throws a {@link ThriftError} on + * malformed input. + * + * **Gotchas** + * + * - A nested read overwrites {@link fieldId}, {@link elementType}, + * {@link keyType} and {@link valueType}: copy one into a local first. + * - {@link readBinary}, {@link readUuid} and the binary values of + * {@link readValue} are views of {@link bytes}: copy them before mutating + * or reusing the input. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export class Reader { + /** The input. */ + readonly bytes: Uint8Array + /** The id of the field whose header was read last. */ + fieldId = 0 + /** The element type of the list or set whose header was read last. */ + elementType: Type = Type.Stop + /** The key type of the map whose header was read last. */ + keyType: Type = Type.Stop + /** The value type of the map whose header was read last. */ + valueType: Type = Type.Stop + + private pos = 0 + private readonly view: DataView + private readonly end: number + private readonly maxDepth: number + private readonly maxBinaryLength: number + private readonly maxContainerSize: number + private lastFieldId = 0 + private readonly fieldIds: Array = [] + private depth = 0 + /** The compact type of a bool field header not yet read: 1 true, 2 false, 0 none. */ + private bool = 0 + /** Set by {@link varint64}: whether the zigzag value is negative. */ + private negative = false + + constructor(bytes: Uint8Array, options?: ReaderOptions) { + this.bytes = bytes + this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + this.end = bytes.length + this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 1000, 64) + this.maxBinaryLength = integerOption("maxBinaryLength", options?.maxBinaryLength, 0, Infinity, Infinity) + this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity, Infinity) + } + + /** + * Where the next read starts in {@link bytes}. Set it to read from + * elsewhere; a position outside the input is a `Usage` error, so check + * offsets read from the input first. + */ + get position(): number { + return this.pos + } + set position(position: number) { + this.pos = checkInteger("position", position, 0, this.end) + this.bool = 0 + } + + private truncated(what: string): never { + return fail("Truncated", `the input ends inside ${what} at byte ${this.pos}`) + } + + private byte(what: string): number { + if (this.pos >= this.end) this.truncated(what) + return this.bytes[this.pos++] + } + + private advance(n: number, what: string): number { + const at = this.pos + if (n > this.end - at) this.truncated(what) + this.pos = at + n + return at + } + + /** An unsigned varint of at most 32 bits. */ + private varint32(what: string): number { + const bytes = this.bytes + const end = this.end + let pos = this.pos + if (pos < end) { + let b = bytes[pos++] + if (b < 0x80) { + this.pos = pos + return b + } + let result = b & 0x7f + for (let shift = 7; shift < 28; shift += 7) { + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + result |= (b & 0x7f) << shift + if (b < 0x80) { + this.pos = pos + return result + } + } + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + if (b > 0x0f) return fail("Malformed", `${what} at byte ${this.pos} is longer than 32 bits`) + this.pos = pos + return (result | (b << 28)) >>> 0 + } + return this.truncated(what) + } + + /** A zigzag varint of at most 16 bits. */ + private varint16(what: string): number { + const n = this.varint32(what) + if (n > 0xffff) fail("Malformed", `${what} before byte ${this.pos} is longer than 16 bits`) + return unzigzag(n) + } + + /** + * A zigzag varint of at most 64 bits as a number: returns `v` for v >= 0 or + * `-v - 1` for v < 0 (exact below 2^53) and sets {@link negative}. + */ + private varint64(what: string): number { + const bytes = this.bytes + const end = this.end + const start = this.pos + let pos = start + if (pos >= end) return this.truncated(what) + let b = bytes[pos++] + this.negative = (b & 1) === 1 + let half = (b & 0x7f) >>> 1 + let scale = 64 + while (b >= 0x80) { + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + if (pos - start === 10 && b > 0x01) { + return fail("Malformed", `${what} at byte ${start} is longer than 64 bits`) + } + half += (b & 0x7f) * scale + scale *= 128 + } + this.pos = pos + return half + } + + /** Checks a container size; every element takes at least `bytesPerElement` bytes. */ + private checkSize(size: number, bytesPerElement: number, what: string): void { + if (size > this.maxContainerSize) { + fail("Limit", `${what} of ${size} elements exceeds the limit of ${this.maxContainerSize}`) + } + if (size * bytesPerElement > this.end - this.pos) { + fail("Truncated", `${what} of ${size} elements at byte ${this.pos} is larger than the remaining input`) + } + } + + /** Reads a binary length and moves past the bytes; returns where they start. */ + private binary(what: string): number { + const length = this.varint32(`${what} length`) + if (length > this.maxBinaryLength) { + fail("Limit", `${what} of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) + } + return this.advance(length, `${what} of ${length} bytes`) + } + + private listHeader(what: string): number { + const header = this.byte(`${what} header`) + let size = header >> 4 + if (size === 15) size = this.varint32(`${what} size`) + this.elementType = this.elementOf(header & 0x0f, size, what) + this.checkSize(size, 1, what) + return size + } + + private enter(): void { + if (++this.depth > this.maxDepth) { + fail("Limit", `the input nests deeper than ${this.maxDepth} at byte ${this.pos}`) + } + } + + private elementOf(nibble: number, size: number, what: string): Type { + const type = fromCompact[nibble] + if (type === undefined || (type === Type.Stop && size > 0)) { + return fail("Malformed", `${what} at byte ${this.pos} has element type code ${nibble}`) + } + return type + } + + /** Starts reading a struct. Read its fields with {@link readFieldBegin}. */ + readStructBegin(): void { + this.enter() + this.fieldIds.push(this.lastFieldId) + this.lastFieldId = 0 + } + + /** Finishes the struct started with {@link readStructBegin}, after its `Stop`. */ + readStructEnd(): void { + if (this.fieldIds.length === 0) fail("Usage", "readStructEnd without readStructBegin") + this.depth-- + this.lastFieldId = this.fieldIds.pop()! + } + + /** + * Reads a field header: returns the field's type and sets {@link fieldId}. + * Returns `Type.Stop` at the end of the struct. + */ + readFieldBegin(): Type { + const header = this.byte("a field header") + this.bool = 0 + const nibble = header & 0x0f + if (nibble === 0) return Type.Stop + const delta = header >> 4 + let id: number + if (delta === 0) { + id = this.varint16("a field id") + } else { + id = this.lastFieldId + delta + if (id > 32767) fail("Malformed", `the field header at byte ${this.pos - 1} has id ${id}, over 32767`) + } + this.lastFieldId = id + this.fieldId = id + const type = fromCompact[nibble] + if (type === undefined) { + return fail("Malformed", `the field header at byte ${this.pos - 1} has type code ${nibble}`) + } + if (nibble <= 2) this.bool = nibble + return type + } + + /** Reads a list header: returns the list's size and sets {@link elementType}. */ + readListBegin(): number { + return this.listHeader("a list") + } + + /** Reads a set header: returns the set's size and sets {@link elementType}. */ + readSetBegin(): number { + return this.listHeader("a set") + } + + /** Reads a map header: returns the map's size and sets {@link keyType} and {@link valueType}. */ + readMapBegin(): number { + const size = this.varint32("a map size") + if (size === 0) { + this.keyType = Type.Stop + this.valueType = Type.Stop + return 0 + } + const types = this.byte("a map header") + this.keyType = this.elementOf(types >> 4, size, "a map") + this.valueType = this.elementOf(types & 0x0f, size, "a map") + this.checkSize(size, 2, "a map") + return size + } + + /** Reads a `bool`: the value of a `Bool` field header just read, or a byte inside a container. */ + readBool(): boolean { + const bool = this.bool + if (bool !== 0) { + this.bool = 0 + return bool === 1 + } + return this.byte("a bool") === 1 + } + + /** Reads an `i8` (`byte`). */ + readI8(): number { + return (this.byte("an i8") << 24) >> 24 + } + + /** Reads an `i16`. */ + readI16(): number { + return this.varint16("an i16") + } + + /** Reads an `i32`. */ + readI32(): number { + return unzigzag(this.varint32("an i32")) + } + + /** Reads an `i64` as a `bigint`. */ + readI64(): bigint { + const start = this.pos + const half = this.varint64("an i64") + if (half < MAX_SAFE) return BigInt(this.negative ? -half - 1 : half) + let u = BigInt(0) + let shift = BigInt(0) + for (let pos = start; pos < this.pos; pos++) { + u |= (BigInt(this.bytes[pos]) & BIGINT_0x7f) << shift + shift += BIGINT_7 + } + u = BigInt.asUintN(64, u) + return (u >> BIGINT_1) ^ -(u & BIGINT_1) + } + + /** + * Reads an `i64` as a `number`. Fails with kind `Range` when the value is + * not a safe integer, leaving {@link position} at the value so that + * {@link readI64} can read it. + */ + readI64Number(): number { + const start = this.pos + const half = this.varint64("an i64") + if (half < MAX_SAFE || (half === MAX_SAFE && !this.negative)) return this.negative ? -half - 1 : half + this.pos = start + return fail("Range", `the i64 at byte ${start} is not a safe integer`) + } + + /** Reads a `double` (little-endian, as the compact protocol stores it). */ + readDouble(): number { + return this.view.getFloat64(this.advance(8, "a double"), true) + } + + /** Reads a `binary` value as a view of {@link bytes}, not a copy. */ + readBinary(): Uint8Array { + return this.bytes.subarray(this.binary("a binary value"), this.pos) + } + + /** Reads a `string`: a `binary` value decoded as UTF-8. */ + readString(): string { + const at = this.binary("a string") + return Utf8.decode(this.bytes, at, this.pos) ?? fail("InvalidUtf8", `the string at byte ${at} is not valid UTF-8`) + } + + /** Reads a `uuid`: 16 bytes, as a view of {@link bytes}. */ + readUuid(): Uint8Array { + const at = this.advance(16, "a uuid") + return this.bytes.subarray(at, at + 16) + } + + /** + * Moves past a value of the given type: the value of a field whose header + * was just read, or a container element. + */ + skip(type: Type): void { + switch (type) { + case Type.Bool: + this.readBool() + return + case Type.I8: + this.advance(1, "an i8") + return + case Type.I16: + this.varint16("an i16") + return + case Type.I32: + this.varint32("an i32") + return + case Type.I64: + this.varint64("an i64") + return + case Type.Double: + this.advance(8, "a double") + return + case Type.Binary: + this.binary("a binary value") + return + case Type.Uuid: + this.advance(16, "a uuid") + return + case Type.Struct: + this.readStructBegin() + for (let t = this.readFieldBegin(); t !== Type.Stop; t = this.readFieldBegin()) this.skip(t) + this.readStructEnd() + return + case Type.List: + case Type.Set: { + this.enter() + const size = type === Type.List ? this.readListBegin() : this.readSetBegin() + const element = this.elementType + for (let i = 0; i < size; i++) this.skip(element) + this.depth-- + return + } + case Type.Map: { + this.enter() + const size = this.readMapBegin() + const key = this.keyType + const value = this.valueType + for (let i = 0; i < size; i++) { + this.skip(key) + this.skip(value) + } + this.depth-- + return + } + default: + fail("Usage", `can't skip a value of type ${describe(type)}`) + } + } + + /** + * Reads a value of the given type as a {@link Value}: the value of a field + * whose header was just read, a container element, or a whole input such + * as a struct. + */ + readValue(type: Type): Value { + switch (type) { + case Type.Bool: + return { _tag: "Bool", value: this.readBool() } + case Type.I8: + return { _tag: "I8", value: this.readI8() } + case Type.I16: + return { _tag: "I16", value: this.readI16() } + case Type.I32: + return { _tag: "I32", value: this.readI32() } + case Type.I64: + return { _tag: "I64", value: this.readI64() } + case Type.Double: + return { _tag: "Double", value: this.readDouble() } + case Type.Binary: + return { _tag: "Binary", value: this.readBinary() } + case Type.Uuid: + return { _tag: "Uuid", value: this.readUuid() } + case Type.Struct: { + const fields: Array = [] + this.readStructBegin() + for (let t = this.readFieldBegin(); t !== Type.Stop; t = this.readFieldBegin()) { + const id = this.fieldId + fields.push({ id, value: this.readValue(t) }) + } + this.readStructEnd() + return { _tag: "Struct", fields } + } + case Type.List: + case Type.Set: { + this.enter() + const size = type === Type.List ? this.readListBegin() : this.readSetBegin() + const element = this.elementType + const elements: Array = [] + for (let i = 0; i < size; i++) elements.push(this.readValue(element)) + this.depth-- + return { _tag: type === Type.List ? "List" : "Set", elementType: typeNames[element], elements } + } + case Type.Map: { + this.enter() + const size = this.readMapBegin() + const key = this.keyType + const value = this.valueType + const entries: Array = [] + for (let i = 0; i < size; i++) { + const k = this.readValue(key) + entries.push([k, this.readValue(value)]) + } + this.depth-- + return { _tag: "Map", keyType: typeNames[key], valueType: typeNames[value], entries } + } + default: + return fail("Usage", `can't read a value of type ${describe(type)}`) + } + } +} + +/** + * Creates a {@link Reader} at the start of `bytes`, which it reads in place. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const makeReader = (bytes: Uint8Array, options?: ReaderOptions): Reader => new Reader(bytes, options) + +/** + * Reads `bytes` with `read`, which gets a {@link Reader} at the start of + * them. Malformed input becomes a failed `Result` instead of an exception. + * + * **Gotchas** + * + * `Usage` errors and errors other than {@link ThriftError} are rethrown: + * they are bugs in `read` or invalid options, not bad input. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const decode = ( + bytes: Uint8Array, + read: (reader: Reader) => A, + options?: ReaderOptions +): Result.Result => { + try { + return Result.succeed(read(new Reader(bytes, options))) + } catch (cause) { + if (isThriftError(cause) && cause.kind !== "Usage") return Result.fail(cause) + throw cause + } +} + +const encoder = new TextEncoder() + +/** + * Writes compact-protocol bytes into a buffer that grows as needed. The + * calls mirror {@link Reader}'s. + * + * **Details** + * + * Write a struct as {@link writeStructBegin}, then for each field + * {@link writeFieldBegin} and its value, then {@link writeFieldStop} and + * {@link writeStructEnd}. + * + * Containers are a header ({@link writeListBegin}, {@link writeSetBegin}, + * {@link writeMapBegin}) followed by exactly that many elements (for a map, + * key then value). + * + * Integers outside their type's range, values of no {@link Type}, and calls + * out of order throw a {@link ThriftError} of kind `Usage`. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export class Writer { + private bytes: Uint8Array + private view: DataView + private position = 0 + private lastFieldId = 0 + private readonly fieldIds: Array = [] + private boolPending = false + private boolFieldId = 0 + + constructor(options?: { readonly initialSize?: number | undefined }) { + this.bytes = new Uint8Array(Math.max(16, integerOption("initialSize", options?.initialSize, 0, Infinity, 1024))) + this.view = new DataView(this.bytes.buffer) + } + + /** The number of bytes written. */ + get length(): number { + return this.position + } + + private reserve(n: number): void { + if (this.position + n <= this.bytes.length) return + let size = this.bytes.length * 2 + while (size < this.position + n) size *= 2 + const next = new Uint8Array(size) + next.set(this.bytes.subarray(0, this.position)) + this.bytes = next + this.view = new DataView(next.buffer) + } + + private checkNoBool(what: string): void { + if (this.boolPending) fail("Usage", `${what} after writeFieldBegin(Type.Bool) instead of writeBool`) + } + + private writeByte(b: number): void { + this.reserve(1) + this.bytes[this.position++] = b + } + + /** An unsigned varint of at most 32 bits. */ + private varint32(n: number): void { + this.reserve(5) + const bytes = this.bytes + let pos = this.position + while (n > 0x7f) { + bytes[pos++] = (n & 0x7f) | 0x80 + n >>>= 7 + } + bytes[pos++] = n + this.position = pos + } + + private raw(value: Uint8Array): void { + this.reserve(value.length) + this.bytes.set(value, this.position) + this.position += value.length + } + + private integer(value: number, min: number, max: number, what: string): void { + this.checkNoBool(what) + checkInteger(what, value, min, max) + } + + private elementNibble(type: number, size: number, what: string): number { + const nibble = toCompact[type] + if (nibble === undefined || (type === Type.Stop && size > 0)) { + return fail("Usage", `${describe(type)} is not a valid ${what}`) + } + return nibble + } + + private fieldHeader(nibble: number, id: number): void { + const delta = id - this.lastFieldId + if (delta > 0 && delta <= 15) { + this.writeByte((delta << 4) | nibble) + } else { + this.writeByte(nibble) + this.varint32(zigzag(id)) + } + this.lastFieldId = id + } + + /** Starts a struct. Write its fields, then {@link writeFieldStop} and {@link writeStructEnd}. */ + writeStructBegin(): void { + this.checkNoBool("writeStructBegin") + this.fieldIds.push(this.lastFieldId) + this.lastFieldId = 0 + } + + /** Finishes the struct started with {@link writeStructBegin}. */ + writeStructEnd(): void { + this.checkNoBool("writeStructEnd") + if (this.fieldIds.length === 0) fail("Usage", "writeStructEnd without writeStructBegin") + this.lastFieldId = this.fieldIds.pop()! + } + + /** Writes a field header. For a `Bool` field, the header is written by the {@link writeBool} that follows. */ + writeFieldBegin(type: Type, id: number): void { + this.checkNoBool("writeFieldBegin") + if (this.fieldIds.length === 0) fail("Usage", "writeFieldBegin outside writeStructBegin/writeStructEnd") + checkInteger("field id", id, -32768, 32767) + if (type === Type.Bool) { + this.boolPending = true + this.boolFieldId = id + return + } + const nibble = toCompact[type] + if (nibble === undefined || type === Type.Stop) { + fail("Usage", `${describe(type)} is not a field type`) + } + this.fieldHeader(nibble, id) + } + + /** Writes the `Stop` that ends a struct's fields. */ + writeFieldStop(): void { + this.checkNoBool("writeFieldStop") + this.writeByte(0) + } + + private listHeader(elementType: Type, size: number, method: string): void { + this.checkNoBool(method) + checkInteger("container size", size, 0, 0xffffffff) + const nibble = this.elementNibble(elementType, size, "element type") + if (size < 15) { + this.writeByte((size << 4) | nibble) + } else { + this.writeByte(0xf0 | nibble) + this.varint32(size) + } + } + + /** Writes a list header; write `size` elements of `elementType` next. */ + writeListBegin(elementType: Type, size: number): void { + this.listHeader(elementType, size, "writeListBegin") + } + + /** Writes a set header; write `size` elements of `elementType` next. */ + writeSetBegin(elementType: Type, size: number): void { + this.listHeader(elementType, size, "writeSetBegin") + } + + /** Writes a map header; write `size` key-value pairs next. */ + writeMapBegin(keyType: Type, valueType: Type, size: number): void { + this.checkNoBool("writeMapBegin") + checkInteger("container size", size, 0, 0xffffffff) + const key = this.elementNibble(keyType, size, "map key type") + const value = this.elementNibble(valueType, size, "map value type") + if (size === 0) { + this.writeByte(0) + } else { + this.varint32(size) + this.writeByte((key << 4) | value) + } + } + + /** Writes a `bool`: the header of the `Bool` field just begun, or a byte inside a container. */ + writeBool(value: boolean): void { + if (this.boolPending) { + this.boolPending = false + this.fieldHeader(value ? 1 : 2, this.boolFieldId) + } else { + this.writeByte(value ? 1 : 2) + } + } + + /** Writes an `i8` (`byte`). */ + writeI8(value: number): void { + this.integer(value, -128, 127, "i8") + this.writeByte(value & 0xff) + } + + /** Writes an `i16`. */ + writeI16(value: number): void { + this.integer(value, -32768, 32767, "i16") + this.varint32(zigzag(value)) + } + + /** Writes an `i32`. */ + writeI32(value: number): void { + this.integer(value, -2147483648, 2147483647, "i32") + this.varint32(zigzag(value)) + } + + /** Writes an `i64` given as a safe-integer `number`. */ + writeI64Number(value: number): void { + this.integer(value, -MAX_SAFE, MAX_SAFE, "i64") + const negative = value < 0 + let half = negative ? -value - 1 : value + this.reserve(10) + const bytes = this.bytes + let pos = this.position + // The first byte holds the sign bit and the low six bits of the magnitude. + let b = ((half % 64) * 2) | (negative ? 1 : 0) + half = Math.floor(half / 64) + while (half > 0) { + bytes[pos++] = b | 0x80 + b = half % 128 + half = Math.floor(half / 128) + } + bytes[pos++] = b + this.position = pos + } + + /** Writes an `i64`. */ + writeI64(value: bigint): void { + this.checkNoBool("writeI64") + if (value >= -MAX_SAFE && value <= MAX_SAFE) return this.writeI64Number(Number(value)) + if (BigInt.asIntN(64, value) !== value) fail("Usage", `i64 ${value} is out of range`) + let u = BigInt.asUintN(64, (value << BIGINT_1) ^ (value >> BIGINT_63)) + this.reserve(10) + while (u > BIGINT_0x7f) { + this.bytes[this.position++] = Number(u & BIGINT_0x7f) | 0x80 + u >>= BIGINT_7 + } + this.bytes[this.position++] = Number(u) + } + + /** Writes a `double` (little-endian). */ + writeDouble(value: number): void { + this.checkNoBool("writeDouble") + this.reserve(8) + this.view.setFloat64(this.position, value, true) + this.position += 8 + } + + /** Writes a `binary` value. */ + writeBinary(value: Uint8Array): void { + this.checkNoBool("writeBinary") + this.varint32(value.length) + this.raw(value) + } + + /** Writes a `string` as UTF-8 `binary`. Lone surrogates, which UTF-8 can't encode, become U+FFFD. */ + writeString(value: string): void { + this.checkNoBool("writeString") + const length = value.length + if (length < 128) { + // One length byte; ASCII strings are written directly. + this.reserve(1 + length) + const bytes = this.bytes + const start = this.position + 1 + let i = 0 + for (; i < length; i++) { + const c = value.charCodeAt(i) + if (c >= 0x80) break + bytes[start + i] = c + } + if (i === length) { + bytes[this.position] = length + this.position = start + length + return + } + } + this.writeBinary(encoder.encode(value)) + } + + /** Writes a `uuid`: exactly 16 bytes. */ + writeUuid(value: Uint8Array): void { + this.checkNoBool("writeUuid") + if (value.length !== 16) fail("Usage", `a uuid is 16 bytes, got ${value.length}`) + this.raw(value) + } + + /** Writes a {@link Value}: a field's value after its header, a container element, or a whole value. */ + writeValue(value: Value): void { + switch (value._tag) { + case "Bool": + return this.writeBool(value.value) + case "I8": + return this.writeI8(value.value) + case "I16": + return this.writeI16(value.value) + case "I32": + return this.writeI32(value.value) + case "I64": + return this.writeI64(value.value) + case "Double": + return this.writeDouble(value.value) + case "Binary": + return this.writeBinary(value.value) + case "Uuid": + return this.writeUuid(value.value) + case "Struct": + this.writeStructBegin() + for (const field of value.fields) { + this.writeFieldBegin(Type[field.value._tag], field.id) + this.writeValue(field.value) + } + this.writeFieldStop() + return this.writeStructEnd() + case "List": + case "Set": { + const type = Type[value.elementType] + if (value._tag === "List") this.writeListBegin(type, value.elements.length) + else this.writeSetBegin(type, value.elements.length) + for (const element of value.elements) { + if (element._tag !== value.elementType) { + fail("Usage", `a ${value._tag} of ${value.elementType} contains a ${element._tag}`) + } + this.writeValue(element) + } + return + } + case "Map": { + this.writeMapBegin(Type[value.keyType], Type[value.valueType], value.entries.length) + for (const [k, v] of value.entries) { + if (k._tag !== value.keyType || v._tag !== value.valueType) { + fail("Usage", `a Map of ${value.keyType} to ${value.valueType} contains ${k._tag} to ${v._tag}`) + } + this.writeValue(k) + this.writeValue(v) + } + return + } + default: + fail("Usage", `${String((value as { readonly _tag: unknown })._tag)} is not a Thrift value`) + } + } + + /** A copy of the bytes written so far. Fails when a struct or `Bool` field is unfinished. */ + finish(): Uint8Array { + this.checkNoBool("finish") + if (this.fieldIds.length > 0) fail("Usage", `finish with ${this.fieldIds.length} unfinished structs`) + return this.bytes.slice(0, this.position) + } +} + +/** + * Creates a {@link Writer}. `initialSize` is the first buffer size in bytes, + * a non-negative integer (default 1024); the buffer grows as needed. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export const makeWriter = (options?: { readonly initialSize?: number | undefined }): Writer => new Writer(options) diff --git a/packages/effect/src/encoding/index.ts b/packages/effect/src/encoding/index.ts index dc7605ca67d..8247d26b903 100644 --- a/packages/effect/src/encoding/index.ts +++ b/packages/effect/src/encoding/index.ts @@ -83,6 +83,12 @@ export * as Sse from "./Sse.ts" */ export * as Tar from "./Tar.ts" +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Thrift from "./Thrift.ts" + /** * @stability unstable * @since 4.0.0 diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts new file mode 100644 index 00000000000..a50da6b98bb --- /dev/null +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -0,0 +1,418 @@ +import { assert, describe, it } from "@effect/vitest" +import { Result, Schema } from "effect" +import * as Hex from "effect/encoding/Hex" +import * as Thrift from "effect/encoding/Thrift" +import { readFileSync } from "node:fs" + +// Expected bytes and trees come from the official Apache Thrift Python +// library; see fixtures/thrift/README.md +type Json = + | { readonly _tag: "Bool"; readonly value: boolean } + | { readonly _tag: "I8" | "I16" | "I32"; readonly value: number } + | { readonly _tag: "I64"; readonly value: string } + | { readonly _tag: "Double"; readonly value: number | string } + | { readonly _tag: "Binary" | "Uuid"; readonly value: string } + | { readonly _tag: "Struct"; readonly fields: ReadonlyArray<{ readonly id: number; readonly value: Json }> } + | { readonly _tag: "List" | "Set"; readonly elementType: Thrift.TypeName; readonly elements: ReadonlyArray } + | { + readonly _tag: "Map" + readonly keyType: Thrift.TypeName + readonly valueType: Thrift.TypeName + readonly entries: ReadonlyArray + } +interface Case { + readonly id: string + readonly type: Thrift.TypeName + readonly value: Json + readonly bytes: string +} + +const gold: { readonly cases: ReadonlyArray; readonly parquet: ReadonlyArray> } = JSON + .parse(readFileSync(new URL("../fixtures/thrift/gold.json", import.meta.url), "utf8")) +const corpus: ReadonlyArray = [...gold.cases, ...gold.parquet.map((c) => ({ ...c, type: "Struct" as const }))] +const parquet = (id: string) => fromHex(gold.parquet.find((c) => c.id === id)!.bytes) + +const fromHex = (hex: string) => Result.getOrThrow(Hex.decode(hex)) + +const fromJson = (v: Json): Thrift.Value => { + switch (v._tag) { + case "I64": + return { _tag: "I64", value: BigInt(v.value) } + case "Double": + return { _tag: "Double", value: Number(v.value) } + case "Binary": + case "Uuid": + return { _tag: v._tag, value: fromHex(v.value) } + case "Struct": + return { _tag: "Struct", fields: v.fields.map((f) => ({ id: f.id, value: fromJson(f.value) })) } + case "List": + case "Set": + return { ...v, elements: v.elements.map(fromJson) } + case "Map": + return { ...v, entries: v.entries.map(([k, x]) => [fromJson(k), fromJson(x)] as const) } + default: + return v + } +} + +const readAll = (bytes: Uint8Array, type: Thrift.TypeName, options?: Thrift.ReaderOptions) => + Thrift.decode(bytes, (r) => { + const value = r.readValue(Thrift.Type[type]) + assert.strictEqual(r.position, bytes.length) + return value + }, options) + +const write = (f: (w: Thrift.Writer) => void): Uint8Array => { + const w = Thrift.makeWriter() + f(w) + return w.finish() +} + +const kindOf = (result: Result.Result): Thrift.ThriftError["kind"] | undefined => + Result.isFailure(result) ? result.failure.kind : undefined + +const thrown = (f: () => unknown): Thrift.ThriftError["kind"] | undefined => { + try { + f() + } catch (error) { + assert.isTrue(Thrift.isThriftError(error), String(error)) + return (error as Thrift.ThriftError).kind + } + return undefined +} + +describe("Thrift", () => { + describe("official library", () => { + it("reads, skips and writes every gold case as the official library does", () => { + for (const c of corpus) { + const bytes = fromHex(c.bytes) + const value = fromJson(c.value) + assert.deepStrictEqual(Result.getOrThrow(readAll(bytes, c.type)), value, c.id) + const r = Thrift.makeReader(bytes) + r.skip(Thrift.Type[c.type]) + assert.strictEqual(r.position, bytes.length, c.id) + assert.strictEqual(Hex.encode(write((w) => w.writeValue(value))), c.bytes, c.id) + } + }) + + it("reads a Parquet footer field by field", () => { + // The fields of parquet.thrift's FileMetaData that pyarrow reports, read + // the way a typed reader does: by field id, skipping the rest. + const struct = (r: Thrift.Reader, field: (id: number, type: Thrift.Type) => void) => { + r.readStructBegin() + for (let t = r.readFieldBegin(); t !== Thrift.Type.Stop; t = r.readFieldBegin()) field(r.fieldId, t) + r.readStructEnd() + } + const list = (r: Thrift.Reader, element: (index: number) => void) => { + for (let n = r.readListBegin(), i = 0; i < n; i++) element(i) + } + const footer = (bytes: Uint8Array) => + Result.getOrThrow(Thrift.decode(bytes, (r) => { + const out = { numRows: 0, columns: [] as Array, createdBy: "", dataPageOffsets: [] as Array } + // ColumnChunk → 3: ColumnMetaData → 9: data_page_offset + const columnChunk = () => + struct(r, (id, t) => { + if (id !== 3) return r.skip(t) + struct(r, (id, t) => id === 9 ? out.dataPageOffsets.push(r.readI64Number()) : r.skip(t)) + }) + // RowGroup → 1: list + const rowGroup = () => struct(r, (id, t) => id === 1 ? list(r, columnChunk) : r.skip(t)) + // SchemaElement → 4: name; element 0 is the root + const schemaElement = (i: number) => + struct(r, (id, t) => id === 4 && i > 0 ? out.columns.push(r.readString()) : r.skip(t)) + struct(r, (id, t) => { + if (id === 2) list(r, schemaElement) + else if (id === 3) out.numRows = r.readI64Number() + else if (id === 4) list(r, rowGroup) + else if (id === 6) out.createdBy = r.readString() + else r.skip(t) + }) + assert.strictEqual(r.position, bytes.length) + return out + })) + const bloom = footer(parquet("data_index_bloom_encoding_stats.parquet/footer")) + assert.strictEqual(bloom.numRows, 14) + assert.deepStrictEqual(bloom.columns, ["String"]) + assert.deepStrictEqual(bloom.dataPageOffsets, [4]) + assert.match(bloom.createdBy, /^parquet-mr version 1\.13\.0-SNAPSHOT/) + const tiny = footer(parquet("alltypes_tiny_pages.parquet/footer")) + assert.strictEqual(tiny.numRows, 7300) + assert.deepStrictEqual(tiny.columns, [ + "id", + "bool_col", + "tinyint_col", + "smallint_col", + "int_col", + "bigint_col", + "float_col", + "double_col", + "date_string_col", + "string_col", + "timestamp_col", + "year", + "month" + ]) + assert.strictEqual(tiny.dataPageOffsets[0], 4) + }) + }) + + describe("Reader", () => { + const roundTrip = (writeOne: (w: Thrift.Writer) => void, readOne: (r: Thrift.Reader) => A): A => { + const bytes = write(writeOne) + const r = Thrift.makeReader(bytes) + const back = readOne(r) + assert.strictEqual(r.position, bytes.length) + return back + } + + it.prop( + "i64 round-trips as bigint", + [Schema.BigInt.check( + Schema.isBetweenBigInt({ minimum: -(BigInt(1) << BigInt(63)), maximum: (BigInt(1) << BigInt(63)) - BigInt(1) }) + )], + ([n]) => { + assert.strictEqual(roundTrip((w) => w.writeI64(n), (r) => r.readI64()), n) + } + ) + + it.prop("i64 round-trips as number", [Schema.Int], ([n]) => { + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64Number()), n) + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64()), BigInt(n)) + }) + + it.prop("string round-trips through UTF-8", [Schema.String], ([s]) => { + // Lone surrogates have no UTF-8 encoding and become U+FFFD. + const expected = new TextDecoder("utf-8", { ignoreBOM: true }).decode(new TextEncoder().encode(s)) + assert.strictEqual(roundTrip((w) => w.writeString(s), (r) => r.readString()), expected) + }) + + it("keeps a leading byte order mark in a string", () => { + assert.strictEqual(roundTrip((w) => w.writeString("\uFEFFa"), (r) => r.readString()), "\uFEFFa") + }) + + it("reads i64 as a number only when it is a safe integer", () => { + const max = BigInt(Number.MAX_SAFE_INTEGER) + const asNumber = (n: bigint) => Thrift.decode(write((w) => w.writeI64(n)), (r) => r.readI64Number()) + assert.strictEqual(Result.getOrThrow(asNumber(max)), Number.MAX_SAFE_INTEGER) + assert.strictEqual(Result.getOrThrow(asNumber(-max)), -Number.MAX_SAFE_INTEGER) + assert.strictEqual(kindOf(asNumber(max + BigInt(1))), "Range") + assert.strictEqual(kindOf(asNumber(-max - BigInt(1))), "Range") + }) + + it("reads from a position set by the caller", () => { + // A Parquet page header sits at an offset inside the file. + const r = Thrift.makeReader(Uint8Array.of(0xff, 0xff, 0x15, 0x02, 0x00)) + r.position = 2 + assert.deepStrictEqual(r.readValue(Thrift.Type.Struct), { + _tag: "Struct", + fields: [{ id: 1, value: { _tag: "I32", value: 1 } }] + }) + assert.strictEqual(r.position, 5) + }) + + it("forgets a bool field header when the position moves", () => { + const r = Thrift.makeReader(Uint8Array.of(0x11, 0x02)) + r.readStructBegin() + assert.strictEqual(r.readFieldBegin(), Thrift.Type.Bool) + r.position = 1 + assert.isFalse(r.readBool()) + assert.strictEqual(r.position, 2) + }) + + it("reads binary as a view of the input", () => { + const bytes = Uint8Array.of(2, 7, 8) + const view = Thrift.makeReader(bytes).readBinary() + assert.strictEqual(view.buffer, bytes.buffer) + assert.deepStrictEqual(Array.from(view), [7, 8]) + }) + + it("reads a bool list element type written as 2", () => { + // Some writers use the FALSE code for the element type of list. + const r = Thrift.makeReader(Uint8Array.of(0x12, 0x01)) + assert.strictEqual(r.readListBegin(), 1) + assert.strictEqual(r.elementType, Thrift.Type.Bool) + assert.isTrue(r.readBool()) + }) + }) + + describe("hostile input", () => { + const struct = (r: Thrift.Reader) => r.readValue(Thrift.Type.Struct) + const table: ReadonlyArray< + readonly [string, string, (r: Thrift.Reader) => unknown, Thrift.ThriftError["kind"], Thrift.ReaderOptions?] + > = [ + ["empty input", "", struct, "Truncated"], + ["a struct without its stop", "1502", struct, "Truncated"], + ["an i32 varint over 5 bytes", "808080808001", (r) => r.readI32(), "Malformed"], + ["an i32 varint over 32 bits", "ffffffff1f", (r) => r.readI32(), "Malformed"], + ["an i64 varint over 10 bytes", "ffffffffffffffffff8101", (r) => r.readI64(), "Malformed"], + ["an i64 varint over 64 bits", "ffffffffffffffffff02", (r) => r.readI64(), "Malformed"], + // Other readers wrap these to 16 bits or keep them, so fields would alias. + ["an i16 over 16 bits", "808004", (r) => r.readI16(), "Malformed"], + ["a field id over 16 bits", "0580800400", struct, "Malformed"], + ["a field id delta past 32767", "05feff0300f50000", struct, "Malformed"], + ["an unknown field type", "1e", struct, "Malformed"], + ["an unknown list element type", "1e", (r) => r.readListBegin(), "Malformed"], + ["a non-empty list of Stop", "10", (r) => r.readListBegin(), "Malformed"], + ["a list larger than the input", "f5ffff03", (r) => r.readListBegin(), "Truncated"], + ["a map larger than the input", "0355020406", (r) => r.readMapBegin(), "Truncated"], + ["a binary longer than the input", "0561", (r) => r.readBinary(), "Truncated"], + ["a string that isn't UTF-8", "02c328", (r) => r.readString(), "InvalidUtf8"], + ["a binary over maxBinaryLength", "03616263", (r) => r.readBinary(), "Limit", { maxBinaryLength: 2 }], + ["a skipped binary over maxBinaryLength", "1803616263", (r) => r.skip(Thrift.Type.Struct), "Limit", { + maxBinaryLength: 2 + }], + ["a list over maxContainerSize", "35", (r) => r.readListBegin(), "Limit", { maxContainerSize: 2 }], + ["a struct over maxDepth", "1c1c000000", struct, "Limit", { maxDepth: 2 }], + ["lists over maxDepth", "1919191500", (r) => r.skip(Thrift.Type.List), "Limit", { maxDepth: 2 }] + ] + for (const [name, hex, read, kind, options] of table) { + it(`fails on ${name}`, () => assert.strictEqual(kindOf(Thrift.decode(fromHex(hex), read, options)), kind)) + } + + it("limits nesting to 64 by default and 1000 at most", () => { + const nest = (depth: number): Thrift.Value => { + let v: Thrift.Value = { _tag: "Struct", fields: [] } + for (let i = 1; i < depth; i++) v = { _tag: "Struct", fields: [{ id: 1, value: v }] } + return v + } + const bytesOf = (depth: number) => write((w) => w.writeValue(nest(depth))) + assert.isTrue(Result.isSuccess(readAll(bytesOf(64), "Struct"))) + assert.strictEqual(kindOf(readAll(bytesOf(65), "Struct")), "Limit") + assert.strictEqual(kindOf(Thrift.decode(bytesOf(65), (r) => r.skip(Thrift.Type.Struct))), "Limit") + assert.strictEqual(kindOf(readAll(bytesOf(1001), "Struct", { maxDepth: 1000 })), "Limit") + }) + + it("fails every truncated gold case with Truncated", () => { + for (const c of corpus) { + const bytes = fromHex(c.bytes) + for (let n = 0; n < bytes.length; n += Math.max(1, Math.floor(bytes.length / 16))) { + assert.strictEqual(kindOf(readAll(bytes.subarray(0, n), c.type)), "Truncated", `${c.id} at ${n}`) + } + } + }) + + it("only ever fails with ThriftError on corrupted input", () => { + // Deterministic xorshift so failures reproduce. + let seed = 0x5eed + const random = () => { + seed ^= seed << 13 + seed ^= seed >>> 17 + seed ^= seed << 5 + return (seed >>> 0) / 0x100000000 + } + let failures = 0 + for (const c of corpus) { + const original = fromHex(c.bytes) + const type = Thrift.Type[c.type] + for (let round = 0; round < 50; round++) { + const bytes = original.slice() + const flips = 1 + Math.floor(random() * 4) + for (let i = 0; i < flips && bytes.length > 0; i++) { + bytes[Math.floor(random() * bytes.length)] = Math.floor(random() * 256) + } + if (thrown(() => Thrift.makeReader(bytes).readValue(type)) !== undefined) failures++ + if (thrown(() => Thrift.makeReader(bytes).skip(type)) !== undefined) failures++ + } + } + assert.isAbove(failures, 0) + }) + + it("rejects limits that aren't integers in range", () => { + // NaN compares false with everything, so it would switch a limit off. + for ( + const options of [ + { maxDepth: NaN }, + { maxDepth: 0 }, + { maxDepth: 1.5 }, + { maxDepth: 1001 }, + { maxBinaryLength: -1 }, + { maxBinaryLength: NaN }, + { maxContainerSize: Infinity } + ] + ) { + assert.strictEqual( + thrown(() => Thrift.makeReader(new Uint8Array(0), options)), + "Usage", + Object.entries(options).join() + ) + } + }) + + it("decode rethrows errors that aren't ThriftErrors", () => { + assert.throws(() => + Thrift.decode(Uint8Array.of(0), () => { + throw new Error("boom") + }), "boom") + }) + + it("decode rethrows Usage errors, which are the caller's", () => { + const misuse: ReadonlyArray<(r: Thrift.Reader) => unknown> = [ + (r) => r.readStructEnd(), + (r) => r.position = -1, + (r) => r.position = 2, + (r) => r.position = 0.5, + (r) => r.skip(Thrift.Type.Stop), + (r) => r.readValue(7 as Thrift.Type) + ] + for (const f of misuse) { + assert.strictEqual(thrown(() => Thrift.decode(Uint8Array.of(0), f)), "Usage", String(f)) + } + }) + }) + + describe("Writer", () => { + const usage: ReadonlyArray void]> = [ + ["an i8 out of range", (w) => w.writeI8(128)], + ["a fractional i16", (w) => w.writeI16(1.5)], + ["an i32 out of range", (w) => w.writeI32(2 ** 31)], + ["an unsafe i64 number", (w) => w.writeI64Number(2 ** 53)], + ["an i64 out of range", (w) => w.writeI64(BigInt(1) << BigInt(63))], + ["a uuid that isn't 16 bytes", (w) => w.writeUuid(new Uint8Array(15))], + ["a field outside a struct", (w) => w.writeFieldBegin(Thrift.Type.I32, 1)], + ["a field id out of range", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.I32, 40000) + }], + ["a Stop field", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.Stop, 1) + }], + ["a bool field without writeBool", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.Bool, 1) + w.writeI32(1) + }], + ["writeStructEnd without writeStructBegin", (w) => w.writeStructEnd()], + ["finish inside a struct", (w) => { + w.writeStructBegin() + w.finish() + }], + ["a non-empty list of Stop", (w) => w.writeListBegin(Thrift.Type.Stop, 1)], + ["a negative list size", (w) => w.writeListBegin(Thrift.Type.I32, -1)], + [ + "a list element of another type", + (w) => w.writeValue({ _tag: "List", elementType: "I32", elements: [{ _tag: "I8", value: 1 }] }) + ], + ["a value of no Thrift type", (w) => w.writeValue({ _tag: "Text", value: "a" } as unknown as Thrift.Value)] + ] + for (const [name, f] of usage) { + it(`rejects ${name}`, () => assert.strictEqual(thrown(() => f(Thrift.makeWriter())), "Usage")) + } + + it("rejects an initialSize that isn't a non-negative integer", () => { + // A NaN size would never satisfy the loop that grows the buffer. + for (const initialSize of [NaN, Infinity, -1, 1.5]) { + assert.strictEqual(thrown(() => Thrift.makeWriter({ initialSize })), "Usage", String(initialSize)) + } + }) + + it("grows from a small buffer and returns a copy", () => { + const bytes = parquet("alltypes_tiny_pages.parquet/footer") + const w = Thrift.makeWriter({ initialSize: 1 }) + w.writeValue(Result.getOrThrow(readAll(bytes, "Struct"))) + const out = w.finish() + assert.deepStrictEqual(out, bytes) + assert.strictEqual(out.buffer.byteLength, out.length) + }) + }) +}) diff --git a/packages/effect/test/fixtures/thrift/README.md b/packages/effect/test/fixtures/thrift/README.md new file mode 100644 index 00000000000..5be9093390d --- /dev/null +++ b/packages/effect/test/fixtures/thrift/README.md @@ -0,0 +1,24 @@ +# Thrift fixtures + +`gold.json` is generated by `generate.py` and read by +`test/encoding/Thrift.test.ts`. Each case is a value tree and the bytes the +official Apache Thrift Python library (`thrift` 0.25.0 from PyPI, +`TCompactProtocol`) writes for it: one hand-written case per rule of the +compact protocol, then 20 small seeded random structs. + +The `parquet` section holds compact-protocol structs cut from two files of +[apache/parquet-testing](https://github.com/apache/parquet-testing) (commit +`bcefdfd`): the `FileMetaData` footers of `data_index_bloom_encoding_stats.parquet` +and `alltypes_tiny_pages.parquet`, a page header from each, and the bloom +filter header, offset index and column index of the first. Each comes with the +tree the Python reader reads from it. The row counts, column names and +offsets the test checks are the ones pyarrow reports for those files. + +Regenerate from `packages/effect`; the output is byte for byte the same: + +```sh +PARQUET_TESTING=/path/to/parquet-testing \ + uv run --with thrift==0.25.0 python test/fixtures/thrift/generate.py +``` + +`PARQUET_TESTING` needs only the two files above, under `data/`. diff --git a/packages/effect/test/fixtures/thrift/generate.py b/packages/effect/test/fixtures/thrift/generate.py new file mode 100644 index 00000000000..b0ce460fb88 --- /dev/null +++ b/packages/effect/test/fixtures/thrift/generate.py @@ -0,0 +1,460 @@ +""" +Generates gold.json, the gold corpus for effect/encoding/Thrift: values and +the bytes the official Apache Thrift Python library's TCompactProtocol writes +for them. README.md describes the corpus and how to regenerate it. + +Values are JSON trees shaped like Thrift.Value: + {"_tag": "Bool", "value": true} + {"_tag": "I8" | "I16" | "I32", "value": 5} + {"_tag": "I64", "value": "-5"} decimal string + {"_tag": "Double", "value": 1.5} or "NaN", "Infinity", "-Infinity", "-0" + {"_tag": "Binary" | "Uuid", "value": "00ff"} hex + {"_tag": "Struct", "fields": [{"id": 1, "value": ...}]} + {"_tag": "List" | "Set", "elementType": "I32", "elements": [...]} + {"_tag": "Map", "keyType": "I32", "valueType": "Bool", "entries": [[k, v]]} +A case is {"id", "type", "value", "bytes"}: `type` is the Type name of the +top-level value. Every case is also read back here with the Python reader to +check the generator. The "parquet" cases are structs and carry no `type`. +""" + +import json +import math +import os +import random +import struct +import uuid +from pathlib import Path + +from thrift.protocol.TCompactProtocol import TCompactProtocol +from thrift.Thrift import TType +from thrift.transport.TTransport import TMemoryBuffer + +HERE = Path(__file__).parent + +TTYPES = { + "Bool": TType.BOOL, + "I8": TType.BYTE, + "I16": TType.I16, + "I32": TType.I32, + "I64": TType.I64, + "Double": TType.DOUBLE, + "Binary": TType.STRING, + "Uuid": TType.UUID, + "Struct": TType.STRUCT, + "List": TType.LIST, + "Set": TType.SET, + "Map": TType.MAP, + "Stop": TType.STOP, +} +NAMES = {v: k for k, v in TTYPES.items()} + +# TCompactProtocol only writes scalars inside a struct or container; these +# are its internal states for "a value comes next". +VALUE_WRITE = 2 +VALUE_READ = 7 + + +def double_json(d): + if math.isnan(d): + return "NaN" + if math.isinf(d): + return "Infinity" if d > 0 else "-Infinity" + if d == 0 and math.copysign(1, d) < 0: + return "-0" + return d + + +def write(p, v): + tag = v["_tag"] + if tag == "Bool": + p.writeBool(v["value"]) + elif tag == "I8": + p.writeByte(v["value"]) + elif tag == "I16": + p.writeI16(v["value"]) + elif tag == "I32": + p.writeI32(v["value"]) + elif tag == "I64": + p.writeI64(int(v["value"])) + elif tag == "Double": + p.writeDouble(float(v["value"])) + elif tag == "Binary": + p.writeBinary(bytes.fromhex(v["value"])) + elif tag == "Uuid": + p.writeUuid(uuid.UUID(bytes=bytes.fromhex(v["value"]))) + elif tag == "Struct": + p.writeStructBegin(None) + for f in v["fields"]: + p.writeFieldBegin(None, TTYPES[f["value"]["_tag"]], f["id"]) + write(p, f["value"]) + p.writeFieldEnd() + p.writeFieldStop() + p.writeStructEnd() + elif tag in ("List", "Set"): + begin = p.writeListBegin if tag == "List" else p.writeSetBegin + begin(TTYPES[v["elementType"]], len(v["elements"])) + for e in v["elements"]: + write(p, e) + p.writeListEnd() + elif tag == "Map": + p.writeMapBegin(TTYPES[v["keyType"]], TTYPES[v["valueType"]], len(v["entries"])) + for k, x in v["entries"]: + write(p, k) + write(p, x) + p.writeMapEnd() + else: + raise ValueError(tag) + + +def read(p, t): + name = NAMES[t] + if name == "Bool": + return {"_tag": name, "value": p.readBool()} + if name == "I8": + return {"_tag": name, "value": p.readByte()} + if name == "I16": + return {"_tag": name, "value": p.readI16()} + if name == "I32": + return {"_tag": name, "value": p.readI32()} + if name == "I64": + return {"_tag": name, "value": str(p.readI64())} + if name == "Double": + return {"_tag": name, "value": double_json(p.readDouble())} + if name == "Binary": + return {"_tag": name, "value": p.readBinary().hex()} + if name == "Uuid": + return {"_tag": name, "value": p.readUuid().bytes.hex()} + if name == "Struct": + fields = [] + p.readStructBegin() + while True: + _, ft, fid = p.readFieldBegin() + if ft == TType.STOP: + break + fields.append({"id": fid, "value": read(p, ft)}) + p.readFieldEnd() + p.readStructEnd() + return {"_tag": name, "fields": fields} + if name in ("List", "Set"): + et, n = p.readListBegin() + elements = [read(p, et) for _ in range(n)] + p.readListEnd() + return {"_tag": name, "elementType": NAMES[et], "elements": elements} + if name == "Map": + kt, vt, n = p.readMapBegin() + entries = [] + for _ in range(n): + k = read(p, kt) + entries.append([k, read(p, vt)]) + p.readMapEnd() + return {"_tag": name, "keyType": NAMES[kt], "valueType": NAMES[vt], "entries": entries} + raise ValueError(name) + + +def encode(v): + buf = TMemoryBuffer() + p = TCompactProtocol(buf) + if v["_tag"] != "Struct": + p.state = VALUE_WRITE + write(p, v) + return buf.getvalue() + + +def decode(data, type_name, offset=0): + """Reads one value at `offset`; returns it and the number of bytes read.""" + buf = TMemoryBuffer(data[offset:]) + p = TCompactProtocol(buf) + if type_name != "Struct": + p.state = VALUE_READ + v = read(p, TTYPES[type_name]) + return v, buf._buffer.tell() + + +cases = [] + + +def case(id, v): + data = encode(v) + back, n = decode(data, v["_tag"]) + assert back == v, (id, back, v) + assert n == len(data), id + cases.append({"id": id, "type": v["_tag"], "value": v, "bytes": data.hex()}) + + +# --- value constructors ------------------------------------------------------- + + +def b(x): + return {"_tag": "Bool", "value": x} + + +def i8(x): + return {"_tag": "I8", "value": x} + + +def i16(x): + return {"_tag": "I16", "value": x} + + +def i32(x): + return {"_tag": "I32", "value": x} + + +def i64(x): + return {"_tag": "I64", "value": str(x)} + + +def dbl(x): + return {"_tag": "Double", "value": double_json(x)} + + +def binary(x): + return {"_tag": "Binary", "value": bytes(x).hex()} + + +def text(s): + return binary(s.encode("utf-8")) + + +def uid(x): + return {"_tag": "Uuid", "value": bytes(x).hex()} + + +def struct_(*fields): + return {"_tag": "Struct", "fields": [{"id": i, "value": v} for i, v in fields]} + + +def lst(t, *es): + return {"_tag": "List", "elementType": t, "elements": list(es)} + + +def st(t, *es): + return {"_tag": "Set", "elementType": t, "elements": list(es)} + + +def mp(k, v, *entries): + if not entries: + k = v = "Stop" + return {"_tag": "Map", "keyType": k, "valueType": v, "entries": [list(e) for e in entries]} + + +# --- hand-written cases ------------------------------------------------------- + +case("struct/empty", struct_()) +case("bool/fields", struct_((1, b(True)), (2, b(False)), (3, b(True)))) +case("bool/long-form", struct_((20, b(True)), (-1, b(False)))) +case("field/delta-1", struct_((1, i32(1)), (2, i32(2)))) +case("field/delta-15", struct_((15, i32(1)), (30, i32(2)))) +case("field/delta-16", struct_((16, i32(1)), (32, i32(2)))) +case("field/id-0", struct_((0, i32(1)))) +case("field/decreasing", struct_((5, i32(1)), (3, i32(2)), (4, i32(3)))) +case("field/repeated", struct_((5, i32(1)), (5, i32(2)))) +case("field/negative", struct_((-1, i32(1)), (-32768, i32(2)), (1, i32(3)))) +case("field/max", struct_((32767, i32(1)))) +case("field/nested-resets", struct_((5, struct_((1, i8(1)), (2, i8(2)))), (6, i8(3)))) +case("i8/bounds", struct_((1, i8(-128)), (2, i8(127)), (3, i8(0)), (4, i8(-1)))) +case("i16/bounds", struct_((1, i16(-32768)), (2, i16(32767)), (3, i16(0)), (4, i16(-1)), (5, i16(64)), (6, i16(-65)))) +case( + "i32/bounds", + struct_( + (1, i32(-(2**31))), (2, i32(2**31 - 1)), (3, i32(0)), (4, i32(-1)), (5, i32(63)), (6, i32(64)), + (7, i32(8191)), (8, i32(8192)), (9, i32(-8193)) + ), +) +case( + "i64/bounds", + struct_(*enumerate([ + i64(0), i64(-1), i64(1), i64(2**31), i64(-(2**31) - 1), i64(2**53 - 1), i64(-(2**53 - 1)), i64(2**53), + i64(-(2**53)), i64(2**53 + 1), i64(-(2**53) - 1), i64(2**62), i64(2**63 - 1), i64(-(2**63)), + i64(2**56 - 1), i64(1 << 49) + ], 1)), +) +case( + "double/values", + struct_(*enumerate([ + dbl(0.0), dbl(-0.0), dbl(1.5), dbl(float("nan")), dbl(float("inf")), dbl(float("-inf")), dbl(5e-324), + dbl(1.7976931348623157e308), dbl(math.pi), dbl(-2.5) + ], 1)), +) +case("binary/lengths", struct_((1, binary(b"")), (2, binary(b"\x00")), (3, binary(bytes(range(128)))))) +case("binary/utf8", struct_((1, text("héllo 🌍")), (2, text("ascii")))) +case("uuid", struct_((1, uid(range(16))), (2, uid(bytes(16))))) +case("list/empty", struct_((1, lst("I32")), (2, lst("Struct")), (3, lst("Bool")))) +case("list/size-14", struct_((1, lst("I32", *[i32(i) for i in range(14)])))) +case("list/size-15", struct_((1, lst("I32", *[i32(i) for i in range(15)])))) +case("list/bool", struct_((1, lst("Bool", b(True), b(False), b(True))))) +case("list/i8", struct_((1, lst("I8", i8(-1), i8(0), i8(127))))) +case("list/i64", struct_((1, lst("I64", i64(-(2**63)), i64(0), i64(2**63 - 1))))) +case("list/double", struct_((1, lst("Double", dbl(1.0), dbl(float("nan")))))) +case("list/binary", struct_((1, lst("Binary", text("a"), text(""), text("bc"))))) +case("list/uuid", struct_((1, lst("Uuid", uid(range(16)), uid(range(16, 32)))))) +case("list/struct", struct_((1, lst("Struct", struct_((1, b(True))), struct_(), struct_((2, i32(7))))))) +case("list/list", struct_((1, lst("List", lst("I32", i32(1)), lst("I32"), lst("Bool", b(False)))))) +case("list/map", struct_((1, lst("Map", mp("I32", "I32", (i32(1), i32(2))), mp("I32", "I32"))))) +case("set/values", struct_((1, st("I32", i32(3), i32(1))), (2, st("Binary")), (3, st("Bool", b(True))))) +case("set/size-15", struct_((1, st("I8", *[i8(i) for i in range(15)])))) +case("map/empty", struct_((1, mp("I32", "I32")))) +case("map/one", struct_((1, mp("Binary", "I64", (text("k"), i64(-5)))))) +case("map/bools", struct_((1, mp("Bool", "Bool", (b(True), b(False)), (b(False), b(True)))))) +case("map/i32-struct", struct_((1, mp("I32", "Struct", (i32(1), struct_((1, b(True)))), (i32(2), struct_()))))) +case("map/nested", struct_((1, mp("I32", "List", (i32(1), lst("I32", i32(2), i32(3))))))) +case("map/uuid", struct_((1, mp("Uuid", "Uuid", (uid(range(16)), uid(bytes(16))))))) + + +def nest(depth): + v = struct_((1, i32(depth))) + for d in range(depth): + v = struct_((1, v), (2, b(d % 2 == 0))) + return v + + +case("struct/deep-5", nest(5)) +case("top/list", lst("I32", i32(1), i32(-1))) +case("top/i64", i64(-(2**40))) +case("top/binary", text("top")) +case("top/map", mp("Binary", "Binary", (text("a"), text("b")))) +case("top/set", st("Struct", struct_((1, i8(1))))) +case("top/double", dbl(0.1)) + +# --- seeded random structs ---------------------------------------------------- + +rng = random.Random(0x7487) +SCALARS = ["Bool", "I8", "I16", "I32", "I64", "Double", "Binary", "Uuid"] +ALL = SCALARS + ["Struct", "List", "Set", "Map"] + + +def rand_int(bits): + r = rng.random() + if r < 0.3: + return rng.randint(-64, 64) + if r < 0.4: + return rng.choice([-(2 ** (bits - 1)), 2 ** (bits - 1) - 1, 0, -1]) + w = rng.randint(1, bits - 1) + return rng.randint(-(2**w), 2**w - 1) + + +def rand_double(): + r = rng.random() + if r < 0.1: + return rng.choice([0.0, -0.0, float("inf"), float("-inf"), float("nan"), 5e-324]) + if r < 0.4: + return float(rng.randint(-1000, 1000)) + return struct.unpack("= 3 or rng.random() < 0.6 else ALL) + + +def rand_struct(depth): + n = rng.randint(0, 3 if depth > 2 else 8) + ids = [] + last = 0 + for _ in range(n): + r = rng.random() + if r < 0.6: + last += rng.randint(1, 15) + elif r < 0.8: + last += rng.randint(16, 300) + elif r < 0.9: + last = rng.randint(-32768, 32767) + else: + last = rng.randint(-5, 5) + last = max(-32768, min(32767, last)) + ids.append(last) + return struct_(*[(i, rand_value(rand_type(depth), depth)) for i in ids]) + + +# Small ones only: they mix every type, size form and id form at low cost. +n = 0 +while n < 20: + v = rand_struct(0) + if len(encode(v)) <= 128: + case(f"random/{n}", v) + n += 1 + +# --- Parquet structs ---------------------------------------------------------- + + +def field(v, fid): + for f in v["fields"]: + if f["id"] == fid: + return f["value"] + return None + + +parquet = [] + + +def cut(file, data, offset, what): + v, n = decode(data, "Struct", offset) + parquet.append({ + "id": f"{file}/{what}", + "value": v, + "bytes": data[offset:offset + n].hex(), + }) + return v + + +root = os.environ.get("PARQUET_TESTING") +assert root, "set PARQUET_TESTING to a checkout of https://github.com/apache/parquet-testing" +# The tiny-pages file's indexes have thousands of entries; its footer is enough. +for file, indexes in [("data_index_bloom_encoding_stats.parquet", True), ("alltypes_tiny_pages.parquet", False)]: + data = (Path(root) / "data" / file).read_bytes() + assert data[-4:] == b"PAR1" + length = struct.unpack("