From a1f4aa1245493e4bc084d2c7611cebed54bc0ba8 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 15 Nov 2026 16:36:44 +1300 Subject: [PATCH 01/32] Fix OpenAI WebSocket error decoding and preserve provider details (#10645) --- .changeset/openai-websocket-error-details.md | 5 ++ packages/ai/openai/src/OpenAiClient.ts | 21 ++++--- packages/ai/openai/test/OpenAiClient.test.ts | 65 ++++++++++++++++++++ 3 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 .changeset/openai-websocket-error-details.md diff --git a/.changeset/openai-websocket-error-details.md b/.changeset/openai-websocket-error-details.md new file mode 100644 index 00000000000..879f105bc0a --- /dev/null +++ b/.changeset/openai-websocket-error-details.md @@ -0,0 +1,5 @@ +--- +"@effect/ai-openai": patch +--- + +Fix OpenAI WebSocket streams hanging on errors with missing messages or nullable type/code fields, and preserve provider error details. diff --git a/packages/ai/openai/src/OpenAiClient.ts b/packages/ai/openai/src/OpenAiClient.ts index f4a44fda860..a71ff430dc9 100644 --- a/packages/ai/openai/src/OpenAiClient.ts +++ b/packages/ai/openai/src/OpenAiClient.ts @@ -602,12 +602,12 @@ const makeSocket = Effect.gen(function*() { if (event.type === "error") { const status = "status" in event ? Number(event.status) : NaN const error = ("error" in event ? event.error : event) as { - readonly type?: string - readonly code?: string - readonly message: string + readonly type?: string | null + readonly code?: string | null + readonly message?: string } const errorType = error.code ?? - (error.type === "api_error" && /^gRPC error: Response with id=\S+ not found$/.test(error.message) + (error.type === "api_error" && /^gRPC error: Response with id=\S+ not found$/.test(error.message ?? "") ? "previous_response_not_found" : error.type ?? "unknown") const json = JSON.stringify(error) @@ -736,11 +736,14 @@ const ErrorEvent = Schema.Struct({ ), status: Schema.optionalKey(Schema.Int), // xAI sends `code` in place of `type`, e.g. `previous_response_not_found` - error: Schema.Struct({ - type: Schema.optional(Schema.String), - code: Schema.optional(Schema.String), - message: Schema.String - }) + error: Schema.StructWithRest( + Schema.Struct({ + type: Schema.optionalKey(Schema.NullOr(Schema.String)), + code: Schema.optionalKey(Schema.NullOr(Schema.String)), + message: Schema.optionalKey(Schema.String) + }), + [Schema.Record(Schema.String, Schema.Unknown)] + ) }) const errorTypeToStatus: Record = { diff --git a/packages/ai/openai/test/OpenAiClient.test.ts b/packages/ai/openai/test/OpenAiClient.test.ts index 0912b4863f6..6e7f17435aa 100644 --- a/packages/ai/openai/test/OpenAiClient.test.ts +++ b/packages/ai/openai/test/OpenAiClient.test.ts @@ -545,6 +545,63 @@ describe("OpenAiClient", () => { }) describe("createResponseStream", () => { + it.effect("preserves the HTTP usage-limit error body", () => + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const result = yield* client.createResponseStream({ model: "gpt-4o", input: "test" }).pipe(Effect.flip) + assert.strictEqual(result.reason._tag, "RateLimitError") + if (result.reason._tag !== "RateLimitError") return + assert.deepStrictEqual(JSON.parse(result.reason.http!.body!), { error: usageLimitError }) + }).pipe(Effect.provide(makeTestLayer(undefined, { + _tag: "Json", + status: 429, + body: { error: usageLimitError } + })))) + + it.live.each([ + { name: "explicit status", status: 429, error: usageLimitError, reason: "RateLimitError" }, + { name: "no status", error: usageLimitError, reason: "InternalProviderError" }, + { + name: "no message", + status: 429, + error: { type: "usage_limit_reached", plan_type: "pro" }, + reason: "RateLimitError" + }, + { name: "null code", status: 429, error: { ...usageLimitError, code: null }, reason: "RateLimitError" }, + { name: "null type", status: 429, error: { ...usageLimitError, type: null }, reason: "RateLimitError" }, + { name: "mapped type", error: { ...usageLimitError, type: "rate_limit_error" }, reason: "RateLimitError" }, + { name: "api error without message", error: { type: "api_error" }, reason: "InternalProviderError" } + ])("preserves WebSocket error details with $name", (frame) => + Effect.gen(function*() { + const server = yield* Effect.acquireRelease( + Effect.sync(() => + new WS(`wss://errors.test/${encodeURIComponent(frame.name)}/responses`, { jsonProtocol: true }) + ), + (server) => Effect.sync(() => server.close()) + ) + yield* OpenAiClient.withWebSocketMode( + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const [, stream] = yield* client.createResponseStream({ model: "gpt-4o", input: "test" }) + const [result] = yield* Effect.all([ + Stream.runDrain(stream).pipe(Effect.flip), + nextWebSocketCreate(server).pipe( + Effect.tap(() => Effect.sync(() => server.send({ type: "error", ...frame }))) + ) + ], { concurrency: "unbounded" }) + assert.strictEqual(result._tag, "AiError") + assert.strictEqual(result.reason._tag, frame.reason) + if (result.reason._tag !== "RateLimitError" && result.reason._tag !== "InternalProviderError") return + assert.deepStrictEqual(result.reason.metadata, frame.error) + assert.deepStrictEqual(JSON.parse(result.reason.http!.body!), frame.error) + }) + ).pipe( + Effect.provide(makeTestLayer({ apiUrl: `https://errors.test/${encodeURIComponent(frame.name)}` })), + Effect.provideService(Socket.WebSocketConstructor, (url) => new globalThis.WebSocket(url)), + Effect.timeout("1 second") + ) + })) + it.effect("emits response.completed before [DONE] in the same SSE chunk", () => Effect.gen(function*() { const client = yield* OpenAiClient.OpenAiClient @@ -1376,3 +1433,11 @@ const makeResponse = ( } }) } + +const usageLimitError = { + type: "usage_limit_reached", + message: "Usage limit reached", + resets_at: 1800000000, + plan_type: "pro", + limit_window_minutes: 300 +} From 1b807a063982a2463017d055ef6fc36240222217 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 15 Nov 2026 20:21:49 +1300 Subject: [PATCH 02/32] Fix LayerRef error inference when preload is omitted (#10646) --- .changeset/fix-layer-ref-preload-default.md | 5 ++++ packages/effect/src/LayerRef.ts | 4 +-- packages/effect/typetest/LayerRef.tst.ts | 29 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .changeset/fix-layer-ref-preload-default.md create mode 100644 packages/effect/typetest/LayerRef.tst.ts diff --git a/.changeset/fix-layer-ref-preload-default.md b/.changeset/fix-layer-ref-preload-default.md new file mode 100644 index 00000000000..c9abaaae7ba --- /dev/null +++ b/.changeset/fix-layer-ref-preload-default.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Fix `LayerRef.make` and `LayerRef.Service` exposing layer build errors during creation when `preload` is omitted. diff --git a/packages/effect/src/LayerRef.ts b/packages/effect/src/LayerRef.ts index f850c5e72a8..d795751f160 100644 --- a/packages/effect/src/LayerRef.ts +++ b/packages/effect/src/LayerRef.ts @@ -134,7 +134,7 @@ export interface LayerRef { * @since 4.0.0 */ export const make = Effect.fnUntraced( - function*( + function*( layer: Layer.Layer, options?: { /** @@ -317,7 +317,7 @@ export const Service = () => R, X, const Deps extends ReadonlyArray> = [], - const Preload extends boolean = never, + const Preload extends boolean = false, SE = never, SR = never >( diff --git a/packages/effect/typetest/LayerRef.tst.ts b/packages/effect/typetest/LayerRef.tst.ts new file mode 100644 index 00000000000..dad278f32fd --- /dev/null +++ b/packages/effect/typetest/LayerRef.tst.ts @@ -0,0 +1,29 @@ +import { type Effect, type Layer, LayerRef, type Scope } from "effect" +import { expect, it } from "tstyche" + +declare const failing: Layer.Layer + +it("make only exposes build errors when preloading", () => { + expect(LayerRef.make(failing)).type.toBe, never, Scope.Scope>>() + expect(LayerRef.make(failing, { idleTimeToLive: "Infinity" })).type.toBe< + Effect.Effect, never, Scope.Scope> + >() + expect(LayerRef.make(failing, { preload: false })).type.toBe< + Effect.Effect, never, Scope.Scope> + >() + expect(LayerRef.make(failing, { preload: true })).type.toBe< + Effect.Effect, "BuildError", Scope.Scope> + >() +}) + +it("Service only exposes build errors in its layer when preloading", () => { + class Lazy extends LayerRef.Service()("Lazy", { layer: failing }) {} + class ExplicitLazy extends LayerRef.Service()("ExplicitLazy", { layer: failing, preload: false }) {} + class Preloaded extends LayerRef.Service()("Preloaded", { layer: failing, preload: true }) {} + + expect(Lazy.layer).type.toBe>() + expect(Lazy.layerNoDeps).type.toBe>() + expect(Lazy.get).type.toBe>() + expect(ExplicitLazy.layer).type.toBe>() + expect(Preloaded.layer).type.toBe>() +}) From 277161b159d71cb140a683a770c40ed988490320 Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Sun, 15 Nov 2026 12:21:58 +0000 Subject: [PATCH 03/32] Add effect/encoding/Thrift Reads and writes the Apache Thrift compact protocol without an IDL. Reader is a TProtocol-shaped cursor: readFieldBegin/readListBegin/ readMapBegin, scalar reads, skip(type), and readValue into a typed Value tree. Every read is bounds-checked; lengths and container sizes larger than the remaining input fail before allocating, nesting is limited by maxDepth, and decode returns a Result with a ThriftError. Binary values are views of the input. i64 reads as bigint, or as a checked number with readI64Number. Writer mirrors the reader. The fixtures hold 349 value trees with the bytes the official Python thrift 0.25.0 TCompactProtocol writes for them, plus seven structs cut from apache/parquet-testing files (footers, page headers, bloom filter header, offset and column index). The tests check that our bytes are identical, read every case back, and fuzz corrupted input. --- .changeset/encoding-thrift.md | 5 + packages/effect/src/encoding/Thrift.ts | 1128 +++++++++++++++++ packages/effect/src/encoding/index.ts | 6 + packages/effect/test/encoding/Thrift.test.ts | 500 ++++++++ .../effect/test/fixtures/thrift/README.md | 21 + .../effect/test/fixtures/thrift/generate.py | 473 +++++++ .../effect/test/fixtures/thrift/gold.json | 1 + 7 files changed, 2134 insertions(+) create mode 100644 .changeset/encoding-thrift.md create mode 100644 packages/effect/src/encoding/Thrift.ts create mode 100644 packages/effect/test/encoding/Thrift.test.ts create mode 100644 packages/effect/test/fixtures/thrift/README.md create mode 100644 packages/effect/test/fixtures/thrift/generate.py create mode 100644 packages/effect/test/fixtures/thrift/gold.json diff --git a/.changeset/encoding-thrift.md b/.changeset/encoding-thrift.md new file mode 100644 index 00000000000..8c98c74101d --- /dev/null +++ b/.changeset/encoding-thrift.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add `effect/encoding/Thrift` for reading and writing the Apache Thrift compact protocol without an IDL. The reader is a bounds-checked cursor that reads binary values without copying, and `decode` returns a `Result`. The writer produces the same bytes as the official Thrift library. diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts new file mode 100644 index 00000000000..932574645c9 --- /dev/null +++ b/packages/effect/src/encoding/Thrift.ts @@ -0,0 +1,1128 @@ +/** + * Reads and writes the Apache Thrift compact protocol: structs, lists, sets, + * maps and scalars, without an IDL. + * + * **Mental model** + * + * - A Thrift struct is a sequence of fields, each a numbered value of some + * {@link Type}, ended by a stop marker. Lists, sets and maps hold values of + * one element type. Nothing on the wire names a field: the IDL that says + * field 3 is `num_rows` lives with the caller. + * - This module works at the protocol level, as generated Thrift code does. + * A {@link Reader} is a cursor over the bytes: read a field header, look at + * its id and type, read the value or {@link Reader.skip} it. A + * {@link Writer} writes the same calls back. + * - The method names and their meaning follow Thrift's `TProtocol`, so code + * written against another Thrift implementation reads the same: a struct + * is `readStructBegin`, then `readFieldBegin` until it returns + * `Type.Stop`, then `readStructEnd`. + * - {@link Reader.readValue} and {@link Writer.writeValue} read and write any + * value as a {@link Value} tree, for debugging and tests. + * + * **Example** (Writing and reading a struct) + * + * ```ts import.meta.vitest + * import { Thrift } from "effect/encoding" + * import { Result } from "effect" + * + * // struct Person { 1: string name; 2: i64 age; 3: list tags } + * const w = Thrift.makeWriter() + * w.writeStructBegin() + * w.writeFieldBegin(Thrift.Type.Binary, 1) + * w.writeString("Ada") + * w.writeFieldBegin(Thrift.Type.I64, 2) + * w.writeI64Number(36) + * w.writeFieldBegin(Thrift.Type.List, 3) + * w.writeListBegin(Thrift.Type.Binary, 1) + * w.writeString("math") + * w.writeFieldStop() + * w.writeStructEnd() + * const bytes = w.finish() + * + * const person = Thrift.decode(bytes, (r) => { + * let name = "" + * let age = 0 + * const tags: Array = [] + * r.readStructBegin() + * for (let type = r.readFieldBegin(); type !== Thrift.Type.Stop; type = r.readFieldBegin()) { + * const id = r.fieldId + * if (id === 1 && type === Thrift.Type.Binary) name = r.readString() + * else if (id === 2 && type === Thrift.Type.I64) age = r.readI64Number() + * else if (id === 3 && type === Thrift.Type.List) { + * for (let n = r.readListBegin(); n > 0; n--) tags.push(r.readString()) + * } else r.skip(type) // unknown fields, or fields of an unexpected type + * } + * r.readStructEnd() + * return { name, age, tags } + * }) + * + * Result.getOrThrow(person) // => { name: "Ada", age: 36, tags: ["math"] } + * ``` + * + * **Details** + * + * Every read is bounds-checked: malformed or hostile input throws a + * {@link ThriftError} (or makes {@link decode} fail) instead of reading past + * the bytes. A length or container size larger than the bytes that remain + * fails before anything is allocated, and nesting is limited by + * {@link Options.maxDepth}. Binary values are views into the input, not + * copies. Strings must be valid UTF-8. + * + * `i64` values read as `bigint` ({@link Reader.readI64}) or, when they are + * known to be safe integers such as offsets and counts, as `number` + * ({@link Reader.readI64Number}). + * + * The binary and JSON protocols and the RPC message envelope are not + * supported. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Data from "../Data.ts" +import * as Predicate from "../Predicate.ts" +import * as Result from "../Result.ts" + +const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" + +/** + * Error raised for malformed input or a misused {@link Writer}. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `Truncated`: the input ends inside a value, or a length or container + * size is larger than the bytes that remain. + * - `Malformed`: an unknown type code, a varint that is too long, or a value + * of type `Stop`. + * - `InvalidUtf8`: a string is not valid UTF-8. + * - `Range`: an `i64` read with {@link Reader.readI64Number} is not a safe + * integer. + * - `Limit`: the input nests deeper than {@link Options.maxDepth}, or a + * length or size exceeds {@link Options.maxBinaryLength} or + * {@link Options.maxContainerSize}. + * - `Usage`: the reader or writer was called out of order, or a value is + * out of range for its type. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class ThriftError extends Data.TaggedError("ThriftError")<{ + readonly kind: "Truncated" | "Malformed" | "InvalidUtf8" | "Range" | "Limit" | "Usage" + readonly message: string +}> { + /** + * Marks this value as a Thrift error for runtime guards. + * + * @stability experimental + * @since 4.1.5 + */ + readonly [ThriftErrorTypeId] = ThriftErrorTypeId +} + +/** + * Checks whether a value is a {@link ThriftError}. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isThriftError = (u: unknown): u is ThriftError => Predicate.hasProperty(u, ThriftErrorTypeId) + +const fail = (kind: ThriftError["kind"], message: string): never => { + throw new ThriftError({ kind, message }) +} + +// ----------------------------------------------------------------------------- +// Types +// ----------------------------------------------------------------------------- + +/** + * The type of a field or container element. The codes are Thrift's `TType` + * codes, the same in every protocol; `Stop` ends a struct. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export const Type = { + Stop: 0, + Bool: 2, + I8: 3, + Double: 4, + I16: 6, + I32: 8, + I64: 10, + Binary: 11, + Struct: 12, + Map: 13, + Set: 14, + List: 15, + Uuid: 16 +} as const + +/** + * A {@link Type} code. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Type = typeof Type[keyof typeof Type] + +/** + * The name of a {@link Type}, as used by {@link Value}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TypeName = keyof typeof Type + +const INVALID = 255 + +/** Compact type nibble → TType. */ +const fromCompact = [0, 2, 2, 3, 6, 8, 10, 4, 11, 15, 14, 13, 12, 16, INVALID, INVALID] + +/** TType → compact type nibble (bool as an element; fields encode the value). */ +const toCompact: Array = new Array(17).fill(INVALID) +toCompact[0] = 0 +for (let nibble = 1; nibble < 14; nibble++) { + if (nibble !== 2) toCompact[fromCompact[nibble]] = nibble +} + +const typeNames: Array = [] +for (const name of Object.keys(Type) as Array) typeNames[Type[name]] = name + +const describe = (type: number): string => typeNames[type] ?? String(type) + +// ----------------------------------------------------------------------------- +// Values +// ----------------------------------------------------------------------------- + +/** + * Any Thrift value, tagged with the name of its {@link Type}: what + * {@link Reader.readValue} returns and {@link Writer.writeValue} writes. + * + * **Details** + * + * The wire does not tell strings from binary, so both read as `Binary`. An + * empty map carries no key and value types on the wire; they read as + * `"Stop"`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Value = + | { readonly _tag: "Bool"; readonly value: boolean } + | { readonly _tag: "I8" | "I16" | "I32" | "Double"; readonly value: number } + | { readonly _tag: "I64"; readonly value: bigint } + | { readonly _tag: "Binary" | "Uuid"; readonly value: Uint8Array } + | { readonly _tag: "Struct"; readonly fields: ReadonlyArray } + | { readonly _tag: "List" | "Set"; readonly elementType: TypeName; readonly elements: ReadonlyArray } + | { + readonly _tag: "Map" + readonly keyType: TypeName + readonly valueType: TypeName + readonly entries: ReadonlyArray + } + +/** + * A field of a `Struct` {@link Value}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Field { + readonly id: number + readonly value: Value +} + +// ----------------------------------------------------------------------------- +// Reading +// ----------------------------------------------------------------------------- + +/** + * Limits for reading untrusted input. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Options { + /** + * How deep structs and containers may nest, counting each struct and each + * container entered by {@link Reader.readStructBegin}, + * {@link Reader.skip} or {@link Reader.readValue}. Defaults to 64. + */ + readonly maxDepth?: number | undefined + /** The longest binary or string value, in bytes. No limit by default. */ + readonly maxBinaryLength?: number | undefined + /** The largest list, set or map. No limit by default. */ + readonly maxContainerSize?: number | undefined +} + +const utf8 = new TextDecoder("utf-8", { fatal: true }) +const MAX_SAFE = Number.MAX_SAFE_INTEGER +const BIGINT_1 = BigInt(1) +const BIGINT_7 = BigInt(7) +const BIGINT_63 = BigInt(63) +const BIGINT_0x7f = BigInt(0x7f) +const I64_MIN = -(BigInt(1) << BIGINT_63) +const I64_MAX = (BigInt(1) << BIGINT_63) - BIGINT_1 + +/** + * A cursor over compact-protocol bytes. Each `read*` method reads one value + * at {@link position} and moves past it; reads throw a {@link ThriftError} + * on malformed input. + * + * **Details** + * + * After {@link readFieldBegin}, {@link fieldId} holds the field's id; after + * {@link readListBegin} or {@link readSetBegin}, {@link elementType} holds + * the element type; after {@link readMapBegin}, {@link keyType} and + * {@link valueType}. Copy the property into a local before reading a nested + * value, which overwrites it. + * + * A `bool` field keeps its value in the field header: {@link readBool} right + * after a `Bool` field header returns it without reading a byte. Inside a + * container, each `bool` is a byte. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export class Reader { + /** The input. */ + readonly bytes: Uint8Array + /** The id of the field whose header was read last. */ + fieldId = 0 + /** The element type of the list or set whose header was read last. */ + elementType: Type = Type.Stop + /** The key type of the map whose header was read last. */ + keyType: Type = Type.Stop + /** The value type of the map whose header was read last. */ + valueType: Type = Type.Stop + + private pos = 0 + private readonly view: DataView + private readonly end: number + private readonly maxDepth: number + private readonly maxBinaryLength: number + private readonly maxContainerSize: number + private lastFieldId = 0 + private readonly fieldIds: Array = [] + private structs = 0 + private depth = 0 + /** The compact type of a bool field header not yet read: 1 true, 2 false, 0 none. */ + private bool = 0 + /** Set by {@link varint64}: whether the zigzag value is negative. */ + private negative = false + + constructor(bytes: Uint8Array, options?: Options) { + this.bytes = bytes + this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + this.end = bytes.length + this.maxDepth = options?.maxDepth ?? 64 + this.maxBinaryLength = options?.maxBinaryLength ?? Infinity + this.maxContainerSize = options?.maxContainerSize ?? Infinity + } + + /** Where the next read starts in {@link bytes}. Set it to read from elsewhere. */ + get position(): number { + return this.pos + } + set position(position: number) { + if (!Number.isInteger(position) || position < 0 || position > this.end) { + fail("Usage", `position ${position} is outside the ${this.end}-byte input`) + } + this.pos = position + } + + private truncated(what: string): never { + return fail("Truncated", `the input ends inside ${what} at byte ${this.pos}`) + } + + private byte(what: string): number { + if (this.pos >= this.end) this.truncated(what) + return this.bytes[this.pos++] + } + + private advance(n: number, what: string): number { + const at = this.pos + if (n > this.end - at) this.truncated(what) + this.pos = at + n + return at + } + + /** An unsigned varint of at most 32 bits. */ + private varint32(what: string): number { + const bytes = this.bytes + const end = this.end + let pos = this.pos + if (pos < end) { + let b = bytes[pos++] + if (b < 0x80) { + this.pos = pos + return b + } + let result = b & 0x7f + for (let shift = 7; shift < 28; shift += 7) { + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + result |= (b & 0x7f) << shift + if (b < 0x80) { + this.pos = pos + return result + } + } + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + if (b > 0x0f) return fail("Malformed", `${what} at byte ${this.pos} is longer than 32 bits`) + this.pos = pos + return (result | (b << 28)) >>> 0 + } + return this.truncated(what) + } + + /** + * A zigzag varint of at most 64 bits as a number: returns the magnitude + * `|v|` for v >= 0 or `|v| - 1` for v < 0 (exact up to 2^53) and sets + * {@link negative}. + */ + private varint64(what: string): number { + const bytes = this.bytes + const end = this.end + const start = this.pos + let pos = start + if (pos >= end) return this.truncated(what) + let b = bytes[pos++] + this.negative = (b & 1) === 1 + let half = (b & 0x7f) >>> 1 + let scale = 64 + while (b >= 0x80) { + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + if (pos - start === 10) { + if (b > 0x01) return fail("Malformed", `${what} at byte ${start} is longer than 64 bits`) + } + half += (b & 0x7f) * scale + scale *= 128 + } + this.pos = pos + return half + } + + private checkSize(size: number, bytesPerElement: number, what: string): void { + if (size * bytesPerElement > this.end - this.pos) { + fail("Truncated", `${what} of ${size} elements at byte ${this.pos} is larger than the remaining input`) + } + if (size > this.maxContainerSize) { + fail("Limit", `${what} of ${size} elements exceeds the limit of ${this.maxContainerSize}`) + } + } + + private enter(): void { + if (++this.depth > this.maxDepth) { + fail("Limit", `the input nests deeper than ${this.maxDepth} at byte ${this.pos}`) + } + } + + private elementOf(nibble: number, size: number, what: string): Type { + const type = fromCompact[nibble] + if (type === INVALID || (type === Type.Stop && size > 0)) { + return fail("Malformed", `${what} at byte ${this.pos} has element type code ${nibble}`) + } + return type as Type + } + + /** Starts reading a struct. Read its fields with {@link readFieldBegin}. */ + readStructBegin(): void { + this.enter() + this.fieldIds[this.structs++] = this.lastFieldId + this.lastFieldId = 0 + } + + /** Finishes the struct started with {@link readStructBegin}, after its `Stop`. */ + readStructEnd(): void { + if (this.structs === 0) fail("Usage", "readStructEnd without readStructBegin") + this.depth-- + this.lastFieldId = this.fieldIds[--this.structs] + } + + /** + * Reads a field header: returns the field's type and sets {@link fieldId}. + * Returns `Type.Stop` at the end of the struct. + */ + readFieldBegin(): Type { + const header = this.byte("a field header") + this.bool = 0 + const nibble = header & 0x0f + if (nibble === 0) return Type.Stop + const delta = header >> 4 + let id: number + if (delta === 0) { + const n = this.varint32("a field id") + id = (((n >>> 1) ^ -(n & 1)) << 16) >> 16 + } else { + id = this.lastFieldId + delta + } + this.lastFieldId = id + this.fieldId = id + const type = fromCompact[nibble] + if (type === INVALID) { + return fail("Malformed", `the field header at byte ${this.pos - 1} has type code ${nibble}`) + } + if (nibble <= 2) this.bool = nibble + return type as Type + } + + /** Reads a list header: returns the list's size and sets {@link elementType}. */ + readListBegin(): number { + const header = this.byte("a list header") + let size = header >> 4 + if (size === 15) size = this.varint32("a list size") + this.elementType = this.elementOf(header & 0x0f, size, "a list") + this.checkSize(size, 1, "a list") + return size + } + + /** Reads a set header: returns the set's size and sets {@link elementType}. */ + readSetBegin(): number { + const header = this.byte("a set header") + let size = header >> 4 + if (size === 15) size = this.varint32("a set size") + this.elementType = this.elementOf(header & 0x0f, size, "a set") + this.checkSize(size, 1, "a set") + return size + } + + /** Reads a map header: returns the map's size and sets {@link keyType} and {@link valueType}. */ + readMapBegin(): number { + const size = this.varint32("a map size") + if (size === 0) { + this.keyType = Type.Stop + this.valueType = Type.Stop + return 0 + } + const types = this.byte("a map header") + this.keyType = this.elementOf(types >> 4, size, "a map") + this.valueType = this.elementOf(types & 0x0f, size, "a map") + this.checkSize(size, 2, "a map") + return size + } + + /** Reads a `bool`: the value of a `Bool` field header just read, or a byte inside a container. */ + readBool(): boolean { + const bool = this.bool + if (bool !== 0) { + this.bool = 0 + return bool === 1 + } + return this.byte("a bool") === 1 + } + + /** Reads an `i8` (`byte`). */ + readI8(): number { + return (this.byte("an i8") << 24) >> 24 + } + + /** Reads an `i16`. */ + readI16(): number { + const n = this.varint32("an i16") + return (((n >>> 1) ^ -(n & 1)) << 16) >> 16 + } + + /** Reads an `i32`. */ + readI32(): number { + const n = this.varint32("an i32") + return (n >>> 1) ^ -(n & 1) + } + + /** Reads an `i64` as a `bigint`. */ + readI64(): bigint { + const start = this.pos + const half = this.varint64("an i64") + if (half < MAX_SAFE) return BigInt(this.negative ? -half - 1 : half) + let u = BigInt(0) + let shift = BigInt(0) + for (let pos = start; pos < this.pos; pos++) { + u |= (BigInt(this.bytes[pos]) & BIGINT_0x7f) << shift + shift += BIGINT_7 + } + u = BigInt.asUintN(64, u) + return (u >> BIGINT_1) ^ -(u & BIGINT_1) + } + + /** + * Reads an `i64` as a `number`. Fails with kind `Range` when the value is + * not a safe integer. + */ + readI64Number(): number { + const start = this.pos + const half = this.varint64("an i64") + if (half < MAX_SAFE || (half === MAX_SAFE && !this.negative)) return this.negative ? -half - 1 : half + this.pos = start + return fail("Range", `the i64 at byte ${start} is not a safe integer`) + } + + /** Reads a `double` (little-endian, as the compact protocol stores it). */ + readDouble(): number { + return this.view.getFloat64(this.advance(8, "a double"), true) + } + + /** Reads a `binary` value as a view of {@link bytes}, not a copy. */ + readBinary(): Uint8Array { + const length = this.varint32("a binary length") + if (length > this.maxBinaryLength) { + fail("Limit", `a binary value of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) + } + const at = this.advance(length, `a binary value of ${length} bytes`) + return this.bytes.subarray(at, at + length) + } + + /** Reads a `string`: a `binary` value decoded as UTF-8. */ + readString(): string { + const length = this.varint32("a string length") + if (length > this.maxBinaryLength) { + fail("Limit", `a string of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) + } + const at = this.advance(length, `a string of ${length} bytes`) + try { + return utf8.decode(this.bytes.subarray(at, at + length)) + } catch { + return fail("InvalidUtf8", `the string at byte ${at} is not valid UTF-8`) + } + } + + /** Reads a `uuid`: 16 bytes, as a view of {@link bytes}. */ + readUuid(): Uint8Array { + const at = this.advance(16, "a uuid") + return this.bytes.subarray(at, at + 16) + } + + /** + * Moves past a value of the given type: the value of a field whose header + * was just read, or a container element. + */ + skip(type: Type): void { + switch (type) { + case Type.Bool: + if (this.bool !== 0) this.bool = 0 + else this.advance(1, "a bool") + return + case Type.I8: + this.advance(1, "an i8") + return + case Type.I16: + case Type.I32: + this.varint32("an integer") + return + case Type.I64: + this.varint64("an i64") + return + case Type.Double: + this.advance(8, "a double") + return + case Type.Binary: { + const length = this.varint32("a binary length") + if (length > this.maxBinaryLength) { + fail("Limit", `a binary value of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) + } + this.advance(length, `a binary value of ${length} bytes`) + return + } + case Type.Uuid: + this.advance(16, "a uuid") + return + case Type.Struct: + this.readStructBegin() + for (let t = this.readFieldBegin(); t !== Type.Stop; t = this.readFieldBegin()) this.skip(t) + this.readStructEnd() + return + case Type.List: + case Type.Set: { + this.enter() + const size = type === Type.List ? this.readListBegin() : this.readSetBegin() + const element = this.elementType + for (let i = 0; i < size; i++) this.skip(element) + this.depth-- + return + } + case Type.Map: { + this.enter() + const size = this.readMapBegin() + const key = this.keyType + const value = this.valueType + for (let i = 0; i < size; i++) { + this.skip(key) + this.skip(value) + } + this.depth-- + return + } + default: + fail("Malformed", `can't skip a value of type ${describe(type)} at byte ${this.pos}`) + } + } + + /** + * Reads a value of the given type as a {@link Value}: the value of a field + * whose header was just read, a container element, or a whole input such + * as a struct. + */ + readValue(type: Type): Value { + switch (type) { + case Type.Bool: + return { _tag: "Bool", value: this.readBool() } + case Type.I8: + return { _tag: "I8", value: this.readI8() } + case Type.I16: + return { _tag: "I16", value: this.readI16() } + case Type.I32: + return { _tag: "I32", value: this.readI32() } + case Type.I64: + return { _tag: "I64", value: this.readI64() } + case Type.Double: + return { _tag: "Double", value: this.readDouble() } + case Type.Binary: + return { _tag: "Binary", value: this.readBinary() } + case Type.Uuid: + return { _tag: "Uuid", value: this.readUuid() } + case Type.Struct: { + const fields: Array = [] + this.readStructBegin() + for (let t = this.readFieldBegin(); t !== Type.Stop; t = this.readFieldBegin()) { + const id = this.fieldId + fields.push({ id, value: this.readValue(t) }) + } + this.readStructEnd() + return { _tag: "Struct", fields } + } + case Type.List: + case Type.Set: { + this.enter() + const size = type === Type.List ? this.readListBegin() : this.readSetBegin() + const element = this.elementType + const elements: Array = [] + for (let i = 0; i < size; i++) elements.push(this.readValue(element)) + this.depth-- + return { _tag: type === Type.List ? "List" : "Set", elementType: typeNames[element], elements } + } + case Type.Map: { + this.enter() + const size = this.readMapBegin() + const key = this.keyType + const value = this.valueType + const entries: Array = [] + for (let i = 0; i < size; i++) { + const k = this.readValue(key) + entries.push([k, this.readValue(value)]) + } + this.depth-- + return { _tag: "Map", keyType: typeNames[key], valueType: typeNames[value], entries } + } + default: + return fail("Malformed", `can't read a value of type ${describe(type)} at byte ${this.pos}`) + } + } +} + +/** + * A {@link Reader} at the start of `bytes`. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const makeReader = (bytes: Uint8Array, options?: Options): Reader => new Reader(bytes, options) + +/** + * Reads `bytes` with `read`, which gets a {@link Reader} at the start of + * them. Malformed input becomes a failed `Result` instead of an exception. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const decode = ( + bytes: Uint8Array, + read: (reader: Reader) => A, + options?: Options +): Result.Result => { + try { + return Result.succeed(read(new Reader(bytes, options))) + } catch (cause) { + if (isThriftError(cause)) return Result.fail(cause) + throw cause + } +} + +// ----------------------------------------------------------------------------- +// Writing +// ----------------------------------------------------------------------------- + +const encoder = new TextEncoder() + +/** + * Writes compact-protocol bytes into a buffer that grows as needed. The + * calls mirror {@link Reader}'s. + * + * **Details** + * + * Write a struct as {@link writeStructBegin}, then for each field + * {@link writeFieldBegin} and its value, then {@link writeFieldStop} and + * {@link writeStructEnd}. A `bool` field's value goes into its header: + * `writeFieldBegin(Type.Bool, id)` writes nothing until {@link writeBool}. + * + * Containers are a header ({@link writeListBegin}, {@link writeSetBegin}, + * {@link writeMapBegin}) followed by exactly that many elements (for a map, + * key then value). + * + * Integers outside their type's range, and calls out of order, throw a + * {@link ThriftError} of kind `Usage`. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export class Writer { + private bytes: Uint8Array + private view: DataView + private position = 0 + private lastFieldId = 0 + private readonly fieldIds: Array = [] + private boolPending = false + private boolFieldId = 0 + + constructor(options?: { readonly initialSize?: number | undefined }) { + this.bytes = new Uint8Array(Math.max(16, options?.initialSize ?? 1024)) + this.view = new DataView(this.bytes.buffer) + } + + /** The number of bytes written. */ + get length(): number { + return this.position + } + + private reserve(n: number): void { + if (this.position + n <= this.bytes.length) return + let size = this.bytes.length * 2 + while (size < this.position + n) size *= 2 + const next = new Uint8Array(size) + next.set(this.bytes.subarray(0, this.position)) + this.bytes = next + this.view = new DataView(next.buffer) + } + + private checkNoBool(what: string): void { + if (this.boolPending) fail("Usage", `${what} after writeFieldBegin(Type.Bool) instead of writeBool`) + } + + private writeByte(b: number): void { + this.reserve(1) + this.bytes[this.position++] = b + } + + /** An unsigned varint of at most 32 bits. */ + private varint32(n: number): void { + this.reserve(5) + const bytes = this.bytes + let pos = this.position + while (n > 0x7f) { + bytes[pos++] = (n & 0x7f) | 0x80 + n >>>= 7 + } + bytes[pos++] = n + this.position = pos + } + + private integer(value: number, min: number, max: number, what: string): void { + this.checkNoBool(what) + if (!Number.isInteger(value) || value < min || value > max) { + fail("Usage", `${what} ${value} is not an integer from ${min} to ${max}`) + } + } + + private checkSize(size: number, what: string): void { + if (!Number.isInteger(size) || size < 0 || size > 0xffffffff) { + fail("Usage", `${what} ${size} is not an integer from 0 to ${0xffffffff}`) + } + } + + private elementNibble(type: number, size: number, what: string): number { + const nibble = toCompact[type] + if (nibble === undefined || nibble === INVALID || (type === Type.Stop && size > 0)) { + return fail("Usage", `${describe(type)} is not a valid ${what}`) + } + return nibble + } + + private fieldHeader(nibble: number, id: number): void { + const delta = id - this.lastFieldId + if (delta > 0 && delta <= 15) { + this.writeByte((delta << 4) | nibble) + } else { + this.writeByte(nibble) + this.varint32(((id << 1) ^ (id >> 31)) >>> 0) + } + this.lastFieldId = id + } + + /** Starts a struct. Write its fields, then {@link writeFieldStop} and {@link writeStructEnd}. */ + writeStructBegin(): void { + this.checkNoBool("writeStructBegin") + this.fieldIds.push(this.lastFieldId) + this.lastFieldId = 0 + } + + /** Finishes the struct started with {@link writeStructBegin}. */ + writeStructEnd(): void { + this.checkNoBool("writeStructEnd") + if (this.fieldIds.length === 0) fail("Usage", "writeStructEnd without writeStructBegin") + this.lastFieldId = this.fieldIds.pop()! + } + + /** Writes a field header. For a `Bool` field, the header is written by the {@link writeBool} that follows. */ + writeFieldBegin(type: Type, id: number): void { + this.checkNoBool("writeFieldBegin") + if (this.fieldIds.length === 0) fail("Usage", "writeFieldBegin outside writeStructBegin/writeStructEnd") + if (!Number.isInteger(id) || id < -32768 || id > 32767) { + fail("Usage", `field id ${id} is not an integer from -32768 to 32767`) + } + if (type === Type.Bool) { + this.boolPending = true + this.boolFieldId = id + return + } + const nibble = toCompact[type] + if (nibble === undefined || nibble === INVALID || type === Type.Stop) { + fail("Usage", `${describe(type)} is not a field type`) + } + this.fieldHeader(nibble, id) + } + + /** Writes the `Stop` that ends a struct's fields. */ + writeFieldStop(): void { + this.checkNoBool("writeFieldStop") + this.writeByte(0) + } + + /** Writes a list header; write `size` elements of `elementType` next. */ + writeListBegin(elementType: Type, size: number): void { + this.checkNoBool("writeListBegin") + this.checkSize(size, "list size") + const nibble = this.elementNibble(elementType, size, "list element type") + if (size < 15) { + this.writeByte((size << 4) | nibble) + } else { + this.writeByte(0xf0 | nibble) + this.varint32(size) + } + } + + /** Writes a set header; write `size` elements of `elementType` next. */ + writeSetBegin(elementType: Type, size: number): void { + this.writeListBegin(elementType, size) + } + + /** Writes a map header; write `size` key-value pairs next. */ + writeMapBegin(keyType: Type, valueType: Type, size: number): void { + this.checkNoBool("writeMapBegin") + this.checkSize(size, "map size") + const key = this.elementNibble(keyType, size, "map key type") + const value = this.elementNibble(valueType, size, "map value type") + if (size === 0) { + this.writeByte(0) + } else { + this.varint32(size) + this.writeByte((key << 4) | value) + } + } + + /** Writes a `bool`: the header of the `Bool` field just begun, or a byte inside a container. */ + writeBool(value: boolean): void { + if (this.boolPending) { + this.boolPending = false + this.fieldHeader(value ? 1 : 2, this.boolFieldId) + } else { + this.writeByte(value ? 1 : 2) + } + } + + /** Writes an `i8` (`byte`). */ + writeI8(value: number): void { + this.integer(value, -128, 127, "i8") + this.writeByte(value & 0xff) + } + + /** Writes an `i16`. */ + writeI16(value: number): void { + this.integer(value, -32768, 32767, "i16") + this.varint32(((value << 1) ^ (value >> 31)) >>> 0) + } + + /** Writes an `i32`. */ + writeI32(value: number): void { + this.integer(value, -2147483648, 2147483647, "i32") + this.varint32(((value << 1) ^ (value >> 31)) >>> 0) + } + + /** Writes an `i64` given as a safe-integer `number`. */ + writeI64Number(value: number): void { + this.integer(value, -MAX_SAFE, MAX_SAFE, "i64") + const negative = value < 0 + let half = negative ? -value - 1 : value + this.reserve(10) + const bytes = this.bytes + let pos = this.position + // The first byte holds the sign bit and the low six bits of the magnitude. + let b = ((half % 64) * 2) | (negative ? 1 : 0) + half = Math.floor(half / 64) + while (half > 0) { + bytes[pos++] = b | 0x80 + b = half % 128 + half = Math.floor(half / 128) + } + bytes[pos++] = b + this.position = pos + } + + /** Writes an `i64`. */ + writeI64(value: bigint): void { + this.checkNoBool("writeI64") + if (value < I64_MIN || value > I64_MAX) fail("Usage", `i64 ${value} is out of range`) + if (value >= -MAX_SAFE && value <= MAX_SAFE) return this.writeI64Number(Number(value)) + let u = BigInt.asUintN(64, (value << BIGINT_1) ^ (value >> BIGINT_63)) + this.reserve(10) + while (u > BIGINT_0x7f) { + this.bytes[this.position++] = Number(u & BIGINT_0x7f) | 0x80 + u >>= BIGINT_7 + } + this.bytes[this.position++] = Number(u) + } + + /** Writes a `double` (little-endian). */ + writeDouble(value: number): void { + this.checkNoBool("writeDouble") + this.reserve(8) + this.view.setFloat64(this.position, value, true) + this.position += 8 + } + + /** Writes a `binary` value. */ + writeBinary(value: Uint8Array): void { + this.checkNoBool("writeBinary") + this.varint32(value.length) + this.reserve(value.length) + this.bytes.set(value, this.position) + this.position += value.length + } + + /** Writes a `string` as UTF-8 `binary`. Lone surrogates, which UTF-8 can't encode, become U+FFFD. */ + writeString(value: string): void { + this.checkNoBool("writeString") + const length = value.length + if (length < 128) { + // One length byte; ASCII strings are written directly. + this.reserve(1 + length) + const bytes = this.bytes + const start = this.position + 1 + let i = 0 + for (; i < length; i++) { + const c = value.charCodeAt(i) + if (c >= 0x80) break + bytes[start + i] = c + } + if (i === length) { + bytes[this.position] = length + this.position = start + length + return + } + } + this.writeBinary(encoder.encode(value)) + } + + /** Writes a `uuid`: exactly 16 bytes. */ + writeUuid(value: Uint8Array): void { + this.checkNoBool("writeUuid") + if (value.length !== 16) fail("Usage", `a uuid is 16 bytes, got ${value.length}`) + this.reserve(16) + this.bytes.set(value, this.position) + this.position += 16 + } + + /** Writes a {@link Value}: a field's value after its header, a container element, or a whole value. */ + writeValue(value: Value): void { + switch (value._tag) { + case "Bool": + return this.writeBool(value.value) + case "I8": + return this.writeI8(value.value) + case "I16": + return this.writeI16(value.value) + case "I32": + return this.writeI32(value.value) + case "I64": + return this.writeI64(value.value) + case "Double": + return this.writeDouble(value.value) + case "Binary": + return this.writeBinary(value.value) + case "Uuid": + return this.writeUuid(value.value) + case "Struct": + this.writeStructBegin() + for (const field of value.fields) { + this.writeFieldBegin(Type[field.value._tag], field.id) + this.writeValue(field.value) + } + this.writeFieldStop() + return this.writeStructEnd() + case "List": + case "Set": { + const type = Type[value.elementType] + if (value._tag === "List") this.writeListBegin(type, value.elements.length) + else this.writeSetBegin(type, value.elements.length) + for (const element of value.elements) { + if (element._tag !== value.elementType) { + fail("Usage", `a ${value._tag} of ${value.elementType} contains a ${element._tag}`) + } + this.writeValue(element) + } + return + } + case "Map": { + this.writeMapBegin(Type[value.keyType], Type[value.valueType], value.entries.length) + for (const [k, v] of value.entries) { + if (k._tag !== value.keyType || v._tag !== value.valueType) { + fail("Usage", `a Map of ${value.keyType} to ${value.valueType} contains ${k._tag} to ${v._tag}`) + } + this.writeValue(k) + this.writeValue(v) + } + return + } + } + } + + /** A copy of the bytes written so far. Fails when a struct or `Bool` field is unfinished. */ + finish(): Uint8Array { + this.checkNoBool("finish") + if (this.fieldIds.length > 0) fail("Usage", `finish with ${this.fieldIds.length} unfinished structs`) + return this.bytes.slice(0, this.position) + } +} + +/** + * A new {@link Writer}. `initialSize` is the first buffer size; it grows as + * needed. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export const makeWriter = (options?: { readonly initialSize?: number | undefined }): Writer => new Writer(options) diff --git a/packages/effect/src/encoding/index.ts b/packages/effect/src/encoding/index.ts index dc7605ca67d..8247d26b903 100644 --- a/packages/effect/src/encoding/index.ts +++ b/packages/effect/src/encoding/index.ts @@ -83,6 +83,12 @@ export * as Sse from "./Sse.ts" */ export * as Tar from "./Tar.ts" +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Thrift from "./Thrift.ts" + /** * @stability unstable * @since 4.0.0 diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts new file mode 100644 index 00000000000..3ee4977eb25 --- /dev/null +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -0,0 +1,500 @@ +import { assert, describe, it } from "@effect/vitest" +import { Result, Schema } from "effect" +import * as Thrift from "effect/encoding/Thrift" +import { readFileSync } from "node:fs" + +// Expected bytes were written by the official Apache Thrift Python library; +// see fixtures/thrift/README.md +type Json = + | { readonly _tag: "Bool"; readonly value: boolean } + | { readonly _tag: "I8" | "I16" | "I32"; readonly value: number } + | { readonly _tag: "I64"; readonly value: string } + | { readonly _tag: "Double"; readonly value: number | string } + | { readonly _tag: "Binary" | "Uuid"; readonly value: string } + | { readonly _tag: "Struct"; readonly fields: ReadonlyArray<{ readonly id: number; readonly value: Json }> } + | { readonly _tag: "List" | "Set"; readonly elementType: Thrift.TypeName; readonly elements: ReadonlyArray } + | { + readonly _tag: "Map" + readonly keyType: Thrift.TypeName + readonly valueType: Thrift.TypeName + readonly entries: ReadonlyArray + } +interface Case { + readonly id: string + readonly type: Thrift.TypeName + readonly value: Json + readonly bytes: string +} + +const gold: { readonly cases: ReadonlyArray; readonly parquet: ReadonlyArray> } = JSON + .parse(readFileSync(new URL("../fixtures/thrift/gold.json", import.meta.url), "utf8")) + +const fromHex = (hex: string) => Uint8Array.from(hex.match(/../g) ?? [], (b) => parseInt(b, 16)) +const toHex = (bytes: Uint8Array) => Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("") + +const fromJson = (v: Json): Thrift.Value => { + switch (v._tag) { + case "I64": + return { _tag: "I64", value: BigInt(v.value) } + case "Double": + return { _tag: "Double", value: Number(v.value) } + case "Binary": + case "Uuid": + return { _tag: v._tag, value: fromHex(v.value) } + case "Struct": + return { _tag: "Struct", fields: v.fields.map((f) => ({ id: f.id, value: fromJson(f.value) })) } + case "List": + case "Set": + return { ...v, elements: v.elements.map(fromJson) } + case "Map": + return { ...v, entries: v.entries.map(([k, x]) => [fromJson(k), fromJson(x)] as const) } + default: + return v + } +} + +const toJson = (v: Thrift.Value): Json => { + switch (v._tag) { + case "I64": + return { _tag: "I64", value: String(v.value) } + case "Double": { + const d = v.value + return { + _tag: "Double", + value: Number.isFinite(d) && !Object.is(d, -0) ? d : Object.is(d, -0) ? "-0" : String(d) + } + } + case "Binary": + case "Uuid": + return { _tag: v._tag, value: toHex(v.value) } + case "Struct": + return { _tag: "Struct", fields: v.fields.map((f) => ({ id: f.id, value: toJson(f.value) })) } + case "List": + case "Set": + return { ...v, elements: v.elements.map(toJson) } + case "Map": + return { ...v, entries: v.entries.map(([k, x]) => [toJson(k), toJson(x)] as const) } + default: + return v + } +} + +const readAll = (bytes: Uint8Array, type: Thrift.TypeName, options?: Thrift.Options) => + Thrift.decode(bytes, (r) => { + const value = r.readValue(Thrift.Type[type]) + assert.strictEqual(r.position, bytes.length) + return value + }, options) + +const write = (f: (w: Thrift.Writer) => void): Uint8Array => { + const w = Thrift.makeWriter() + f(w) + return w.finish() +} + +const kindOf = (result: Result.Result): Thrift.ThriftError["kind"] | undefined => + Result.isFailure(result) ? result.failure.kind : undefined + +const nest = (depth: number): Thrift.Value => { + let v: Thrift.Value = { _tag: "Struct", fields: [] } + for (let i = 0; i < depth - 1; i++) v = { _tag: "Struct", fields: [{ id: 1, value: v }] } + return v +} + +describe("Thrift", () => { + describe("gold", () => { + it("reads the official library's bytes", () => { + assert.isAbove(gold.cases.length, 300) + for (const c of gold.cases) { + assert.deepStrictEqual(toJson(Result.getOrThrow(readAll(fromHex(c.bytes), c.type))), c.value, c.id) + } + }) + + it("writes the official library's bytes", () => { + for (const c of gold.cases) { + assert.strictEqual(toHex(write((w) => w.writeValue(fromJson(c.value)))), c.bytes, c.id) + } + }) + + it("skips every value to its end", () => { + for (const c of gold.cases) { + const bytes = fromHex(c.bytes) + const r = Thrift.makeReader(bytes) + r.skip(Thrift.Type[c.type]) + assert.strictEqual(r.position, bytes.length, c.id) + } + }) + + it("reads and writes structs cut from Parquet files", () => { + assert.strictEqual(gold.parquet.length, 7) + for (const c of gold.parquet) { + const value = Result.getOrThrow(readAll(fromHex(c.bytes), "Struct")) + assert.deepStrictEqual(toJson(value), c.value, c.id) + assert.strictEqual(toHex(write((w) => w.writeValue(value))), c.bytes, c.id) + } + }) + }) + + it("reads a Parquet footer field by field", () => { + // The fields of parquet.thrift's FileMetaData that pyarrow reports, read + // the way a typed reader does: by field id, skipping the rest. + const struct = (r: Thrift.Reader, field: (id: number, type: Thrift.Type) => void) => { + r.readStructBegin() + for (let t = r.readFieldBegin(); t !== Thrift.Type.Stop; t = r.readFieldBegin()) field(r.fieldId, t) + r.readStructEnd() + } + const list = (r: Thrift.Reader, element: (index: number) => void) => { + for (let n = r.readListBegin(), i = 0; i < n; i++) element(i) + } + const footer = (bytes: Uint8Array) => + Thrift.decode(bytes, (r) => { + const out = { numRows: 0, columns: [] as Array, createdBy: "", dataPageOffsets: [] as Array } + // ColumnChunk → 3: ColumnMetaData → 9: data_page_offset + const columnChunk = () => + struct(r, (id, t) => { + if (id !== 3) return r.skip(t) + struct(r, (id, t) => id === 9 ? out.dataPageOffsets.push(r.readI64Number()) : r.skip(t)) + }) + // RowGroup → 1: list + const rowGroup = () => struct(r, (id, t) => id === 1 ? list(r, columnChunk) : r.skip(t)) + // SchemaElement → 4: name; element 0 is the root + const schemaElement = (i: number) => + struct(r, (id, t) => id === 4 && i > 0 ? out.columns.push(r.readString()) : r.skip(t)) + struct(r, (id, t) => { + if (id === 2) list(r, schemaElement) + else if (id === 3) out.numRows = r.readI64Number() + else if (id === 4) list(r, rowGroup) + else if (id === 6) out.createdBy = r.readString() + else r.skip(t) + }) + assert.strictEqual(r.position, bytes.length) + return out + }) + const bloom = Result.getOrThrow(footer(fromHex(gold.parquet[0].bytes))) + assert.strictEqual(bloom.numRows, 14) + assert.deepStrictEqual(bloom.columns, ["String"]) + assert.deepStrictEqual(bloom.dataPageOffsets, [4]) + assert.match(bloom.createdBy, /^parquet-mr version 1\.13\.0-SNAPSHOT/) + const tiny = Result.getOrThrow(footer(fromHex(gold.parquet[5].bytes))) + assert.strictEqual(tiny.numRows, 7300) + assert.deepStrictEqual(tiny.columns, [ + "id", + "bool_col", + "tinyint_col", + "smallint_col", + "int_col", + "bigint_col", + "float_col", + "double_col", + "date_string_col", + "string_col", + "timestamp_col", + "year", + "month" + ]) + assert.strictEqual(tiny.dataPageOffsets[0], 4) + }) + + describe("encoding", () => { + // One example per rule of the compact protocol spec. + const s = (...fields: Array): Thrift.Value => ({ + _tag: "Struct", + fields: fields.map(([id, value]) => ({ id, value })) + }) + const i32 = (value: number): Thrift.Value => ({ _tag: "I32", value }) + const table: ReadonlyArray = [ + ["a field id delta packs into the header", s([1, i32(1)], [3, i32(-1)]), "1502250100"], + ["a delta over 15 uses the long form", s([16, i32(0)]), "05200000"], + ["a decreasing id uses the long form", s([2, i32(0)], [1, i32(0)]), "250005020000"], + [ + "a bool field's value is its header type", + s([1, { _tag: "Bool", value: true }], [2, { _tag: "Bool", value: false }]), + "111200" + ], + [ + "a nested struct counts ids from zero", + s([5, s([1, i32(0)])], [6, i32(0)]), + "5c150000150000" + ], + ["integers are zigzag varints", s([1, i32(-65)], [2, i32(64)]), "15810115800100"], + ["i8 is a raw byte", { _tag: "I8", value: -1 }, "ff"], + ["i64 is a zigzag varint", { _tag: "I64", value: BigInt(-1) << BigInt(63) }, "ffffffffffffffffff01"], + ["double is little-endian", { _tag: "Double", value: 1 }, "000000000000f03f"], + ["binary is a length and bytes", { _tag: "Binary", value: Uint8Array.of(0x61, 0x62) }, "026162"], + ["a list of up to 14 packs its size", { _tag: "List", elementType: "I32", elements: [i32(1), i32(2)] }, "250204"], + [ + "a list of 15 or more has a varint size", + { _tag: "List", elementType: "I8", elements: Array.from({ length: 15 }, () => ({ _tag: "I8", value: 0 })) }, + "f30f" + "00".repeat(15) + ], + [ + "list bools are bytes 1 and 2", + { + _tag: "List", + elementType: "Bool", + elements: [{ _tag: "Bool", value: true }, { _tag: "Bool", value: false }] + }, + "210102" + ], + ["an empty map is one byte", { _tag: "Map", keyType: "Stop", valueType: "Stop", entries: [] }, "00"], + ["a map packs key and value types", { + _tag: "Map", + keyType: "I32", + valueType: "I32", + entries: [[i32(1), i32(2)]] + }, "01550204"] + ] + for (const [name, value, hex] of table) { + it(name, () => { + assert.strictEqual(toHex(write((w) => w.writeValue(value))), hex) + assert.deepStrictEqual(Result.getOrThrow(readAll(fromHex(hex), value._tag)), value) + }) + } + + it("reads a bool list element type written as 2", () => { + // Some writers use the FALSE code for the element type of list. + const r = Thrift.makeReader(Uint8Array.of(0x12, 0x01)) + assert.strictEqual(r.readListBegin(), 1) + assert.strictEqual(r.elementType, Thrift.Type.Bool) + assert.isTrue(r.readBool()) + }) + }) + + describe("scalars", () => { + const roundTrip = (writeOne: (w: Thrift.Writer, a: A) => void, readOne: (r: Thrift.Reader) => A, a: A) => { + const bytes = write((w) => writeOne(w, a)) + const r = Thrift.makeReader(bytes) + const back = readOne(r) + assert.strictEqual(r.position, bytes.length) + return back + } + + it.prop( + "i64 round-trips as bigint", + [Schema.BigInt.check( + Schema.isBetweenBigInt({ minimum: -(BigInt(1) << BigInt(63)), maximum: (BigInt(1) << BigInt(63)) - BigInt(1) }) + )], + ([n]) => { + assert.strictEqual(roundTrip((w, a) => w.writeI64(a), (r) => r.readI64(), n), n) + } + ) + + it.prop("i64 round-trips as number", [Schema.Int], ([n]) => { + assert.strictEqual(roundTrip((w, a) => w.writeI64Number(a), (r) => r.readI64Number(), n), n) + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64(), n), BigInt(n)) + }) + + it.prop( + "i32 round-trips", + [Schema.Int.check(Schema.isBetween({ minimum: -(2 ** 31), maximum: 2 ** 31 - 1 }))], + ([n]) => { + assert.strictEqual(roundTrip((w, a) => w.writeI32(a), (r) => r.readI32(), n), n) + } + ) + + it.prop("double round-trips", [Schema.Number], ([n]) => { + assert.isTrue(Object.is(roundTrip((w, a) => w.writeDouble(a), (r) => r.readDouble(), n), n)) + }) + + it.prop("string round-trips through UTF-8", [Schema.String], ([s]) => { + // Lone surrogates have no UTF-8 encoding and become U+FFFD. + const expected = new TextDecoder().decode(new TextEncoder().encode(s)) + assert.strictEqual(roundTrip((w, a) => w.writeString(a), (r) => r.readString(), s), expected) + }) + + it.prop("binary round-trips", [Schema.Uint8Array], ([bytes]) => { + assert.deepStrictEqual(roundTrip((w, a) => w.writeBinary(a), (r) => r.readBinary(), bytes), bytes) + }) + + it("reads i64 limits", () => { + const max = BigInt(Number.MAX_SAFE_INTEGER) + const bytesOf = (n: bigint) => write((w) => w.writeI64(n)) + const asNumber = (n: bigint) => Thrift.decode(bytesOf(n), (r) => r.readI64Number()) + assert.strictEqual(Result.getOrThrow(asNumber(max)), Number.MAX_SAFE_INTEGER) + assert.strictEqual(Result.getOrThrow(asNumber(-max)), -Number.MAX_SAFE_INTEGER) + assert.strictEqual(kindOf(asNumber(max + BigInt(1))), "Range") + assert.strictEqual(kindOf(asNumber(-max - BigInt(1))), "Range") + for (const n of [max, max + BigInt(1), -max - BigInt(1), -max - BigInt(2)]) { + assert.strictEqual(Result.getOrThrow(Thrift.decode(bytesOf(n), (r) => r.readI64())), n) + } + }) + + it("reads from a position set by the caller", () => { + // A Parquet page header sits at an offset inside the file. + const r = Thrift.makeReader(Uint8Array.of(0xff, 0xff, 0x15, 0x02, 0x00)) + r.position = 2 + assert.deepStrictEqual(r.readValue(Thrift.Type.Struct), { + _tag: "Struct", + fields: [{ id: 1, value: { _tag: "I32", value: 1 } }] + }) + assert.strictEqual(r.position, 5) + }) + + it("reads binary as a view of the input", () => { + const bytes = Uint8Array.of(2, 7, 8) + const view = Thrift.makeReader(bytes).readBinary() + assert.strictEqual(view.buffer, bytes.buffer) + assert.deepStrictEqual(Array.from(view), [7, 8]) + }) + + it("reads lengths with a three-byte varint", () => { + const value = new Uint8Array(20000).fill(7) + const bytes = write((w) => w.writeBinary(value)) + assert.strictEqual(bytes.length, 3 + value.length) + assert.deepStrictEqual(Thrift.makeReader(bytes).readBinary(), value) + }) + }) + + describe("malformed input", () => { + const fails = (hex: string, read: (r: Thrift.Reader) => unknown, options?: Thrift.Options) => + kindOf(Thrift.decode(fromHex(hex), read, options)) + const struct = (r: Thrift.Reader) => r.readValue(Thrift.Type.Struct) + const table: ReadonlyArray< + readonly [string, string, (r: Thrift.Reader) => unknown, Thrift.ThriftError["kind"], Thrift.Options?] + > = [ + ["empty input", "", struct, "Truncated"], + ["a struct without its stop", "1502", struct, "Truncated"], + ["an i32 varint over 5 bytes", "808080808001", (r) => r.readI32(), "Malformed"], + ["an i32 varint over 32 bits", "ffffffff1f", (r) => r.readI32(), "Malformed"], + ["an i64 varint over 10 bytes", "ffffffffffffffffff8101", (r) => r.readI64(), "Malformed"], + ["an i64 varint over 64 bits", "ffffffffffffffffff02", (r) => r.readI64(), "Malformed"], + ["an unknown field type", "1e", struct, "Malformed"], + ["an unknown list element type", "1e", (r) => r.readListBegin(), "Malformed"], + ["a non-empty list of Stop", "10", (r) => r.readListBegin(), "Malformed"], + ["a list larger than the input", "f5ffff03", (r) => r.readListBegin(), "Truncated"], + ["a map larger than the input", "0355020406", (r) => r.readMapBegin(), "Truncated"], + ["a binary longer than the input", "0561", (r) => r.readBinary(), "Truncated"], + ["a string that isn't UTF-8", "02c328", (r) => r.readString(), "InvalidUtf8"], + ["a long string that isn't UTF-8", "41" + "61".repeat(64) + "ff", (r) => r.readString(), "InvalidUtf8"], + ["skipping Stop", "00", (r) => r.skip(Thrift.Type.Stop), "Malformed"], + ["readStructEnd without readStructBegin", "", (r) => r.readStructEnd(), "Usage"], + ["a position before the input", "00", (r) => r.position = -1, "Usage"], + ["a position past the input", "00", (r) => r.position = 2, "Usage"], + ["a binary over maxBinaryLength", "03616263", (r) => r.readBinary(), "Limit", { maxBinaryLength: 2 }], + ["a skipped binary over maxBinaryLength", "1803616263", (r) => r.skip(Thrift.Type.Struct), "Limit", { + maxBinaryLength: 2 + }], + ["a list over maxContainerSize", "35010203", (r) => r.readListBegin(), "Limit", { maxContainerSize: 2 }], + ["a struct over maxDepth", "1c1c000000", struct, "Limit", { maxDepth: 2 }], + ["lists over maxDepth", "1919191500", (r) => r.skip(Thrift.Type.List), "Limit", { maxDepth: 2 }] + ] + for (const [name, hex, read, kind, options] of table) { + it(name, () => assert.strictEqual(fails(hex, read, options), kind)) + } + + it("limits nesting to 64 by default", () => { + const bytesOf = (depth: number) => write((w) => w.writeValue(nest(depth))) + assert.isTrue(Result.isSuccess(readAll(bytesOf(64), "Struct"))) + assert.strictEqual(kindOf(readAll(bytesOf(65), "Struct")), "Limit") + assert.strictEqual(kindOf(Thrift.decode(bytesOf(65), (r) => r.skip(Thrift.Type.Struct))), "Limit") + }) + + it("fails every truncated gold case", () => { + for (const c of gold.cases) { + const bytes = fromHex(c.bytes) + for (let n = 0; n < bytes.length; n += Math.max(1, Math.floor(bytes.length / 16))) { + assert.strictEqual(kindOf(readAll(bytes.subarray(0, n), c.type)), "Truncated", `${c.id} at ${n}`) + } + } + }) + + it("decode rethrows errors that aren't ThriftErrors", () => { + assert.throws(() => + Thrift.decode(Uint8Array.of(0), () => { + throw new Error("boom") + }), "boom") + }) + + it("only ever fails with ThriftError on corrupted input", () => { + // Deterministic xorshift so failures reproduce. + let seed = 0x5eed + const random = () => { + seed ^= seed << 13 + seed ^= seed >>> 17 + seed ^= seed << 5 + return (seed >>> 0) / 0x100000000 + } + let failures = 0 + for (const c of [...gold.cases, ...gold.parquet.map((p) => ({ ...p, type: "Struct" as const }))]) { + const original = fromHex(c.bytes) + for (let round = 0; round < 10; round++) { + const bytes = original.slice() + const flips = 1 + Math.floor(random() * 4) + for (let i = 0; i < flips && bytes.length > 0; i++) { + bytes[Math.floor(random() * bytes.length)] = Math.floor(random() * 256) + } + for ( + const read of [ + (r: Thrift.Reader) => r.readValue(Thrift.Type[c.type]), + (r: Thrift.Reader) => r.skip(Thrift.Type[c.type]) + ] + ) { + try { + read(Thrift.makeReader(bytes, { maxContainerSize: 1 << 16 })) + } catch (error) { + assert.isTrue(Thrift.isThriftError(error), `${c.id}: ${String(error)}`) + failures++ + } + } + } + } + assert.isAbove(failures, 0) + }) + }) + + describe("Writer", () => { + const usage: ReadonlyArray void]> = [ + ["an i8 out of range", (w) => w.writeI8(128)], + ["a fractional i16", (w) => w.writeI16(1.5)], + ["an i32 out of range", (w) => w.writeI32(2 ** 31)], + ["an unsafe i64 number", (w) => w.writeI64Number(2 ** 53)], + ["an i64 out of range", (w) => w.writeI64(BigInt(1) << BigInt(63))], + ["a uuid that isn't 16 bytes", (w) => w.writeUuid(new Uint8Array(15))], + ["a field outside a struct", (w) => w.writeFieldBegin(Thrift.Type.I32, 1)], + ["a field id out of range", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.I32, 40000) + }], + ["a Stop field", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.Stop, 1) + }], + ["a bool field without writeBool", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.Bool, 1) + w.writeI32(1) + }], + ["writeStructEnd without writeStructBegin", (w) => w.writeStructEnd()], + ["finish inside a struct", (w) => { + w.writeStructBegin() + w.finish() + }], + ["a non-empty list of Stop", (w) => w.writeListBegin(Thrift.Type.Stop, 1)], + ["a negative list size", (w) => w.writeListBegin(Thrift.Type.I32, -1)], + [ + "a list element of another type", + (w) => w.writeValue({ _tag: "List", elementType: "I32", elements: [{ _tag: "I8", value: 1 }] }) + ] + ] + for (const [name, f] of usage) { + it(`rejects ${name}`, () => { + try { + f(Thrift.makeWriter()) + assert.fail("expected a ThriftError") + } catch (error) { + assert.isTrue(Thrift.isThriftError(error)) + assert.strictEqual((error as Thrift.ThriftError).kind, "Usage") + } + }) + } + + it("grows from a small buffer and returns a copy", () => { + const w = Thrift.makeWriter({ initialSize: 1 }) + w.writeValue(fromJson(gold.cases.at(-1)!.value)) + const bytes = w.finish() + assert.strictEqual(bytes.length, w.length) + assert.strictEqual(bytes.byteOffset, 0) + assert.strictEqual(bytes.buffer.byteLength, bytes.length) + }) + }) +}) diff --git a/packages/effect/test/fixtures/thrift/README.md b/packages/effect/test/fixtures/thrift/README.md new file mode 100644 index 00000000000..0dd7f038b1f --- /dev/null +++ b/packages/effect/test/fixtures/thrift/README.md @@ -0,0 +1,21 @@ +# Thrift fixtures + +`gold.json` is read by `test/encoding/Thrift.test.ts`. Each case is a value +tree and the bytes the official Apache Thrift Python library (`thrift` +0.25.0 from PyPI, `TCompactProtocol`) writes for it: hand-written cases for +each feature of the compact protocol, then 300 seeded random structs. + +The `parquet` section holds compact-protocol structs cut from two files of +[apache/parquet-testing](https://github.com/apache/parquet-testing) (commit +`bcefdfd`): the `FileMetaData` footers of `data_index_bloom_encoding_stats.parquet` +and `alltypes_tiny_pages.parquet`, a page header from each, and the bloom +filter header, offset index and column index of the first. Each comes with the +tree the Python reader reads from it. The row counts, column names and +offsets the test checks are the ones pyarrow reports for those files. + +Regenerate with: + +```sh +pip install thrift==0.25.0 +PARQUET_TESTING=/path/to/parquet-testing python test/fixtures/thrift/generate.py +``` diff --git a/packages/effect/test/fixtures/thrift/generate.py b/packages/effect/test/fixtures/thrift/generate.py new file mode 100644 index 00000000000..c49eb787885 --- /dev/null +++ b/packages/effect/test/fixtures/thrift/generate.py @@ -0,0 +1,473 @@ +""" +Gold corpus for effect/encoding/Thrift: values written by the official Apache +Thrift Python library's TCompactProtocol (pip install thrift), and the bytes +it produces. The TypeScript test reads every case back into a Thrift.Value, +expects the same tree, and writes the tree back, expecting the same bytes. + + PARQUET_TESTING=/path/to/apache/parquet-testing \ + python test/fixtures/thrift/generate.py # writes gold.json + +Values are JSON trees shaped like Thrift.Value: + {"_tag": "Bool", "value": true} + {"_tag": "I8" | "I16" | "I32", "value": 5} + {"_tag": "I64", "value": "-5"} decimal string + {"_tag": "Double", "value": 1.5} or "NaN", "Infinity", "-Infinity", "-0" + {"_tag": "Binary" | "Uuid", "value": "00ff"} hex + {"_tag": "Struct", "fields": [{"id": 1, "value": ...}]} + {"_tag": "List" | "Set", "elementType": "I32", "elements": [...]} + {"_tag": "Map", "keyType": "I32", "valueType": "Bool", "entries": [[k, v]]} +A case is {"id", "type", "value", "bytes"}: `type` is the Type name of the +top-level value. Every case is also read back here with the Python reader to +check the generator. + +The "parquet" cases are Thrift structs cut from parquet-testing files (a +footer, page headers, a bloom filter header, column and offset indexes), +with the tree the Python reader reads from them. +""" + +import json +import math +import os +import random +import struct +import uuid +from pathlib import Path + +from thrift.protocol.TCompactProtocol import TCompactProtocol +from thrift.Thrift import TType +from thrift.transport.TTransport import TMemoryBuffer + +HERE = Path(__file__).parent + +TTYPES = { + "Bool": TType.BOOL, + "I8": TType.BYTE, + "I16": TType.I16, + "I32": TType.I32, + "I64": TType.I64, + "Double": TType.DOUBLE, + "Binary": TType.STRING, + "Uuid": TType.UUID, + "Struct": TType.STRUCT, + "List": TType.LIST, + "Set": TType.SET, + "Map": TType.MAP, + "Stop": TType.STOP, +} +NAMES = {v: k for k, v in TTYPES.items()} + +# TCompactProtocol only writes scalars inside a struct or container; these +# are its internal states for "a value comes next". +VALUE_WRITE = 2 +VALUE_READ = 7 + + +def double_json(d): + if math.isnan(d): + return "NaN" + if math.isinf(d): + return "Infinity" if d > 0 else "-Infinity" + if d == 0 and math.copysign(1, d) < 0: + return "-0" + return d + + +def double_py(v): + return float(v) if isinstance(v, str) else float(v) + + +def write(p, v): + tag = v["_tag"] + if tag == "Bool": + p.writeBool(v["value"]) + elif tag == "I8": + p.writeByte(v["value"]) + elif tag == "I16": + p.writeI16(v["value"]) + elif tag == "I32": + p.writeI32(v["value"]) + elif tag == "I64": + p.writeI64(int(v["value"])) + elif tag == "Double": + p.writeDouble(double_py(v["value"])) + elif tag == "Binary": + p.writeBinary(bytes.fromhex(v["value"])) + elif tag == "Uuid": + p.writeUuid(uuid.UUID(bytes=bytes.fromhex(v["value"]))) + elif tag == "Struct": + p.writeStructBegin(None) + for f in v["fields"]: + p.writeFieldBegin(None, TTYPES[f["value"]["_tag"]], f["id"]) + write(p, f["value"]) + p.writeFieldEnd() + p.writeFieldStop() + p.writeStructEnd() + elif tag in ("List", "Set"): + begin = p.writeListBegin if tag == "List" else p.writeSetBegin + begin(TTYPES[v["elementType"]], len(v["elements"])) + for e in v["elements"]: + write(p, e) + p.writeListEnd() + elif tag == "Map": + p.writeMapBegin(TTYPES[v["keyType"]], TTYPES[v["valueType"]], len(v["entries"])) + for k, x in v["entries"]: + write(p, k) + write(p, x) + p.writeMapEnd() + else: + raise ValueError(tag) + + +def read(p, t): + name = NAMES[t] + if name == "Bool": + return {"_tag": name, "value": p.readBool()} + if name == "I8": + return {"_tag": name, "value": p.readByte()} + if name == "I16": + return {"_tag": name, "value": p.readI16()} + if name == "I32": + return {"_tag": name, "value": p.readI32()} + if name == "I64": + return {"_tag": name, "value": str(p.readI64())} + if name == "Double": + return {"_tag": name, "value": double_json(p.readDouble())} + if name == "Binary": + return {"_tag": name, "value": p.readBinary().hex()} + if name == "Uuid": + return {"_tag": name, "value": p.readUuid().bytes.hex()} + if name == "Struct": + fields = [] + p.readStructBegin() + while True: + _, ft, fid = p.readFieldBegin() + if ft == TType.STOP: + break + fields.append({"id": fid, "value": read(p, ft)}) + p.readFieldEnd() + p.readStructEnd() + return {"_tag": name, "fields": fields} + if name in ("List", "Set"): + et, n = p.readListBegin() + elements = [read(p, et) for _ in range(n)] + p.readListEnd() + return {"_tag": name, "elementType": NAMES[et], "elements": elements} + if name == "Map": + kt, vt, n = p.readMapBegin() + entries = [] + for _ in range(n): + k = read(p, kt) + entries.append([k, read(p, vt)]) + p.readMapEnd() + return {"_tag": name, "keyType": NAMES[kt], "valueType": NAMES[vt], "entries": entries} + raise ValueError(name) + + +def encode(v): + buf = TMemoryBuffer() + p = TCompactProtocol(buf) + if v["_tag"] != "Struct": + p.state = VALUE_WRITE + write(p, v) + return buf.getvalue() + + +def decode(data, type_name, offset=0): + """Reads one value at `offset`; returns it and the number of bytes read.""" + buf = TMemoryBuffer(data[offset:]) + p = TCompactProtocol(buf) + if type_name != "Struct": + p.state = VALUE_READ + v = read(p, TTYPES[type_name]) + return v, buf._buffer.tell() + + +cases = [] + + +def case(id, v): + data = encode(v) + back, n = decode(data, v["_tag"]) + assert back == v, (id, back, v) + assert n == len(data), id + cases.append({"id": id, "type": v["_tag"], "value": v, "bytes": data.hex()}) + + +# --- value constructors ------------------------------------------------------- + + +def b(x): + return {"_tag": "Bool", "value": x} + + +def i8(x): + return {"_tag": "I8", "value": x} + + +def i16(x): + return {"_tag": "I16", "value": x} + + +def i32(x): + return {"_tag": "I32", "value": x} + + +def i64(x): + return {"_tag": "I64", "value": str(x)} + + +def dbl(x): + return {"_tag": "Double", "value": double_json(x)} + + +def binary(x): + return {"_tag": "Binary", "value": bytes(x).hex()} + + +def text(s): + return binary(s.encode("utf-8")) + + +def uid(x): + return {"_tag": "Uuid", "value": bytes(x).hex()} + + +def struct_(*fields): + return {"_tag": "Struct", "fields": [{"id": i, "value": v} for i, v in fields]} + + +def lst(t, *es): + return {"_tag": "List", "elementType": t, "elements": list(es)} + + +def st(t, *es): + return {"_tag": "Set", "elementType": t, "elements": list(es)} + + +def mp(k, v, *entries): + if not entries: + k = v = "Stop" + return {"_tag": "Map", "keyType": k, "valueType": v, "entries": [list(e) for e in entries]} + + +# --- hand-written cases ------------------------------------------------------- + +case("struct/empty", struct_()) +case("bool/fields", struct_((1, b(True)), (2, b(False)), (3, b(True)))) +case("bool/long-form", struct_((20, b(True)), (-1, b(False)))) +case("field/delta-1", struct_((1, i32(1)), (2, i32(2)))) +case("field/delta-15", struct_((15, i32(1)), (30, i32(2)))) +case("field/delta-16", struct_((16, i32(1)), (32, i32(2)))) +case("field/id-0", struct_((0, i32(1)))) +case("field/decreasing", struct_((5, i32(1)), (3, i32(2)), (4, i32(3)))) +case("field/repeated", struct_((5, i32(1)), (5, i32(2)))) +case("field/negative", struct_((-1, i32(1)), (-32768, i32(2)), (1, i32(3)))) +case("field/max", struct_((32767, i32(1)))) +case("field/nested-resets", struct_((5, struct_((1, i8(1)), (2, i8(2)))), (6, i8(3)))) +case("i8/bounds", struct_((1, i8(-128)), (2, i8(127)), (3, i8(0)), (4, i8(-1)))) +case("i16/bounds", struct_((1, i16(-32768)), (2, i16(32767)), (3, i16(0)), (4, i16(-1)), (5, i16(64)), (6, i16(-65)))) +case( + "i32/bounds", + struct_( + (1, i32(-(2**31))), (2, i32(2**31 - 1)), (3, i32(0)), (4, i32(-1)), (5, i32(63)), (6, i32(64)), + (7, i32(8191)), (8, i32(8192)), (9, i32(-8193)) + ), +) +case( + "i64/bounds", + struct_(*enumerate([ + i64(0), i64(-1), i64(1), i64(2**31), i64(-(2**31) - 1), i64(2**53 - 1), i64(-(2**53 - 1)), i64(2**53), + i64(-(2**53)), i64(2**53 + 1), i64(-(2**53) - 1), i64(2**62), i64(2**63 - 1), i64(-(2**63)), + i64(2**56 - 1), i64(1 << 49) + ], 1)), +) +case( + "double/values", + struct_(*enumerate([ + dbl(0.0), dbl(-0.0), dbl(1.5), dbl(float("nan")), dbl(float("inf")), dbl(float("-inf")), dbl(5e-324), + dbl(1.7976931348623157e308), dbl(math.pi), dbl(-2.5) + ], 1)), +) +case( + "binary/lengths", + struct_((1, binary(b"")), (2, binary(b"\x00")), (3, binary(bytes(range(127)))), (4, binary(bytes(128))), + (5, binary(bytes(i % 256 for i in range(300))))), +) +case("binary/utf8", struct_((1, text("héllo 🌍")), (2, text("")), (3, text("ascii")), (4, text("日本語" * 30)))) +case("uuid", struct_((1, uid(range(16))), (2, uid(bytes(16))))) +case("list/empty", struct_((1, lst("I32")), (2, lst("Struct")), (3, lst("Bool")))) +case("list/size-14", struct_((1, lst("I32", *[i32(i) for i in range(14)])))) +case("list/size-15", struct_((1, lst("I32", *[i32(i) for i in range(15)])))) +case("list/size-200", struct_((1, lst("I16", *[i16(i - 100) for i in range(200)])))) +case("list/bool", struct_((1, lst("Bool", b(True), b(False), b(True))))) +case("list/i8", struct_((1, lst("I8", i8(-1), i8(0), i8(127))))) +case("list/i64", struct_((1, lst("I64", i64(-(2**63)), i64(0), i64(2**63 - 1))))) +case("list/double", struct_((1, lst("Double", dbl(1.0), dbl(float("nan")))))) +case("list/binary", struct_((1, lst("Binary", text("a"), text(""), text("bc"))))) +case("list/uuid", struct_((1, lst("Uuid", uid(range(16)), uid(range(16, 32)))))) +case("list/struct", struct_((1, lst("Struct", struct_((1, b(True))), struct_(), struct_((2, i32(7))))))) +case("list/list", struct_((1, lst("List", lst("I32", i32(1)), lst("I32"), lst("Bool", b(False)))))) +case("list/map", struct_((1, lst("Map", mp("I32", "I32", (i32(1), i32(2))), mp("I32", "I32"))))) +case("set/values", struct_((1, st("I32", i32(3), i32(1))), (2, st("Binary")), (3, st("Bool", b(True))))) +case("set/size-20", struct_((1, st("I64", *[i64(i * 1000) for i in range(20)])))) +case("map/empty", struct_((1, mp("I32", "I32")))) +case("map/one", struct_((1, mp("Binary", "I64", (text("k"), i64(-5)))))) +case("map/bools", struct_((1, mp("Bool", "Bool", (b(True), b(False)), (b(False), b(True)))))) +case("map/i32-struct", struct_((1, mp("I32", "Struct", (i32(1), struct_((1, b(True)))), (i32(2), struct_()))))) +case("map/size-20", struct_((1, mp("I16", "Double", *[(i16(i), dbl(i / 2)) for i in range(20)])))) +case("map/nested", struct_((1, mp("I32", "List", (i32(1), lst("I32", i32(2), i32(3))))))) +case("map/uuid", struct_((1, mp("Uuid", "Uuid", (uid(range(16)), uid(bytes(16))))))) + + +def nest(depth): + v = struct_((1, i32(depth))) + for d in range(depth): + v = struct_((1, v), (2, b(d % 2 == 0))) + return v + + +case("struct/deep-20", nest(20)) +case("top/list", lst("I32", i32(1), i32(-1))) +case("top/i64", i64(-(2**40))) +case("top/binary", text("top")) +case("top/map", mp("Binary", "Binary", (text("a"), text("b")))) +case("top/set", st("Struct", struct_((1, i8(1))))) +case("top/double", dbl(0.1)) + +# --- seeded random structs ---------------------------------------------------- + +rng = random.Random(0x7487) +SCALARS = ["Bool", "I8", "I16", "I32", "I64", "Double", "Binary", "Uuid"] +ALL = SCALARS + ["Struct", "List", "Set", "Map"] + + +def rand_int(bits): + r = rng.random() + if r < 0.3: + return rng.randint(-64, 64) + if r < 0.4: + return rng.choice([-(2 ** (bits - 1)), 2 ** (bits - 1) - 1, 0, -1]) + w = rng.randint(1, bits - 1) + return rng.randint(-(2**w), 2**w - 1) + + +def rand_double(): + r = rng.random() + if r < 0.1: + return rng.choice([0.0, -0.0, float("inf"), float("-inf"), float("nan"), 5e-324]) + if r < 0.4: + return float(rng.randint(-1000, 1000)) + return struct.unpack("= 3 or rng.random() < 0.6 else ALL) + + +def rand_struct(depth): + n = rng.randint(0, 3 if depth > 2 else 8) + ids = [] + last = 0 + for _ in range(n): + r = rng.random() + if r < 0.6: + last += rng.randint(1, 15) + elif r < 0.8: + last += rng.randint(16, 300) + elif r < 0.9: + last = rng.randint(-32768, 32767) + else: + last = rng.randint(-5, 5) + last = max(-32768, min(32767, last)) + ids.append(last) + return struct_(*[(i, rand_value(rand_type(depth), depth)) for i in ids]) + + +for n in range(300): + case(f"random/{n}", rand_struct(0)) + +# --- Parquet structs ---------------------------------------------------------- + + +def field(v, fid): + for f in v["fields"]: + if f["id"] == fid: + return f["value"] + return None + + +parquet = [] + + +def cut(file, data, offset, what): + v, n = decode(data, "Struct", offset) + parquet.append({ + "id": f"{file}/{what}", + "value": v, + "bytes": data[offset:offset + n].hex(), + }) + return v + + +root = os.environ.get("PARQUET_TESTING") +assert root, "set PARQUET_TESTING to a checkout of https://github.com/apache/parquet-testing" +# The tiny-pages file's indexes have thousands of entries; its footer is enough. +for file, indexes in [("data_index_bloom_encoding_stats.parquet", True), ("alltypes_tiny_pages.parquet", False)]: + data = (Path(root) / "data" / file).read_bytes() + assert data[-4:] == b"PAR1" + length = struct.unpack(" Date: Sun, 15 Nov 2026 12:34:32 +0000 Subject: [PATCH 04/32] Focus Thrift tests and shrink the gold fixture --- dprint.json | 1 + packages/effect/test/encoding/Thrift.test.ts | 352 ++++++------------ .../effect/test/fixtures/thrift/README.md | 17 +- .../effect/test/fixtures/thrift/generate.py | 27 +- .../effect/test/fixtures/thrift/gold.json | 2 +- 5 files changed, 141 insertions(+), 258 deletions(-) diff --git a/dprint.json b/dprint.json index 0622d2e2642..d8e26de0455 100644 --- a/dprint.json +++ b/dprint.json @@ -22,6 +22,7 @@ "**/CHANGELOG.md", "packages/effect/src/StandardSchema.ts", "packages/tools/graphql-generator/test/generated/**/*.graphql.ts", + "packages/effect/test/fixtures/thrift/gold.json", "!scratchpad/**/*", "**/.*", "!.agents", diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 3ee4977eb25..f70525c00ae 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -3,8 +3,8 @@ import { Result, Schema } from "effect" import * as Thrift from "effect/encoding/Thrift" import { readFileSync } from "node:fs" -// Expected bytes were written by the official Apache Thrift Python library; -// see fixtures/thrift/README.md +// Expected bytes and trees come from the official Apache Thrift Python +// library; see fixtures/thrift/README.md type Json = | { readonly _tag: "Bool"; readonly value: boolean } | { readonly _tag: "I8" | "I16" | "I32"; readonly value: number } @@ -28,6 +28,8 @@ interface Case { const gold: { readonly cases: ReadonlyArray; readonly parquet: ReadonlyArray> } = JSON .parse(readFileSync(new URL("../fixtures/thrift/gold.json", import.meta.url), "utf8")) +const corpus: ReadonlyArray = [...gold.cases, ...gold.parquet.map((c) => ({ ...c, type: "Struct" as const }))] +const parquet = (id: string) => fromHex(gold.parquet.find((c) => c.id === id)!.bytes) const fromHex = (hex: string) => Uint8Array.from(hex.match(/../g) ?? [], (b) => parseInt(b, 16)) const toHex = (bytes: Uint8Array) => Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("") @@ -57,13 +59,11 @@ const toJson = (v: Thrift.Value): Json => { switch (v._tag) { case "I64": return { _tag: "I64", value: String(v.value) } - case "Double": { - const d = v.value + case "Double": return { _tag: "Double", - value: Number.isFinite(d) && !Object.is(d, -0) ? d : Object.is(d, -0) ? "-0" : String(d) + value: Object.is(v.value, -0) ? "-0" : Number.isFinite(v.value) ? v.value : String(v.value) } - } case "Binary": case "Uuid": return { _tag: v._tag, value: toHex(v.value) } @@ -95,174 +95,93 @@ const write = (f: (w: Thrift.Writer) => void): Uint8Array => { const kindOf = (result: Result.Result): Thrift.ThriftError["kind"] | undefined => Result.isFailure(result) ? result.failure.kind : undefined -const nest = (depth: number): Thrift.Value => { - let v: Thrift.Value = { _tag: "Struct", fields: [] } - for (let i = 0; i < depth - 1; i++) v = { _tag: "Struct", fields: [{ id: 1, value: v }] } - return v +const thrown = (f: () => unknown): Thrift.ThriftError["kind"] | undefined => { + try { + f() + } catch (error) { + assert.isTrue(Thrift.isThriftError(error), String(error)) + return (error as Thrift.ThriftError).kind + } + return undefined } describe("Thrift", () => { - describe("gold", () => { - it("reads the official library's bytes", () => { - assert.isAbove(gold.cases.length, 300) - for (const c of gold.cases) { - assert.deepStrictEqual(toJson(Result.getOrThrow(readAll(fromHex(c.bytes), c.type))), c.value, c.id) - } - }) - - it("writes the official library's bytes", () => { - for (const c of gold.cases) { - assert.strictEqual(toHex(write((w) => w.writeValue(fromJson(c.value)))), c.bytes, c.id) - } - }) - - it("skips every value to its end", () => { - for (const c of gold.cases) { + describe("official library", () => { + it("reads, skips and writes every gold case as the official library does", () => { + for (const c of corpus) { const bytes = fromHex(c.bytes) + assert.deepStrictEqual(toJson(Result.getOrThrow(readAll(bytes, c.type))), c.value, c.id) const r = Thrift.makeReader(bytes) r.skip(Thrift.Type[c.type]) assert.strictEqual(r.position, bytes.length, c.id) + assert.strictEqual(toHex(write((w) => w.writeValue(fromJson(c.value)))), c.bytes, c.id) } }) - it("reads and writes structs cut from Parquet files", () => { - assert.strictEqual(gold.parquet.length, 7) - for (const c of gold.parquet) { - const value = Result.getOrThrow(readAll(fromHex(c.bytes), "Struct")) - assert.deepStrictEqual(toJson(value), c.value, c.id) - assert.strictEqual(toHex(write((w) => w.writeValue(value))), c.bytes, c.id) + it("reads a Parquet footer field by field", () => { + // The fields of parquet.thrift's FileMetaData that pyarrow reports, read + // the way a typed reader does: by field id, skipping the rest. + const struct = (r: Thrift.Reader, field: (id: number, type: Thrift.Type) => void) => { + r.readStructBegin() + for (let t = r.readFieldBegin(); t !== Thrift.Type.Stop; t = r.readFieldBegin()) field(r.fieldId, t) + r.readStructEnd() } - }) - }) - - it("reads a Parquet footer field by field", () => { - // The fields of parquet.thrift's FileMetaData that pyarrow reports, read - // the way a typed reader does: by field id, skipping the rest. - const struct = (r: Thrift.Reader, field: (id: number, type: Thrift.Type) => void) => { - r.readStructBegin() - for (let t = r.readFieldBegin(); t !== Thrift.Type.Stop; t = r.readFieldBegin()) field(r.fieldId, t) - r.readStructEnd() - } - const list = (r: Thrift.Reader, element: (index: number) => void) => { - for (let n = r.readListBegin(), i = 0; i < n; i++) element(i) - } - const footer = (bytes: Uint8Array) => - Thrift.decode(bytes, (r) => { - const out = { numRows: 0, columns: [] as Array, createdBy: "", dataPageOffsets: [] as Array } - // ColumnChunk → 3: ColumnMetaData → 9: data_page_offset - const columnChunk = () => + const list = (r: Thrift.Reader, element: (index: number) => void) => { + for (let n = r.readListBegin(), i = 0; i < n; i++) element(i) + } + const footer = (bytes: Uint8Array) => + Result.getOrThrow(Thrift.decode(bytes, (r) => { + const out = { numRows: 0, columns: [] as Array, createdBy: "", dataPageOffsets: [] as Array } + // ColumnChunk → 3: ColumnMetaData → 9: data_page_offset + const columnChunk = () => + struct(r, (id, t) => { + if (id !== 3) return r.skip(t) + struct(r, (id, t) => id === 9 ? out.dataPageOffsets.push(r.readI64Number()) : r.skip(t)) + }) + // RowGroup → 1: list + const rowGroup = () => struct(r, (id, t) => id === 1 ? list(r, columnChunk) : r.skip(t)) + // SchemaElement → 4: name; element 0 is the root + const schemaElement = (i: number) => + struct(r, (id, t) => id === 4 && i > 0 ? out.columns.push(r.readString()) : r.skip(t)) struct(r, (id, t) => { - if (id !== 3) return r.skip(t) - struct(r, (id, t) => id === 9 ? out.dataPageOffsets.push(r.readI64Number()) : r.skip(t)) + if (id === 2) list(r, schemaElement) + else if (id === 3) out.numRows = r.readI64Number() + else if (id === 4) list(r, rowGroup) + else if (id === 6) out.createdBy = r.readString() + else r.skip(t) }) - // RowGroup → 1: list - const rowGroup = () => struct(r, (id, t) => id === 1 ? list(r, columnChunk) : r.skip(t)) - // SchemaElement → 4: name; element 0 is the root - const schemaElement = (i: number) => - struct(r, (id, t) => id === 4 && i > 0 ? out.columns.push(r.readString()) : r.skip(t)) - struct(r, (id, t) => { - if (id === 2) list(r, schemaElement) - else if (id === 3) out.numRows = r.readI64Number() - else if (id === 4) list(r, rowGroup) - else if (id === 6) out.createdBy = r.readString() - else r.skip(t) - }) - assert.strictEqual(r.position, bytes.length) - return out - }) - const bloom = Result.getOrThrow(footer(fromHex(gold.parquet[0].bytes))) - assert.strictEqual(bloom.numRows, 14) - assert.deepStrictEqual(bloom.columns, ["String"]) - assert.deepStrictEqual(bloom.dataPageOffsets, [4]) - assert.match(bloom.createdBy, /^parquet-mr version 1\.13\.0-SNAPSHOT/) - const tiny = Result.getOrThrow(footer(fromHex(gold.parquet[5].bytes))) - assert.strictEqual(tiny.numRows, 7300) - assert.deepStrictEqual(tiny.columns, [ - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col", - "year", - "month" - ]) - assert.strictEqual(tiny.dataPageOffsets[0], 4) - }) - - describe("encoding", () => { - // One example per rule of the compact protocol spec. - const s = (...fields: Array): Thrift.Value => ({ - _tag: "Struct", - fields: fields.map(([id, value]) => ({ id, value })) - }) - const i32 = (value: number): Thrift.Value => ({ _tag: "I32", value }) - const table: ReadonlyArray = [ - ["a field id delta packs into the header", s([1, i32(1)], [3, i32(-1)]), "1502250100"], - ["a delta over 15 uses the long form", s([16, i32(0)]), "05200000"], - ["a decreasing id uses the long form", s([2, i32(0)], [1, i32(0)]), "250005020000"], - [ - "a bool field's value is its header type", - s([1, { _tag: "Bool", value: true }], [2, { _tag: "Bool", value: false }]), - "111200" - ], - [ - "a nested struct counts ids from zero", - s([5, s([1, i32(0)])], [6, i32(0)]), - "5c150000150000" - ], - ["integers are zigzag varints", s([1, i32(-65)], [2, i32(64)]), "15810115800100"], - ["i8 is a raw byte", { _tag: "I8", value: -1 }, "ff"], - ["i64 is a zigzag varint", { _tag: "I64", value: BigInt(-1) << BigInt(63) }, "ffffffffffffffffff01"], - ["double is little-endian", { _tag: "Double", value: 1 }, "000000000000f03f"], - ["binary is a length and bytes", { _tag: "Binary", value: Uint8Array.of(0x61, 0x62) }, "026162"], - ["a list of up to 14 packs its size", { _tag: "List", elementType: "I32", elements: [i32(1), i32(2)] }, "250204"], - [ - "a list of 15 or more has a varint size", - { _tag: "List", elementType: "I8", elements: Array.from({ length: 15 }, () => ({ _tag: "I8", value: 0 })) }, - "f30f" + "00".repeat(15) - ], - [ - "list bools are bytes 1 and 2", - { - _tag: "List", - elementType: "Bool", - elements: [{ _tag: "Bool", value: true }, { _tag: "Bool", value: false }] - }, - "210102" - ], - ["an empty map is one byte", { _tag: "Map", keyType: "Stop", valueType: "Stop", entries: [] }, "00"], - ["a map packs key and value types", { - _tag: "Map", - keyType: "I32", - valueType: "I32", - entries: [[i32(1), i32(2)]] - }, "01550204"] - ] - for (const [name, value, hex] of table) { - it(name, () => { - assert.strictEqual(toHex(write((w) => w.writeValue(value))), hex) - assert.deepStrictEqual(Result.getOrThrow(readAll(fromHex(hex), value._tag)), value) - }) - } - - it("reads a bool list element type written as 2", () => { - // Some writers use the FALSE code for the element type of list. - const r = Thrift.makeReader(Uint8Array.of(0x12, 0x01)) - assert.strictEqual(r.readListBegin(), 1) - assert.strictEqual(r.elementType, Thrift.Type.Bool) - assert.isTrue(r.readBool()) + assert.strictEqual(r.position, bytes.length) + return out + })) + const bloom = footer(parquet("data_index_bloom_encoding_stats.parquet/footer")) + assert.strictEqual(bloom.numRows, 14) + assert.deepStrictEqual(bloom.columns, ["String"]) + assert.deepStrictEqual(bloom.dataPageOffsets, [4]) + assert.match(bloom.createdBy, /^parquet-mr version 1\.13\.0-SNAPSHOT/) + const tiny = footer(parquet("alltypes_tiny_pages.parquet/footer")) + assert.strictEqual(tiny.numRows, 7300) + assert.deepStrictEqual(tiny.columns, [ + "id", + "bool_col", + "tinyint_col", + "smallint_col", + "int_col", + "bigint_col", + "float_col", + "double_col", + "date_string_col", + "string_col", + "timestamp_col", + "year", + "month" + ]) + assert.strictEqual(tiny.dataPageOffsets[0], 4) }) }) - describe("scalars", () => { - const roundTrip = (writeOne: (w: Thrift.Writer, a: A) => void, readOne: (r: Thrift.Reader) => A, a: A) => { - const bytes = write((w) => writeOne(w, a)) + describe("Reader", () => { + const roundTrip = (writeOne: (w: Thrift.Writer) => void, readOne: (r: Thrift.Reader) => A): A => { + const bytes = write(writeOne) const r = Thrift.makeReader(bytes) const back = readOne(r) assert.strictEqual(r.position, bytes.length) @@ -275,48 +194,28 @@ describe("Thrift", () => { Schema.isBetweenBigInt({ minimum: -(BigInt(1) << BigInt(63)), maximum: (BigInt(1) << BigInt(63)) - BigInt(1) }) )], ([n]) => { - assert.strictEqual(roundTrip((w, a) => w.writeI64(a), (r) => r.readI64(), n), n) + assert.strictEqual(roundTrip((w) => w.writeI64(n), (r) => r.readI64()), n) } ) it.prop("i64 round-trips as number", [Schema.Int], ([n]) => { - assert.strictEqual(roundTrip((w, a) => w.writeI64Number(a), (r) => r.readI64Number(), n), n) - assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64(), n), BigInt(n)) - }) - - it.prop( - "i32 round-trips", - [Schema.Int.check(Schema.isBetween({ minimum: -(2 ** 31), maximum: 2 ** 31 - 1 }))], - ([n]) => { - assert.strictEqual(roundTrip((w, a) => w.writeI32(a), (r) => r.readI32(), n), n) - } - ) - - it.prop("double round-trips", [Schema.Number], ([n]) => { - assert.isTrue(Object.is(roundTrip((w, a) => w.writeDouble(a), (r) => r.readDouble(), n), n)) + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64Number()), n) + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64()), BigInt(n)) }) it.prop("string round-trips through UTF-8", [Schema.String], ([s]) => { // Lone surrogates have no UTF-8 encoding and become U+FFFD. const expected = new TextDecoder().decode(new TextEncoder().encode(s)) - assert.strictEqual(roundTrip((w, a) => w.writeString(a), (r) => r.readString(), s), expected) - }) - - it.prop("binary round-trips", [Schema.Uint8Array], ([bytes]) => { - assert.deepStrictEqual(roundTrip((w, a) => w.writeBinary(a), (r) => r.readBinary(), bytes), bytes) + assert.strictEqual(roundTrip((w) => w.writeString(s), (r) => r.readString()), expected) }) - it("reads i64 limits", () => { + it("reads i64 as a number only when it is a safe integer", () => { const max = BigInt(Number.MAX_SAFE_INTEGER) - const bytesOf = (n: bigint) => write((w) => w.writeI64(n)) - const asNumber = (n: bigint) => Thrift.decode(bytesOf(n), (r) => r.readI64Number()) + const asNumber = (n: bigint) => Thrift.decode(write((w) => w.writeI64(n)), (r) => r.readI64Number()) assert.strictEqual(Result.getOrThrow(asNumber(max)), Number.MAX_SAFE_INTEGER) assert.strictEqual(Result.getOrThrow(asNumber(-max)), -Number.MAX_SAFE_INTEGER) assert.strictEqual(kindOf(asNumber(max + BigInt(1))), "Range") assert.strictEqual(kindOf(asNumber(-max - BigInt(1))), "Range") - for (const n of [max, max + BigInt(1), -max - BigInt(1), -max - BigInt(2)]) { - assert.strictEqual(Result.getOrThrow(Thrift.decode(bytesOf(n), (r) => r.readI64())), n) - } }) it("reads from a position set by the caller", () => { @@ -337,17 +236,16 @@ describe("Thrift", () => { assert.deepStrictEqual(Array.from(view), [7, 8]) }) - it("reads lengths with a three-byte varint", () => { - const value = new Uint8Array(20000).fill(7) - const bytes = write((w) => w.writeBinary(value)) - assert.strictEqual(bytes.length, 3 + value.length) - assert.deepStrictEqual(Thrift.makeReader(bytes).readBinary(), value) + it("reads a bool list element type written as 2", () => { + // Some writers use the FALSE code for the element type of list. + const r = Thrift.makeReader(Uint8Array.of(0x12, 0x01)) + assert.strictEqual(r.readListBegin(), 1) + assert.strictEqual(r.elementType, Thrift.Type.Bool) + assert.isTrue(r.readBool()) }) }) - describe("malformed input", () => { - const fails = (hex: string, read: (r: Thrift.Reader) => unknown, options?: Thrift.Options) => - kindOf(Thrift.decode(fromHex(hex), read, options)) + describe("hostile input", () => { const struct = (r: Thrift.Reader) => r.readValue(Thrift.Type.Struct) const table: ReadonlyArray< readonly [string, string, (r: Thrift.Reader) => unknown, Thrift.ThriftError["kind"], Thrift.Options?] @@ -365,11 +263,6 @@ describe("Thrift", () => { ["a map larger than the input", "0355020406", (r) => r.readMapBegin(), "Truncated"], ["a binary longer than the input", "0561", (r) => r.readBinary(), "Truncated"], ["a string that isn't UTF-8", "02c328", (r) => r.readString(), "InvalidUtf8"], - ["a long string that isn't UTF-8", "41" + "61".repeat(64) + "ff", (r) => r.readString(), "InvalidUtf8"], - ["skipping Stop", "00", (r) => r.skip(Thrift.Type.Stop), "Malformed"], - ["readStructEnd without readStructBegin", "", (r) => r.readStructEnd(), "Usage"], - ["a position before the input", "00", (r) => r.position = -1, "Usage"], - ["a position past the input", "00", (r) => r.position = 2, "Usage"], ["a binary over maxBinaryLength", "03616263", (r) => r.readBinary(), "Limit", { maxBinaryLength: 2 }], ["a skipped binary over maxBinaryLength", "1803616263", (r) => r.skip(Thrift.Type.Struct), "Limit", { maxBinaryLength: 2 @@ -379,18 +272,23 @@ describe("Thrift", () => { ["lists over maxDepth", "1919191500", (r) => r.skip(Thrift.Type.List), "Limit", { maxDepth: 2 }] ] for (const [name, hex, read, kind, options] of table) { - it(name, () => assert.strictEqual(fails(hex, read, options), kind)) + it(`fails on ${name}`, () => assert.strictEqual(kindOf(Thrift.decode(fromHex(hex), read, options)), kind)) } it("limits nesting to 64 by default", () => { + const nest = (depth: number): Thrift.Value => { + let v: Thrift.Value = { _tag: "Struct", fields: [] } + for (let i = 1; i < depth; i++) v = { _tag: "Struct", fields: [{ id: 1, value: v }] } + return v + } const bytesOf = (depth: number) => write((w) => w.writeValue(nest(depth))) assert.isTrue(Result.isSuccess(readAll(bytesOf(64), "Struct"))) assert.strictEqual(kindOf(readAll(bytesOf(65), "Struct")), "Limit") assert.strictEqual(kindOf(Thrift.decode(bytesOf(65), (r) => r.skip(Thrift.Type.Struct))), "Limit") }) - it("fails every truncated gold case", () => { - for (const c of gold.cases) { + it("fails every truncated gold case with Truncated", () => { + for (const c of corpus) { const bytes = fromHex(c.bytes) for (let n = 0; n < bytes.length; n += Math.max(1, Math.floor(bytes.length / 16))) { assert.strictEqual(kindOf(readAll(bytes.subarray(0, n), c.type)), "Truncated", `${c.id} at ${n}`) @@ -398,13 +296,6 @@ describe("Thrift", () => { } }) - it("decode rethrows errors that aren't ThriftErrors", () => { - assert.throws(() => - Thrift.decode(Uint8Array.of(0), () => { - throw new Error("boom") - }), "boom") - }) - it("only ever fails with ThriftError on corrupted input", () => { // Deterministic xorshift so failures reproduce. let seed = 0x5eed @@ -415,31 +306,28 @@ describe("Thrift", () => { return (seed >>> 0) / 0x100000000 } let failures = 0 - for (const c of [...gold.cases, ...gold.parquet.map((p) => ({ ...p, type: "Struct" as const }))]) { + for (const c of corpus) { const original = fromHex(c.bytes) - for (let round = 0; round < 10; round++) { + const type = Thrift.Type[c.type] + for (let round = 0; round < 50; round++) { const bytes = original.slice() const flips = 1 + Math.floor(random() * 4) for (let i = 0; i < flips && bytes.length > 0; i++) { bytes[Math.floor(random() * bytes.length)] = Math.floor(random() * 256) } - for ( - const read of [ - (r: Thrift.Reader) => r.readValue(Thrift.Type[c.type]), - (r: Thrift.Reader) => r.skip(Thrift.Type[c.type]) - ] - ) { - try { - read(Thrift.makeReader(bytes, { maxContainerSize: 1 << 16 })) - } catch (error) { - assert.isTrue(Thrift.isThriftError(error), `${c.id}: ${String(error)}`) - failures++ - } - } + if (thrown(() => Thrift.makeReader(bytes).readValue(type)) !== undefined) failures++ + if (thrown(() => Thrift.makeReader(bytes).skip(type)) !== undefined) failures++ } } assert.isAbove(failures, 0) }) + + it("decode rethrows errors that aren't ThriftErrors", () => { + assert.throws(() => + Thrift.decode(Uint8Array.of(0), () => { + throw new Error("boom") + }), "boom") + }) }) describe("Writer", () => { @@ -477,24 +365,16 @@ describe("Thrift", () => { ] ] for (const [name, f] of usage) { - it(`rejects ${name}`, () => { - try { - f(Thrift.makeWriter()) - assert.fail("expected a ThriftError") - } catch (error) { - assert.isTrue(Thrift.isThriftError(error)) - assert.strictEqual((error as Thrift.ThriftError).kind, "Usage") - } - }) + it(`rejects ${name}`, () => assert.strictEqual(thrown(() => f(Thrift.makeWriter())), "Usage")) } it("grows from a small buffer and returns a copy", () => { + const bytes = parquet("alltypes_tiny_pages.parquet/footer") const w = Thrift.makeWriter({ initialSize: 1 }) - w.writeValue(fromJson(gold.cases.at(-1)!.value)) - const bytes = w.finish() - assert.strictEqual(bytes.length, w.length) - assert.strictEqual(bytes.byteOffset, 0) - assert.strictEqual(bytes.buffer.byteLength, bytes.length) + w.writeValue(Result.getOrThrow(readAll(bytes, "Struct"))) + const out = w.finish() + assert.deepStrictEqual(out, bytes) + assert.strictEqual(out.buffer.byteLength, out.length) }) }) }) diff --git a/packages/effect/test/fixtures/thrift/README.md b/packages/effect/test/fixtures/thrift/README.md index 0dd7f038b1f..5be9093390d 100644 --- a/packages/effect/test/fixtures/thrift/README.md +++ b/packages/effect/test/fixtures/thrift/README.md @@ -1,9 +1,10 @@ # Thrift fixtures -`gold.json` is read by `test/encoding/Thrift.test.ts`. Each case is a value -tree and the bytes the official Apache Thrift Python library (`thrift` -0.25.0 from PyPI, `TCompactProtocol`) writes for it: hand-written cases for -each feature of the compact protocol, then 300 seeded random structs. +`gold.json` is generated by `generate.py` and read by +`test/encoding/Thrift.test.ts`. Each case is a value tree and the bytes the +official Apache Thrift Python library (`thrift` 0.25.0 from PyPI, +`TCompactProtocol`) writes for it: one hand-written case per rule of the +compact protocol, then 20 small seeded random structs. The `parquet` section holds compact-protocol structs cut from two files of [apache/parquet-testing](https://github.com/apache/parquet-testing) (commit @@ -13,9 +14,11 @@ filter header, offset index and column index of the first. Each comes with the tree the Python reader reads from it. The row counts, column names and offsets the test checks are the ones pyarrow reports for those files. -Regenerate with: +Regenerate from `packages/effect`; the output is byte for byte the same: ```sh -pip install thrift==0.25.0 -PARQUET_TESTING=/path/to/parquet-testing python test/fixtures/thrift/generate.py +PARQUET_TESTING=/path/to/parquet-testing \ + uv run --with thrift==0.25.0 python test/fixtures/thrift/generate.py ``` + +`PARQUET_TESTING` needs only the two files above, under `data/`. diff --git a/packages/effect/test/fixtures/thrift/generate.py b/packages/effect/test/fixtures/thrift/generate.py index c49eb787885..1478066dbc1 100644 --- a/packages/effect/test/fixtures/thrift/generate.py +++ b/packages/effect/test/fixtures/thrift/generate.py @@ -1,11 +1,11 @@ """ Gold corpus for effect/encoding/Thrift: values written by the official Apache -Thrift Python library's TCompactProtocol (pip install thrift), and the bytes +Thrift Python library's TCompactProtocol (thrift 0.25.0), and the bytes it produces. The TypeScript test reads every case back into a Thrift.Value, expects the same tree, and writes the tree back, expecting the same bytes. PARQUET_TESTING=/path/to/apache/parquet-testing \ - python test/fixtures/thrift/generate.py # writes gold.json + uv run --with thrift==0.25.0 python test/fixtures/thrift/generate.py Values are JSON trees shaped like Thrift.Value: {"_tag": "Bool", "value": true} @@ -288,17 +288,12 @@ def mp(k, v, *entries): dbl(1.7976931348623157e308), dbl(math.pi), dbl(-2.5) ], 1)), ) -case( - "binary/lengths", - struct_((1, binary(b"")), (2, binary(b"\x00")), (3, binary(bytes(range(127)))), (4, binary(bytes(128))), - (5, binary(bytes(i % 256 for i in range(300))))), -) -case("binary/utf8", struct_((1, text("héllo 🌍")), (2, text("")), (3, text("ascii")), (4, text("日本語" * 30)))) +case("binary/lengths", struct_((1, binary(b"")), (2, binary(b"\x00")), (3, binary(bytes(range(128)))))) +case("binary/utf8", struct_((1, text("héllo 🌍")), (2, text("ascii")))) case("uuid", struct_((1, uid(range(16))), (2, uid(bytes(16))))) case("list/empty", struct_((1, lst("I32")), (2, lst("Struct")), (3, lst("Bool")))) case("list/size-14", struct_((1, lst("I32", *[i32(i) for i in range(14)])))) case("list/size-15", struct_((1, lst("I32", *[i32(i) for i in range(15)])))) -case("list/size-200", struct_((1, lst("I16", *[i16(i - 100) for i in range(200)])))) case("list/bool", struct_((1, lst("Bool", b(True), b(False), b(True))))) case("list/i8", struct_((1, lst("I8", i8(-1), i8(0), i8(127))))) case("list/i64", struct_((1, lst("I64", i64(-(2**63)), i64(0), i64(2**63 - 1))))) @@ -309,12 +304,11 @@ def mp(k, v, *entries): case("list/list", struct_((1, lst("List", lst("I32", i32(1)), lst("I32"), lst("Bool", b(False)))))) case("list/map", struct_((1, lst("Map", mp("I32", "I32", (i32(1), i32(2))), mp("I32", "I32"))))) case("set/values", struct_((1, st("I32", i32(3), i32(1))), (2, st("Binary")), (3, st("Bool", b(True))))) -case("set/size-20", struct_((1, st("I64", *[i64(i * 1000) for i in range(20)])))) +case("set/size-15", struct_((1, st("I8", *[i8(i) for i in range(15)])))) case("map/empty", struct_((1, mp("I32", "I32")))) case("map/one", struct_((1, mp("Binary", "I64", (text("k"), i64(-5)))))) case("map/bools", struct_((1, mp("Bool", "Bool", (b(True), b(False)), (b(False), b(True)))))) case("map/i32-struct", struct_((1, mp("I32", "Struct", (i32(1), struct_((1, b(True)))), (i32(2), struct_()))))) -case("map/size-20", struct_((1, mp("I16", "Double", *[(i16(i), dbl(i / 2)) for i in range(20)])))) case("map/nested", struct_((1, mp("I32", "List", (i32(1), lst("I32", i32(2), i32(3))))))) case("map/uuid", struct_((1, mp("Uuid", "Uuid", (uid(range(16)), uid(bytes(16))))))) @@ -326,7 +320,7 @@ def nest(depth): return v -case("struct/deep-20", nest(20)) +case("struct/deep-5", nest(5)) case("top/list", lst("I32", i32(1), i32(-1))) case("top/i64", i64(-(2**40))) case("top/binary", text("top")) @@ -424,8 +418,13 @@ def rand_struct(depth): return struct_(*[(i, rand_value(rand_type(depth), depth)) for i in ids]) -for n in range(300): - case(f"random/{n}", rand_struct(0)) +# Small ones only: they mix every type, size form and id form at low cost. +n = 0 +while n < 20: + v = rand_struct(0) + if len(encode(v)) <= 128: + case(f"random/{n}", v) + n += 1 # --- Parquet structs ---------------------------------------------------------- diff --git a/packages/effect/test/fixtures/thrift/gold.json b/packages/effect/test/fixtures/thrift/gold.json index 827f72caac8..a042e68d4c7 100644 --- a/packages/effect/test/fixtures/thrift/gold.json +++ b/packages/effect/test/fixtures/thrift/gold.json @@ -1 +1 @@ -{"cases":[{"id":"struct/empty","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"bool/fields","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}},{"id":2,"value":{"_tag":"Bool","value":false}},{"id":3,"value":{"_tag":"Bool","value":true}}]},"bytes":"11121100"},{"id":"bool/long-form","type":"Struct","value":{"_tag":"Struct","fields":[{"id":20,"value":{"_tag":"Bool","value":true}},{"id":-1,"value":{"_tag":"Bool","value":false}}]},"bytes":"0128020100"},{"id":"field/delta-1","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"I32","value":2}}]},"bytes":"1502150400"},{"id":"field/delta-15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"I32","value":1}},{"id":30,"value":{"_tag":"I32","value":2}}]},"bytes":"f502f50400"},{"id":"field/delta-16","type":"Struct","value":{"_tag":"Struct","fields":[{"id":16,"value":{"_tag":"I32","value":1}},{"id":32,"value":{"_tag":"I32","value":2}}]},"bytes":"05200205400400"},{"id":"field/id-0","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"I32","value":1}}]},"bytes":"05000200"},{"id":"field/decreasing","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":2}},{"id":4,"value":{"_tag":"I32","value":3}}]},"bytes":"5502050604150600"},{"id":"field/repeated","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I32","value":1}},{"id":5,"value":{"_tag":"I32","value":2}}]},"bytes":"5502050a0400"},{"id":"field/negative","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1,"value":{"_tag":"I32","value":1}},{"id":-32768,"value":{"_tag":"I32","value":2}},{"id":1,"value":{"_tag":"I32","value":3}}]},"bytes":"05010205ffff030405020600"},{"id":"field/max","type":"Struct","value":{"_tag":"Struct","fields":[{"id":32767,"value":{"_tag":"I32","value":1}}]},"bytes":"05feff030200"},{"id":"field/nested-resets","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":1}},{"id":2,"value":{"_tag":"I8","value":2}}]}},{"id":6,"value":{"_tag":"I8","value":3}}]},"bytes":"5c1301130200130300"},{"id":"i8/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":-128}},{"id":2,"value":{"_tag":"I8","value":127}},{"id":3,"value":{"_tag":"I8","value":0}},{"id":4,"value":{"_tag":"I8","value":-1}}]},"bytes":"1380137f130013ff00"},{"id":"i16/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I16","value":-32768}},{"id":2,"value":{"_tag":"I16","value":32767}},{"id":3,"value":{"_tag":"I16","value":0}},{"id":4,"value":{"_tag":"I16","value":-1}},{"id":5,"value":{"_tag":"I16","value":64}},{"id":6,"value":{"_tag":"I16","value":-65}}]},"bytes":"14ffff0314feff031400140114800114810100"},{"id":"i32/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":-2147483648}},{"id":2,"value":{"_tag":"I32","value":2147483647}},{"id":3,"value":{"_tag":"I32","value":0}},{"id":4,"value":{"_tag":"I32","value":-1}},{"id":5,"value":{"_tag":"I32","value":63}},{"id":6,"value":{"_tag":"I32","value":64}},{"id":7,"value":{"_tag":"I32","value":8191}},{"id":8,"value":{"_tag":"I32","value":8192}},{"id":9,"value":{"_tag":"I32","value":-8193}}]},"bytes":"15ffffffff0f15feffffff0f15001501157e15800115fe7f158080011581800100"},{"id":"i64/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I64","value":"0"}},{"id":2,"value":{"_tag":"I64","value":"-1"}},{"id":3,"value":{"_tag":"I64","value":"1"}},{"id":4,"value":{"_tag":"I64","value":"2147483648"}},{"id":5,"value":{"_tag":"I64","value":"-2147483649"}},{"id":6,"value":{"_tag":"I64","value":"9007199254740991"}},{"id":7,"value":{"_tag":"I64","value":"-9007199254740991"}},{"id":8,"value":{"_tag":"I64","value":"9007199254740992"}},{"id":9,"value":{"_tag":"I64","value":"-9007199254740992"}},{"id":10,"value":{"_tag":"I64","value":"9007199254740993"}},{"id":11,"value":{"_tag":"I64","value":"-9007199254740993"}},{"id":12,"value":{"_tag":"I64","value":"4611686018427387904"}},{"id":13,"value":{"_tag":"I64","value":"9223372036854775807"}},{"id":14,"value":{"_tag":"I64","value":"-9223372036854775808"}},{"id":15,"value":{"_tag":"I64","value":"72057594037927935"}},{"id":16,"value":{"_tag":"I64","value":"562949953421312"}}]},"bytes":"16001601160216808080801016818080801016feffffffffffff1f16fdffffffffffff1f16808080808080802016ffffffffffffff1f168280808080808020168180808080808020168080808080808080800116feffffffffffffffff0116ffffffffffffffffff0116feffffffffffffff0116808080808080800200"},{"id":"double/values","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Double","value":0.0}},{"id":2,"value":{"_tag":"Double","value":"-0"}},{"id":3,"value":{"_tag":"Double","value":1.5}},{"id":4,"value":{"_tag":"Double","value":"NaN"}},{"id":5,"value":{"_tag":"Double","value":"Infinity"}},{"id":6,"value":{"_tag":"Double","value":"-Infinity"}},{"id":7,"value":{"_tag":"Double","value":5e-324}},{"id":8,"value":{"_tag":"Double","value":1.7976931348623157e+308}},{"id":9,"value":{"_tag":"Double","value":3.141592653589793}},{"id":10,"value":{"_tag":"Double","value":-2.5}}]},"bytes":"17000000000000000017000000000000008017000000000000f83f17000000000000f87f17000000000000f07f17000000000000f0ff17010000000000000017ffffffffffffef7f17182d4454fb2109401700000000000004c000"},{"id":"binary/lengths","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":""}},{"id":2,"value":{"_tag":"Binary","value":"00"}},{"id":3,"value":{"_tag":"Binary","value":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e"}},{"id":4,"value":{"_tag":"Binary","value":"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}},{"id":5,"value":{"_tag":"Binary","value":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b"}}]},"bytes":"1800180100187f000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e188001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000018ac02000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b00"},{"id":"binary/utf8","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"68c3a96c6c6f20f09f8c8d"}},{"id":2,"value":{"_tag":"Binary","value":""}},{"id":3,"value":{"_tag":"Binary","value":"6173636969"}},{"id":4,"value":{"_tag":"Binary","value":"e697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9e"}}]},"bytes":"180b68c3a96c6c6f20f09f8c8d180018056173636969188e02e697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9ee697a5e69cace8aa9e00"},{"id":"uuid","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Uuid","value":"000102030405060708090a0b0c0d0e0f"}},{"id":2,"value":{"_tag":"Uuid","value":"00000000000000000000000000000000"}}]},"bytes":"1d000102030405060708090a0b0c0d0e0f1d0000000000000000000000000000000000"},{"id":"list/empty","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I32","elements":[]}},{"id":2,"value":{"_tag":"List","elementType":"Struct","elements":[]}},{"id":3,"value":{"_tag":"List","elementType":"Bool","elements":[]}}]},"bytes":"1905190c190100"},{"id":"list/size-14","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":1},{"_tag":"I32","value":2},{"_tag":"I32","value":3},{"_tag":"I32","value":4},{"_tag":"I32","value":5},{"_tag":"I32","value":6},{"_tag":"I32","value":7},{"_tag":"I32","value":8},{"_tag":"I32","value":9},{"_tag":"I32","value":10},{"_tag":"I32","value":11},{"_tag":"I32","value":12},{"_tag":"I32","value":13}]}}]},"bytes":"19e500020406080a0c0e10121416181a00"},{"id":"list/size-15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":1},{"_tag":"I32","value":2},{"_tag":"I32","value":3},{"_tag":"I32","value":4},{"_tag":"I32","value":5},{"_tag":"I32","value":6},{"_tag":"I32","value":7},{"_tag":"I32","value":8},{"_tag":"I32","value":9},{"_tag":"I32","value":10},{"_tag":"I32","value":11},{"_tag":"I32","value":12},{"_tag":"I32","value":13},{"_tag":"I32","value":14}]}}]},"bytes":"19f50f00020406080a0c0e10121416181a1c00"},{"id":"list/size-200","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":-100},{"_tag":"I16","value":-99},{"_tag":"I16","value":-98},{"_tag":"I16","value":-97},{"_tag":"I16","value":-96},{"_tag":"I16","value":-95},{"_tag":"I16","value":-94},{"_tag":"I16","value":-93},{"_tag":"I16","value":-92},{"_tag":"I16","value":-91},{"_tag":"I16","value":-90},{"_tag":"I16","value":-89},{"_tag":"I16","value":-88},{"_tag":"I16","value":-87},{"_tag":"I16","value":-86},{"_tag":"I16","value":-85},{"_tag":"I16","value":-84},{"_tag":"I16","value":-83},{"_tag":"I16","value":-82},{"_tag":"I16","value":-81},{"_tag":"I16","value":-80},{"_tag":"I16","value":-79},{"_tag":"I16","value":-78},{"_tag":"I16","value":-77},{"_tag":"I16","value":-76},{"_tag":"I16","value":-75},{"_tag":"I16","value":-74},{"_tag":"I16","value":-73},{"_tag":"I16","value":-72},{"_tag":"I16","value":-71},{"_tag":"I16","value":-70},{"_tag":"I16","value":-69},{"_tag":"I16","value":-68},{"_tag":"I16","value":-67},{"_tag":"I16","value":-66},{"_tag":"I16","value":-65},{"_tag":"I16","value":-64},{"_tag":"I16","value":-63},{"_tag":"I16","value":-62},{"_tag":"I16","value":-61},{"_tag":"I16","value":-60},{"_tag":"I16","value":-59},{"_tag":"I16","value":-58},{"_tag":"I16","value":-57},{"_tag":"I16","value":-56},{"_tag":"I16","value":-55},{"_tag":"I16","value":-54},{"_tag":"I16","value":-53},{"_tag":"I16","value":-52},{"_tag":"I16","value":-51},{"_tag":"I16","value":-50},{"_tag":"I16","value":-49},{"_tag":"I16","value":-48},{"_tag":"I16","value":-47},{"_tag":"I16","value":-46},{"_tag":"I16","value":-45},{"_tag":"I16","value":-44},{"_tag":"I16","value":-43},{"_tag":"I16","value":-42},{"_tag":"I16","value":-41},{"_tag":"I16","value":-40},{"_tag":"I16","value":-39},{"_tag":"I16","value":-38},{"_tag":"I16","value":-37},{"_tag":"I16","value":-36},{"_tag":"I16","value":-35},{"_tag":"I16","value":-34},{"_tag":"I16","value":-33},{"_tag":"I16","value":-32},{"_tag":"I16","value":-31},{"_tag":"I16","value":-30},{"_tag":"I16","value":-29},{"_tag":"I16","value":-28},{"_tag":"I16","value":-27},{"_tag":"I16","value":-26},{"_tag":"I16","value":-25},{"_tag":"I16","value":-24},{"_tag":"I16","value":-23},{"_tag":"I16","value":-22},{"_tag":"I16","value":-21},{"_tag":"I16","value":-20},{"_tag":"I16","value":-19},{"_tag":"I16","value":-18},{"_tag":"I16","value":-17},{"_tag":"I16","value":-16},{"_tag":"I16","value":-15},{"_tag":"I16","value":-14},{"_tag":"I16","value":-13},{"_tag":"I16","value":-12},{"_tag":"I16","value":-11},{"_tag":"I16","value":-10},{"_tag":"I16","value":-9},{"_tag":"I16","value":-8},{"_tag":"I16","value":-7},{"_tag":"I16","value":-6},{"_tag":"I16","value":-5},{"_tag":"I16","value":-4},{"_tag":"I16","value":-3},{"_tag":"I16","value":-2},{"_tag":"I16","value":-1},{"_tag":"I16","value":0},{"_tag":"I16","value":1},{"_tag":"I16","value":2},{"_tag":"I16","value":3},{"_tag":"I16","value":4},{"_tag":"I16","value":5},{"_tag":"I16","value":6},{"_tag":"I16","value":7},{"_tag":"I16","value":8},{"_tag":"I16","value":9},{"_tag":"I16","value":10},{"_tag":"I16","value":11},{"_tag":"I16","value":12},{"_tag":"I16","value":13},{"_tag":"I16","value":14},{"_tag":"I16","value":15},{"_tag":"I16","value":16},{"_tag":"I16","value":17},{"_tag":"I16","value":18},{"_tag":"I16","value":19},{"_tag":"I16","value":20},{"_tag":"I16","value":21},{"_tag":"I16","value":22},{"_tag":"I16","value":23},{"_tag":"I16","value":24},{"_tag":"I16","value":25},{"_tag":"I16","value":26},{"_tag":"I16","value":27},{"_tag":"I16","value":28},{"_tag":"I16","value":29},{"_tag":"I16","value":30},{"_tag":"I16","value":31},{"_tag":"I16","value":32},{"_tag":"I16","value":33},{"_tag":"I16","value":34},{"_tag":"I16","value":35},{"_tag":"I16","value":36},{"_tag":"I16","value":37},{"_tag":"I16","value":38},{"_tag":"I16","value":39},{"_tag":"I16","value":40},{"_tag":"I16","value":41},{"_tag":"I16","value":42},{"_tag":"I16","value":43},{"_tag":"I16","value":44},{"_tag":"I16","value":45},{"_tag":"I16","value":46},{"_tag":"I16","value":47},{"_tag":"I16","value":48},{"_tag":"I16","value":49},{"_tag":"I16","value":50},{"_tag":"I16","value":51},{"_tag":"I16","value":52},{"_tag":"I16","value":53},{"_tag":"I16","value":54},{"_tag":"I16","value":55},{"_tag":"I16","value":56},{"_tag":"I16","value":57},{"_tag":"I16","value":58},{"_tag":"I16","value":59},{"_tag":"I16","value":60},{"_tag":"I16","value":61},{"_tag":"I16","value":62},{"_tag":"I16","value":63},{"_tag":"I16","value":64},{"_tag":"I16","value":65},{"_tag":"I16","value":66},{"_tag":"I16","value":67},{"_tag":"I16","value":68},{"_tag":"I16","value":69},{"_tag":"I16","value":70},{"_tag":"I16","value":71},{"_tag":"I16","value":72},{"_tag":"I16","value":73},{"_tag":"I16","value":74},{"_tag":"I16","value":75},{"_tag":"I16","value":76},{"_tag":"I16","value":77},{"_tag":"I16","value":78},{"_tag":"I16","value":79},{"_tag":"I16","value":80},{"_tag":"I16","value":81},{"_tag":"I16","value":82},{"_tag":"I16","value":83},{"_tag":"I16","value":84},{"_tag":"I16","value":85},{"_tag":"I16","value":86},{"_tag":"I16","value":87},{"_tag":"I16","value":88},{"_tag":"I16","value":89},{"_tag":"I16","value":90},{"_tag":"I16","value":91},{"_tag":"I16","value":92},{"_tag":"I16","value":93},{"_tag":"I16","value":94},{"_tag":"I16","value":95},{"_tag":"I16","value":96},{"_tag":"I16","value":97},{"_tag":"I16","value":98},{"_tag":"I16","value":99}]}}]},"bytes":"19f4c801c701c501c301c101bf01bd01bb01b901b701b501b301b101af01ad01ab01a901a701a501a301a1019f019d019b01990197019501930191018f018d018b01890187018501830181017f7d7b79777573716f6d6b69676563615f5d5b59575553514f4d4b49474543413f3d3b39373533312f2d2b29272523211f1d1b19171513110f0d0b090705030100020406080a0c0e10121416181a1c1e20222426282a2c2e30323436383a3c3e40424446484a4c4e50525456585a5c5e60626466686a6c6e70727476787a7c7e800182018401860188018a018c018e01900192019401960198019a019c019e01a001a201a401a601a801aa01ac01ae01b001b201b401b601b801ba01bc01be01c001c201c401c60100"},{"id":"list/bool","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}}]},"bytes":"193101020100"},{"id":"list/i8","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":-1},{"_tag":"I8","value":0},{"_tag":"I8","value":127}]}}]},"bytes":"1933ff007f00"},{"id":"list/i64","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-9223372036854775808"},{"_tag":"I64","value":"0"},{"_tag":"I64","value":"9223372036854775807"}]}}]},"bytes":"1936ffffffffffffffffff0100feffffffffffffffff0100"},{"id":"list/double","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":1.0},{"_tag":"Double","value":"NaN"}]}}]},"bytes":"1927000000000000f03f000000000000f87f00"},{"id":"list/binary","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"61"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"6263"}]}}]},"bytes":"193801610002626300"},{"id":"list/uuid","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"000102030405060708090a0b0c0d0e0f"},{"_tag":"Uuid","value":"101112131415161718191a1b1c1d1e1f"}]}}]},"bytes":"192d000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f00"},{"id":"list/struct","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}}]},{"_tag":"Struct","fields":[]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I32","value":7}}]}]}}]},"bytes":"193c110000250e0000"},{"id":"list/list","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":1}]},{"_tag":"List","elementType":"I32","elements":[]},{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false}]}]}}]},"bytes":"1939150205110200"},{"id":"list/map","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Map","elements":[{"_tag":"Map","keyType":"I32","valueType":"I32","entries":[[{"_tag":"I32","value":1},{"_tag":"I32","value":2}]]},{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}]}}]},"bytes":"192b015502040000"},{"id":"set/values","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":1}]}},{"id":2,"value":{"_tag":"Set","elementType":"Binary","elements":[]}},{"id":3,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true}]}}]},"bytes":"1a2506021a081a110100"},{"id":"set/size-20","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"0"},{"_tag":"I64","value":"1000"},{"_tag":"I64","value":"2000"},{"_tag":"I64","value":"3000"},{"_tag":"I64","value":"4000"},{"_tag":"I64","value":"5000"},{"_tag":"I64","value":"6000"},{"_tag":"I64","value":"7000"},{"_tag":"I64","value":"8000"},{"_tag":"I64","value":"9000"},{"_tag":"I64","value":"10000"},{"_tag":"I64","value":"11000"},{"_tag":"I64","value":"12000"},{"_tag":"I64","value":"13000"},{"_tag":"I64","value":"14000"},{"_tag":"I64","value":"15000"},{"_tag":"I64","value":"16000"},{"_tag":"I64","value":"17000"},{"_tag":"I64","value":"18000"},{"_tag":"I64","value":"19000"}]}}]},"bytes":"1af61400d00fa01ff02ec03e904ee05db06d807dd08c01a09c01f0ab01c0bb0190cb01e0da01b0ea0180fa01d08902a09902f0a80200"},{"id":"map/empty","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}}]},"bytes":"1b0000"},{"id":"map/one","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Binary","valueType":"I64","entries":[[{"_tag":"Binary","value":"6b"},{"_tag":"I64","value":"-5"}]]}}]},"bytes":"1b0186016b0900"},{"id":"map/bools","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Bool","valueType":"Bool","entries":[[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false}],[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]]}}]},"bytes":"1b02110102020100"},{"id":"map/i32-struct","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"I32","valueType":"Struct","entries":[[{"_tag":"I32","value":1},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}}]}],[{"_tag":"I32","value":2},{"_tag":"Struct","fields":[]}]]}}]},"bytes":"1b025c021100040000"},{"id":"map/size-20","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"I16","valueType":"Double","entries":[[{"_tag":"I16","value":0},{"_tag":"Double","value":0.0}],[{"_tag":"I16","value":1},{"_tag":"Double","value":0.5}],[{"_tag":"I16","value":2},{"_tag":"Double","value":1.0}],[{"_tag":"I16","value":3},{"_tag":"Double","value":1.5}],[{"_tag":"I16","value":4},{"_tag":"Double","value":2.0}],[{"_tag":"I16","value":5},{"_tag":"Double","value":2.5}],[{"_tag":"I16","value":6},{"_tag":"Double","value":3.0}],[{"_tag":"I16","value":7},{"_tag":"Double","value":3.5}],[{"_tag":"I16","value":8},{"_tag":"Double","value":4.0}],[{"_tag":"I16","value":9},{"_tag":"Double","value":4.5}],[{"_tag":"I16","value":10},{"_tag":"Double","value":5.0}],[{"_tag":"I16","value":11},{"_tag":"Double","value":5.5}],[{"_tag":"I16","value":12},{"_tag":"Double","value":6.0}],[{"_tag":"I16","value":13},{"_tag":"Double","value":6.5}],[{"_tag":"I16","value":14},{"_tag":"Double","value":7.0}],[{"_tag":"I16","value":15},{"_tag":"Double","value":7.5}],[{"_tag":"I16","value":16},{"_tag":"Double","value":8.0}],[{"_tag":"I16","value":17},{"_tag":"Double","value":8.5}],[{"_tag":"I16","value":18},{"_tag":"Double","value":9.0}],[{"_tag":"I16","value":19},{"_tag":"Double","value":9.5}]]}}]},"bytes":"1b144700000000000000000002000000000000e03f04000000000000f03f06000000000000f83f0800000000000000400a00000000000004400c00000000000008400e0000000000000c401000000000000010401200000000000012401400000000000014401600000000000016401800000000000018401a0000000000001a401c0000000000001c401e0000000000001e4020000000000000204022000000000000214024000000000000224026000000000000234000"},{"id":"map/nested","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"I32","valueType":"List","entries":[[{"_tag":"I32","value":1},{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":2},{"_tag":"I32","value":3}]}]]}}]},"bytes":"1b01590225040600"},{"id":"map/uuid","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Uuid","valueType":"Uuid","entries":[[{"_tag":"Uuid","value":"000102030405060708090a0b0c0d0e0f"},{"_tag":"Uuid","value":"00000000000000000000000000000000"}]]}}]},"bytes":"1b01dd000102030405060708090a0b0c0d0e0f0000000000000000000000000000000000"},{"id":"struct/deep-20","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":20}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]},"bytes":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c15280011001200110012001100120011001200110012001100120011001200110012001100120011001200"},{"id":"top/list","type":"List","value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":1},{"_tag":"I32","value":-1}]},"bytes":"250201"},{"id":"top/i64","type":"I64","value":{"_tag":"I64","value":"-1099511627776"},"bytes":"ffffffffff3f"},{"id":"top/binary","type":"Binary","value":{"_tag":"Binary","value":"746f70"},"bytes":"03746f70"},{"id":"top/map","type":"Map","value":{"_tag":"Map","keyType":"Binary","valueType":"Binary","entries":[[{"_tag":"Binary","value":"61"},{"_tag":"Binary","value":"62"}]]},"bytes":"018801610162"},{"id":"top/set","type":"Set","value":{"_tag":"Set","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":1}}]}]},"bytes":"1c130100"},{"id":"top/double","type":"Double","value":{"_tag":"Double","value":0.1},"bytes":"9a9999999999b93f"},{"id":"random/0","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"667520796ac3bc712f"},{"_tag":"Binary","value":"2890ded869d8"},{"_tag":"Binary","value":"d6802952c60f48aa3965c213d4115119de944833"},{"_tag":"Binary","value":"9295fa11d46b7960a8b4e028d3a9587f87ebd2321e71355ef26752740ce50811a232231bd2c02a2283be9bab3d48aec54d052e4af28f94267f5565e52bd81662b990cdee57dab1efaacbd2fdc5380f85f0ab18ec1e174d7704377f3b422afcb50e55c8e1334ef9588593938c049fba7d22d144cfdc6657e842ad3fdbe4469ef2"},{"_tag":"Binary","value":"2b9176b883bc"},{"_tag":"Binary","value":"95000147"}]}},{"id":-1,"value":{"_tag":"I16","value":-245}},{"id":9,"value":{"_tag":"Binary","value":"9c17630e"}},{"id":18,"value":{"_tag":"Uuid","value":"bbf1f49349579c312d3354148ac0dc55"}}]},"bytes":"4a6809667520796ac3bc712f062890ded869d814d6802952c60f48aa3965c213d4115119de94483380019295fa11d46b7960a8b4e028d3a9587f87ebd2321e71355ef26752740ce50811a232231bd2c02a2283be9bab3d48aec54d052e4af28f94267f5565e52bd81662b990cdee57dab1efaacbd2fdc5380f85f0ab18ec1e174d7704377f3b422afcb50e55c8e1334ef9588593938c049fba7d22d144cfdc6657e842ad3fdbe4469ef2062b9176b883bc04950001470401e903a8049c17630e9dbbf1f49349579c312d3354148ac0dc5500"},{"id":"random/1","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I32","value":-2}},{"id":287,"value":{"_tag":"I8","value":50}},{"id":354,"value":{"_tag":"Uuid","value":"1c542cdb430d6d3790b1b6e4170601e2"}},{"id":617,"value":{"_tag":"Binary","value":"e14f5dc15575b39f263e3a509a92ab8bc32fd786c0e80b7247c55dfdcf9ac8f835b20e75f786cf89e729ffc3cf1fc9c7def4b8775a38e5c76a754d751067e0e8c92d24272eb4ea8dfade5cc5b904a29321b62d30ea55fe689db5602a499fb8066853bb6ec547fd40e403eeddfd68fa5e062f87a655d053bd440d073922fb06"}}]},"bytes":"e50303be04320dc4051c542cdb430d6d3790b1b6e4170601e208d2097fe14f5dc15575b39f263e3a509a92ab8bc32fd786c0e80b7247c55dfdcf9ac8f835b20e75f786cf89e729ffc3cf1fc9c7def4b8775a38e5c76a754d751067e0e8c92d24272eb4ea8dfade5cc5b904a29321b62d30ea55fe689db5602a499fb8066853bb6ec547fd40e403eeddfd68fa5e062f87a655d053bd440d073922fb0600"},{"id":"random/2","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"I64","value":"-29"}},{"id":14,"value":{"_tag":"I16","value":-59}},{"id":23,"value":{"_tag":"Double","value":1.4115744041294328e+88}}]},"bytes":"060039e475971730f51829623c5200"},{"id":"random/3","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I16","value":-3320}},{"id":-4,"value":{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-56},{"_tag":"I16","value":61},{"_tag":"I16","value":1},{"_tag":"I16","value":263},{"_tag":"I16","value":2},{"_tag":"I16","value":60},{"_tag":"I16","value":-19},{"_tag":"I16","value":-8},{"_tag":"I16","value":13},{"_tag":"I16","value":39},{"_tag":"I16","value":2},{"_tag":"I16","value":32767},{"_tag":"I16","value":514},{"_tag":"I16","value":9}]}}]},"bytes":"e4ef330a07e46f7a028e040478250f1a4e04feff0384081200"},{"id":"random/4","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-29451,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"cec1ac0bf57cef30a5082be06f325c79"},{"_tag":"Uuid","value":"092897dae3c87bf715c8600adb36d5f5"},{"_tag":"Uuid","value":"a32829ea541c8063a7ab79c1abf328d3"},{"_tag":"Uuid","value":"45db9230ad34c2b629c5e8a9cf30b96a"},{"_tag":"Uuid","value":"6d8da58bcda9ddd0875eeb9381d9a522"}]}}]},"bytes":"0995cc035dcec1ac0bf57cef30a5082be06f325c79092897dae3c87bf715c8600adb36d5f5a32829ea541c8063a7ab79c1abf328d345db9230ad34c2b629c5e8a9cf30b96a6d8da58bcda9ddd0875eeb9381d9a52200"},{"id":"random/5","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I8","value":4}},{"id":7,"value":{"_tag":"I8","value":-4}},{"id":8,"value":{"_tag":"I32","value":-8197731}},{"id":-3,"value":{"_tag":"I16","value":11}},{"id":90,"value":{"_tag":"Map","keyType":"Double","valueType":"I16","entries":[[{"_tag":"Double","value":-26380.583127614926},{"_tag":"I16","value":13}],[{"_tag":"Double","value":-239.0},{"_tag":"I16","value":-12}],[{"_tag":"Double","value":-5.969692745028776e-185},{"_tag":"I16","value":-19965}],[{"_tag":"Double","value":-258737.13185530005},{"_tag":"I16","value":-81}]]}},{"id":96,"value":{"_tag":"Binary","value":"677768"}},{"id":108,"value":{"_tag":"I32","value":0}},{"id":112,"value":{"_tag":"Struct","fields":[{"id":-8192,"value":{"_tag":"I64","value":"-34"}},{"id":-8181,"value":{"_tag":"I64","value":"56"}},{"id":-8167,"value":{"_tag":"I32","value":-1047514659}},{"id":-8166,"value":{"_tag":"Uuid","value":"e26c925eab7f1c6a0c0fd61ec2749a09"}},{"id":8426,"value":{"_tag":"I64","value":"-71"}}]}}]},"bytes":"230453fc15c5d9e8070405160bb4010474e07cf65125c3d9c01a0000000000e06dc01761e38ba6f03bb099f9b702cc260a0e89950fc1a1016803677768c5004c06ff7f43b670e5c5b8fee6071de26c925eab7f1c6a0c0fd61ec2749a0906d483018d010000"},{"id":"random/6","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":-855}},{"id":16,"value":{"_tag":"Uuid","value":"a958bb6c386785bcad834ad8fcbf2e67"}},{"id":92,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"6916868375171133"}]}},{"id":102,"value":{"_tag":"Map","keyType":"I16","valueType":"I32","entries":[[{"_tag":"I16","value":10},{"_tag":"I32","value":-44}],[{"_tag":"I16","value":-41},{"_tag":"I32","value":17077}],[{"_tag":"I16","value":-15},{"_tag":"I32","value":74}],[{"_tag":"I16","value":-45},{"_tag":"I32","value":23}],[{"_tag":"I16","value":-32768},{"_tag":"I32","value":499905923}],[{"_tag":"I16","value":8790},{"_tag":"I32","value":32}],[{"_tag":"I16","value":7121},{"_tag":"I32","value":-1}],[{"_tag":"I16","value":-18},{"_tag":"I32","value":-4866846}],[{"_tag":"I16","value":44},{"_tag":"I32","value":-64635}],[{"_tag":"I16","value":-36},{"_tag":"I32","value":-49}],[{"_tag":"I16","value":27},{"_tag":"I32","value":-9}],[{"_tag":"I16","value":-18},{"_tag":"I32","value":-26465}],[{"_tag":"I16","value":-8181},{"_tag":"I32","value":15}],[{"_tag":"I16","value":1943},{"_tag":"I32","value":-2524193}],[{"_tag":"I16","value":49},{"_tag":"I32","value":-138126707}],[{"_tag":"I16","value":-4340},{"_tag":"I32","value":0}]]}},{"id":110,"value":{"_tag":"I8","value":3}},{"id":119,"value":{"_tag":"Bool","value":true}},{"id":131,"value":{"_tag":"List","elementType":"Uuid","elements":[]}},{"id":132,"value":{"_tag":"Uuid","value":"31c6135e3ddd116a08793a1decc0fc25"}}]},"bytes":"34ad0ddda958bb6c386785bcad834ad8fcbf2e6709b80116fac0aea7ddb6c918ab1045145751ea8a021d9401592effff0386d6dfdc03ac890140a26f0123bb8cd20458f5f1074761361123c19d03e97f1eae1ec190b40262e595dd8301e74300830391c90d1d31c6135e3ddd116a08793a1decc0fc2500"},{"id":"random/7","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"54"}},{"id":12,"value":{"_tag":"I16","value":188}},{"id":-4,"value":{"_tag":"I64","value":"39998194313015362"}},{"id":106,"value":{"_tag":"Binary","value":"78787a6b6f716b76676f2d2fc3bc786d747320"}},{"id":116,"value":{"_tag":"Uuid","value":"684da0ae1a924e10be8d41da3c18d6be"}},{"id":5,"value":{"_tag":"Binary","value":"64db49b29f7e3a42d407"}},{"id":15,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"473234873"},{"_tag":"I64","value":"-33"},{"_tag":"I64","value":"38"},{"_tag":"I64","value":"-408447675439872"}]}},{"id":22,"value":{"_tag":"Struct","fields":[{"id":32,"value":{"_tag":"Map","keyType":"Struct","valueType":"I64","entries":[[{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Uuid","value":"a8d3de826c1c439b645346ab3c7c3025"}},{"id":23,"value":{"_tag":"Double","value":5e-324}},{"id":262,"value":{"_tag":"Double","value":992100.7174823971}}]},{"_tag":"I64","value":"-4"}],[{"_tag":"Struct","fields":[{"id":98,"value":{"_tag":"I16","value":54}},{"id":105,"value":{"_tag":"Double","value":-762.0}}]},{"_tag":"I64","value":"-1"}],[{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"I16","value":4}}]},{"_tag":"I64","value":"2965627937"}],[{"_tag":"Struct","fields":[{"id":192,"value":{"_tag":"I64","value":"-3158819"}}]},{"_tag":"I64","value":"-7"}],[{"_tag":"Struct","fields":[{"id":208,"value":{"_tag":"I32","value":-7}},{"id":214,"value":{"_tag":"Double","value":588.0}}]},{"_tag":"I64","value":"-406773756535660067"}],[{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Binary","value":"702e9b1e0c7250f6792e6d1d29"}},{"id":21,"value":{"_tag":"Uuid","value":"836163e7fe42ef479a479625fe8810b1"}},{"id":25,"value":{"_tag":"I64","value":"-32261"}}]},{"_tag":"I64","value":"2055328139"}],[{"_tag":"Struct","fields":[]},{"_tag":"I64","value":"-9223372036854775808"}],[{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Bool","value":true}},{"id":11,"value":{"_tag":"I64","value":"7345"}}]},{"_tag":"I64","value":"0"}],[{"_tag":"Struct","fields":[{"id":153,"value":{"_tag":"I32","value":57}},{"id":154,"value":{"_tag":"I64","value":"-1"}},{"id":1880,"value":{"_tag":"I16","value":-47}}]},{"_tag":"I64","value":"-95"}],[{"_tag":"Struct","fields":[{"id":287,"value":{"_tag":"I8","value":-30}}]},{"_tag":"I64","value":"-1091920970931208884"}],[{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"I64","value":"2"}}]},{"_tag":"I64","value":"459871726993153846"}],[{"_tag":"Struct","fields":[]},{"_tag":"I64","value":"0"}],[{"_tag":"Struct","fields":[{"id":140,"value":{"_tag":"I64","value":"52523286064329301"}}]},{"_tag":"I64","value":"8"}],[{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Bool","value":false}},{"id":215,"value":{"_tag":"I32","value":-2147483648}},{"id":465,"value":{"_tag":"I16","value":1004}}]},{"_tag":"I64","value":"5"}],[{"_tag":"Struct","fields":[{"id":207,"value":{"_tag":"Uuid","value":"ff30a038363123c3412d27353bbbc317"}},{"id":220,"value":{"_tag":"I32","value":38232476}}]},{"_tag":"I64","value":"684540"}],[{"_tag":"Struct","fields":[]},{"_tag":"I64","value":"-60"}]]}},{"id":47,"value":{"_tag":"I16","value":-56}},{"id":1,"value":{"_tag":"Uuid","value":"7865773d200e3137b6d89ef8198acc5c"}},{"id":5,"value":{"_tag":"I8","value":1}}]}}]},"bytes":"266ca4f802060784c1f9baaa898d8e0108d4011378787a6b6f716b76676f2d2fc3bc786d747320ad684da0ae1a924e10be8d41da3c18d6be080a0a64db49b29f7e3a42d407aa46f2f6a7c303414cffebd4a9e4deb9017c0b4010c69da8d3de826c1c439b645346ab3c7c3025e70100000000000000078c044eda596fc9462e41000704c4016c770000000000d087c00001b40800c2909f8c16068003c5cc8103000d05a0030d67000000000060824000c5f892e4f1a493a50bc80d702e9b1e0c7250f6792e6d1d299d836163e7fe42ef479a479625fe8810b14689f8030096c68ea80f00ffffffffffffffffff019126e272000005b20272160104b01d5d00bd0103be04e200e78abcb2cf95a4a71e760400ecfce09ac6b9e5e10c0000069802aac9e784d0e9ccba0100104205ae03ffffffff0f04a207d80f000a0d9e03ff30a038363123c3412d27353bbbc317d5b886bb2400f8c7530077f46f0d027865773d200e3137b6d89ef8198acc5c43010000"},{"id":"random/8","type":"Struct","value":{"_tag":"Struct","fields":[{"id":204,"value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Uuid","value":"6a478a59537e811e2e07a5b9d7c429ed"}},{"id":21,"value":{"_tag":"Uuid","value":"bda43bd788101b4086e0391b2638a10f"}},{"id":307,"value":{"_tag":"I32","value":2147483647}},{"id":-13984,"value":{"_tag":"I32","value":2147483647}}]}},{"id":218,"value":{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":9},{"_tag":"I16","value":-135},{"_tag":"I16","value":-35},{"_tag":"I16","value":-5161},{"_tag":"I16","value":-10446},{"_tag":"I16","value":3}]}},{"id":232,"value":{"_tag":"I16","value":3492}},{"id":246,"value":{"_tag":"I32","value":-3408983}},{"id":297,"value":{"_tag":"I64","value":"36"}},{"id":5,"value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Uuid","value":"94888325fe9ebd29dfd40af3c35c6f8e"}},{"id":64,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}},{"id":-14403,"value":{"_tag":"Map","keyType":"Binary","valueType":"Bool","entries":[[{"_tag":"Binary","value":"fc340b"},{"_tag":"Bool","value":true}],[{"_tag":"Binary","value":""},{"_tag":"Bool","value":true}],[{"_tag":"Binary","value":""},{"_tag":"Bool","value":true}]]}},{"id":-14394,"value":{"_tag":"Bool","value":true}},{"id":-14338,"value":{"_tag":"I64","value":"-9223372036854775808"}},{"id":-14253,"value":{"_tag":"I32","value":2}},{"id":7079,"value":{"_tag":"I64","value":"-38"}},{"id":7094,"value":{"_tag":"Bool","value":true}}]}}]},"bytes":"0c9803bd6a478a59537e811e2e07a5b9d7c429edadbda43bd788101b4086e0391b2638a10f05e604feffffff0f05bfda01feffffff0f00e964128d0245d1509ba30106e4c836e5ad91a00306d204480c0a6d94888325fe9ebd29dfd40af3c35c6f8e0a8001f11e0201010101010102020201010201020102010202020102010202020202010b85e101038103fc340b0100010001910683e001ffffffffffffffffff0105d9de010406ce6e4bf10000"},{"id":"random/9","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Double","value":651.0}},{"id":12,"value":{"_tag":"Bool","value":false}},{"id":21,"value":{"_tag":"Bool","value":true}},{"id":33,"value":{"_tag":"Uuid","value":"b61b7133dd5c9e8d585c922bfa35c6fe"}},{"id":247,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"I16","elements":[]}]}},{"id":257,"value":{"_tag":"Map","keyType":"Bool","valueType":"Double","entries":[[{"_tag":"Bool","value":true},{"_tag":"Double","value":"-Infinity"}]]}}]},"bytes":"3700000000005884409291cdb61b7133dd5c9e8d585c922bfa35c6fe09ee031904ab011701000000000000f0ff00"},{"id":"random/10","type":"Struct","value":{"_tag":"Struct","fields":[{"id":206,"value":{"_tag":"Double","value":658410.8958999449}},{"id":219,"value":{"_tag":"I16","value":-32}},{"id":225,"value":{"_tag":"I16","value":-54}}]},"bytes":"079c03c865b3cad5172441d43f646b00"},{"id":"random/11","type":"Struct","value":{"_tag":"Struct","fields":[{"id":129,"value":{"_tag":"I8","value":-2}},{"id":1,"value":{"_tag":"I32","value":-80551}},{"id":13,"value":{"_tag":"Double","value":744831.811722348}},{"id":18,"value":{"_tag":"Bool","value":true}},{"id":24,"value":{"_tag":"I32","value":3287}}]},"bytes":"038202fe0502cdea09c754129a9fffba26415165ae3300"},{"id":"random/12","type":"Struct","value":{"_tag":"Struct","fields":[{"id":114,"value":{"_tag":"I8","value":13}},{"id":129,"value":{"_tag":"Double","value":-73.0}},{"id":139,"value":{"_tag":"I8","value":0}},{"id":7,"value":{"_tag":"Binary","value":"7a"}},{"id":19,"value":{"_tag":"Double","value":724235.1509789564}}]},"bytes":"03e4010df700000000004052c0a300080e017ac7201d4d4d161a264100"},{"id":"random/13","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Double","value":165979.1439026415}},{"id":10,"value":{"_tag":"Binary","value":"30ea7eab88d0d981f184df"}},{"id":20,"value":{"_tag":"Binary","value":"736e6b2df09f8c8de697a57a6e6370206e775f69696e6668756575676870c3a964c3bc7665e697a5f09f8c8de697a5787377722d72e697a57661c3bc6c752e776d6775645f746a65e697a5696ee697a5632e61c3bc612f73716a636165c3a97463726d74c3a920776966e697a5205f62795f5f2d72797473652d6220666e6d71656e7375782e776f737969746a666f6b2f676778722d632e79c3bc"}},{"id":31,"value":{"_tag":"Bool","value":false}}]},"bytes":"17986db626d9420441980b30ea7eab88d0d981f184dfa89b01736e6b2df09f8c8de697a57a6e6370206e775f69696e6668756575676870c3a964c3bc7665e697a5f09f8c8de697a5787377722d72e697a57661c3bc6c752e776d6775645f746a65e697a5696ee697a5632e61c3bc612f73716a636165c3a97463726d74c3a920776966e697a5205f62795f5f2d72797473652d6220666e6d71656e7375782e776f737969746a666f6b2f676778722d632e79c3bcb200"},{"id":"random/14","type":"Struct","value":{"_tag":"Struct","fields":[{"id":231,"value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Bool","value":true}},{"id":12,"value":{"_tag":"Uuid","value":"612360392bfdc4be16e62bd8bf1ec79e"}},{"id":17,"value":{"_tag":"I64","value":"-1623860795"}},{"id":23,"value":{"_tag":"Binary","value":"5450a788777788558a96"}},{"id":29,"value":{"_tag":"Binary","value":"782062"}},{"id":173,"value":{"_tag":"I8","value":10}}]}},{"id":437,"value":{"_tag":"Bool","value":true}},{"id":-4,"value":{"_tag":"I64","value":"-1"}},{"id":11,"value":{"_tag":"Bool","value":false}},{"id":14,"value":{"_tag":"Bool","value":false}},{"id":19,"value":{"_tag":"Binary","value":"6d6f78e697a52d792f2e2d752f"}},{"id":-5481,"value":{"_tag":"Binary","value":"74717a"}},{"id":-5479,"value":{"_tag":"Bool","value":false}}]},"bytes":"0cce03517d612360392bfdc4be16e62bd8bf1ec79e56f598d18c0c680a5450a788777788558a96680378206203da020a0001ea06060701f232580d6d6f78e697a52d792f2e2d752f08d1550374717a2200"},{"id":"random/15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":43,"value":{"_tag":"Double","value":-825.0}},{"id":-10189,"value":{"_tag":"Binary","value":"cd39bc"}},{"id":-10027,"value":{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":5.384133272083983e+208},{"_tag":"Double","value":-313926.1086485876},{"_tag":"Double","value":-765168.6597299525},{"_tag":"Double","value":640.0},{"_tag":"Double","value":"-Infinity"}]}},{"id":-9729,"value":{"_tag":"Uuid","value":"62b7ae846d41664daf7c1ed29c606fd5"}},{"id":-9725,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false}]}}]},"bytes":"07560000000000c889c008999f0103cd39bc09d59c0157270e047978f6446b4a93416f182913c1d41fc851e15927c10000000000008440000000000000f0ff0d81980162b7ae846d41664daf7c1ed29c606fd5496102020102020200"},{"id":"random/16","type":"Struct","value":{"_tag":"Struct","fields":[{"id":30,"value":{"_tag":"I8","value":47}},{"id":-9656,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"e78449ac4d4095dea487cbc9e510289e6ea28de8"},{"_tag":"Binary","value":"f1"},{"_tag":"Binary","value":"f68af3c0d95be0726dfa73e868"},{"_tag":"Binary","value":"f22f06"},{"_tag":"Binary","value":"70"},{"_tag":"Binary","value":"202f5fc3bc71c3bc2f6e62756a73676a6a7565707262636774e697a57a7273e697a56a666b7073f09f8c8d7378786175716bc3bc762d7577c3a9f09f8c8d7867767974687066e697a5766e2e7a7469745f2ec3bc6b2e6477f09f8c8d2065647377616868766a7a5f2d7972626f206f20716b6d74696f646168722f722e67617120696d71726e6b61c3a9642d67666d5ff09f8c8d765f77c3bc737669696be697a5c3bc75645f696764c3bc61e697a57a6c796a686967c3bc6969756c206f6a697772636275697370c3bc71696d6a675f6b796e6a6f647374c3a977726e5fc3a974777861e697a5f09f8c8d7671672d72"},{"_tag":"Binary","value":"37a2c4bddb636eb00ea3260e3b7cf2ad"},{"_tag":"Binary","value":"6c73782d776a65c3a9207073c3bc"},{"_tag":"Binary","value":"c61cd860f5c528391e3d29c86b6b"},{"_tag":"Binary","value":"6de697a5747263e697a578686263c3bcc3bc67736f727862c3bc766d62716d746d6e7420742e766964c3a9716d6a7462766a646a792d2d61752f20202061f09f8c8d657378c3bc64786e62656b75617078677a6370f09f8c8d7964f09f8c8d7a797a6771706f656a626c706220c3a9627a6b6c6f656170626a772d73632e6571612f7520705f6375622f71647964c3a96c6d7a"},{"_tag":"Binary","value":"7bf206435d99"},{"_tag":"Binary","value":"2b68372ab204a5f02d1eb740d2c019aeaa6e7a5861dc4e0473827509d4d994b2c88f6c836ff55d8189449275fdb7a83f91b55fb62c9a9dcb5a6ba8423873083e2649f710054351126b07632ea77489f489d5b957fbe257cef59d50fe210df94ea7f1920d27ee12fa7d80319fa2d0f398b1a5c9233fb12b178a93598dec0d33c685c08675004a17bd9edeec5095239cf683a829a3ef5c8d10a1f23a0970be3bbbeb4fd1622441178ddcd7a510c9de0d8b6f93cf80f50ff5cd6d3a92c83bd7cd195ee5d069f45fad2a"},{"_tag":"Binary","value":"6f6c766e75"},{"_tag":"Binary","value":"e2ea3b5f45476fc06c8937e5f303aa71"},{"_tag":"Binary","value":"e1a68d2e41e034"},{"_tag":"Binary","value":"c62a05720e383e"},{"_tag":"Binary","value":"8fcaee08d0aab6"},{"_tag":"Binary","value":"58475b3919b867b3d2041e39c5"},{"_tag":"Binary","value":"7167e697a5"},{"_tag":"Binary","value":"72636e65"},{"_tag":"Binary","value":"6478"},{"_tag":"Binary","value":"d77c"},{"_tag":"Binary","value":"6d726b6c2e2e76616d2f2e66"},{"_tag":"Binary","value":"f09f8c8d6778"},{"_tag":"Binary","value":"75"},{"_tag":"Binary","value":"6b64"},{"_tag":"Binary","value":"6472"},{"_tag":"Binary","value":"ef"},{"_tag":"Binary","value":"3b10"},{"_tag":"Binary","value":"79"}]}},{"id":-9482,"value":{"_tag":"Double","value":-924540.261637397}}]},"bytes":"033c2f09ef9601f81e14e78449ac4d4095dea487cbc9e510289e6ea28de801f10df68af3c0d95be0726dfa73e86803f22f060170f001202f5fc3bc71c3bc2f6e62756a73676a6a7565707262636774e697a57a7273e697a56a666b7073f09f8c8d7378786175716bc3bc762d7577c3a9f09f8c8d7867767974687066e697a5766e2e7a7469745f2ec3bc6b2e6477f09f8c8d2065647377616868766a7a5f2d7972626f206f20716b6d74696f646168722f722e67617120696d71726e6b61c3a9642d67666d5ff09f8c8d765f77c3bc737669696be697a5c3bc75645f696764c3bc61e697a57a6c796a686967c3bc6969756c206f6a697772636275697370c3bc71696d6a675f6b796e6a6f647374c3a977726e5fc3a974777861e697a5f09f8c8d7671672d721037a2c4bddb636eb00ea3260e3b7cf2ad0e6c73782d776a65c3a9207073c3bc0ec61cd860f5c528391e3d29c86b6b93016de697a5747263e697a578686263c3bcc3bc67736f727862c3bc766d62716d746d6e7420742e766964c3a9716d6a7462766a646a792d2d61752f20202061f09f8c8d657378c3bc64786e62656b75617078677a6370f09f8c8d7964f09f8c8d7a797a6771706f656a626c706220c3a9627a6b6c6f656170626a772d73632e6571612f7520705f6375622f71647964c3a96c6d7a067bf206435d99c8012b68372ab204a5f02d1eb740d2c019aeaa6e7a5861dc4e0473827509d4d994b2c88f6c836ff55d8189449275fdb7a83f91b55fb62c9a9dcb5a6ba8423873083e2649f710054351126b07632ea77489f489d5b957fbe257cef59d50fe210df94ea7f1920d27ee12fa7d80319fa2d0f398b1a5c9233fb12b178a93598dec0d33c685c08675004a17bd9edeec5095239cf683a829a3ef5c8d10a1f23a0970be3bbbeb4fd1622441178ddcd7a510c9de0d8b6f93cf80f50ff5cd6d3a92c83bd7cd195ee5d069f45fad2a056f6c766e7510e2ea3b5f45476fc06c8937e5f303aa7107e1a68d2e41e03407c62a05720e383e078fcaee08d0aab60d58475b3919b867b3d2041e39c5057167e697a50472636e6502647802d77c0c6d726b6c2e2e76616d2f2e6606f09f8c8d67780175026b6402647201ef023b100179079394013f56f585f8362cc100"},{"id":"random/17","type":"Struct","value":{"_tag":"Struct","fields":[{"id":117,"value":{"_tag":"Uuid","value":"de0f8fb87c624fd02cfbd4b5fb1c96d8"}},{"id":298,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"e2ff82b33a40e35600786aaa74dc319d6103163ff49fd5189c3bfb6e5a53504edbd13e03b7b29076927daf7b7d2e54971a6b01adb0044d02f45b02295d409d6f62fd3f42559d5a04101102b80c0fc125f27df6de4a54af78ca8a3564f0d90380da30fd4262741274864254fc09097c74db3d6203f739b79a1b1e617c440d2aa0"},{"_tag":"Binary","value":"6de688a0b7f1bef594c5fd0eb1c8213e156538"},{"_tag":"Binary","value":"9d"},{"_tag":"Binary","value":"ddea0237f0e166b2647de1967310"},{"_tag":"Binary","value":"655f64"},{"_tag":"Binary","value":"716c68e697a5706d63"},{"_tag":"Binary","value":"83"},{"_tag":"Binary","value":"e3342e6b7fcf2f3e9768d33446"},{"_tag":"Binary","value":"fb1c"},{"_tag":"Binary","value":"560cd50c5d"},{"_tag":"Binary","value":"2d"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"616164"},{"_tag":"Binary","value":"202e645f2d6e6f65757061c3a967725f69205f762e7270642f65c3a97677637a692f7372746b766b77672f667566686c72696cc3bc6a67c3a9727a775f6f752f2e76796377787a765f736b7379f09f8c8d726ce697a56779666d757765772dc3bc6ef09f8c8d732d6ac3a96978677569656c67762d66c3bc6cf09f8c8d6e6f2d20742f5f2d742e207965f09f8c8d6b776f657562"},{"_tag":"Binary","value":"07f85ba88275"},{"_tag":"Binary","value":"5f64c3a97a20642f74c3bc6d76797477617320745f74732d79677967756e5fc3a96ff09f8c8d666d78c3a9646f20662f20792f20767770c3bce697a55f777af09f8c8d5f6b7479c3bc69746b6f6a6f2dc3a92ec3a96e647166f09f8c8d6c617761772f616120727779c3a9756e686e73f09f8c8d792066c3bc2e77786f746f75c3a96e6d6e70656e6c2f767177732f616d7861682d706a2f"}]}},{"id":312,"value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Uuid","value":"01f92a3919133ce4c80f69374bc2ecf2"}},{"id":116,"value":{"_tag":"I64","value":"-584338652"}},{"id":119,"value":{"_tag":"I32","value":-35}}]}},{"id":410,"value":{"_tag":"I64","value":"-237465511953055068"}},{"id":418,"value":{"_tag":"Uuid","value":"13d1efd0541a6ddefd051e5da22a6d7e"}}]},"bytes":"0dea01de0f8fb87c624fd02cfbd4b5fb1c96d809d404f8108001e2ff82b33a40e35600786aaa74dc319d6103163ff49fd5189c3bfb6e5a53504edbd13e03b7b29076927daf7b7d2e54971a6b01adb0044d02f45b02295d409d6f62fd3f42559d5a04101102b80c0fc125f27df6de4a54af78ca8a3564f0d90380da30fd4262741274864254fc09097c74db3d6203f739b79a1b1e617c440d2aa0136de688a0b7f1bef594c5fd0eb1c8213e156538019d0eddea0237f0e166b2647de196731003655f6409716c68e697a5706d6301830de3342e6b7fcf2f3e9768d3344602fb1c05560cd50c5d012d00036161649401202e645f2d6e6f65757061c3a967725f69205f762e7270642f65c3a97677637a692f7372746b766b77672f667566686c72696cc3bc6a67c3a9727a775f6f752f2e76796377787a765f736b7379f09f8c8d726ce697a56779666d757765772dc3bc6ef09f8c8d732d6ac3a96978677569656c67762d66c3bc6cf09f8c8d6e6f2d20742f5f2d742e207965f09f8c8d6b776f6575620607f85ba8827598015f64c3a97a20642f74c3bc6d76797477617320745f74732d79677967756e5fc3a96ff09f8c8d666d78c3a9646f20662f20792f20767770c3bce697a55f777af09f8c8d5f6b7479c3bc69746b6f6a6f2dc3a92ec3a96e647166f09f8c8d6c617761772f616120727779c3a9756e686e73f09f8c8d792066c3bc2e77786f746f75c3a96e6d6e70656e6c2f767177732f616d7861682d706a2fecbd01f92a3919133ce4c80f69374bc2ecf206e801b7b3a2ad0435450006b406b7f5f1e5ffe7d2cb068d13d1efd0541a6ddefd051e5da22a6d7e00"},{"id":"random/18","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Binary","value":"6a7661642f6b79702dc3bc6f64707377"}},{"id":-2,"value":{"_tag":"Bool","value":true}},{"id":6,"value":{"_tag":"I8","value":-17}},{"id":31203,"value":{"_tag":"Double","value":527667.6184021656}},{"id":7810,"value":{"_tag":"Bool","value":false}},{"id":7876,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Binary","value":"332d"}},{"id":16,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"62"},{"_tag":"Binary","value":"636f6f2e6163617a796c2ef09f8c8d747ac3a9712f6ac3a9"}]}},{"id":31,"value":{"_tag":"Uuid","value":"b8b2a7a5ce3bebd218f91766697e64a1"}},{"id":290,"value":{"_tag":"Uuid","value":"f0dc62129edd1e72f7dd4306cf593cb5"}},{"id":326,"value":{"_tag":"I32","value":42}}]}}]},"bytes":"58106a7661642f6b79702dc3bc6f64707377010383ef07c6e7036a359f3c671a204102847a0c887b3802332dda28016218636f6f2e6163617a796c2ef09f8c8d747ac3a9712f6ac3a9fdb8b2a7a5ce3bebd218f91766697e64a10dc404f0dc62129edd1e72f7dd4306cf593cb5058c05540000"},{"id":"random/19","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I32","value":40}}]},"bytes":"855000"},{"id":"random/20","type":"Struct","value":{"_tag":"Struct","fields":[{"id":279,"value":{"_tag":"Bool","value":true}},{"id":281,"value":{"_tag":"I16","value":81}}]},"bytes":"01ae0424a20100"},{"id":"random/21","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"696c69"}]}},{"id":10,"value":{"_tag":"I32","value":-63}},{"id":19,"value":{"_tag":"I8","value":6}},{"id":33,"value":{"_tag":"Binary","value":"7a6c646a76"}}]},"bytes":"491803696c69657d9306e8057a6c646a7600"},{"id":"random/22","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-11701,"value":{"_tag":"Double","value":-6.001937545422097e+177}},{"id":-11698,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":-1},{"_tag":"I32","value":0},{"_tag":"I32","value":-5}]}},{"id":10974,"value":{"_tag":"I8","value":-12}},{"id":10983,"value":{"_tag":"I8","value":-2}},{"id":10994,"value":{"_tag":"Double","value":421711.43568380247}},{"id":0,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"29"}]}}]},"bytes":"07e9b60162de986ab8b2d7e43a450001000903bcab01f493feb70ce523be3dbd19410a00163a00"},{"id":"random/23","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Double","value":78889.97803132748}}]},"bytes":"a7602d04a69f42f34000"},{"id":"random/24","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":26}},{"id":16,"value":{"_tag":"I16","value":6}}]},"bytes":"3434d40c00"},{"id":"random/25","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I32","value":-94716}},{"id":12,"value":{"_tag":"I8","value":2}},{"id":-2,"value":{"_tag":"Binary","value":"3a10b02f41d2bb4e7c8323cc14443e5a8d35c0edbfeb2331ddbf0642d9bab462cc502a036ca2ca76cd5a375ada143a984fc7fb150526fb10a75598646c8d8d85f091f83c958085b2a1c382790bbefb4466b460aa52e2dc6ebfedae4cf2ae8f11c79edc670036bcada9544068ee288c0621106af3dcb9e24c144e17773fc1711be1ac82202765d08caec79fa77fdae198660ba3d12f008d2e58e56e2b5e49ad9e721fd296aa0a3f084d0172961c95be2d23f72868717c4bcec52e2e502b036410875a8ebcd422a522"}},{"id":5,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"4c27a54884b8e2390edfd48704479ee9"},{"_tag":"Uuid","value":"6f865395eb1aadad5e5f0d5fb0968bd7"}]}}]},"bytes":"35f7c70b93020803c8013a10b02f41d2bb4e7c8323cc14443e5a8d35c0edbfeb2331ddbf0642d9bab462cc502a036ca2ca76cd5a375ada143a984fc7fb150526fb10a75598646c8d8d85f091f83c958085b2a1c382790bbefb4466b460aa52e2dc6ebfedae4cf2ae8f11c79edc670036bcada9544068ee288c0621106af3dcb9e24c144e17773fc1711be1ac82202765d08caec79fa77fdae198660ba3d12f008d2e58e56e2b5e49ad9e721fd296aa0a3f084d0172961c95be2d23f72868717c4bcec52e2e502b036410875a8ebcd422a5227a2d4c27a54884b8e2390edfd48704479ee96f865395eb1aadad5e5f0d5fb0968bd700"},{"id":"random/26","type":"Struct","value":{"_tag":"Struct","fields":[{"id":272,"value":{"_tag":"Binary","value":"6b66762d6870636e"}},{"id":287,"value":{"_tag":"I16","value":-33}},{"id":579,"value":{"_tag":"I8","value":-41}},{"id":16972,"value":{"_tag":"I32","value":16}},{"id":-17242,"value":{"_tag":"I8","value":-22}},{"id":-17229,"value":{"_tag":"Bool","value":false}},{"id":27585,"value":{"_tag":"I64","value":"56"}},{"id":-5705,"value":{"_tag":"I16","value":22846}}]},"bytes":"08a004086b66762d6870636ef441038609d7059889022003b38d02ead20682af0370049159fce40200"},{"id":"random/27","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Binary","value":"657362682d63646f736571626870662d642e2f6c6c676cc3bc20797569c3bc2f69c3bc5f716620f09f8c8d7a6874712f202077796563676268666c6364616179745f6c2d2f6771727af09f8c8d70706772c3bc2f5f69626b76716e2f687765c3bc726870f09f8c8d2f752d6a2e6465207776667373646e786f702f62686b626f206c6d20c3a9756e7a7a68716b646a786b77f09f8c8d735f6566f09f8c8d65652e6c6bf09f8c8d2e7ae697a56464786c6cc3bc69777a6362627878732f772f20702f6773f09f8c8d686d726e752d647178616e75636cf09f8c8d7a6c2d696e71752e77c3bc636962206a"}},{"id":162,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-45558"},{"_tag":"I64","value":"-56"},{"_tag":"I64","value":"-66721570"},{"_tag":"I64","value":"-64"},{"_tag":"I64","value":"2"},{"_tag":"I64","value":"-57352949770"}]}},{"id":-4,"value":{"_tag":"Uuid","value":"beb1ebc38992fa76d39459acd256d877"}}]},"bytes":"78ea01657362682d63646f736571626870662d642e2f6c6c676cc3bc20797569c3bc2f69c3bc5f716620f09f8c8d7a6874712f202077796563676268666c6364616179745f6c2d2f6771727af09f8c8d70706772c3bc2f5f69626b76716e2f687765c3bc726870f09f8c8d2f752d6a2e6465207776667373646e786f702f62686b626f206c6d20c3a9756e7a7a68716b646a786b77f09f8c8d735f6566f09f8c8d65652e6c6bf09f8c8d2e7ae697a56464786c6cc3bc69777a6362627878732f772f20702f6773f09f8c8d686d726e752d647178616e75636cf09f8c8d7a6c2d696e71752e77c3bc636962206a0ac40266ebc7056fc3dcd03f7f0493c084a8ab030d07beb1ebc38992fa76d39459acd256d87700"},{"id":"random/28","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I8","value":12}},{"id":12,"value":{"_tag":"Binary","value":"73f09f8c8d6f76752d71c3bc6466646d6a72e697a565686a67c3bc6b61766a6a747a79c3bc612d6a2d7272612f6361637671686a67c3a973e697a579c3bc7971706e62f09f8c8df09f8c8d6a6e6c746720745f6b2e682f7973716261e697a5616f776174726e7974c3bc6b632d65616572615f6d68e697a5c3bc6220f09f8c8d696276e697a56d6965756b2e2f71686a6c65e697a5762e5f7277656d2e682d69637468c3a976686776747320642d5f207468c3bc76786c20626f6e6a787ae697a5756964f09f8c8d69742e61647971c3bc742f79702f6f5f2e6a73796c7268637374207268c3a96bc3a972f09f8c8d6f78636b6c"}},{"id":15,"value":{"_tag":"Uuid","value":"41c76ce9772b0bfacf66b221c6a8f974"}},{"id":-31446,"value":{"_tag":"Bool","value":false}}]},"bytes":"530c78f40173f09f8c8d6f76752d71c3bc6466646d6a72e697a565686a67c3bc6b61766a6a747a79c3bc612d6a2d7272612f6361637671686a67c3a973e697a579c3bc7971706e62f09f8c8df09f8c8d6a6e6c746720745f6b2e682f7973716261e697a5616f776174726e7974c3bc6b632d65616572615f6d68e697a5c3bc6220f09f8c8d696276e697a56d6965756b2e2f71686a6c65e697a5762e5f7277656d2e682d69637468c3a976686776747320642d5f207468c3bc76786c20626f6e6a787ae697a5756964f09f8c8d69742e61647971c3bc742f79702f6f5f2e6a73796c7268637374207268c3a96bc3a972f09f8c8d6f78636b6c3d41c76ce9772b0bfacf66b221c6a8f97402abeb0300"},{"id":"random/29","type":"Struct","value":{"_tag":"Struct","fields":[{"id":206,"value":{"_tag":"Double","value":499840.97379794903}},{"id":-23604,"value":{"_tag":"Bool","value":false}}]},"bytes":"079c03204a2be503821e4102e7f00200"},{"id":"random/30","type":"Struct","value":{"_tag":"Struct","fields":[{"id":114,"value":{"_tag":"Bool","value":false}},{"id":259,"value":{"_tag":"I16","value":20}}]},"bytes":"02e4010486042800"},{"id":"random/31","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"I32","value":-514580040}},{"id":-3,"value":{"_tag":"I64","value":"-38"}},{"id":-1,"value":{"_tag":"I64","value":"3"}},{"id":2,"value":{"_tag":"Double","value":-787615.9346157521}},{"id":-4,"value":{"_tag":"I16","value":6968}},{"id":10,"value":{"_tag":"Bool","value":true}},{"id":-1,"value":{"_tag":"I8","value":25}}]},"bytes":"458ff9deea0306054b260637b3f485de3f0928c10407f06ce103011900"},{"id":"random/32","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/33","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"I8","value":-1}},{"id":101,"value":{"_tag":"Bool","value":false}}]},"bytes":"63ff02ca0100"},{"id":"random/34","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Uuid","value":"959248c437d06d74042cca74c0b3828b"}},{"id":26,"value":{"_tag":"Bool","value":false}},{"id":36,"value":{"_tag":"I16","value":637}},{"id":17459,"value":{"_tag":"Uuid","value":"c390924029d97c11a31e09abb66bca02"}},{"id":17464,"value":{"_tag":"Uuid","value":"84bad09388141a4b04d7adee82436946"}},{"id":17517,"value":{"_tag":"I64","value":"-3087935"}},{"id":17531,"value":{"_tag":"I64","value":"-40762707"}}]},"bytes":"cd959248c437d06d74042cca74c0b3828be2a4fa090de69002c390924029d97c11a31e09abb66bca025d84bad09388141a4b04d7adee8243694606da9102fdf8f802e6a5f5ef2600"},{"id":"random/35","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":119822}},{"id":-3,"value":{"_tag":"Double","value":2.299034654899588e+18}},{"id":-1,"value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Map","keyType":"Double","valueType":"Binary","entries":[[{"_tag":"Double","value":-581.0},{"_tag":"Binary","value":"6d206965726a69636278"}],[{"_tag":"Double","value":-921467.248649394},{"_tag":"Binary","value":"e697a5"}],[{"_tag":"Double","value":-227473.0759702979},{"_tag":"Binary","value":"f2c609dff39f70cd39c5"}],[{"_tag":"Double","value":1.616783381002012e-258},{"_tag":"Binary","value":""}],[{"_tag":"Double","value":-308.0},{"_tag":"Binary","value":"bf"}]]}},{"id":24,"value":{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":-2},{"_tag":"I16","value":-29},{"_tag":"I16","value":-40},{"_tag":"I16","value":-2}]}},{"id":265,"value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I8","value":0}},{"id":161,"value":{"_tag":"Double","value":-335.0}},{"id":169,"value":{"_tag":"I8","value":-14}}]}},{"id":530,"value":{"_tag":"I64","value":"-547177435482408509"}}]}},{"id":2,"value":{"_tag":"I32","value":-2147483648}},{"id":17,"value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I32","value":2147483647}},{"id":15,"value":{"_tag":"Double","value":-2.1753671863594602e-278}},{"id":21,"value":{"_tag":"Bool","value":false}},{"id":-10829,"value":{"_tag":"I8","value":0}}]}},{"id":175,"value":{"_tag":"I8","value":11}}]},"bytes":"159cd00e07054c9a7bd6cfe7bf432cbb057800000000002882c00a6d206965726a696362782ff94e7ff61e2cc103e697a5c850969b88c40bc10af2c609dff39f70cd39c598cc8c99d2db680a0000000000004073c001bfd94403394f030c9204930007c2020000000000f074c083f20006a408f998cef0cfbefb970f0035ffffffff0ffce5feffffff0f17dcd7e5e4fbad4886620399a901000003de020b00"},{"id":"random/36","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Double","value":"NaN"}},{"id":15,"value":{"_tag":"Uuid","value":"9b5dc6870568cc3eea5646a9838ea00e"}},{"id":4,"value":{"_tag":"Double","value":531976.6544860012}},{"id":3,"value":{"_tag":"Uuid","value":"c8c8eb56cbf2fed83d951f508d7c52f0"}},{"id":18,"value":{"_tag":"I64","value":"23"}},{"id":30,"value":{"_tag":"Bool","value":true}},{"id":23873,"value":{"_tag":"I16","value":-19819}},{"id":23881,"value":{"_tag":"Binary","value":"6f"}}]},"bytes":"c7000000000000f87f3d9b5dc6870568cc3eea5646a9838ea00e070806ca184f113c20410d06c8c8eb56cbf2fed83d951f508d7c52f0f62ec10482f502d5b50288016f00"},{"id":"random/37","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Bool","value":false}},{"id":17,"value":{"_tag":"I32","value":-14}},{"id":24,"value":{"_tag":"Bool","value":false}}]},"bytes":"22f51b7200"},{"id":"random/38","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I32","value":-211}},{"id":241,"value":{"_tag":"I64","value":"62339106847078"}},{"id":245,"value":{"_tag":"Bool","value":false}},{"id":-3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}},{"id":246,"value":{"_tag":"Double","value":-670704.9840457938}},{"id":256,"value":{"_tag":"I8","value":-2}},{"id":-21477,"value":{"_tag":"Bool","value":true}},{"id":-21469,"value":{"_tag":"Uuid","value":"d343cde3951ec6bd87056152fe385e33"}},{"id":-21455,"value":{"_tag":"I64","value":"108485739046"}}]}}]},"bytes":"25a50306e203cce58dc5ceac1c420c051107ec03acd9d4f7e17724c1a3fe01c9cf028dd343cde3951ec6bd87056152fe385e33e6cc8888a4a8060000"},{"id":"random/39","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"9223372036854775807"}},{"id":12,"value":{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":0},{"_tag":"I8","value":10}]}},{"id":18,"value":{"_tag":"Map","keyType":"I16","valueType":"I32","entries":[[{"_tag":"I16","value":20},{"_tag":"I32","value":-7172}],[{"_tag":"I16","value":-1},{"_tag":"I32","value":-9}],[{"_tag":"I16","value":-32768},{"_tag":"I32","value":-590}],[{"_tag":"I16","value":-4},{"_tag":"I32","value":-51142}],[{"_tag":"I16","value":378},{"_tag":"I32","value":1615499830}],[{"_tag":"I16","value":-32768},{"_tag":"I32","value":12926}],[{"_tag":"I16","value":51},{"_tag":"I32","value":-1}],[{"_tag":"I16","value":-953},{"_tag":"I32","value":0}],[{"_tag":"I16","value":0},{"_tag":"I32","value":-5828}],[{"_tag":"I16","value":95},{"_tag":"I32","value":754}],[{"_tag":"I16","value":20},{"_tag":"I32","value":33}],[{"_tag":"I16","value":32767},{"_tag":"I32","value":-7}],[{"_tag":"I16","value":6391},{"_tag":"I32","value":-3}],[{"_tag":"I16","value":-168},{"_tag":"I32","value":-10695}],[{"_tag":"I16","value":-64},{"_tag":"I32","value":723877647}],[{"_tag":"I16","value":4},{"_tag":"I32","value":-71}]]}},{"id":-2,"value":{"_tag":"I16","value":-1}},{"id":-2,"value":{"_tag":"Bool","value":false}},{"id":13,"value":{"_tag":"I32","value":-12033326}},{"id":23,"value":{"_tag":"I32","value":-18}}]},"bytes":"26feffffffffffffffff01aa23000a6b10452887700111ffff039b09078b9f06f405ecc8d4840cffff03fcc9016601f10e0000875bbe01e40b2842feff030dee6305cf028da7017f9efcabb205088d010403010203f5dbf4bc0ba52300"},{"id":"random/40","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"I16","value":-257}},{"id":21,"value":{"_tag":"Double","value":978.0}}]},"bytes":"c48104970000000000908e4000"},{"id":"random/41","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/42","type":"Struct","value":{"_tag":"Struct","fields":[{"id":28785,"value":{"_tag":"I8","value":60}},{"id":28791,"value":{"_tag":"Uuid","value":"2153209bd2561bf0d59751275d3e00e6"}},{"id":29052,"value":{"_tag":"Map","keyType":"Bool","valueType":"Uuid","entries":[[{"_tag":"Bool","value":true},{"_tag":"Uuid","value":"fe7642a3579ee94dbe595126a8ac6fa2"}],[{"_tag":"Bool","value":true},{"_tag":"Uuid","value":"cb2cfca6cb4508e3df7c740b94006fb6"}],[{"_tag":"Bool","value":true},{"_tag":"Uuid","value":"d964d6a2e7aa433b5a28e16f720697d3"}],[{"_tag":"Bool","value":false},{"_tag":"Uuid","value":"1ecc41b9282b59b3089725c5113f1973"}]]}},{"id":29059,"value":{"_tag":"I16","value":-65}}]},"bytes":"03e2c1033c6d2153209bd2561bf0d59751275d3e00e60bf8c503041d01fe7642a3579ee94dbe595126a8ac6fa201cb2cfca6cb4508e3df7c740b94006fb601d964d6a2e7aa433b5a28e16f720697d3021ecc41b9282b59b3089725c5113f197374810100"},{"id":"random/43","type":"Struct","value":{"_tag":"Struct","fields":[{"id":186,"value":{"_tag":"Binary","value":"77766c74f09f8c8d7075c3a9706c62f09f8c8dc3a9615f63637566"}},{"id":-3,"value":{"_tag":"Uuid","value":"68f67725a59fd86a51b7350510535b66"}},{"id":295,"value":{"_tag":"I16","value":-12}},{"id":385,"value":{"_tag":"Bool","value":true}},{"id":1,"value":{"_tag":"Binary","value":"6703c055"}},{"id":9,"value":{"_tag":"Bool","value":true}}]},"bytes":"08f4021b77766c74f09f8c8d7075c3a9706c62f09f8c8dc3a9615f636375660d0568f67725a59fd86a51b7350510535b6604ce04170182060802046703c0558100"},{"id":"random/44","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-18637,"value":{"_tag":"I32","value":248848940}},{"id":22617,"value":{"_tag":"Map","keyType":"Binary","valueType":"I16","entries":[[{"_tag":"Binary","value":"6b2f74"},{"_tag":"I16","value":7451}],[{"_tag":"Binary","value":"8b0bb72857f0"},{"_tag":"I16","value":-1}],[{"_tag":"Binary","value":"3f3af1fce6e0e36b0d"},{"_tag":"I16","value":24646}],[{"_tag":"Binary","value":"5f676862726275677078"},{"_tag":"I16","value":-13}],[{"_tag":"Binary","value":"a6"},{"_tag":"I16","value":-16}]]}}]},"bytes":"0599a302d888a9ed010bb2e1020584036b2f74b674068b0bb72857f001093f3af1fce6e0e36b0d8c81030a5f6768627262756770781901a61f00"},{"id":"random/45","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/46","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Double","value":-332.0}},{"id":14,"value":{"_tag":"I16","value":0}}]},"bytes":"c70000000000c074c0240000"},{"id":"random/47","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"Uuid","value":"bffada6ed56a6d25b646f5e3ff3396b0"}},{"id":-30305,"value":{"_tag":"Double","value":442.0}}]},"bytes":"8dbffada6ed56a6d25b646f5e3ff3396b007c1d9030000000000a07b4000"},{"id":"random/48","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I8","value":52}}]},"bytes":"533400"},{"id":"random/49","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/50","type":"Struct","value":{"_tag":"Struct","fields":[{"id":108,"value":{"_tag":"I8","value":-128}}]},"bytes":"03d8018000"},{"id":"random/51","type":"Struct","value":{"_tag":"Struct","fields":[{"id":182,"value":{"_tag":"I8","value":2}},{"id":352,"value":{"_tag":"Double","value":644607.3121819485}}]},"bytes":"03ec020207c005f64fd69ffeab234100"},{"id":"random/52","type":"Struct","value":{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"I8","value":-9}},{"id":25,"value":{"_tag":"Bool","value":false}},{"id":40,"value":{"_tag":"Binary","value":"6bf09f8c8d"}},{"id":86,"value":{"_tag":"I64","value":"-51"}},{"id":99,"value":{"_tag":"I32","value":-25341415}}]},"bytes":"d3f7c2f8056bf09f8c8d06ac0165d5cdb7951800"},{"id":"random/53","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Map","keyType":"I64","valueType":"Map","entries":[[{"_tag":"I64","value":"-29117"},{"_tag":"Map","keyType":"Binary","valueType":"Bool","entries":[[{"_tag":"Binary","value":"c7a7c60636ec3ab2a4b411ea"},{"_tag":"Bool","value":false}],[{"_tag":"Binary","value":"6e766d6a6268726ec3bcc3a9616e79727a612df09f8c8d63"},{"_tag":"Bool","value":true}]]}]]}},{"id":17,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":19,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"52"},{"_tag":"Binary","value":"070b4a60eed1fadb9d11ffece8d2f0af5df03ca5afa507fffd1057306ae3e40a54bf556ebd151aee568cd067c33ab875ff0e4d31063d73b4195a100951d8deea4f8c36bf7315979a141be01c4d3f60bd2fc329af9f4a18b5d11b8e0d1ed5830830aa98121f8be7c38e27b0465a79f59d6793363faf1491c933f8b3091e69a5ec"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"202d7866"},{"_tag":"Binary","value":"b0"},{"_tag":"Binary","value":"57ed911621ff90"},{"_tag":"Binary","value":"4c19f50da641fbbf8fe1c9de54562b8c41"},{"_tag":"Binary","value":"f077e93c753405"},{"_tag":"Binary","value":"c95e302389d72a76fb3fda76693d"},{"_tag":"Binary","value":"a4607259dc7f822300a44e41a9c3"},{"_tag":"Binary","value":"3124"},{"_tag":"Binary","value":"c42bcf11464dfd914567"},{"_tag":"Binary","value":"ed6f0a5ff048646aebbb"},{"_tag":"Binary","value":"bafe42d558fd851abec95821bedf4093695f03d96b42ff80d0fb5e4f09052ea4b6160eb8c79a77fd8573b6412428a759d24f5cfd730c7eee396764d80a5b761d84f459fca78054bc6208116a1c46a5b4957cc9a7a4f43874e55b47a8abc159b409cb139fc0482454c73f6ed6d2fc52bb020218e9e8a19155c575236375b85ebb"},{"_tag":"Binary","value":"3628418c75de1fc6a201"}]}},{"id":33,"value":{"_tag":"Uuid","value":"52fc7cf8e5e53848ca2fc53fe7f1fe6f"}},{"id":42,"value":{"_tag":"I64","value":"131495536282534711"}},{"id":51,"value":{"_tag":"Uuid","value":"399f0eb99cce1b1d2911721bd38483b6"}},{"id":62,"value":{"_tag":"I16","value":-9}}]},"bytes":"ab016bf9c60302810cc7a7c60636ec3ab2a4b411ea02186e766d6a6268726ec3bcc3a9616e79727a612df09f8c8d63017b0029f80f01528001070b4a60eed1fadb9d11ffece8d2f0af5df03ca5afa507fffd1057306ae3e40a54bf556ebd151aee568cd067c33ab875ff0e4d31063d73b4195a100951d8deea4f8c36bf7315979a141be01c4d3f60bd2fc329af9f4a18b5d11b8e0d1ed5830830aa98121f8be7c38e27b0465a79f59d6793363faf1491c933f8b3091e69a5ec0004202d786601b00757ed911621ff90114c19f50da641fbbf8fe1c9de54562b8c4107f077e93c7534050ec95e302389d72a76fb3fda76693d0ea4607259dc7f822300a44e41a9c30231240ac42bcf11464dfd9145670aed6f0a5ff048646aebbb8001bafe42d558fd851abec95821bedf4093695f03d96b42ff80d0fb5e4f09052ea4b6160eb8c79a77fd8573b6412428a759d24f5cfd730c7eee396764d80a5b761d84f459fca78054bc6208116a1c46a5b4957cc9a7a4f43874e55b47a8abc159b409cb139fc0482454c73f6ed6d2fc52bb020218e9e8a19155c575236375b85ebb0a3628418c75de1fc6a201ed52fc7cf8e5e53848ca2fc53fe7f1fe6f96ee8cdc9acb9f95d3039d399f0eb99cce1b1d2911721bd38483b6b41100"},{"id":"random/54","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"Uuid","value":"255e1a57b9c7487e7399e7f4520c3516"}},{"id":216,"value":{"_tag":"I64","value":"652"}}]},"bytes":"ed255e1a57b9c7487e7399e7f4520c351606b003980a00"},{"id":"random/55","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15299,"value":{"_tag":"Binary","value":"7f4fc57b82722980fef040dbcdf53e16"}},{"id":15305,"value":{"_tag":"Uuid","value":"46031347439ad95d5b72aa4765fef3d1"}},{"id":-4949,"value":{"_tag":"I8","value":-25}},{"id":-7142,"value":{"_tag":"Binary","value":"50ccd1fc7115ea3b1915bffae505f1826711f771"}},{"id":-7135,"value":{"_tag":"I16","value":31}},{"id":-7127,"value":{"_tag":"I16","value":21}},{"id":-6953,"value":{"_tag":"Double","value":185.0}},{"id":-6941,"value":{"_tag":"Binary","value":"7069c3a9"}}]},"bytes":"0886ef01107f4fc57b82722980fef040dbcdf53e166d46031347439ad95d5b72aa4765fef3d103a94de708cb6f1450ccd1fc7115ea3b1915bffae505f1826711f771743e842a07d16c0000000000206740c8047069c3a900"},{"id":"random/56","type":"Struct","value":{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"I32","value":-2}},{"id":21,"value":{"_tag":"I32","value":1639426}},{"id":29,"value":{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":-2},{"_tag":"I16","value":62}]}},{"id":5,"value":{"_tag":"I64","value":"-961"}}]},"bytes":"d503858490c8018924037c060a810f00"},{"id":"random/57","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I64","value":"-1"}},{"id":48,"value":{"_tag":"I64","value":"86904975"}},{"id":63,"value":{"_tag":"I16","value":27}},{"id":326,"value":{"_tag":"Uuid","value":"6ac656e8b210d1b40a6d36803be7cfbf"}},{"id":338,"value":{"_tag":"I32","value":51}},{"id":350,"value":{"_tag":"I32","value":-526353}}]},"bytes":"160106609ec2f052f4360d8c056ac656e8b210d1b40a6d36803be7cfbfc566c5a1a04000"},{"id":"random/58","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Binary","value":"6f65e697a56566e697a56ec3bc2f"}},{"id":22,"value":{"_tag":"Uuid","value":"a06272b190616070dbe43f4149998d75"}},{"id":37,"value":{"_tag":"I32","value":2147483647}}]},"bytes":"c80e6f65e697a56566e697a56ec3bc2fada06272b190616070dbe43f4149998d75f5feffffff0f00"},{"id":"random/59","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1,"value":{"_tag":"I16","value":8}},{"id":3,"value":{"_tag":"List","elementType":"Set","elements":[{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-94082596},{"_tag":"I32","value":-51643},{"_tag":"I32","value":3},{"_tag":"I32","value":-58},{"_tag":"I32","value":47}]}]}},{"id":10,"value":{"_tag":"Bool","value":true}},{"id":19,"value":{"_tag":"Uuid","value":"6bc3067caa065d6276af54d9b819b26a"}}]},"bytes":"040110491a55c7d8dc59f5a60606735e719d6bc3067caa065d6276af54d9b819b26a00"},{"id":"random/60","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"86e111e22155f6a0d5e41d2d64df64ec"},{"_tag":"Uuid","value":"f4715dba4effc072925385e98c0f96a3"}]}},{"id":18,"value":{"_tag":"Bool","value":false}},{"id":22,"value":{"_tag":"Double","value":915773.6677056516}},{"id":30,"value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Uuid","value":"4b7a1da83d78be433e55509cb18a8f16"}},{"id":20,"value":{"_tag":"I64","value":"25"}},{"id":275,"value":{"_tag":"I64","value":"-41638222480"}},{"id":565,"value":{"_tag":"I8","value":19}}]}},{"id":43,"value":{"_tag":"Double","value":658.0}},{"id":56,"value":{"_tag":"I16","value":-33}},{"id":-30618,"value":{"_tag":"Binary","value":"0a448b0d392592e9d002c1"}},{"id":-30554,"value":{"_tag":"Uuid","value":"17e535706212c71ab127ccccb61eb299"}}]},"bytes":"692d86e111e22155f6a0d5e41d2d64df64ecf4715dba4effc072925385e98c0f96a3c247e283dd557bf22b418ccd4b7a1da83d78be433e55509cb18a8f16863206a6049feaa69db60203ea081300d70000000000908440d44108b3de030b0a448b0d392592e9d002c10db3dd0317e535706212c71ab127ccccb61eb29900"},{"id":"random/61","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Double","value":-8.164277870875512e-228}},{"id":21,"value":{"_tag":"Map","keyType":"I64","valueType":"Binary","entries":[[{"_tag":"I64","value":"21"},{"_tag":"Binary","value":"2f7971"}],[{"_tag":"I64","value":"-932873"},{"_tag":"Binary","value":"7a666f2d75696a"}],[{"_tag":"I64","value":"4512217"},{"_tag":"Binary","value":"6965736a66736c75205f766420c3a9"}]]}},{"id":32,"value":{"_tag":"I16","value":31}},{"id":39,"value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"Uuid","value":"807044b086b6d86bd888e28c0a652e0e"}},{"id":8,"value":{"_tag":"Binary","value":"04cbb9d9ab9e29f5889f23b9e05af0ac9917"}},{"id":10,"value":{"_tag":"I16","value":2311}}]}}]},"bytes":"a7c397983498c1c890bb03682a032f797191f071077a666f2d75696ab2e7a6040f6965736a66736c75205f766420c3a9b43e7c0d07807044b086b6d86bd888e28c0a652e0ec81204cbb9d9ab9e29f5889f23b9e05af0ac9917248e240000"},{"id":"random/62","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"Double","value":-4.0}},{"id":17,"value":{"_tag":"I16","value":-57}},{"id":26,"value":{"_tag":"I16","value":-60}},{"id":29,"value":{"_tag":"Uuid","value":"0d0ad7dcca6220ecea71a8b9b5ba7449"}},{"id":30,"value":{"_tag":"I32","value":32291}}]},"bytes":"e700000000000010c0347194773d0d0ad7dcca6220ecea71a8b9b5ba744915c6f80300"},{"id":"random/63","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"I32","value":-1469856044}},{"id":10,"value":{"_tag":"I64","value":"-4775688667787"}},{"id":16,"value":{"_tag":"I32","value":0}}]},"bytes":"75d7e4e1f90a36958ad0d3fd9502650000"},{"id":"random/64","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"I32","value":-7731375}},{"id":285,"value":{"_tag":"Binary","value":"7a742d79772d76"}},{"id":300,"value":{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":2.684064426090369e+181},{"_tag":"Double","value":-2.821675305316185e+39},{"_tag":"Double","value":534827.1868791913},{"_tag":"Double","value":239283.1897893271},{"_tag":"Double","value":-650954.6095114024},{"_tag":"Double","value":-665.0},{"_tag":"Double","value":-542941.9176623522},{"_tag":"Double","value":324673.5389539916},{"_tag":"Double","value":983434.8458527278},{"_tag":"Double","value":628251.3983450965},{"_tag":"Double","value":-9.355421038001027e+265},{"_tag":"Double","value":0.0},{"_tag":"Double","value":-851824.601377438},{"_tag":"Double","value":-593.0},{"_tag":"Double","value":5e-324},{"_tag":"Double","value":-89.0}]}},{"id":-29818,"value":{"_tag":"I16","value":32767}},{"id":-29811,"value":{"_tag":"I16","value":6596}},{"id":-29799,"value":{"_tag":"I32","value":-253}}]},"bytes":"65dde2af0708ba04077a742d79772d76faf71004c9c60d9fdf996593edf2ca959520c81ea1ae5f565220414844b08499350d41e8e0113895dd23c10000000000c884c0ffd6d7d5bb9120c1208ee32706d11341d69b13b115032e4174e3f3cb362c2341be9c34a50f3627f700000000000000005abee733e1fe29c100000000008882c0010000000000000000000000004056c004f3d103feff03748867c5f90300"},{"id":"random/65","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1,"value":{"_tag":"I16","value":8}},{"id":12,"value":{"_tag":"Bool","value":false}},{"id":25,"value":{"_tag":"I8","value":25}},{"id":1379,"value":{"_tag":"Map","keyType":"I8","valueType":"Uuid","entries":[[{"_tag":"I8","value":56},{"_tag":"Uuid","value":"5b8cf4f3cab8453699ecf679671b58bb"}],[{"_tag":"I8","value":-26},{"_tag":"Uuid","value":"fe11a362b30e9df0a53820caebcad4bc"}]]}}]},"bytes":"040110d2d3190bc615023d385b8cf4f3cab8453699ecf679671b58bbe6fe11a362b30e9df0a53820caebcad4bc00"},{"id":"random/66","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Double","value":"-Infinity"}},{"id":16306,"value":{"_tag":"I64","value":"-39"}},{"id":16588,"value":{"_tag":"Uuid","value":"a867c2e377c3e7780647b7efa03ad7a5"}},{"id":16695,"value":{"_tag":"Binary","value":"625f2f676f6b74712f726b68"}},{"id":16697,"value":{"_tag":"Double","value":243526.03060886706}},{"id":16710,"value":{"_tag":"I32","value":59}},{"id":16713,"value":{"_tag":"I32","value":-3}}]},"bytes":"f7000000000000f0ff06e4fe014d0d988302a867c2e377c3e7780647b7efa03ad7a508ee84020c625f2f676f6b74712f726b682798dcaf3e30ba0d41d576350500"},{"id":"random/67","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/68","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Binary","value":"61e697a5796b6d5fe697a5f09f8c8d6ac3a9762e63702e7168f09f8c8d2d6d66762f2f2f7220207267726b616dc3bc787476726b7971f09f8c8d7a5f6e6d2079e697a566726d70e697a56a2e7a687a2d7a63687863672f7a712fc3a977635f7263c3bc78666fe697a5756a7278706ac3a96f2d682f62666a64716866766e63c3bc6b5f797070f09f8c8d69702e2fe697a579676c7573736264c3a9696a7a20"}},{"id":5053,"value":{"_tag":"Binary","value":"627ac3bc7470616b782062"}}]},"bytes":"489f0161e697a5796b6d5fe697a5f09f8c8d6ac3a9762e63702e7168f09f8c8d2d6d66762f2f2f7220207267726b616dc3bc787476726b7971f09f8c8d7a5f6e6d2079e697a566726d70e697a56a2e7a687a2d7a63687863672f7a712fc3a977635f7263c3bc78666fe697a5756a7278706ac3a96f2d682f62666a64716866766e63c3bc6b5f797070f09f8c8d69702e2fe697a579676c7573736264c3a9696a7a2008fa4e0b627ac3bc7470616b78206200"},{"id":"random/69","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I32","value":49}},{"id":14,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":448}]}},{"id":80,"value":{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-3},{"_tag":"I8","value":-2}]}},{"id":90,"value":{"_tag":"I16","value":-1}},{"id":161,"value":{"_tag":"Binary","value":"7164"}},{"id":171,"value":{"_tag":"I64","value":"-5746"}}]},"bytes":"3562ba1580070aa00123fdfea40108c202027164a6e35900"},{"id":"random/70","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"I16","value":-2}},{"id":-24176,"value":{"_tag":"I16","value":-55}},{"id":-24161,"value":{"_tag":"Bool","value":true}},{"id":-24091,"value":{"_tag":"Bool","value":true}},{"id":-24085,"value":{"_tag":"Double","value":924022.5692087875}},{"id":-23916,"value":{"_tag":"Struct","fields":[{"id":-11820,"value":{"_tag":"Struct","fields":[{"id":-20348,"value":{"_tag":"Bool","value":true}},{"id":-20340,"value":{"_tag":"Uuid","value":"e99290dd078c0c4fdde4ddb8b5ab817a"}},{"id":-20329,"value":{"_tag":"Uuid","value":"2233046f39662876456fa2e44fc1b8ce"}},{"id":-1,"value":{"_tag":"List","elementType":"I64","elements":[]}},{"id":14,"value":{"_tag":"I16","value":-90}},{"id":26,"value":{"_tag":"I8","value":13}}]}},{"id":-25262,"value":{"_tag":"Uuid","value":"00742e6d75daac8c2e71bb62150a4ffa"}},{"id":-25248,"value":{"_tag":"I32","value":-13553}},{"id":-25234,"value":{"_tag":"Bool","value":false}},{"id":2,"value":{"_tag":"Binary","value":"615f636a2d736e69"}},{"id":13,"value":{"_tag":"Uuid","value":"c6c2efad6e690c705624a46cc47a3859"}},{"id":28260,"value":{"_tag":"Binary","value":"6c78746f6c7961677468f09f8c8d706276766c792e5f"}}]}},{"id":-23912,"value":{"_tag":"Uuid","value":"16496442c39907fc5a19e56cb8e11796"}}]},"bytes":"640304dff9026df101b5f802678e556f23ed322c410cd7f5020cd7b80101f7bd028de99290dd078c0c4fdde4ddb8b5ab817abd2233046f39662876456fa2e44fc1b8ce090106f4b301c30d000ddb8a0300742e6d75daac8c2e71bb62150a4ffae5e1d301e2080408615f636a2d736e69bdc6c2efad6e690c705624a46cc47a385908c8b903166c78746f6c7961677468f09f8c8d706276766c792e5f004d16496442c39907fc5a19e56cb8e1179600"},{"id":"random/71","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/72","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Set","elementType":"I64","elements":[]}},{"id":11,"value":{"_tag":"I16","value":0}},{"id":24,"value":{"_tag":"Bool","value":false}},{"id":27,"value":{"_tag":"Uuid","value":"9ad475790d0f55085c5ac60cf42972c1"}},{"id":103,"value":{"_tag":"I32","value":-1925748}},{"id":399,"value":{"_tag":"I32","value":70607}}]},"bytes":"9a062400d23d9ad475790d0f55085c5ac60cf42972c105ce01e789eb01059e069ecf0800"},{"id":"random/73","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Map","keyType":"I32","valueType":"Binary","entries":[[{"_tag":"I32","value":7199510},{"_tag":"Binary","value":"71"}],[{"_tag":"I32","value":56},{"_tag":"Binary","value":"353341aaa7fc108fbc569007c8ea87de"}]]}},{"id":13,"value":{"_tag":"I16","value":1373}},{"id":21,"value":{"_tag":"Binary","value":"1037373c57f77c02b0609fddc1a2267ef742"}}]},"bytes":"6b0258acecee0601717010353341aaa7fc108fbc569007c8ea87de74ba1588121037373c57f77c02b0609fddc1a2267ef74200"},{"id":"random/74","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Double","value":3.6471950971736565e+190}},{"id":9,"value":{"_tag":"I16","value":12}},{"id":21,"value":{"_tag":"Uuid","value":"fe15198acc3467b25da1f54844f95709"}},{"id":88,"value":{"_tag":"I8","value":32}},{"id":18163,"value":{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-6},{"_tag":"I16","value":-2},{"_tag":"I16","value":-1126},{"_tag":"I16","value":-6},{"_tag":"I16","value":-38}]}},{"id":18173,"value":{"_tag":"I64","value":"-8109013519055"}}]},"bytes":"2757ad64a7225f80677418cdfe15198acc3467b25da1f54844f9570903b001200ae69b02540b03cb110b4ba69dabf6ed80d80300"},{"id":"random/75","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"I16","value":3330}}]},"bytes":"b4843400"},{"id":"random/76","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"I64","value":"-43"}},{"id":24,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"63ceb8e2f9262bc7ffb66108d805cadf"},{"_tag":"Uuid","value":"13066b18f2d713bbbd4a330df375ce78"}]}}]},"bytes":"c655ca2d63ceb8e2f9262bc7ffb66108d805cadf13066b18f2d713bbbd4a330df375ce7800"},{"id":"random/77","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":459}},{"id":6,"value":{"_tag":"Map","keyType":"I16","valueType":"I32","entries":[[{"_tag":"I16","value":55},{"_tag":"I32","value":238424}]]}},{"id":17,"value":{"_tag":"I64","value":"6069050364"}},{"id":223,"value":{"_tag":"I64","value":"9223372036854775807"}},{"id":227,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":9},{"_tag":"I8","value":30},{"_tag":"I8","value":20},{"_tag":"I8","value":-58}]},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":3.835371675925545e+218},{"_tag":"Double","value":-594350.3861339506},{"_tag":"Double","value":-124810.69882578927},{"_tag":"Double","value":-889571.5071748449},{"_tag":"Double","value":-4.785664977996344e+160}]},{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":-4808},{"_tag":"I16","value":-24},{"_tag":"I16","value":1},{"_tag":"I16","value":130},{"_tag":"I16","value":61},{"_tag":"I16","value":20}]},{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":-64},{"_tag":"I16","value":-657}]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-151"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"-463960596872"},{"_tag":"I64","value":"21"},{"_tag":"I64","value":"22"},{"_tag":"I64","value":"-45"},{"_tag":"I64","value":"-17359312541805"},{"_tag":"I64","value":"0"},{"_tag":"I64","value":"-16"},{"_tag":"I64","value":"-55"},{"_tag":"I64","value":"-106231495"},{"_tag":"I64","value":"270924"},{"_tag":"I64","value":"63"},{"_tag":"I64","value":"6232"},{"_tag":"I64","value":"34725"}]},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"46b9413ca97dfb6fea46934f146e63d9"},{"_tag":"Uuid","value":"6444e322dc9989a024b659d4d5156358"},{"_tag":"Uuid","value":"562b733bafc0a176eadbe4d591364d82"},{"_tag":"Uuid","value":"cf878ccc97c5592aa751ae2158cbfeb2"}]},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":482.0}]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-24732829068236"},{"_tag":"I64","value":"168313698771"}]},{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":32767},{"_tag":"I16","value":110},{"_tag":"I16","value":0},{"_tag":"I16","value":-2},{"_tag":"I16","value":56},{"_tag":"I16","value":-2},{"_tag":"I16","value":-19},{"_tag":"I16","value":290},{"_tag":"I16","value":8},{"_tag":"I16","value":-42},{"_tag":"I16","value":1415},{"_tag":"I16","value":-42},{"_tag":"I16","value":-2},{"_tag":"I16","value":-18},{"_tag":"I16","value":-80}]},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"30a6a01e553e39385b191bc6c401f029"},{"_tag":"Uuid","value":"75c28bfe12508429696853dfe3f05725"},{"_tag":"Uuid","value":"e0cfbd27f04d0ac14429c51e590c8b11"},{"_tag":"Uuid","value":"0728a2ea5a4f499db9426187dd326cf2"}]},{"_tag":"List","elementType":"I64","elements":[]},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":626575.0398540336},{"_tag":"Double","value":887240.2347959871}]},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"8b410d5e2ec087ba507364a2ddf59e78"}]},{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":-9},{"_tag":"I16","value":241},{"_tag":"I16","value":-16}]}]}},{"id":230,"value":{"_tag":"Map","keyType":"I8","valueType":"I64","entries":[[{"_tag":"I8","value":-2},{"_tag":"I64","value":"837"}],[{"_tag":"I8","value":48},{"_tag":"I64","value":"-25"}],[{"_tag":"I8","value":127},{"_tag":"I64","value":"2074032031996127"}],[{"_tag":"I8","value":-25},{"_tag":"I64","value":"49"}]]}},{"id":384,"value":{"_tag":"Binary","value":"6e"}}]},"bytes":"3496073b01456eb08d1db6f8eff29b2d06be03feffffffffffffffff0149e943091e14c657db4dde1d5062516d6359b3c55c2322c168f3632eab78fec0d86bac03c7252bc1e6756600503b4be1648f4b2f0284027a28247fa10af60fad02feffffffffffffffff018f86a7e3801b2a2c59d9e1c2fab8f207001f6d8ddba7659889217eb061ca9e044d46b9413ca97dfb6fea46934f146e63d96444e322dc9989a024b659d4d5156358562b733bafc0a176eadbe4d591364d82cf878ccc97c5592aa751ae2158cbfeb2170000000000207e402697afedf7d19f0ba6e7b984e609f40ffeff03dc010003700325c40410538e165303239f014d30a6a01e553e39385b191bc6c401f02975c28bfe12508429696853dfe3f05725e0cfbd27f04d0ac14429c51e590c8b110728a2ea5a4f499db9426187dd326cf2062776bf67141e1f2341fc2d377890132b411d8b410d5e2ec087ba507364a2ddf59e783411e2031f3b0436fe8a0d30317fbee3a8e8c694af07e762088006016e00"},{"id":"random/78","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-5,"value":{"_tag":"Uuid","value":"f632b17bbb1ac22bb112e61caa39ba15"}},{"id":9,"value":{"_tag":"Binary","value":"69206f"}},{"id":-4,"value":{"_tag":"I16","value":-4}},{"id":0,"value":{"_tag":"Double","value":0.0}}]},"bytes":"0d09f632b17bbb1ac22bb112e61caa39ba15e80369206f04070747000000000000000000"},{"id":"random/79","type":"Struct","value":{"_tag":"Struct","fields":[{"id":202,"value":{"_tag":"I16","value":4945}},{"id":322,"value":{"_tag":"Binary","value":"6d6f636362747a6269762e6b727a6273626be697a576"}},{"id":324,"value":{"_tag":"Bool","value":false}},{"id":332,"value":{"_tag":"I8","value":-56}},{"id":415,"value":{"_tag":"I16","value":0}},{"id":592,"value":{"_tag":"Bool","value":false}},{"id":858,"value":{"_tag":"Double","value":-1.133524750797728e-70}},{"id":872,"value":{"_tag":"I64","value":"-3"}}]},"bytes":"049403a24d088405166d6f636362747a6269762e6b727a6273626be697a5762283c804be060002a00907b40d59531cfdd80869b1e60500"},{"id":"random/80","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Bool","value":false}},{"id":14,"value":{"_tag":"Bool","value":false}},{"id":-4,"value":{"_tag":"I32","value":-1}},{"id":9,"value":{"_tag":"I16","value":-36}},{"id":22,"value":{"_tag":"I8","value":2}},{"id":35,"value":{"_tag":"Uuid","value":"a31b42d7cb48c0048f99202265b76d70"}},{"id":39,"value":{"_tag":"Bool","value":false}}]},"bytes":"32b2050701d447d302dda31b42d7cb48c0048f99202265b76d704200"},{"id":"random/81","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"47765642043028957"},{"_tag":"I64","value":"-1"}]}},{"id":27,"value":{"_tag":"I16","value":-6153}},{"id":293,"value":{"_tag":"Bool","value":false}},{"id":434,"value":{"_tag":"I8","value":7}},{"id":447,"value":{"_tag":"Bool","value":false}},{"id":3,"value":{"_tag":"Bool","value":false}}]},"bytes":"6a26bad78be7a5a6d9a901010436916002ca0403e40607d2020600"},{"id":"random/82","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"I16","value":50}},{"id":171,"value":{"_tag":"I8","value":117}},{"id":-1,"value":{"_tag":"Bool","value":true}},{"id":9,"value":{"_tag":"List","elementType":"Map","elements":[{"_tag":"Map","keyType":"I64","valueType":"Binary","entries":[[{"_tag":"I64","value":"54440042"},{"_tag":"Binary","value":"766778726261f09f8c8d2065c3bc20717972c3a9696171c3a9"}],[{"_tag":"I64","value":"-26480"},{"_tag":"Binary","value":"6a62766c5f61"}],[{"_tag":"I64","value":"-1221152155342"},{"_tag":"Binary","value":"d9b82427cb"}]]},{"_tag":"Map","keyType":"Double","valueType":"I16","entries":[[{"_tag":"Double","value":-522.0},{"_tag":"I16","value":3}],[{"_tag":"Double","value":-827.0},{"_tag":"I16","value":55}],[{"_tag":"Double","value":-701.0},{"_tag":"I16","value":-14}],[{"_tag":"Double","value":-514541.9891183709},{"_tag":"I16","value":-28}]]},{"_tag":"Map","keyType":"I8","valueType":"Map","entries":[[{"_tag":"I8","value":53},{"_tag":"Map","keyType":"Bool","valueType":"Uuid","entries":[[{"_tag":"Bool","value":true},{"_tag":"Uuid","value":"e96d20f12ab3e878c1d11d0e2e469399"}]]}]]},{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"Map","keyType":"I16","valueType":"Bool","entries":[[{"_tag":"I16","value":101},{"_tag":"Bool","value":true}],[{"_tag":"I16","value":-44},{"_tag":"Bool","value":true}]]}]}}]},"bytes":"b46403d602750101a95b0368d4c1f53319766778726261f09f8c8d2065c3bc20717972c3a9696171c3a9df9d03066a62766c5f619b9bb9a58a4705d9b82427cb047400000000005080c0060000000000d889c06e0000000000e885c01b3c72dbf4b7671fc137013b35011d01e96d20f12ab3e878c1d11d0e2e469399000241ca0101570100"},{"id":"random/83","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"29506"}]}},{"id":23,"value":{"_tag":"Struct","fields":[{"id":-22126,"value":{"_tag":"I64","value":"3"}},{"id":-22115,"value":{"_tag":"Struct","fields":[{"id":108,"value":{"_tag":"Map","keyType":"I32","valueType":"Binary","entries":[[{"_tag":"I32","value":-245089950},{"_tag":"Binary","value":"6562752d62"}],[{"_tag":"I32","value":-1},{"_tag":"Binary","value":"27bb"}]]}},{"id":309,"value":{"_tag":"Map","keyType":"I64","valueType":"Bool","entries":[[{"_tag":"I64","value":"36"},{"_tag":"Bool","value":false}]]}},{"id":315,"value":{"_tag":"I32","value":30}},{"id":323,"value":{"_tag":"Binary","value":"0f5f52772f"}},{"id":549,"value":{"_tag":"I32","value":91417}},{"id":555,"value":{"_tag":"Map","keyType":"I64","valueType":"I16","entries":[[{"_tag":"I64","value":"4486"},{"_tag":"I16","value":56}],[{"_tag":"I64","value":"-52"},{"_tag":"I16","value":-39}],[{"_tag":"I64","value":"-38351"},{"_tag":"I16","value":-4}],[{"_tag":"I64","value":"-19278222380813833"},{"_tag":"I16","value":1}],[{"_tag":"I64","value":"44"},{"_tag":"I16","value":1027}],[{"_tag":"I64","value":"5647322"},{"_tag":"I16","value":32}],[{"_tag":"I64","value":"-1611468281773217"},{"_tag":"I16","value":-46}],[{"_tag":"I64","value":"-8"},{"_tag":"I16","value":-62}],[{"_tag":"I64","value":"56852848701451"},{"_tag":"I16","value":166}],[{"_tag":"I64","value":"-1"},{"_tag":"I16","value":1711}],[{"_tag":"I64","value":"6600706563680"},{"_tag":"I16","value":6011}],[{"_tag":"I64","value":"59"},{"_tag":"I16","value":-37}],[{"_tag":"I64","value":"-6684547752354082"},{"_tag":"I16","value":-21}],[{"_tag":"I64","value":"2074067946872156"},{"_tag":"I16","value":125}],[{"_tag":"I64","value":"-685439128867511583"},{"_tag":"I16","value":-47}],[{"_tag":"I64","value":"29"},{"_tag":"I16","value":9}]]}}]}},{"id":-22015,"value":{"_tag":"Double","value":183.0}},{"id":-21914,"value":{"_tag":"I16","value":3}},{"id":-9272,"value":{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":229.0},{"_tag":"Double","value":-649.0},{"_tag":"Double","value":663.0},{"_tag":"Double","value":-251.0},{"_tag":"Double","value":408.0}]}},{"id":-9073,"value":{"_tag":"I32","value":10093734}}]}},{"id":30,"value":{"_tag":"I64","value":"-93502508875"}},{"id":290,"value":{"_tag":"I64","value":"4708961988877"}},{"id":371,"value":{"_tag":"Double","value":-935.0}},{"id":653,"value":{"_tag":"I32","value":-2147483648}}]},"bytes":"991684cd03ec06dbd90206bc0bd8010258bb9adee901056562752d62010227bb0bea0401614802653c88050f5f52772f05ca08b2940b6b10648c4670674d9dd7040791b8c6ce9ddcbe4402588610b4afb10540c1f283a8e6e7dc055b0f7b96c0cfdda2ed19cc0201de1ac0f9a88c9b8003f65d7649c3849be4f1e3df1729b8f5b4b3d296af07fa01bdf4e8f4b1d09583135d3a120007fdd7020000000000e0664004b3d602060aef9001570000000000a06c4000000000004884c00000000000b884400000000000606fc0000000000080794005e18d01cc92d009007695bdf8d2b80506c4049ac48dc08c920207e6050000000000388dc0059a0affffffff0f00"},{"id":"random/84","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":19,"value":{"_tag":"I8","value":-4}},{"id":112,"value":{"_tag":"I8","value":20}},{"id":114,"value":{"_tag":"Map","keyType":"I64","valueType":"Binary","entries":[[{"_tag":"I64","value":"32"},{"_tag":"Binary","value":"61676d6d7af09f8c8d62737675626164"}],[{"_tag":"I64","value":"370712101035466"},{"_tag":"Binary","value":"1bd15a855d614f65b1c37050438a01e1d95c0b8b2f014caaf8802202713522ec497e95875953b012019dc408f62e12518772173740b7ab5b478c896b8e43c1040b50c753308d0718506eebfafd1a7d22793f390cdf79829f29c8b67a1d24a2a1943ecedb0afc47a745ce6d2cd1a788a462bde1129c5eecdbfb175878afba8796"}],[{"_tag":"I64","value":"-3704"},{"_tag":"Binary","value":"656664746be697a562205f626b6d7761636e7a70622d6d79666d206b74c3bc7877756d6976c3bc62656a69615f6a20c3bc78637572786876617174646d6d6768e697a56874c3bc6d746d706464c3a96c79732f2e616a6379666c6f6e6677795f68637371206461c3a9626a636c742f6cc3bc68746f6ef09f8c8d616f762f77c3a96a6470f09f8c8d692fe697a5e697a57564676c786c"}],[{"_tag":"I64","value":"-2201276175782274"},{"_tag":"Binary","value":"bcb234"}],[{"_tag":"I64","value":"-34433597450983453"},{"_tag":"Binary","value":"03ef29a139c03e9611f93c1fa7117590e1"}],[{"_tag":"I64","value":"-1"},{"_tag":"Binary","value":"e697a5757067626b"}],[{"_tag":"I64","value":"-12672"},{"_tag":"Binary","value":"7674666f6d617a71c3bc6d6a686270"}],[{"_tag":"I64","value":"42"},{"_tag":"Binary","value":"77"}],[{"_tag":"I64","value":"40"},{"_tag":"Binary","value":"6d6e7a6469717278682d2f"}],[{"_tag":"I64","value":"17"},{"_tag":"Binary","value":"c3a9"}],[{"_tag":"I64","value":"82896754858"},{"_tag":"Binary","value":"627920636264666c6a6d656c71f09f8c8d79c3bcf09f8c8d"}],[{"_tag":"I64","value":"-26"},{"_tag":"Binary","value":"1c6475d93cc2ebb6a00490819e1a62cb59c00f25"}],[{"_tag":"I64","value":"1803"},{"_tag":"Binary","value":"6c"}],[{"_tag":"I64","value":"9223372036854775807"},{"_tag":"Binary","value":"a3d77c9f724896acd806d6e449bb957a"}],[{"_tag":"I64","value":"-422609559"},{"_tag":"Binary","value":"2f2f7266656bc3a96d6a2e"}],[{"_tag":"I64","value":"-6278219332050"},{"_tag":"Binary","value":"79797a2f7a7664766d752078796e632f706ac3bc7a6a6fe697a5797a646e73717a2f2f706c666ac3bc2d7962202d746a2e61656373f09f8c8df09f8c8d2ff09f8c8d2f742e7575f09f8c8d706d5f68e697a5f09f8c8d6569772d76685fc3a9f09f8c8d7ac3a96f62716f73c3a96d692f696264726e7061716a637a6f77616820c3bc775f207a6673f09f8c8d2e71622e6c776b69612d6f6b2067e697a578776d75692fc3bc776162766e64636d6e7365616f736c2d5f20796b756bc3a9c3bc75647920627175717a72f09f8c8d7768e697a5745f6ac3bc6f6261636ec3bc6b7a2ef09f8c8d6774686c66207564206e6d64687a6372e697a5"}]]}},{"id":-15187,"value":{"_tag":"Bool","value":false}}]},"bytes":"7b00c3fc03e001142b1068401061676d6d7af09f8c8d6273767562616494f7c4b1a4caa80180011bd15a855d614f65b1c37050438a01e1d95c0b8b2f014caaf8802202713522ec497e95875953b012019dc408f62e12518772173740b7ab5b478c896b8e43c1040b50c753308d0718506eebfafd1a7d22793f390cdf79829f29c8b67a1d24a2a1943ecedb0afc47a745ce6d2cd1a788a462bde1129c5eecdbfb175878afba8796ef399601656664746be697a562205f626b6d7761636e7a70622d6d79666d206b74c3bc7877756d6976c3bc62656a69615f6a20c3bc78637572786876617174646d6d6768e697a56874c3bc6d746d706464c3a96c79732f2e616a6379666c6f6e6677795f68637371206461c3a9626a636c742f6cc3bc68746f6ef09f8c8d616f762f77c3a96a6470f09f8c8d692fe697a5e697a57564676c786c83c694c89183e90703bcb234b9e0bc8695cbaa7a1103ef29a139c03e9611f93c1fa7117590e10108e697a5757067626bffc5010f7674666f6d617a71c3bc6d6a686270540177500b6d6e7a6469717278682d2f2202c3a9d4e2c0d0e90418627920636264666c6a6d656c71f09f8c8d79c3bcf09f8c8d33141c6475d93cc2ebb6a00490819e1a62cb59c00f25961c016cfeffffffffffffffff0110a3d77c9f724896acd806d6e449bb957aad8a8493030b2f2f7266656bc3a96d6a2ea3e7c3b0b8ed02f80179797a2f7a7664766d752078796e632f706ac3bc7a6a6fe697a5797a646e73717a2f2f706c666ac3bc2d7962202d746a2e61656373f09f8c8df09f8c8d2ff09f8c8d2f742e7575f09f8c8d706d5f68e697a5f09f8c8d6569772d76685fc3a9f09f8c8d7ac3a96f62716f73c3a96d692f696264726e7061716a637a6f77616820c3bc775f207a6673f09f8c8d2e71622e6c776b69612d6f6b2067e697a578776d75692fc3bc776162766e64636d6e7365616f736c2d5f20796b756bc3a9c3bc75647920627175717a72f09f8c8d7768e697a5745f6ac3bc6f6261636ec3bc6b7a2ef09f8c8d6774686c66207564206e6d64687a6372e697a502a5ed0100"},{"id":"random/85","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I64","value":"1"}},{"id":27,"value":{"_tag":"Double","value":-59.0}},{"id":34,"value":{"_tag":"Bool","value":false}},{"id":39,"value":{"_tag":"Uuid","value":"dba1355d55e42beeae39576de5a39255"}}]},"bytes":"e602d70000000000804dc0725ddba1355d55e42beeae39576de5a3925500"},{"id":"random/86","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Bool","value":true}},{"id":-17715,"value":{"_tag":"I32","value":62}},{"id":-17713,"value":{"_tag":"I16","value":-44}},{"id":3,"value":{"_tag":"I64","value":"10897977086"}},{"id":14,"value":{"_tag":"Uuid","value":"acb1077ed567ddf7c55c20bd5fef5e36"}},{"id":-3064,"value":{"_tag":"I64","value":"22"}},{"id":-3049,"value":{"_tag":"I8","value":-15}}]},"bytes":"f105e594027c24570606fcbb8f9951bdacb1077ed567ddf7c55c20bd5fef5e3606ef2f2cf3f100"},{"id":"random/87","type":"Struct","value":{"_tag":"Struct","fields":[{"id":173,"value":{"_tag":"Double","value":1000.0}}]},"bytes":"07da020000000000408f4000"},{"id":"random/88","type":"Struct","value":{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"Uuid","value":"e6423fda3586575ccc9d29c15bb1d376"}}]},"bytes":"dde6423fda3586575ccc9d29c15bb1d37600"},{"id":"random/89","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"I16","value":32767}},{"id":18,"value":{"_tag":"Uuid","value":"cda9c7b22b7eae7c578ffb9f71990b48"}},{"id":24,"value":{"_tag":"Uuid","value":"12be70fd6cb41a7b022627c20317efd5"}}]},"bytes":"64feff03cdcda9c7b22b7eae7c578ffb9f71990b486d12be70fd6cb41a7b022627c20317efd500"},{"id":"random/90","type":"Struct","value":{"_tag":"Struct","fields":[{"id":16833,"value":{"_tag":"I64","value":"3675201261460296"}},{"id":-18411,"value":{"_tag":"Struct","fields":[{"id":-10919,"value":{"_tag":"Set","elementType":"Map","elements":[{"_tag":"Map","keyType":"I16","valueType":"Binary","entries":[[{"_tag":"I16","value":14},{"_tag":"Binary","value":"66c3a96267c3a96d72726d72"}]]},{"_tag":"Map","keyType":"I32","valueType":"I8","entries":[[{"_tag":"I32","value":-8362760},{"_tag":"I8","value":-1}],[{"_tag":"I32","value":62},{"_tag":"I8","value":-4}],[{"_tag":"I32","value":-2},{"_tag":"I8","value":28}],[{"_tag":"I32","value":131464},{"_tag":"I8","value":20}],[{"_tag":"I32","value":61616},{"_tag":"I8","value":-6}]]},{"_tag":"Map","keyType":"I32","valueType":"I8","entries":[[{"_tag":"I32","value":396782},{"_tag":"I8","value":41}]]},{"_tag":"Map","keyType":"I8","valueType":"I16","entries":[[{"_tag":"I8","value":-5},{"_tag":"I16","value":-170}]]},{"_tag":"Map","keyType":"I32","valueType":"Uuid","entries":[[{"_tag":"I32","value":3985448},{"_tag":"Uuid","value":"e2035a086ffdf7e96c55e64ea7a6252b"}]]}]}},{"id":-10907,"value":{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":-1},{"_tag":"I8","value":-20},{"_tag":"I8","value":8},{"_tag":"I8","value":-31},{"_tag":"I8","value":-49},{"_tag":"I8","value":33},{"_tag":"I8","value":2},{"_tag":"I8","value":-5},{"_tag":"I8","value":-5},{"_tag":"I8","value":6},{"_tag":"I8","value":-16},{"_tag":"I8","value":-18},{"_tag":"I8","value":15},{"_tag":"I8","value":15},{"_tag":"I8","value":15}]}},{"id":-10900,"value":{"_tag":"Uuid","value":"d2b4c7f057624cad412289b48230dbb4"}},{"id":-10889,"value":{"_tag":"Uuid","value":"7a951f36dd3fade75b7fd89df51fd996"}},{"id":31209,"value":{"_tag":"Binary","value":"651e9fe9d52f1e98da0576565c"}},{"id":31221,"value":{"_tag":"Uuid","value":"53e4b9accd2bb02edb6d3b24df865de7"}},{"id":31482,"value":{"_tag":"I8","value":-5}},{"id":2,"value":{"_tag":"Map","keyType":"I8","valueType":"Struct","entries":[[{"_tag":"I8","value":3},{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":-39}},{"id":2,"value":{"_tag":"I64","value":"56"}},{"id":13,"value":{"_tag":"Uuid","value":"fc9ff03d490e7f7955a2d1ca93427dab"}}]}],[{"_tag":"I8","value":-62},{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Bool","value":true}},{"id":19,"value":{"_tag":"I32","value":-41717027}}]}]]}}]}},{"id":-18228,"value":{"_tag":"I32","value":-16}},{"id":3,"value":{"_tag":"Binary","value":"c3a9696871"}}]},"bytes":"0682870290edd69aefa4870d0cd59f020acdaa015b01481c0c66c3a96267c3a96d72726d7205538fecfc07ff7cfc031c90861014e0c207fa0153dcb730290134fbd302015dd0c0e603e2035a086ffdf7e96c55e64ea7a6252bc9f30fffec08e1cf2102fbfb06f0ee0f0f0f7dd2b4c7f057624cad412289b48230dbb4bd7a951f36dd3fade75b7fd89df51fd99608d2e7030d651e9fe9d52f1e98da0576565ccd53e4b9accd2bb02edb6d3b24df865de703f4eb03fb0b04023c03344d060470bdfc9ff03d490e7f7955a2d1ca93427dab00c291a5c5b4e427000005e79c021f080605c3a969687100"},{"id":"random/91","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/92","type":"Struct","value":{"_tag":"Struct","fields":[{"id":267,"value":{"_tag":"I16","value":7}}]},"bytes":"0496040e00"},{"id":"random/93","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10199,"value":{"_tag":"I8","value":-14}},{"id":3,"value":{"_tag":"I8","value":3}},{"id":-17490,"value":{"_tag":"I8","value":3}},{"id":-17482,"value":{"_tag":"List","elementType":"I32","elements":[]}},{"id":-17480,"value":{"_tag":"I32","value":10921823}},{"id":-17475,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":1},{"_tag":"I32","value":12},{"_tag":"I32","value":1961086281},{"_tag":"I32","value":349},{"_tag":"I32","value":4}]}},{"id":-17471,"value":{"_tag":"I8","value":0}},{"id":-17470,"value":{"_tag":"I16","value":-21}}]},"bytes":"03ae9f01f203060303a3910203890525be9db50a5a55021892b59ece0eba05084300142900"},{"id":"random/94","type":"Struct","value":{"_tag":"Struct","fields":[{"id":138,"value":{"_tag":"I64","value":"0"}},{"id":146,"value":{"_tag":"I64","value":"-2636620"}},{"id":155,"value":{"_tag":"I64","value":"-223733589"}},{"id":168,"value":{"_tag":"Binary","value":"10d5d01c4a241abe6456419f3e3a45277e8a87e9e7df18cc2afc40f938f3ff0569ecea28f9d5bd3d294b1411913d9b48463b04342bb450b4129b968a6fa84630447ab0b09013d6b3a382da57012988a9e065adc7f5e1ab4f599b1ba6ca1bbc4be8aab94a7c4fe84149070f0d59b3aef172afd696d2752efcec0938aed991c8"}},{"id":176,"value":{"_tag":"Uuid","value":"400d1f836329e09d19d18234475ccb66"}}]},"bytes":"069402008697edc10296a99dafd501d87f10d5d01c4a241abe6456419f3e3a45277e8a87e9e7df18cc2afc40f938f3ff0569ecea28f9d5bd3d294b1411913d9b48463b04342bb450b4129b968a6fa84630447ab0b09013d6b3a382da57012988a9e065adc7f5e1ab4f599b1ba6ca1bbc4be8aab94a7c4fe84149070f0d59b3aef172afd696d2752efcec0938aed991c88d400d1f836329e09d19d18234475ccb6600"},{"id":"random/95","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I32","value":-43}},{"id":117,"value":{"_tag":"Uuid","value":"ef4a3e06b9e71fd67920ca42bbd8251e"}},{"id":376,"value":{"_tag":"I32","value":2147483647}},{"id":383,"value":{"_tag":"Bool","value":false}}]},"bytes":"85550dea01ef4a3e06b9e71fd67920ca42bbd8251e05f005feffffff0f7200"},{"id":"random/96","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Uuid","value":"adb9fdb63e6011d9b9e8b85ba47ed123"}},{"id":-4,"value":{"_tag":"I32","value":-11450}},{"id":-1,"value":{"_tag":"Binary","value":"b3c78f0fe22d6e473cff1b49010909689c3c4dc5acbb1d250895be63a6fae9524315d90e09c4d030c9a65b41a560216a1067c28afaaf1915d4b49e152f0a4a56d6e365fe8fd494b877694622e4e68a72ef20bfa315bf7901c8c50fcf4151257d449dacc7a0ea2c4b1a1bb604172ecf589ead6f9e9c56bc8f6e2ea233ea431f796293afc0fbe62efe2c11e4446e207cc1cf942d9ed61be8f3211927a3304e778b19297af7d367911c9d63889836b9ef0993ec6d6244689eadb8d75ff4c64481f1fbcb018a0289a3f8"}},{"id":131,"value":{"_tag":"I8","value":39}},{"id":139,"value":{"_tag":"I16","value":29}},{"id":148,"value":{"_tag":"I8","value":0}},{"id":152,"value":{"_tag":"Binary","value":"c3bce697a562"}}]},"bytes":"4dadb9fdb63e6011d9b9e8b85ba47ed1230507f3b20138c801b3c78f0fe22d6e473cff1b49010909689c3c4dc5acbb1d250895be63a6fae9524315d90e09c4d030c9a65b41a560216a1067c28afaaf1915d4b49e152f0a4a56d6e365fe8fd494b877694622e4e68a72ef20bfa315bf7901c8c50fcf4151257d449dacc7a0ea2c4b1a1bb604172ecf589ead6f9e9c56bc8f6e2ea233ea431f796293afc0fbe62efe2c11e4446e207cc1cf942d9ed61be8f3211927a3304e778b19297af7d367911c9d63889836b9ef0993ec6d6244689eadb8d75ff4c64481f1fbcb018a0289a3f803860227843a93004806c3bce697a56200"},{"id":"random/97","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}},{"id":-21142,"value":{"_tag":"Double","value":241426.82980571687}},{"id":-21138,"value":{"_tag":"Double","value":-121.0}},{"id":-20988,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"347775015103739422"},{"_tag":"I64","value":"-33"},{"_tag":"I64","value":"257912"},{"_tag":"I64","value":"-21"},{"_tag":"I64","value":"-129523530869"},{"_tag":"I64","value":"-9223372036854775808"},{"_tag":"I64","value":"3640"},{"_tag":"I64","value":"-17"},{"_tag":"I64","value":"58"},{"_tag":"I64","value":"-18467973079843"},{"_tag":"I64","value":"-12"},{"_tag":"I64","value":"8877831692119038"},{"_tag":"I64","value":"7427819241767595531"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"21824559833"},{"_tag":"I64","value":"-7508"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"151"},{"_tag":"I64","value":"42875854053"},{"_tag":"I64","value":"1022525670865823756"},{"_tag":"I64","value":"-9"},{"_tag":"I64","value":"219559514800202"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"0"},{"_tag":"I64","value":"47"},{"_tag":"I64","value":"-212833"},{"_tag":"I64","value":"8015514931"},{"_tag":"I64","value":"-61"},{"_tag":"I64","value":"-1297243739519081655"},{"_tag":"I64","value":"52"}]},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":-2.236295352843275e-290},{"_tag":"Double","value":0.06540977163678456},{"_tag":"Double","value":-2.563939197218818e-13},{"_tag":"Double","value":1.434816857910204e-225},{"_tag":"Double","value":"NaN"},{"_tag":"Double","value":1.0376492190218768e+84},{"_tag":"Double","value":175.0},{"_tag":"Double","value":500796.5893304264},{"_tag":"Double","value":-2.0132928680774722e-104},{"_tag":"Double","value":0.0},{"_tag":"Double","value":-872.0},{"_tag":"Double","value":-604.0},{"_tag":"Double","value":"NaN"},{"_tag":"Double","value":-537.0},{"_tag":"Double","value":30353895031969.246},{"_tag":"Double","value":-6.374606143053738e-128}]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-5262149736408"},{"_tag":"I64","value":"39"},{"_tag":"I64","value":"1183778"},{"_tag":"I64","value":"-26635535329454"},{"_tag":"I64","value":"-9223372036854775808"},{"_tag":"I64","value":"-9713220609696370"},{"_tag":"I64","value":"-4"},{"_tag":"I64","value":"-545107911650"},{"_tag":"I64","value":"1899615736972887"},{"_tag":"I64","value":"49"},{"_tag":"I64","value":"8761048998097"},{"_tag":"I64","value":"-3327659053766642363"},{"_tag":"I64","value":"-56"},{"_tag":"I64","value":"1671695847474"},{"_tag":"I64","value":"15"},{"_tag":"I64","value":"-3001688636"},{"_tag":"I64","value":"5"},{"_tag":"I64","value":"611331401199550"},{"_tag":"I64","value":"2"},{"_tag":"I64","value":"11080"},{"_tag":"I64","value":"-2696"},{"_tag":"I64","value":"-9223372036854775808"},{"_tag":"I64","value":"-45"},{"_tag":"I64","value":"-247472482777933440"},{"_tag":"I64","value":"-58"},{"_tag":"I64","value":"-14583"},{"_tag":"I64","value":"-54"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"111"},{"_tag":"I64","value":"-7087907"}]},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"a9d9e66223aa709d8cfd3594246fd970"},{"_tag":"Uuid","value":"d96061d0694c9be98d4c2f62490726b7"},{"_tag":"Uuid","value":"21c5ce5fee9f21aaefac837a52d3c9ec"},{"_tag":"Uuid","value":"7deb44827ff0c6642b44f89b61afcc9e"},{"_tag":"Uuid","value":"4231e36316aba2b35297fc62d57a52af"},{"_tag":"Uuid","value":"7f548ac873b3dea54763bcfd0fe0ac6c"},{"_tag":"Uuid","value":"b3cb686df57f21c1e7819e49f595b70d"},{"_tag":"Uuid","value":"39c3ab3ff6c02d85990fca6dc980b787"},{"_tag":"Uuid","value":"70a16a4ebe9d534c61efe10a2bd0f994"},{"_tag":"Uuid","value":"264557e87a69fb716a924dd2eb3ea94e"},{"_tag":"Uuid","value":"a427eecf254996dcb2ba709ffd0f0054"},{"_tag":"Uuid","value":"cd33d014f181d75d563a559fe0f42067"},{"_tag":"Uuid","value":"ed0b258f0d1f04e99d65d56b5a0cc7c6"},{"_tag":"Uuid","value":"90c7f2d4171438f4979b4a79c72ee6e7"}]},{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"6570c3a96f617520736120"},{"_tag":"Binary","value":"2eff92"},{"_tag":"Binary","value":"5e7a12"},{"_tag":"Binary","value":"5f64e697a563637264"},{"_tag":"Binary","value":"79f0be5528a3f6c9b8beed66fa4e28a285dc"}]},{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"25dc0927c5e0b99fcd89ce158a586d6965a6cb11022b55c50f46765243a863a6f830cd0b1942e68c040fe16a70bad7afa0e4241398e40a74f228ffcb14fd58c83e0fed0ebc6918aa5d07bded7e6356d5159f13fcde320c549820f298031324481227847b710d95b45dd660e7d481dde6bc901edc24368782de6e8f3cbf3da947"},{"_tag":"Binary","value":"64"},{"_tag":"Binary","value":"69627861686467c3a9616f"},{"_tag":"Binary","value":"6de697a56a617a687371745f787974767a2f716170"},{"_tag":"Binary","value":"73"},{"_tag":"Binary","value":"6d76686e5f75662d64c3a9"},{"_tag":"Binary","value":"cba6767984e18804536141ad10fef6f9583a"},{"_tag":"Binary","value":"c575"},{"_tag":"Binary","value":"af82"},{"_tag":"Binary","value":"7a6f73"},{"_tag":"Binary","value":"d486572f3118e24a119b96a6b95a5dd01b98f84ec2901f77b68d17eab0213fdad31182c0a31f650d0117815603170b489212a0f814c61bbe4aa22f0bef2d2f6a3c31451d87b146a5fdadebd7be79a205df11b3cc64237fdcb978005a1cec6ab8e8ef3addddbc71a5830881ea5ab452caae5fda4fd0ce459fac9f29dbfe8fa756"},{"_tag":"Binary","value":"a201"},{"_tag":"Binary","value":"9de41fae"},{"_tag":"Binary","value":"5f6d20c3a96ac3a9"},{"_tag":"Binary","value":"0213d15a81baa172dc026b1f95fa83521937766e"},{"_tag":"Binary","value":"676571"},{"_tag":"Binary","value":"20797a6f6a6a766d"},{"_tag":"Binary","value":"662069"},{"_tag":"Binary","value":"57855a712ef2f84e"},{"_tag":"Binary","value":"656979762ec3bc756d66637220f09f8c8d636978207078c3bc2f782f6c20616bc3bc6563616b747670c3a96c7579787273e697a52d6765e697a5616e6e207069752f716667c3a964706f7475792d78202f61c3bc627977736e6c76726571792e20706e6b2f202f6e69666b67686e776c68f09f8c8d72767175c3bcc3bc6f65755f63206bf09f8c8d75c3bc6b6f6573f09f8c8dc3bc64f09f8c8d666b76776f7666f09f8c8d7679f09f8c8de697a579e697a56875c3a97278756276612e20636774e697a57ae697a5e697a5642e64736471c3bc2f6c746178726d656b6f676e7a686f7a73f09f8c8d7a2f627862616f207a6e6174617261752f75"},{"_tag":"Binary","value":"981fea37c901ea3861c160ac037db7a8af"},{"_tag":"Binary","value":"02df2799f3eece87f3fb"},{"_tag":"Binary","value":"781585e9cc437e84272d9cd64abed1c7f4d20bab18dd336409778dc70ad4406dfef3854dc8640c0e911f439231adfb575a2e1a6a2b674259058194fa485cff43921054d05ea1a0eefcc98dabfbe564d99f2585073fdbde62692d208d4238453119526c208376e3905bed2626feae00f16fcbbb424275553ab8c42cfeeed2df38486d0b0d8c19975b5cdec6a66d9b145fd9d7a55979bef969eae555384b60310c3150bf2ca08e87c442602882d170272f5ce4bc383cf77983ba964f3691a6a33af7b6582be9b22888"},{"_tag":"Binary","value":"6b6a656c6d2f6a78747020786961c3bc766ee697a52dc3bce697a52d68736e626378206f7a616bc3bc6cf09f8c8d6774676f6471782f73755f6e206a73626b627767c3a966746e6ec3a96d5f6ce697a5e697a5e697a5647af09f8c8de697a564636d642f61777276616e747579787870715f6b5f70c3bc5f677077f09f8c8dc3bc7a73657472656a716ec3bc6c2d6467686d2e2d7065616a6d7ac3a96bc3bc"},{"_tag":"Binary","value":"c4900782369a0a3865a7b79823905fa0aff91d"},{"_tag":"Binary","value":"ab10b4"},{"_tag":"Binary","value":"626b6f79"},{"_tag":"Binary","value":"a0abf43cb9fbbe0c78b3f4f4b321dd454ab4d48447335f5b36c7da2eaa02f8458ad58602826ff85cc0c83149127f1ca79c1a830e09d0a03da7403ae00acbc939104af36a31c20cd138a3274a317cf5d49f5f19e32a6db530d9b8fb26c1cee58bd6845e1de700f26dc10f7f738b0bf2098ee4d927576271ddb649bb6fa5c4cd00d8266f86f697d7b8e6d23b54b7e58fa10c5e4bc7d0701596f47c24e8d57388706a04f43306c2a756db4ee42d7514ddee0244df59981a2e9bd9f98fb95d37f13aa9884adb90642029"},{"_tag":"Binary","value":"558342d2498c46"},{"_tag":"Binary","value":"2f70c3a97176f09f8c8d637a7164"}]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"19677819164"},{"_tag":"I64","value":"-344884555451088698"},{"_tag":"I64","value":"60"},{"_tag":"I64","value":"479040149509558911"},{"_tag":"I64","value":"-2"}]},{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]},{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3}]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"1894226624520911"},{"_tag":"I64","value":"-3364835076319065916"},{"_tag":"I64","value":"100788808971338490"},{"_tag":"I64","value":"23"},{"_tag":"I64","value":"-6992403138"}]},{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false}]},{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":127},{"_tag":"I8","value":-1},{"_tag":"I8","value":51},{"_tag":"I8","value":-31},{"_tag":"I8","value":-2},{"_tag":"I8","value":2},{"_tag":"I8","value":127},{"_tag":"I8","value":0},{"_tag":"I8","value":0},{"_tag":"I8","value":13},{"_tag":"I8","value":29},{"_tag":"I8","value":-128},{"_tag":"I8","value":13},{"_tag":"I8","value":2},{"_tag":"I8","value":22},{"_tag":"I8","value":-38}]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"4353"},{"_tag":"I64","value":"-128645"},{"_tag":"I64","value":"-34"},{"_tag":"I64","value":"-20"},{"_tag":"I64","value":"-3714613248779457057"}]},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":-2.3051012520996917e+41},{"_tag":"Double","value":833.0}]},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":383914.228845404},{"_tag":"Double","value":-825000.6803769185},{"_tag":"Double","value":-196304.01743804594},{"_tag":"Double","value":-69.0},{"_tag":"Double","value":3.787044442195096e-106},{"_tag":"Double","value":929.0},{"_tag":"Double","value":18.0},{"_tag":"Double","value":959.0},{"_tag":"Double","value":758265.8912543426},{"_tag":"Double","value":317.0},{"_tag":"Double","value":-288.0},{"_tag":"Double","value":3.011095234301387e-36},{"_tag":"Double","value":"-0"},{"_tag":"Double","value":-440419.1074941575},{"_tag":"Double","value":339423.10860250215},{"_tag":"Double","value":-135787.66840763355},{"_tag":"Double","value":810914.6174076165},{"_tag":"Double","value":-993407.6049421505},{"_tag":"Double","value":-949711.0194712102},{"_tag":"Double","value":403035.8090644749},{"_tag":"Double","value":"NaN"},{"_tag":"Double","value":-5.545112439032668e-230},{"_tag":"Double","value":-248.0},{"_tag":"Double","value":221321.6310509385},{"_tag":"Double","value":-545477.4047520197},{"_tag":"Double","value":620.0},{"_tag":"Double","value":834.0},{"_tag":"Double","value":-402082.74684104265},{"_tag":"Double","value":"-Infinity"},{"_tag":"Double","value":-111506.604679515}]}]}}]},"bytes":"1107abca02002e71a396780d41470000000000405ec009f7c702f90ff61ebcd89dd093e2c5d30941f0bd1f29e9f1a183c507ffffffffffffffffff01f0382174c58ccd8ffdb20817fcaf93c6e895c51f96b8f395e08ff594ce01feffffffffffffffff01b2ebc2cda201a77501ae02caf3ccb9bf0298a0e79ac3ebdeb01c11948198ee85ec6301005ec1fd19e6b498dc3b79edf29292db93de802468f7106a1cb3ee42e5cb8343d104deb1beb03ff0fe3941c80a52bdca09c690bdfe4011000000000000f87f9322cf31a41761510000000000e06540706f795bf2901e4185ee524a611667aa00000000000000000000000000408bc00000000000e082c0000000000000f87f0000000000c880c03fa18421519bbb42a998c08fe51786a5f61eafdf8689a6b2024ec4c09001dba2f999b28e0cffffffffffffffffff01e3898781ed87c12207c38fb5afdd1fae99b29398ecdf0662a283b5f3fafd03f5aafc95eb919cae5c6fe4c0b28da7611ef788d1ae160afcbe91f0958096020490ad018f2affffffffffffffffff0159ff99cdbaaeba99ef0673ede3016bfeffffffffffffffff01de01c59ce106eda9d9e66223aa709d8cfd3594246fd970d96061d0694c9be98d4c2f62490726b721c5ce5fee9f21aaefac837a52d3c9ec7deb44827ff0c6642b44f89b61afcc9e4231e36316aba2b35297fc62d57a52af7f548ac873b3dea54763bcfd0fe0ac6cb3cb686df57f21c1e7819e49f595b70d39c3ab3ff6c02d85990fca6dc980b78770a16a4ebe9d534c61efe10a2bd0f994264557e87a69fb716a924dd2eb3ea94ea427eecf254996dcb2ba709ffd0f0054cd33d014f181d75d563a559fe0f42067ed0b258f0d1f04e99d65d56b5a0cc7c690c7f2d4171438f4979b4a79c72ee6e7580b6570c3a96f617520736120032eff92035e7a12095f64e697a5636372641279f0be5528a3f6c9b8beed66fa4e28a285dcf81e800125dc0927c5e0b99fcd89ce158a586d6965a6cb11022b55c50f46765243a863a6f830cd0b1942e68c040fe16a70bad7afa0e4241398e40a74f228ffcb14fd58c83e0fed0ebc6918aa5d07bded7e6356d5159f13fcde320c549820f298031324481227847b710d95b45dd660e7d481dde6bc901edc24368782de6e8f3cbf3da94701640b69627861686467c3a9616f156de697a56a617a687371745f787974767a2f71617001730b6d76686e5f75662d64c3a912cba6767984e18804536141ad10fef6f9583a02c57502af82037a6f738001d486572f3118e24a119b96a6b95a5dd01b98f84ec2901f77b68d17eab0213fdad31182c0a31f650d0117815603170b489212a0f814c61bbe4aa22f0bef2d2f6a3c31451d87b146a5fdadebd7be79a205df11b3cc64237fdcb978005a1cec6ab8e8ef3addddbc71a5830881ea5ab452caae5fda4fd0ce459fac9f29dbfe8fa75602a201049de41fae085f6d20c3a96ac3a9140213d15a81baa172dc026b1f95fa83521937766e036765710820797a6f6a6a766d036620690857855a712ef2f84efa01656979762ec3bc756d66637220f09f8c8d636978207078c3bc2f782f6c20616bc3bc6563616b747670c3a96c7579787273e697a52d6765e697a5616e6e207069752f716667c3a964706f7475792d78202f61c3bc627977736e6c76726571792e20706e6b2f202f6e69666b67686e776c68f09f8c8d72767175c3bcc3bc6f65755f63206bf09f8c8d75c3bc6b6f6573f09f8c8dc3bc64f09f8c8d666b76776f7666f09f8c8d7679f09f8c8de697a579e697a56875c3a97278756276612e20636774e697a57ae697a5e697a5642e64736471c3bc2f6c746178726d656b6f676e7a686f7a73f09f8c8d7a2f627862616f207a6e6174617261752f7511981fea37c901ea3861c160ac037db7a8af0a02df2799f3eece87f3fbc801781585e9cc437e84272d9cd64abed1c7f4d20bab18dd336409778dc70ad4406dfef3854dc8640c0e911f439231adfb575a2e1a6a2b674259058194fa485cff43921054d05ea1a0eefcc98dabfbe564d99f2585073fdbde62692d208d4238453119526c208376e3905bed2626feae00f16fcbbb424275553ab8c42cfeeed2df38486d0b0d8c19975b5cdec6a66d9b145fd9d7a55979bef969eae555384b60310c3150bf2ca08e87c442602882d170272f5ce4bc383cf77983ba964f3691a6a33af7b6582be9b228889f016b6a656c6d2f6a78747020786961c3bc766ee697a52dc3bce697a52d68736e626378206f7a616bc3bc6cf09f8c8d6774676f6471782f73755f6e206a73626b627767c3a966746e6ec3a96d5f6ce697a5e697a5e697a5647af09f8c8de697a564636d642f61777276616e747579787870715f6b5f70c3bc5f677077f09f8c8dc3bc7a73657472656a716ec3bc6c2d6467686d2e2d7065616a6d7ac3a96bc3bc13c4900782369a0a3865a7b79823905fa0aff91d03ab10b404626b6f79c801a0abf43cb9fbbe0c78b3f4f4b321dd454ab4d48447335f5b36c7da2eaa02f8458ad58602826ff85cc0c83149127f1ca79c1a830e09d0a03da7403ae00acbc939104af36a31c20cd138a3274a317cf5d49f5f19e32a6db530d9b8fb26c1cee58bd6845e1de700f26dc10f7f738b0bf2098ee4d927576271ddb649bb6fa5c4cd00d8266f86f697d7b8e6d23b54b7e58fa10c5e4bc7d0701596f47c24e8d57388706a04f43306c2a756db4ee42d7514ddee0244df59981a2e9bd9f98fb95d37f13aa9884adb9064202907558342d2498c460e2f70c3a97176f09f8c8d637a716466feffffffffffffffff01b8c49dce9201f3fcf182a1aba3c90978fec9d68acb9ef2a50d03f11e0201020102020101020201010102010202010102020202020201020102011506569e8ba191c0b2dd06f7dcadbc93eba5b25df48b92c1e8b889e6022e83ebbc8c34f10f020101010201010201020102010102f3107fff33e1fe027f00000d1d800d0216da56824489da0f4327c1c8acb7b4c4fa8c67275edf195a442b85c80000000000088a40f71e187356eaa86e17410c5d5a5c512d29c1e88eb62380f607c100000000004051c0f9e94d653acb0b2a0000000000088d4000000000000032400000000000f88d403c7d52c8f32327410000000000d0734000000000000072c006cb5a8e7d0290380000000000000080ccf2126e8ce11ac18c7e356f7cb71441f419e6585d9300c1e2d91c3c45bf284141faba35ff502ec16621f8099efb2cc1d0657b3c6f991841000000000000f87f5387f61bb08555900000000000006fc0386f640c4d040b411fa83bcf8aa520c100000000006083400000000000108a40f6e5c3fc8a8a18c1000000000000f0ff586dc4ac2939fbc000"},{"id":"random/98","type":"Struct","value":{"_tag":"Struct","fields":[{"id":132,"value":{"_tag":"I8","value":127}},{"id":140,"value":{"_tag":"Double","value":-3547001532.684836}},{"id":152,"value":{"_tag":"Binary","value":"822c861360a9"}},{"id":-19122,"value":{"_tag":"Struct","fields":[{"id":-3,"value":{"_tag":"I8","value":63}},{"id":144,"value":{"_tag":"Bool","value":true}},{"id":151,"value":{"_tag":"I64","value":"-319792"}},{"id":160,"value":{"_tag":"I32","value":2147483647}},{"id":167,"value":{"_tag":"Uuid","value":"bb20f7153582b6b97f00dfcc5fbfd5cf"}}]}},{"id":-19109,"value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":-16},{"_tag":"I8","value":-1}]}},{"id":13,"value":{"_tag":"Binary","value":"eb2449e71812a5"}},{"id":-4213,"value":{"_tag":"Uuid","value":"eb01d1b87fce73ea3e3350846c858e37"}},{"id":-4206,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":-4020,"value":{"_tag":"Binary","value":"bddba839bb01d987a21496a43e"}},{"id":-4009,"value":{"_tag":"I32","value":-2147483648}},{"id":-3886,"value":{"_tag":"Bool","value":false}}]}}]},"bytes":"0388027f872dea95575e6deac1c806822c861360a90ce3aa0203053f01a00276df842795feffffff0f7dbb20f7153582b6b97f00dfcc5fbfd5cf00dcb923f0ff2807eb2449e71812a50de941eb01d1b87fce73ea3e3350846c858e377b0008e73e0dbddba839bb01d987a21496a43eb5ffffffff0f02db3c0000"},{"id":"random/99","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Uuid","value":"e80f1735146faf0680f2ed32d3b4352d"}},{"id":12,"value":{"_tag":"Bool","value":false}}]},"bytes":"9de80f1735146faf0680f2ed32d3b4352d3200"},{"id":"random/100","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-5,"value":{"_tag":"Bool","value":false}},{"id":0,"value":{"_tag":"I64","value":"682984144082"}},{"id":163,"value":{"_tag":"Binary","value":"69182171e5e9f19f4c461fd234a6e910d98c7217b141dd7388a776b62f20ef5cfaea2c2b37eb187848606f0fb1ce39b27ba13aed2197e2cdf02de8aff22671925fffe5009509d3e9fe8d99618018a986d9726ad7b7416096b98eb83dc7166d11e7c148b86b8c2decec7f973afcf03d661ea80e2d3152af4d0553a4765bd5e268"}},{"id":169,"value":{"_tag":"Binary","value":"5f6f7463726b786a6571"}},{"id":181,"value":{"_tag":"I8","value":-2}},{"id":-25638,"value":{"_tag":"I16","value":-2}},{"id":2,"value":{"_tag":"Uuid","value":"583ffe54fc69d940e3450dd16a0a630d"}}]},"bytes":"020956a4f3b7d0e02708c602800169182171e5e9f19f4c461fd234a6e910d98c7217b141dd7388a776b62f20ef5cfaea2c2b37eb187848606f0fb1ce39b27ba13aed2197e2cdf02de8aff22671925fffe5009509d3e9fe8d99618018a986d9726ad7b7416096b98eb83dc7166d11e7c148b86b8c2decec7f973afcf03d661ea80e2d3152af4d0553a4765bd5e268680a5f6f7463726b786a6571c3fe04cb9003030d04583ffe54fc69d940e3450dd16a0a630d00"},{"id":"random/101","type":"Struct","value":{"_tag":"Struct","fields":[{"id":196,"value":{"_tag":"Binary","value":"6b70e697a571e697a52e67686e6b7466682e766e2d"}},{"id":205,"value":{"_tag":"I32","value":-40}},{"id":216,"value":{"_tag":"Bool","value":true}},{"id":225,"value":{"_tag":"Double","value":-582283.399419874}},{"id":236,"value":{"_tag":"Binary","value":"eee60be36cd6f2854221fcdf"}},{"id":242,"value":{"_tag":"Uuid","value":"d0bac8bf82b5e995b57dff1319c05682"}},{"id":249,"value":{"_tag":"I32","value":-50}}]},"bytes":"088803156b70e697a571e697a52e67686e6b7466682e766e2d954fb19700c380cc16c521c1b80ceee60be36cd6f2854221fcdf6dd0bac8bf82b5e995b57dff1319c05682756300"},{"id":"random/102","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/103","type":"Struct","value":{"_tag":"Struct","fields":[{"id":25162,"value":{"_tag":"I32","value":55000}},{"id":25164,"value":{"_tag":"I32","value":-64}},{"id":25175,"value":{"_tag":"Binary","value":"92716f12ccf18834a16d1a6854f3"}},{"id":25363,"value":{"_tag":"Uuid","value":"88f804a0f6fcb348500c15fbd7730ed4"}}]},"bytes":"05948903b0db06257fb80e92716f12ccf18834a16d1a6854f30da68c0388f804a0f6fcb348500c15fbd7730ed400"},{"id":"random/104","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Uuid","value":"cd12a5533a0068073bf478f66ba06f3f"}},{"id":18,"value":{"_tag":"I64","value":"1"}}]},"bytes":"4dcd12a5533a0068073bf478f66ba06f3fe60200"},{"id":"random/105","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"Bool","value":false}}]},"bytes":"020700"},{"id":"random/106","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I16","value":33}},{"id":8,"value":{"_tag":"I8","value":1}},{"id":-1,"value":{"_tag":"Map","keyType":"Map","valueType":"I8","entries":[[{"_tag":"Map","keyType":"I16","valueType":"I16","entries":[[{"_tag":"I16","value":51},{"_tag":"I16","value":-39}]]},{"_tag":"I8","value":-3}],[{"_tag":"Map","keyType":"I8","valueType":"I16","entries":[[{"_tag":"I8","value":127},{"_tag":"I16","value":-1506}],[{"_tag":"I8","value":40},{"_tag":"I16","value":4}]]},{"_tag":"I8","value":-18}],[{"_tag":"Map","keyType":"Binary","valueType":"I16","entries":[[{"_tag":"Binary","value":"65646767"},{"_tag":"I16","value":-56}],[{"_tag":"Binary","value":"5bf859a4357c08ebcb15f7af0374ed0cd9258190"},{"_tag":"I16","value":-1}],[{"_tag":"Binary","value":"e6933350a650810dd1"},{"_tag":"I16","value":356}]]},{"_tag":"I8","value":-2}],[{"_tag":"Map","keyType":"Struct","valueType":"I16","entries":[[{"_tag":"Struct","fields":[]},{"_tag":"I16","value":-385}]]},{"_tag":"I8","value":0}],[{"_tag":"Map","keyType":"Map","valueType":"I16","entries":[[{"_tag":"Map","keyType":"Double","valueType":"I8","entries":[[{"_tag":"Double","value":-201.0},{"_tag":"I8","value":2}],[{"_tag":"Double","value":0.0},{"_tag":"I8","value":-27}],[{"_tag":"Double","value":-31.0},{"_tag":"I8","value":1}],[{"_tag":"Double","value":95828.48037970322},{"_tag":"I8","value":-2}],[{"_tag":"Double","value":122.0},{"_tag":"I8","value":-124}],[{"_tag":"Double","value":-928.0},{"_tag":"I8","value":3}],[{"_tag":"Double","value":142161.35166881979},{"_tag":"I8","value":0}],[{"_tag":"Double","value":997.0},{"_tag":"I8","value":-122}],[{"_tag":"Double","value":-606392.7272509313},{"_tag":"I8","value":-1}],[{"_tag":"Double","value":-587.0},{"_tag":"I8","value":-8}],[{"_tag":"Double","value":5e-324},{"_tag":"I8","value":16}],[{"_tag":"Double","value":386342.95573553606},{"_tag":"I8","value":40}],[{"_tag":"Double","value":-270360.0294415635},{"_tag":"I8","value":-48}],[{"_tag":"Double","value":-597407.185118766},{"_tag":"I8","value":7}],[{"_tag":"Double","value":"-0"},{"_tag":"I8","value":1}],[{"_tag":"Double","value":235866.46381476824},{"_tag":"I8","value":-1}]]},{"_tag":"I16","value":29}],[{"_tag":"Map","keyType":"Uuid","valueType":"I8","entries":[[{"_tag":"Uuid","value":"c248a5a64948a9262e05fd1ab53b35d9"},{"_tag":"I8","value":-2}],[{"_tag":"Uuid","value":"9c1fcacdcd7fc6f4e94a8b4fc8fe340d"},{"_tag":"I8","value":-32}]]},{"_tag":"I16","value":1}],[{"_tag":"Map","keyType":"I8","valueType":"I8","entries":[[{"_tag":"I8","value":14},{"_tag":"I8","value":102}]]},{"_tag":"I16","value":17}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I16","value":-25}],[{"_tag":"Map","keyType":"Double","valueType":"I64","entries":[[{"_tag":"Double","value":-773209.9969369364},{"_tag":"I64","value":"-30"}],[{"_tag":"Double","value":-111.0},{"_tag":"I64","value":"-3030535804273849725"}],[{"_tag":"Double","value":-701.0},{"_tag":"I64","value":"-15"}],[{"_tag":"Double","value":952195.5927252334},{"_tag":"I64","value":"44570552"}]]},{"_tag":"I16","value":12305}],[{"_tag":"Map","keyType":"I8","valueType":"I64","entries":[[{"_tag":"I8","value":-19},{"_tag":"I64","value":"-155340260381438254"}],[{"_tag":"I8","value":0},{"_tag":"I64","value":"7600109313166434"}],[{"_tag":"I8","value":11},{"_tag":"I64","value":"6094904570"}]]},{"_tag":"I16","value":-23114}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I16","value":32767}],[{"_tag":"Map","keyType":"I8","valueType":"I16","entries":[[{"_tag":"I8","value":51},{"_tag":"I16","value":1}],[{"_tag":"I8","value":1},{"_tag":"I16","value":-32768}],[{"_tag":"I8","value":67},{"_tag":"I16","value":539}]]},{"_tag":"I16","value":-1128}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I16","value":2}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I16","value":1}],[{"_tag":"Map","keyType":"I8","valueType":"Bool","entries":[[{"_tag":"I8","value":8},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":1},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-123},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-2},{"_tag":"Bool","value":false}]]},{"_tag":"I16","value":-8}],[{"_tag":"Map","keyType":"I16","valueType":"I64","entries":[[{"_tag":"I16","value":-33},{"_tag":"I64","value":"-191036788755706"}],[{"_tag":"I16","value":-52},{"_tag":"I64","value":"-59"}],[{"_tag":"I16","value":-27},{"_tag":"I64","value":"50"}],[{"_tag":"I16","value":-1},{"_tag":"I64","value":"-70748314078772"}],[{"_tag":"I16","value":-23},{"_tag":"I64","value":"250597992487387"}],[{"_tag":"I16","value":0},{"_tag":"I64","value":"-1246370"}],[{"_tag":"I16","value":5},{"_tag":"I64","value":"25781719674290"}],[{"_tag":"I16","value":-32768},{"_tag":"I64","value":"16665915339"}],[{"_tag":"I16","value":21},{"_tag":"I64","value":"-22"}],[{"_tag":"I16","value":-13521},{"_tag":"I64","value":"-108977660539900720"}],[{"_tag":"I16","value":-8},{"_tag":"I64","value":"-192477678216017521"}],[{"_tag":"I16","value":32767},{"_tag":"I64","value":"339852602"}],[{"_tag":"I16","value":49},{"_tag":"I64","value":"84640"}],[{"_tag":"I16","value":-14},{"_tag":"I64","value":"-725026841944658"}],[{"_tag":"I16","value":-3},{"_tag":"I64","value":"871235948554102665"}],[{"_tag":"I16","value":-27},{"_tag":"I64","value":"39"}]]},{"_tag":"I16","value":-245}],[{"_tag":"Map","keyType":"Binary","valueType":"Uuid","entries":[[{"_tag":"Binary","value":"61726a7570666c7571"},{"_tag":"Uuid","value":"1fccb3e6ed25d6ac62c4e7c24e136546"}]]},{"_tag":"I16","value":34}],[{"_tag":"Map","keyType":"I8","valueType":"Bool","entries":[[{"_tag":"I8","value":5},{"_tag":"Bool","value":true}],[{"_tag":"I8","value":-64},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":2},{"_tag":"Bool","value":true}],[{"_tag":"I8","value":-48},{"_tag":"Bool","value":true}],[{"_tag":"I8","value":-8},{"_tag":"Bool","value":true}],[{"_tag":"I8","value":-12},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-2},{"_tag":"Bool","value":true}],[{"_tag":"I8","value":-2},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-7},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":29},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-56},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":0},{"_tag":"Bool","value":true}],[{"_tag":"I8","value":9},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":37},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-6},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-2},{"_tag":"Bool","value":true}]]},{"_tag":"I16","value":32767}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I16","value":-12}],[{"_tag":"Map","keyType":"Bool","valueType":"Double","entries":[[{"_tag":"Bool","value":false},{"_tag":"Double","value":460681.8485238224}]]},{"_tag":"I16","value":4383}]]},{"_tag":"I8","value":1}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I8","value":21}],[{"_tag":"Map","keyType":"I32","valueType":"Double","entries":[[{"_tag":"I32","value":-55},{"_tag":"Double","value":-252.0}],[{"_tag":"I32","value":19093260},{"_tag":"Double","value":2.105150484778415e-265}],[{"_tag":"I32","value":-24},{"_tag":"Double","value":310536.6712745405}]]},{"_tag":"I8","value":-1}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I8","value":35}],[{"_tag":"Map","keyType":"Bool","valueType":"Binary","entries":[[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"8a2b7f38ff3ec13464359babc9880569de05a7fba5d49c987f854f2c132204ff8672ad7834f8b074fd3b6cba358f945b9903ffa2360337a98c5325ffc00e1d146ae01d9cc3255568942b39fd9269b53ba483742b464bc31cd035a2aa878680a6b2b656cd76d39bbf4ea38a952a7b66d312a5ea4b0f41e3d33a3fa4ffb035a9"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"61716d"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"68ea82009c0dab1eb746e60fea4ba3b1af78fc44"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"4a392d4e94fdcdfdf3165d4327"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"e697a56e73796462e697a56f7674c3a9"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"6b7e964de05b1daeea16ee624568cfbe674beacbc77649cf8b2ade71c7b6d022ba39510917b3d179cd4140d8b6b993e57b8de4f15a8542496460fec302788b28dca213e5c10586232bb1fd333f80858fdeee51b60ff748b5663cb2b8ddde6a02466314444d933d0f33ce2ad70595634453209f8133db779c5326e53e51320875"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"977a1f9678e21310"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"7822664be577887a497cc8d0f1eeae86f1700a"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"67e697a574c3a962745f7220732d"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"64765f5f67e697a56a697278766f77686273706963"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"732d7871e697a56c716c7066f09f8c8d73"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"70626a73747a62766d6e"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"6a7120636171c3bc617a7a726dc3bcf09f8c8d"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"c3bc65696e696f6977646173692d676671"}],[{"_tag":"Bool","value":true},{"_tag":"Binary","value":"782fc3a96e2e705f75726777"}],[{"_tag":"Bool","value":false},{"_tag":"Binary","value":"1cf8ac8bacafce"}]]},{"_tag":"I8","value":0}],[{"_tag":"Map","keyType":"Set","valueType":"I64","entries":[[{"_tag":"Set","elementType":"Binary","elements":[]},{"_tag":"I64","value":"-31752"}],[{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-912.0}]},{"_tag":"I64","value":"23"}],[{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-55},{"_tag":"I8","value":-4}]},{"_tag":"I64","value":"-4541975657008"}]]},{"_tag":"I8","value":127}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I8","value":3}],[{"_tag":"Map","keyType":"Bool","valueType":"Struct","entries":[[{"_tag":"Bool","value":true},{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Double","value":-330359.78606721875}}]}],[{"_tag":"Bool","value":true},{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Binary","value":"6b62736966786b626c6f63636c78686a706f"}},{"id":15,"value":{"_tag":"Double","value":-202348.47070084}}]}]]},{"_tag":"I8","value":-128}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I8","value":-57}],[{"_tag":"Map","keyType":"Uuid","valueType":"Double","entries":[[{"_tag":"Uuid","value":"d89fbda8004e7bf0543e1419a0d423b9"},{"_tag":"Double","value":-727380.6141404309}]]},{"_tag":"I8","value":-1}],[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"I8","value":-48}],[{"_tag":"Map","keyType":"Bool","valueType":"I32","entries":[[{"_tag":"Bool","value":true},{"_tag":"I32","value":-478}],[{"_tag":"Bool","value":true},{"_tag":"I32","value":213073784}],[{"_tag":"Bool","value":true},{"_tag":"I32","value":81}]]},{"_tag":"I8","value":2}]]}},{"id":-32674,"value":{"_tag":"I32","value":312}},{"id":5029,"value":{"_tag":"Double","value":1.5868543616556684e+209}},{"id":5296,"value":{"_tag":"Bool","value":false}}]},"bytes":"144273010b0110b30144664dfd02347fc3172808ee038404656467676f145bf859a4357c08ebcb15f7af0374ed0cd92581900109e6933350a650810dd1c805fe01c40081060010b4107300000000002069c0020000000000000000e50000000000003fc001b0a0a2af4765f740fe0000000000805e40840000000000008dc00300be37d08a5a0141000000000000288f4086ec3b5a74718122c1ff00000000005882c0f80100000000000000101c56acd29b94174128e2ed251e608010c1d00ce3c75e3e3b22c1070000000000000080016884e4b5d3ca0c41ff3a02d3c248a5a64948a9262e05fd1ab53b35d9fe9c1fcacdcd7fc6f4e94a8b4fc8fe340de00201330e662200310476a4846efeb39827c13b0000000000c05bc0f9d5c3cdbd90d08e540000000000e885c01d8aae792f070f2d41f0dec02aa2c0010336eddbc4d1f599c9f0a70400c4b1ecfdca90801b0bf4f3c6b42d93e90200feff030334330201ffff0343b608cf11000400020431080201028502fe020f104641f3b3f6fee6ef566775356401e7b888828c96202db6e796c5dcfa7100c39298010ae4b6f1e3d8dc0bffff0396bfed957c2a2ba1d301dfccf88edba69583030fe1d98fd4b6dbe8ab05feff03f4f48dc40262c0aa0a1ba389eff58edac90205929ef1d1d59ea09718354ee903018d0961726a7570666c75711fccb3e6ed25d6ac62c4e7c24e1365464410310501c0020201d001f801f402fe01fe02f9021d02c802000109022502fa02fe01feff030017011702cc6de364271e1c41be4401001503576d0000000000806fc098dc9a127c186b2edf26fb082fd89762af22f41241ff00231018017f8a2b7f38ff3ec13464359babc9880569de05a7fba5d49c987f854f2c132204ff8672ad7834f8b074fd3b6cba358f945b9903ffa2360337a98c5325ffc00e1d146ae01d9cc3255568942b39fd9269b53ba483742b464bc31cd035a2aa878680a6b2b656cd76d39bbf4ea38a952a7b66d312a5ea4b0f41e3d33a3fa4ffb035a9020361716d011468ea82009c0dab1eb746e60fea4ba3b1af78fc44010d4a392d4e94fdcdfdf3165d43270210e697a56e73796462e697a56f7674c3a90280016b7e964de05b1daeea16ee624568cfbe674beacbc77649cf8b2ade71c7b6d022ba39510917b3d179cd4140d8b6b993e57b8de4f15a8542496460fec302788b28dca213e5c10586232bb1fd333f80858fdeee51b60ff748b5663cb2b8ddde6a02466314444d933d0f33ce2ad70595634453209f8133db779c5326e53e513208750108977a1f9678e2131001137822664be577887a497cc8d0f1eeae86f1700a020e67e697a574c3a962745f7220732d021564765f5f67e697a56a697278766f776862737069630211732d7871e697a56c716c7066f09f8c8d73010a70626a73747a62766d6e02136a7120636171c3bc617a7a726dc3bcf09f8c8d0111c3bc65696e696f6977646173692d676671010c782fc3a96e2e705f7572677702071cf8ac8bacafce0003a6088ff003170000000000808cc02e23c9fcdfe8c3adb088027f0003021c019714ceee24df2914c1000178126b62736966786b626c6f63636c78686a706f8750cdfec363b308c1008000c701d7d89fbda8004e7bf0543e1419a0d423b9549d703aa93226c1ff00d0031501bb0701f0fd99cb0101a2010205c3fe03f00407ca4e6334dc4e3ee45e6b02e05200"},{"id":"random/107","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Binary","value":"2f68c3bc7073736c632ff09f8c8d61706d6d"}},{"id":11,"value":{"_tag":"I16","value":-33}},{"id":20,"value":{"_tag":"I8","value":0}}]},"bytes":"28122f68c3bc7073736c632ff09f8c8d61706d6d9441930000"},{"id":"random/108","type":"Struct","value":{"_tag":"Struct","fields":[{"id":113,"value":{"_tag":"I8","value":-9}},{"id":273,"value":{"_tag":"I64","value":"-341920811678655217"}}]},"bytes":"03e201f706a204e18bdb96d5cadfbe0900"},{"id":"random/109","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/110","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Bool","value":true}},{"id":17,"value":{"_tag":"I16","value":-30}},{"id":32,"value":{"_tag":"I8","value":-8}},{"id":175,"value":{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":80.0},{"_tag":"Double","value":-982.0},{"_tag":"Double","value":"-Infinity"},{"_tag":"Double","value":-3.0768876357028497e-243},{"_tag":"Double","value":"-Infinity"},{"_tag":"Double","value":250.0},{"_tag":"Double","value":-48.0},{"_tag":"Double","value":-691742.8952242313},{"_tag":"Double","value":499226.05052296375},{"_tag":"Double","value":-5.928608825020555e+288},{"_tag":"Double","value":-469019.69309735403},{"_tag":"Double","value":-765.0},{"_tag":"Double","value":-232.0},{"_tag":"Double","value":838.0},{"_tag":"Double","value":512.0},{"_tag":"Double","value":535.0}]}},{"id":328,"value":{"_tag":"Binary","value":"676f74"}},{"id":330,"value":{"_tag":"I64","value":"-9223372036854775808"}},{"id":531,"value":{"_tag":"I32","value":46}}]},"bytes":"41d43bf3f809de02f71000000000000054400000000000b08ec0000000000000f0ff40a477855602958d000000000000f0ff0000000000406f4000000000000048c098d45aca3d1c25c1b44abc3368781e4167db188ba477e3fb1250bbc56ea01cc10000000000e887c00000000000006dc00000000000308a4000000000000080400000000000b8804008900503676f7426ffffffffffffffffff0105a6085c00"},{"id":"random/111","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/112","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I32","value":-318031501}},{"id":8,"value":{"_tag":"I32","value":-334742}},{"id":13,"value":{"_tag":"I64","value":"-54989201926210"}},{"id":245,"value":{"_tag":"Uuid","value":"c17e4143baab9fde2216a084f1aac03e"}},{"id":252,"value":{"_tag":"Bool","value":true}},{"id":260,"value":{"_tag":"Binary","value":"2f7a6974"}}]},"bytes":"55999aa6af0235abee285683a1c7bde580190dea03c17e4143baab9fde2216a084f1aac03e7188042f7a697400"},{"id":"random/113","type":"Struct","value":{"_tag":"Struct","fields":[{"id":27,"value":{"_tag":"Bool","value":true}},{"id":29,"value":{"_tag":"Double","value":-958338.8191866155}},{"id":42,"value":{"_tag":"Bool","value":true}},{"id":-4515,"value":{"_tag":"Bool","value":true}},{"id":-4505,"value":{"_tag":"I8","value":20}},{"id":-2,"value":{"_tag":"I8","value":2}}]},"bytes":"013627966d6ca3053f2dc1d101c546a31403030200"},{"id":"random/114","type":"Struct","value":{"_tag":"Struct","fields":[{"id":126,"value":{"_tag":"I64","value":"-42"}},{"id":-27426,"value":{"_tag":"Binary","value":"725f6ee697a56367676965c3a97a7a64656ff09f8c8d"}},{"id":-27215,"value":{"_tag":"Uuid","value":"a10a5c4286e3e382edf26ecd90dd5f26"}},{"id":-27208,"value":{"_tag":"Uuid","value":"fc33de427d60caf4a60bd9e694f1d6ea"}},{"id":-27194,"value":{"_tag":"Double","value":483952.93523551314}},{"id":-4,"value":{"_tag":"I32","value":-42}}]},"bytes":"06fc015308c3ac0316725f6ee697a56367676965c3a97a7a64656ff09f8c8d0d9da903a10a5c4286e3e382edf26ecd90dd5f267dfc33de427d60caf4a60bd9e694f1d6eae7dc60aebdc3891d4105075300"},{"id":"random/115","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"I64","value":"-22"}},{"id":211,"value":{"_tag":"Double","value":-2.881905017674453e+204}},{"id":-1,"value":{"_tag":"I32","value":-2064556951}},{"id":0,"value":{"_tag":"Uuid","value":"b90e8de2e6ba3999a03725deeeb95197"}},{"id":13,"value":{"_tag":"I8","value":-5}},{"id":28,"value":{"_tag":"I64","value":"1736107"}}]},"bytes":"c62b07a603e6a9643c396262ea0501ad8ef5b00f1db90e8de2e6ba3999a03725deeeb95197d3fbf6d6f6d30100"},{"id":"random/116","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Binary","value":"1976c96e7b7f1d5872334dac255d1011f7ed"}},{"id":13,"value":{"_tag":"Uuid","value":"f1798b0d9262fe15b1d240555840fa70"}},{"id":26,"value":{"_tag":"I32","value":139577}},{"id":38,"value":{"_tag":"I8","value":12}},{"id":41,"value":{"_tag":"Uuid","value":"d5810a9c58fda975ae8f551eaeaac7ef"}},{"id":55,"value":{"_tag":"I8","value":-1}}]},"bytes":"b8121976c96e7b7f1d5872334dac255d1011f7ed2df1798b0d9262fe15b1d240555840fa70d5f28411c30c3dd5810a9c58fda975ae8f551eaeaac7efe3ff00"},{"id":"random/117","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Double","value":453.0}},{"id":4,"value":{"_tag":"I16","value":0}},{"id":13,"value":{"_tag":"Binary","value":"e313"}},{"id":21,"value":{"_tag":"I32","value":-1484160344}},{"id":-3,"value":{"_tag":"I16","value":-2616}}]},"bytes":"770000000000507c400408009802e31385aff5b3870b0405ef2800"},{"id":"random/118","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":254,"value":{"_tag":"Binary","value":"712d62"}}]},"bytes":"6b0008fc0303712d6200"},{"id":"random/119","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I16","value":1665}},{"id":4,"value":{"_tag":"I32","value":-1}},{"id":28,"value":{"_tag":"I8","value":-17}},{"id":40,"value":{"_tag":"Uuid","value":"5223c1812f35995b0eba9e9a34847d16"}},{"id":50,"value":{"_tag":"I16","value":32767}},{"id":214,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}}]},"bytes":"84821a0508010338efcd5223c1812f35995b0eba9e9a34847d16a4feff030bac030000"},{"id":"random/120","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":-1}},{"id":13,"value":{"_tag":"I16","value":-57}},{"id":103,"value":{"_tag":"I8","value":-38}},{"id":106,"value":{"_tag":"Binary","value":"64636d"}},{"id":121,"value":{"_tag":"Binary","value":"6177792d5f76727575786c697469637420f09f8c8d2fc3bc"}},{"id":123,"value":{"_tag":"I8","value":2}}]},"bytes":"13ffc47103ce01da380364636df8186177792d5f76727575786c697469637420f09f8c8d2fc3bc230200"},{"id":"random/121","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1,"value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"I16","value":-4}},{"id":10,"value":{"_tag":"Binary","value":"05773c105991c2"}},{"id":252,"value":{"_tag":"Uuid","value":"52e898b4c7838d07a5c6601b4e737168"}},{"id":551,"value":{"_tag":"Binary","value":"772b4c9c13e9fcae"}},{"id":8103,"value":{"_tag":"Uuid","value":"5805873e512e9a597ffce60234bb3d00"}},{"id":8108,"value":{"_tag":"Binary","value":"2e70756968667a6af09f8c8d77"}},{"id":8114,"value":{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":39},{"_tag":"I16","value":-7162},{"_tag":"I16","value":11},{"_tag":"I16","value":27290},{"_tag":"I16","value":-1},{"_tag":"I16","value":779},{"_tag":"I16","value":45},{"_tag":"I16","value":-450},{"_tag":"I16","value":-6233},{"_tag":"I16","value":-2119},{"_tag":"I16","value":-14},{"_tag":"I16","value":-69},{"_tag":"I16","value":14},{"_tag":"I16","value":-1},{"_tag":"I16","value":20},{"_tag":"I16","value":-921},{"_tag":"I16","value":0},{"_tag":"I16","value":62},{"_tag":"I16","value":-12745},{"_tag":"I16","value":-10},{"_tag":"I16","value":-29},{"_tag":"I16","value":51},{"_tag":"I16","value":0},{"_tag":"I16","value":1},{"_tag":"I16","value":116},{"_tag":"I16","value":773},{"_tag":"I16","value":-9769},{"_tag":"I16","value":-9},{"_tag":"I16","value":524},{"_tag":"I16","value":-35}]}},{"id":8118,"value":{"_tag":"Uuid","value":"6546f6de37befc94b666a08969c958d9"}}]}},{"id":18567,"value":{"_tag":"I8","value":-13}},{"id":21264,"value":{"_tag":"Binary","value":"bd0a7be5613c66a708f6d1ff7376ad8f23"}},{"id":5,"value":{"_tag":"I8","value":-24}}]},"bytes":"0c016407480705773c105991c20df80352e898b4c7838d07a5c6601b4e73716808ce0808772b4c9c13e9fcae0dce7e5805873e512e9a597ffce60234bb3d00580d2e70756968667a6af09f8c8d776af41e4ef36f16b4aa0301960c5a8307b1618d211b89011c0128b10e007c91c7011339660002e8018a0cd19801119808454d6546f6de37befc94b666a08969c958d900038ea202f308a0cc0211bd0a7be5613c66a708f6d1ff7376ad8f23030ae800"},{"id":"random/122","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"1796"}},{"id":12,"value":{"_tag":"Set","elementType":"I64","elements":[]}},{"id":57,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"576799f90565b4123b803d7af588ed3966"},{"_tag":"Binary","value":"c02d8a57ebee31d50e0da1"},{"_tag":"Binary","value":"655f2e2dc3a966f09f8c8d766a735f776b6c6b"},{"_tag":"Binary","value":"6a727220787276726d685f20"}]}},{"id":71,"value":{"_tag":"Double","value":335503.6482437444}},{"id":75,"value":{"_tag":"I8","value":-52}},{"id":-2,"value":{"_tag":"Binary","value":"2f5fc3bc5f756878"}}]},"bytes":"36881c9a060a724811576799f90565b4123b803d7af588ed39660bc02d8a57ebee31d50e0da113655f2e2dc3a966f09f8c8d766a735f776b6c6b0c6a727220787276726d685f20e74835cd973e7a144143cc0803082f5fc3bc5f75687800"},{"id":"random/123","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Double","value":439.0}},{"id":298,"value":{"_tag":"Double","value":3.9488129040481363e+71}},{"id":514,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"67511ef990819b622e539ab66ce05bd9d0688d"},{"_tag":"Binary","value":"408ad059447dd2d5e253d2f7954ea450df2b"},{"_tag":"Binary","value":"f7210a44ce9cdbb9d685"},{"_tag":"Binary","value":"6ec3a9c3a9666d"},{"_tag":"Binary","value":"53ced9e0"}]}},{"id":527,"value":{"_tag":"I8","value":0}},{"id":22181,"value":{"_tag":"Double","value":-405.0}}]},"bytes":"670000000000707b4007d404ac88b4057b9bcc4e0a8408581367511ef990819b622e539ab66ce05bd9d0688d12408ad059447dd2d5e253d2f7954ea450df2b0af7210a44ce9cdbb9d685076ec3a9c3a9666d0453ced9e0d30007cada0200000000005079c000"},{"id":"random/124","type":"Struct","value":{"_tag":"Struct","fields":[{"id":20330,"value":{"_tag":"I8","value":0}},{"id":20603,"value":{"_tag":"I8","value":16}},{"id":5,"value":{"_tag":"Bool","value":false}},{"id":18,"value":{"_tag":"Binary","value":"642f6c796c756477c3a976666971c3a9c3bc67"}}]},"bytes":"03d4bd020003f6c10210020ad813642f6c796c756477c3a976666971c3a9c3bc6700"},{"id":"random/125","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I16","value":-3}},{"id":23,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"1869116202"},{"_tag":"I64","value":"63"},{"_tag":"I64","value":"22844536"},{"_tag":"I64","value":"3"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"-494504"},{"_tag":"I64","value":"54"},{"_tag":"I64","value":"49"},{"_tag":"I64","value":"-22501660198866"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"-3"},{"_tag":"I64","value":"-112857428762735759"},{"_tag":"I64","value":"63866974979220"},{"_tag":"I64","value":"6804919805"},{"_tag":"I64","value":"-6"},{"_tag":"I64","value":"58"},{"_tag":"I64","value":"7"},{"_tag":"I64","value":"-205264969120"},{"_tag":"I64","value":"-39135062"},{"_tag":"I64","value":"-34"},{"_tag":"I64","value":"-48"},{"_tag":"I64","value":"165859"},{"_tag":"I64","value":"-1816332023109056897"},{"_tag":"I64","value":"50422285"},{"_tag":"I64","value":"0"},{"_tag":"I64","value":"50"},{"_tag":"I64","value":"1398883660"},{"_tag":"I64","value":"15543202"},{"_tag":"I64","value":"3177544159"},{"_tag":"I64","value":"9422149338052"}]}},{"id":38,"value":{"_tag":"Double","value":934331.0594845247}},{"id":-7291,"value":{"_tag":"Double","value":"-Infinity"}},{"id":-7281,"value":{"_tag":"Binary","value":"60323a697ded"}},{"id":26147,"value":{"_tag":"Double","value":-926.0}},{"id":26160,"value":{"_tag":"I16","value":-7630}}]},"bytes":"e40599f61ed4ccc3f60d7ef0d1e41506feffffffffffffffff01cfae3c6c62a39fb4b7e29d0afeffffffffffffffff01059dd2d5faf8cef99003a892da85c6851dfad7d6d9320b740ebff6f6abf90bab9da925435fc69f1481e6a7f9c096f4b4329a888b30006498958ab60ac4aee90ebee7abd617888feabdb8a404f770c1741e76832c4107f571000000000000f0ffa80660323a697ded07c698030000000000f08cc0d49b7700"},{"id":"random/126","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"I16","value":-17}},{"id":19,"value":{"_tag":"I16","value":11}},{"id":5,"value":{"_tag":"Double","value":-5315.022468729294}},{"id":10,"value":{"_tag":"Binary","value":"73f09f8c8d"}},{"id":24,"value":{"_tag":"Uuid","value":"bed662fd67cef0f4cf3a3db0c84f4cde"}},{"id":-21025,"value":{"_tag":"I8","value":31}},{"id":-20882,"value":{"_tag":"I16","value":1589}}]},"bytes":"a4219416070a80b982c005c3b4c0580573f09f8c8dedbed662fd67cef0f4cf3a3db0c84f4cde03c1c8021f04a3c602ea1800"},{"id":"random/127","type":"Struct","value":{"_tag":"Struct","fields":[{"id":75,"value":{"_tag":"Double","value":-972095.8141851552}}]},"bytes":"0796016de0dca07faa2dc100"},{"id":"random/128","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I64","value":"-2034114169660"}},{"id":16,"value":{"_tag":"Double","value":"Infinity"}}]},"bytes":"e6f7acb8aab37627000000000000f07f00"},{"id":"random/129","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"Double","value":276.0}},{"id":-3059,"value":{"_tag":"I8","value":-38}},{"id":-3054,"value":{"_tag":"I8","value":-37}},{"id":-2846,"value":{"_tag":"I32","value":34}},{"id":-2821,"value":{"_tag":"I32","value":2147483647}}]},"bytes":"0700000000000040714003e52fda53db05bb2c4405892cfeffffff0f00"},{"id":"random/130","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I8","value":1}},{"id":15,"value":{"_tag":"Bool","value":true}},{"id":17,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"78"}]}},{"id":27,"value":{"_tag":"Binary","value":"66"}},{"id":23597,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"7b8162a041ef569a82a44cfacda29f3c"},{"_tag":"Uuid","value":"e0e96b703d76577fefd7f1e21baf9ae7"},{"_tag":"Uuid","value":"825812bcfda05d38e0b362964a5791a9"},{"_tag":"Uuid","value":"39e16daaf6ec0efe9bf0743325333174"},{"_tag":"Uuid","value":"624f479743af63639a2a3b7e97a69f02"},{"_tag":"Uuid","value":"6963e304590f033ee2c63f627550929a"},{"_tag":"Uuid","value":"d3953f845939fc2cedb0bf93283d8046"},{"_tag":"Uuid","value":"a1817b332b58701206f620595b3f9a54"},{"_tag":"Uuid","value":"8178cd39bab11c7fabb91eb660a98625"},{"_tag":"Uuid","value":"7f477302602f1ffe8339b3590c840c61"},{"_tag":"Uuid","value":"3d09b1e20b6d8a7dbbc93cfc627fdc86"},{"_tag":"Uuid","value":"3f4d11f0006affc4e001fab42e60576b"},{"_tag":"Uuid","value":"99e2a4bff91f55972422bf58d8d8f709"},{"_tag":"Uuid","value":"d37832a2e16497a039d7e6d8a769773c"},{"_tag":"Uuid","value":"314cd270d1a6acb179c579847c504f21"}]}}]},"bytes":"93016129180178a801660adaf002fd0f7b8162a041ef569a82a44cfacda29f3ce0e96b703d76577fefd7f1e21baf9ae7825812bcfda05d38e0b362964a5791a939e16daaf6ec0efe9bf0743325333174624f479743af63639a2a3b7e97a69f026963e304590f033ee2c63f627550929ad3953f845939fc2cedb0bf93283d8046a1817b332b58701206f620595b3f9a548178cd39bab11c7fabb91eb660a986257f477302602f1ffe8339b3590c840c613d09b1e20b6d8a7dbbc93cfc627fdc863f4d11f0006affc4e001fab42e60576b99e2a4bff91f55972422bf58d8d8f709d37832a2e16497a039d7e6d8a769773c314cd270d1a6acb179c579847c504f2100"},{"id":"random/131","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I8","value":-29}},{"id":66,"value":{"_tag":"Double","value":215.0}},{"id":72,"value":{"_tag":"I8","value":-1}},{"id":289,"value":{"_tag":"I64","value":"-1434675400545633"}}]},"bytes":"93e30784010000000000e06a4063ff06c204c1d5f0d98cb58c0500"},{"id":"random/132","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I32","value":-447}},{"id":4,"value":{"_tag":"Double","value":-137506.60427348362}},{"id":104,"value":{"_tag":"Double","value":245.0}},{"id":8973,"value":{"_tag":"Uuid","value":"fa15bda5caf1bc57dfee6986a136d961"}}]},"bytes":"35fd061710568dd514c900c107d0010000000000a06e400d9a8c01fa15bda5caf1bc57dfee6986a136d96100"},{"id":"random/133","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Uuid","value":"6220601feacf9b10c147f14f592e3b9c"}},{"id":2671,"value":{"_tag":"Bool","value":false}},{"id":2852,"value":{"_tag":"I16","value":-47}},{"id":2858,"value":{"_tag":"Binary","value":"f900b0c3181bce8be656012a48a0c5be66"}},{"id":2865,"value":{"_tag":"I64","value":"57"}}]},"bytes":"7d6220601feacf9b10c147f14f592e3b9c02de2904c82c5d6811f900b0c3181bce8be656012a48a0c5be66767200"},{"id":"random/134","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Struct","fields":[{"id":-16719,"value":{"_tag":"Bool","value":true}},{"id":-16625,"value":{"_tag":"I32","value":-991}},{"id":-16617,"value":{"_tag":"Uuid","value":"3d9e0c2f09eff1ac4144928032e30bf4"}},{"id":2,"value":{"_tag":"Double","value":6.18405398914733e+81}},{"id":-26156,"value":{"_tag":"Binary","value":"f09f8c8d6761e697a57a776c6d726b6677756e632e752e65207664716f7566c3bc61202d725f79c3bc2f68616a6a7273616c6b7a6378686c66626b737664762df09f8c8d706f642e705f752e6d6f2f5f76782e65c3a9722f682e675f77c3a96f792d7a7968f09f8c8d6b6b796f642d6764c3bc6f712e6f702e6e7663c3bc782d6c786a61c3bc61f09f8c8d7167785f6867752d6c68"}}]}},{"id":191,"value":{"_tag":"I16","value":1}},{"id":201,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"95346395"},{"_tag":"I64","value":"29"},{"_tag":"I64","value":"1423758"},{"_tag":"I64","value":"-56"},{"_tag":"I64","value":"38360"},{"_tag":"I64","value":"74890798970327521"},{"_tag":"I64","value":"-270934468993"},{"_tag":"I64","value":"-29691756363109"},{"_tag":"I64","value":"-9223372036854775808"},{"_tag":"I64","value":"8"},{"_tag":"I64","value":"-5"},{"_tag":"I64","value":"-1857696435731004"},{"_tag":"I64","value":"-53"},{"_tag":"I64","value":"3213020"},{"_tag":"I64","value":"-9"},{"_tag":"I64","value":"-51"},{"_tag":"I64","value":"-64"},{"_tag":"I64","value":"1895717090151713207"},{"_tag":"I64","value":"-12835435970"},{"_tag":"I64","value":"-6361663468104704"},{"_tag":"I64","value":"-9"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"30"},{"_tag":"I64","value":"208632"},{"_tag":"I64","value":"9223372036854775807"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"-410249948072675"},{"_tag":"I64","value":"1091246"},{"_tag":"I64","value":"-60"},{"_tag":"I64","value":"60"}]}},{"id":215,"value":{"_tag":"Bool","value":false}},{"id":222,"value":{"_tag":"I8","value":14}}]},"bytes":"3c019d850205e18302bd0f8d3d9e0c2f09eff1ac4144928032e30bf4070473b6170bd113ea5008d798039501f09f8c8d6761e697a57a776c6d726b6677756e632e752e65207664716f7566c3bc61202d725f79c3bc2f68616a6a7273616c6b7a6378686c66626b737664762df09f8c8d706f642e705f752e6d6f2f5f76782e65c3a9722f682e675f77c3a96f792d7a7968f09f8c8d6b6b796f642d6764c3bc6f712e6f702e6e7663c3bc782d6c786a61c3bc61f09f8c8d7167785f6867752d6c680004fe0202aaf61eb6fbf65a3a9ce6ad016fb0d704c2c792d99db2888a0281a69fcfe20fc9f5b9e9a4c00dffffffffffffffffff011009f7e8b78295e4cc0669b89b880311657feea68a92c9a9f8ce3483d7e9d05fffaf93b3c6f9cc1611feffffffffffffffff013cf0bb19feffffffffffffffff0101c5abe4a6d8c7ba01dc9a85017778e2730e00"},{"id":"random/135","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/136","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/137","type":"Struct","value":{"_tag":"Struct","fields":[{"id":96,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":-9},{"_tag":"I32","value":172},{"_tag":"I32","value":231617791},{"_tag":"I32","value":-61},{"_tag":"I32","value":26}]}}]}}]},"bytes":"0cc001395511d802fed3f1dc0179340000"},{"id":"random/138","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Binary","value":"79"}},{"id":25,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":40,"value":{"_tag":"Uuid","value":"aa30e83f2b1c79794cca24f2dd2a84c0"}},{"id":55,"value":{"_tag":"Struct","fields":[{"id":-2,"value":{"_tag":"Bool","value":false}},{"id":149,"value":{"_tag":"I64","value":"55"}},{"id":281,"value":{"_tag":"Uuid","value":"042cb0ae0098e1a1d65838e35026bb9b"}}]}}]},"bytes":"f80179ab00fdaa30e83f2b1c79794cca24f2dd2a84c0fc020306aa026e0db204042cb0ae0098e1a1d65838e35026bb9b0000"},{"id":"random/139","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Bool","value":false}},{"id":18,"value":{"_tag":"I16","value":-60}},{"id":26,"value":{"_tag":"Set","elementType":"Bool","elements":[]}},{"id":66,"value":{"_tag":"I64","value":"-1"}},{"id":69,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}},{"id":73,"value":{"_tag":"I32","value":741928}},{"id":-8507,"value":{"_tag":"Double","value":180727.02620518813}}]},"bytes":"52d4778a01068401013951010102020145d0c85a07f58401d010ab35b80f064100"},{"id":"random/140","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Uuid","value":"cfcf00f88bacff7ef5aec0691ce7b06c"}},{"id":-32359,"value":{"_tag":"I32","value":-12868185}},{"id":-28396,"value":{"_tag":"Uuid","value":"1974f0596bbdac7b5431f61f55204478"}},{"id":-28388,"value":{"_tag":"Binary","value":"c3bc692e7271646663c3bcc3bc2d2d76787ac3bc74"}},{"id":-28386,"value":{"_tag":"I16","value":-1}},{"id":-28381,"value":{"_tag":"Double","value":241996.2537309921}}]},"bytes":"bdcfcf00f88bacff7ef5aec0691ce7b06c05cdf903b1e9a20c0dd7bb031974f0596bbdac7b5431f61f552044788815c3bc692e7271646663c3bcc3bc2d2d76787ac3bc74240157481da407628a0d4100"},{"id":"random/141","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-2,"value":{"_tag":"Double","value":4.305365410689644e-283}},{"id":0,"value":{"_tag":"I64","value":"20"}},{"id":129,"value":{"_tag":"Double","value":7.0973821214174716e+131}},{"id":273,"value":{"_tag":"I16","value":-301}},{"id":284,"value":{"_tag":"I8","value":-4}},{"id":299,"value":{"_tag":"I32","value":26706}},{"id":307,"value":{"_tag":"I8","value":-5}}]},"bytes":"070379724f72600150052628078202adfc9b463fff4f5b04a204d904b3fcf5a4a10383fb00"},{"id":"random/142","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"I16","value":31}},{"id":25,"value":{"_tag":"I64","value":"-44"}},{"id":0,"value":{"_tag":"I32","value":-64}},{"id":15,"value":{"_tag":"I32","value":543433}},{"id":27,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false}]}}]},"bytes":"a43ef65705007ff592ab42ca21010200"},{"id":"random/143","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"I32","value":28591}},{"id":11,"value":{"_tag":"I64","value":"-433"}},{"id":231,"value":{"_tag":"Map","keyType":"I32","valueType":"Double","entries":[[{"_tag":"I32","value":328364},{"_tag":"Double","value":-768536.6232949324}],[{"_tag":"I32","value":4},{"_tag":"Double","value":"-Infinity"}],[{"_tag":"I32","value":958880},{"_tag":"Double","value":-167846.59863373253}],[{"_tag":"I32","value":-26126},{"_tag":"Double","value":173040.74541288824}]]}}]},"bytes":"0507debe03f6e1060bce030457d88a286d83203f317427c108000000000000f0ffc086757c7b00ca347d04c19b980348089bf6851f054100"},{"id":"random/144","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I64","value":"-5750973709006040"}},{"id":-22164,"value":{"_tag":"I16","value":4506}}]},"bytes":"86afd3cd90dd9eb71404a7da02b44600"},{"id":"random/145","type":"Struct","value":{"_tag":"Struct","fields":[{"id":18470,"value":{"_tag":"Uuid","value":"0c3b295b65a21b253c1f5e5a0ab8ca95"}},{"id":18484,"value":{"_tag":"Uuid","value":"13b57dd6230d303203ae7c64c3d5fe1f"}},{"id":18490,"value":{"_tag":"I32","value":-6734286}},{"id":18500,"value":{"_tag":"I16","value":-607}},{"id":1,"value":{"_tag":"Uuid","value":"b308125847002c644901a62662462f09"}},{"id":185,"value":{"_tag":"I16","value":37}},{"id":194,"value":{"_tag":"I16","value":-23}}]},"bytes":"0dcca0020c3b295b65a21b253c1f5e5a0ab8ca95ed13b57dd6230d303203ae7c64c3d5fe1f659b87b606a4bd090d02b308125847002c644901a62662462f0904f2024a942d00"},{"id":"random/146","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"Binary","value":"af61cb1f739769b603c003661bbf0799e4afe8b78d32f2995bdc431e7f68fe0ae9daa4e9be8e4708864a7defa32d7cb14aac3d5963f08e269390606d823cd43250fea6395d07551efad0993afad4b56e9345c332bef601a9c1202a41ec8696a23d6c39e9c60f5db9bea0856996d7adb855d4b2b5fa24d9e713f3bf6fd1c9da"}},{"id":175,"value":{"_tag":"I8","value":-60}}]},"bytes":"e87faf61cb1f739769b603c003661bbf0799e4afe8b78d32f2995bdc431e7f68fe0ae9daa4e9be8e4708864a7defa32d7cb14aac3d5963f08e269390606d823cd43250fea6395d07551efad0993afad4b56e9345c332bef601a9c1202a41ec8696a23d6c39e9c60f5db9bea0856996d7adb855d4b2b5fa24d9e713f3bf6fd1c9da03de02c400"},{"id":"random/147","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/148","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Binary","value":"6175f09f8c8d7778e697a578e697a57679616dc3bc"}},{"id":15,"value":{"_tag":"I32","value":57}},{"id":24,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"6c"},{"_tag":"Binary","value":"68762d76e697a575612fc3bc5f726f6c7320"},{"_tag":"Binary","value":"752e69755f616b615f6d726f20726f70"},{"_tag":"Binary","value":"185e0749e06d34086ef7"}]}}]},"bytes":"c8156175f09f8c8d7778e697a578e697a57679616dc3bc35729948016c1268762d76e697a575612fc3bc5f726f6c732010752e69755f616b615f6d726f20726f700a185e0749e06d34086ef700"},{"id":"random/149","type":"Struct","value":{"_tag":"Struct","fields":[{"id":32350,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}},{"id":32355,"value":{"_tag":"Map","keyType":"Bool","valueType":"List","entries":[[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"370c8d9deb32711bac31aa0609adce68"},{"_tag":"Uuid","value":"b6f2e48ee09c54eb055c07f43a699fcc"},{"_tag":"Uuid","value":"d55df967d24216bbe6b448c47f305e91"},{"_tag":"Uuid","value":"5dd0bcb222d8c478da1ccea87577f77f"},{"_tag":"Uuid","value":"a2819b5cb65533434599167a2bc81142"},{"_tag":"Uuid","value":"69067a92882c4c16942c5755409a1799"},{"_tag":"Uuid","value":"a0f8c6875089927802f265633f2a89b8"},{"_tag":"Uuid","value":"107d99e75888a328f906d546525357dc"},{"_tag":"Uuid","value":"9dede288fdeee552b132ba62d8adcd06"},{"_tag":"Uuid","value":"761b66fadc44420f9df7374d5d67ef50"},{"_tag":"Uuid","value":"ed781752249f206e9e07ccae05f73c47"},{"_tag":"Uuid","value":"da60de82419f932a42b770358234d76f"},{"_tag":"Uuid","value":"62cb1e11f7aa8ab4281a633ca0b96905"},{"_tag":"Uuid","value":"25c3cbb78baca7b538cf5717b83fe5ef"},{"_tag":"Uuid","value":"a1427e7ca1943c629702d77e3c550295"},{"_tag":"Uuid","value":"aae613e8e7129fc9594b93ec68df8a98"}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Uuid","elements":[]}],[{"_tag":"Bool","value":false},{"_tag":"List","elementType":"Uuid","elements":[]}],[{"_tag":"Bool","value":false},{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":174641385},{"_tag":"I32","value":-6507559},{"_tag":"I32","value":59},{"_tag":"I32","value":-5},{"_tag":"I32","value":-1824}]}],[{"_tag":"Bool","value":false},{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":23885},{"_tag":"I16","value":66},{"_tag":"I16","value":221},{"_tag":"I16","value":163},{"_tag":"I16","value":-63}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":166},{"_tag":"I32","value":-67506666},{"_tag":"I32","value":13},{"_tag":"I32","value":-33577096},{"_tag":"I32","value":1019253536},{"_tag":"I32","value":1014881},{"_tag":"I32","value":-1632949},{"_tag":"I32","value":-2},{"_tag":"I32","value":465213262},{"_tag":"I32","value":0},{"_tag":"I32","value":-6151891},{"_tag":"I32","value":0},{"_tag":"I32","value":59},{"_tag":"I32","value":-18},{"_tag":"I32","value":-46729},{"_tag":"I32","value":-74796241}]}],[{"_tag":"Bool","value":false},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"ed2689478b24a8ae235cf8ef1c939c3d"}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"fd9f6a98f4d16373a2b75d621038b4e39b1f"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"e697a52d2e6a685f6a66642f616d7561e697a5c3a961736c"},{"_tag":"Binary","value":"626a7a71692d7264"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"00"},{"_tag":"Binary","value":"c7748469840ac4b0dd"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"22cc9b54cc37209fe9"},{"_tag":"Binary","value":"6f"},{"_tag":"Binary","value":"c3bc736b6a6e7367"},{"_tag":"Binary","value":"2dc3bc76f09f8c8d7472746b6c6e77752f76736f6f"},{"_tag":"Binary","value":"6c76716a65c3bcc3a96579f09f8c8d2d"},{"_tag":"Binary","value":"22b66903645b"},{"_tag":"Binary","value":"6a2d666a646474e697a5617af09f8c8d73"},{"_tag":"Binary","value":"56d33edcedc087bd8f7ca3cf5e"}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":-46},{"_tag":"I8","value":-128}]}],[{"_tag":"Bool","value":false},{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"2d"},{"_tag":"Binary","value":"75716e"},{"_tag":"Binary","value":"6364642f5f76696f6e712de697a520e697a5687379666176"},{"_tag":"Binary","value":"686e796d656c68756b6a7062"},{"_tag":"Binary","value":"2394e66363584f91913b6893da20d730207b"},{"_tag":"Binary","value":"8987f22b26f15c51ce9b1f9b7e4c496e0b71b3f9f3daded024a4909753fc33aeb81026b118159993fec5ef169fa552050062e1107c1cc66c3f231e174ed113b1852528db9c146f4f154498de4e62aa3b873cd2c3692621ab303472139fb3772c8d061b16027da680531feba7a53823e997158253a2f55ce142c68886fc63fb61"},{"_tag":"Binary","value":"1c0ae6223e1073"},{"_tag":"Binary","value":"2f2e6b7179657a685fc3a9765f7574"},{"_tag":"Binary","value":"6ff09f8c8d"},{"_tag":"Binary","value":"6f656664656e786d5f6b665f716373666e745f6f69e697a56167796875616669756375777165e697a5c3a97473746674646677c3bc6f5f696ce697a5c3a9622075726c68e697a55f65632f5f6d71c3bc772d6b65c3a96476616b2f666f6e6b62c3bc2f6cf09f8c8d73646cc3bc6c6e75742073725f69c3a96c6c6965686170c3bc6bc3bc71756a6f79f09f8c8d6ec3bc6d7a5f78f09f8c8d2d6b69705f69636f6b61c3a9f09f8c8d69706bf09f8c8d61686a6975746c7a652ec3bc6b726570696164647776f09f8c8d2f676ac3a96c76c3a96b7a78c3bc732d71657579622e616a2d795f756b2d7664c3a96f696b6bc3bc70e697a569"},{"_tag":"Binary","value":"b6c2deb54ab0f7525c1089369f"},{"_tag":"Binary","value":"08bedfb2fbfb9a663883120eee1cadb1fcf5fd37"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"d7b323b94015f16fa7349da58dcddfe6"},{"_tag":"Binary","value":"c5b9bf96d8e4"}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":183.0},{"_tag":"Double","value":741680.8728109484},{"_tag":"Double","value":523.0},{"_tag":"Double","value":718.0},{"_tag":"Double","value":1.1639842560684043e+100},{"_tag":"Double","value":-944.0},{"_tag":"Double","value":-879.0},{"_tag":"Double","value":-940757.4833468479},{"_tag":"Double","value":-331109.59436805756},{"_tag":"Double","value":0.0},{"_tag":"Double","value":1.784653308947836e-283},{"_tag":"Double","value":371.0},{"_tag":"Double","value":584.0},{"_tag":"Double","value":-953.0},{"_tag":"Double","value":-345.0}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Set","elements":[{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-46"},{"_tag":"I64","value":"-1"}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":32767}]},{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-3},{"_tag":"I8","value":2}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-857},{"_tag":"I16","value":-17}]},{"_tag":"Set","elementType":"Binary","elements":[]},{"_tag":"Set","elementType":"I16","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-551.0},{"_tag":"Double","value":578926.1480884259}]},{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"c01f0c"}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-26}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":686070.1130041745},{"_tag":"Double","value":-109894.80767404602}]},{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":""}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-3"},{"_tag":"I64","value":"-36"}]},{"_tag":"Set","elementType":"I16","elements":[]},{"_tag":"Set","elementType":"Binary","elements":[]},{"_tag":"Set","elementType":"I16","elements":[]}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":113.0},{"_tag":"Double","value":-416190.58147995116},{"_tag":"Double","value":899.0},{"_tag":"Double","value":-314479.9263538291},{"_tag":"Double","value":884.0},{"_tag":"Double","value":-984.0},{"_tag":"Double","value":-241865.88807397196},{"_tag":"Double","value":1.4187109754053166e+51},{"_tag":"Double","value":-503.0},{"_tag":"Double","value":-359334.19595420663},{"_tag":"Double","value":-906168.1253700852},{"_tag":"Double","value":-708.0},{"_tag":"Double","value":-7.834221780224412e-13},{"_tag":"Double","value":-127411.07597689715},{"_tag":"Double","value":6.1373481858812804e+85},{"_tag":"Double","value":15.0}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Bool","value":false}}]},{"_tag":"Struct","fields":[{"id":207,"value":{"_tag":"Bool","value":false}},{"id":358,"value":{"_tag":"I16","value":12}}]},{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"Binary","value":"b8855756a46b45"}}]},{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Uuid","value":"c0d2296892cd9712e76cc56fe3514803"}},{"id":18,"value":{"_tag":"I32","value":-209}},{"id":25,"value":{"_tag":"I16","value":22}}]},{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"I64","value":"604255"}},{"id":19,"value":{"_tag":"Double","value":-287261.3529680987}},{"id":33,"value":{"_tag":"Binary","value":"79f09f8c8d71"}}]}]}],[{"_tag":"Bool","value":true},{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":-61},{"_tag":"I8","value":-86},{"_tag":"I8","value":-50},{"_tag":"I8","value":9},{"_tag":"I8","value":19},{"_tag":"I8","value":0},{"_tag":"I8","value":-3},{"_tag":"I8","value":-27},{"_tag":"I8","value":-1},{"_tag":"I8","value":127},{"_tag":"I8","value":0},{"_tag":"I8","value":-49},{"_tag":"I8","value":-1},{"_tag":"I8","value":127},{"_tag":"I8","value":10},{"_tag":"I8","value":28}]}]]}},{"id":32368,"value":{"_tag":"Double","value":2.652226819568778e+178}},{"id":32383,"value":{"_tag":"I16","value":61}},{"id":32393,"value":{"_tag":"I8","value":20}},{"id":0,"value":{"_tag":"I16","value":37}},{"id":2,"value":{"_tag":"I64","value":"1071790"}}]},"bytes":"0abcf903310202015b101901fd10370c8d9deb32711bac31aa0609adce68b6f2e48ee09c54eb055c07f43a699fccd55df967d24216bbe6b448c47f305e915dd0bcb222d8c478da1ccea87577f77fa2819b5cb65533434599167a2bc8114269067a92882c4c16942c5755409a1799a0f8c6875089927802f265633f2a89b8107d99e75888a328f906d546525357dc9dede288fdeee552b132ba62d8adcd06761b66fadc44420f9df7374d5d67ef50ed781752249f206e9e07ccae05f73c47da60de82419f932a42b770358234d76f62cb1e11f7aa8ab4281a633ca0b9690525c3cbb78baca7b538cf5717b83fe5efa1427e7ca1943c629702d77e3c550295aae613e8e7129fc9594b93ec68df8a98010d020d0255d2c3c6a601cdb09a067609bf1c0221020101549af5028401ba03c6027d01f510cc02d3c7b0401a8fe28220c0cc84cc07c2f17be9aac701039cddd4bb0300a5fbee0500762391da05a1b3aa47021ded2689478b24a8ae235cf8ef1c939c3d01f81012fd9f6a98f4d16373a2b75d621038b4e39b1f0018e697a52d2e6a685f6a66642f616d7561e697a5c3a961736c08626a7a71692d726400010009c7748469840ac4b0dd000922cc9b54cc37209fe9016f08c3bc736b6a6e7367152dc3bc76f09f8c8d7472746b6c6e77752f76736f6f106c76716a65c3bcc3a96579f09f8c8d2d0622b66903645b116a2d666a646474e697a5617af09f8c8d730d56d33edcedc087bd8f7ca3cf5e0123d28002f80f012d0375716e186364642f5f76696f6e712de697a520e697a56873796661760c686e796d656c68756b6a7062122394e66363584f91913b6893da20d730207b80018987f22b26f15c51ce9b1f9b7e4c496e0b71b3f9f3daded024a4909753fc33aeb81026b118159993fec5ef169fa552050062e1107c1cc66c3f231e174ed113b1852528db9c146f4f154498de4e62aa3b873cd2c3692621ab303472139fb3772c8d061b16027da680531feba7a53823e997158253a2f55ce142c68886fc63fb61071c0ae6223e10730f2f2e6b7179657a685fc3a9765f7574056ff09f8c8df6016f656664656e786d5f6b665f716373666e745f6f69e697a56167796875616669756375777165e697a5c3a97473746674646677c3bc6f5f696ce697a5c3a9622075726c68e697a55f65632f5f6d71c3bc772d6b65c3a96476616b2f666f6e6b62c3bc2f6cf09f8c8d73646cc3bc6c6e75742073725f69c3a96c6c6965686170c3bc6bc3bc71756a6f79f09f8c8d6ec3bc6d7a5f78f09f8c8d2d6b69705f69636f6b61c3a9f09f8c8d69706bf09f8c8d61686a6975746c7a652ec3bc6b726570696164647776f09f8c8d2f676ac3a96c76c3a96b7a78c3bc732d71657579622e616a2d795f756b2d7664c3a96f696b6bc3bc70e697a5690db6c2deb54ab0f7525c1089369f1408bedfb2fbfb9a663883120eee1cadb1fcf5fd370010d7b323b94015f16fa7349da58dcddfe606c5b9bf96d8e401f70f0000000000e066409e13e1be61a226410000000000588040000000000070864024cc12c76549b5540000000000808dc00000000000788bc0f13c79f7aab52cc12405a260963514c10000000000000000125d91b0c0893a05000000000030774000000000004082400000000000c88dc000000000009075c001fa0f265b0114feff0323fd0224b10d2108042700000000003881c0043fd24bdcaa21411803c01f0c143327e4aedb39ecef2441d89e3bec6cd4fac0180026054704080401f7100000000000405c40f67a6f53fa6619c10000000000188c40221996b4bf3113c10000000000a08b400000000000c08ec0d086c61a4f860dc18a22db82c5558e4a0000000000707fc03438a8c898ee15c10082304070a72bc100000000002086c0d8c16d8d70906bbd088d3337311bffc01e4f2576a497bf510000000000002e40016c002200029e0304cc051800080007b8855756a46b45004dc0d2296892cd9712e76cc56fe3514803e5a103742c00a6bee1499722787069758811c1e80679f09f8c8d710001f310c3aace091300fde5ff7f00cfff7f0a1cd7d8562d95222efa64f47aa31404004a26dcea820100"},{"id":"random/150","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-9713,"value":{"_tag":"Binary","value":"cff68295458a0c2d500545f92374a1530cefb23b2557fbfd756c831575bc7bd435d008cfc8b593a477232a25c2649247e5fe721cf583085fd87b55682b709140c07c591c73b318573729a5e78344a748c0e584aac3f495830863c55d5153643c8098491dfc8ba8cb28ebf4390e789f4e2012a2721eada70a78551ec8d301c900f106789f5c3230479ef4ce61340cbe3bb6c5beb4a99127d77029bcf1ea7feb0a3ee54fd77543be5a7d70903dfc4eb96f9783e020104e57d41597bbaee6dd504a3a7772276ed681ba"}},{"id":-3,"value":{"_tag":"Binary","value":"f396eb99b41a8ea6fcccbdbcbf"}},{"id":0,"value":{"_tag":"I64","value":"-352951611"}},{"id":9,"value":{"_tag":"Bool","value":true}},{"id":98,"value":{"_tag":"Bool","value":true}}]},"bytes":"08e19701c801cff68295458a0c2d500545f92374a1530cefb23b2557fbfd756c831575bc7bd435d008cfc8b593a477232a25c2649247e5fe721cf583085fd87b55682b709140c07c591c73b318573729a5e78344a748c0e584aac3f495830863c55d5153643c8098491dfc8ba8cb28ebf4390e789f4e2012a2721eada70a78551ec8d301c900f106789f5c3230479ef4ce61340cbe3bb6c5beb4a99127d77029bcf1ea7feb0a3ee54fd77543be5a7d70903dfc4eb96f9783e020104e57d41597bbaee6dd504a3a7772276ed681ba08050df396eb99b41a8ea6fcccbdbcbf36f5f4ccd0029101c40100"},{"id":"random/151","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Bool","value":false}},{"id":169,"value":{"_tag":"I8","value":10}},{"id":176,"value":{"_tag":"I8","value":-26}},{"id":2,"value":{"_tag":"Uuid","value":"f7b9d819360c07285de77c7597dbe826"}}]},"bytes":"b203d2020a73e60d04f7b9d819360c07285de77c7597dbe82600"},{"id":"random/152","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I32","value":2147483647}},{"id":5,"value":{"_tag":"I32","value":-2147483648}}]},"bytes":"25feffffff0f35ffffffff0f00"},{"id":"random/153","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Binary","value":"657677785f64645f676f6e2f"}},{"id":15,"value":{"_tag":"Double","value":-7.135360059564528e+267}},{"id":18,"value":{"_tag":"I32","value":-25926402}},{"id":33,"value":{"_tag":"Binary","value":"1f8d3f56"}},{"id":31844,"value":{"_tag":"Double","value":-474.0}}]},"bytes":"380c657677785f64645f676f6e2fc7a795ad083fa98bf73583ecdc18f8041f8d3f5607c8f1030000000000a07dc000"},{"id":"random/154","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Bool","value":false}},{"id":225,"value":{"_tag":"Double","value":630372.9093135498}}]},"bytes":"2207c203ac8b91d1c93c234100"},{"id":"random/155","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Bool","value":true}},{"id":14,"value":{"_tag":"Double","value":1.530766506695314e+198}},{"id":16,"value":{"_tag":"Double","value":-972.0}},{"id":27,"value":{"_tag":"I16","value":-70}},{"id":1,"value":{"_tag":"Uuid","value":"2eb3df3493ba278b9e448d7b15950740"}},{"id":9,"value":{"_tag":"Binary","value":"6f7264f09f8c8d73e697a56a6367"}},{"id":17,"value":{"_tag":"Binary","value":"85"}}]},"bytes":"915739d311566b7a1469270000000000608ec0b48b010d022eb3df3493ba278b9e448d7b15950740880e6f7264f09f8c8d73e697a56a636788018500"},{"id":"random/156","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"Uuid","value":"7a54dd1d13ba5f38f3897cd70ff60497"}},{"id":10,"value":{"_tag":"Set","elementType":"Set","elements":[{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":497},{"_tag":"I32","value":-167},{"_tag":"I32","value":1083566070},{"_tag":"I32","value":23},{"_tag":"I32","value":-1830},{"_tag":"I32","value":-52},{"_tag":"I32","value":-13061},{"_tag":"I32","value":175051576},{"_tag":"I32","value":-45},{"_tag":"I32","value":13815955},{"_tag":"I32","value":-1},{"_tag":"I32","value":-232831946},{"_tag":"I32","value":8080},{"_tag":"I32","value":0}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-29"},{"_tag":"I64","value":"-29"}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":828.0},{"_tag":"Double","value":-102.0},{"_tag":"Double","value":-2.514572695616334e-150},{"_tag":"Double","value":4.720848338143313e-67},{"_tag":"Double","value":-98271.08273001644},{"_tag":"Double","value":122458.93486267305},{"_tag":"Double","value":4.678281229773815e-130},{"_tag":"Double","value":939709.6939371116},{"_tag":"Double","value":293872.5502803142},{"_tag":"Double","value":-104522.2889861872},{"_tag":"Double","value":-410570.1812972933},{"_tag":"Double","value":1.656194072607686e-232},{"_tag":"Double","value":-440.0},{"_tag":"Double","value":703.0},{"_tag":"Double","value":-900985.8576532446}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-3284"},{"_tag":"I64","value":"328998821"}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-1},{"_tag":"I16","value":-639},{"_tag":"I16","value":-4},{"_tag":"I16","value":-1},{"_tag":"I16","value":-64},{"_tag":"I16","value":365},{"_tag":"I16","value":157},{"_tag":"I16","value":19},{"_tag":"I16","value":-8349},{"_tag":"I16","value":-27},{"_tag":"I16","value":52},{"_tag":"I16","value":1},{"_tag":"I16","value":-28},{"_tag":"I16","value":-105},{"_tag":"I16","value":-471}]},{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false}]},{"_tag":"Set","elementType":"Set","elements":[{"_tag":"Set","elementType":"I16","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":0.0}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-264480522668711872"}]},{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"79637866687a5f6b2e64c3bc20772e77676b"},{"_tag":"Binary","value":""}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-31}]},{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"4502a26bdd22ceceb7d37d67aa828e9c"}]},{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"a507254a89499c2f0f5223368f6169c9"}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":364.0}]},{"_tag":"Set","elementType":"Bool","elements":[]},{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"1573738e07b226eed8db85d6b241b721"}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":31},{"_tag":"I32","value":212735420}]},{"_tag":"Set","elementType":"Uuid","elements":[]},{"_tag":"Set","elementType":"Uuid","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":"NaN"},{"_tag":"Double","value":990342.597048346}]},{"_tag":"Set","elementType":"I32","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-5.856790828860446e-95},{"_tag":"Double","value":952.0}]},{"_tag":"Set","elementType":"I8","elements":[]},{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":false}]},{"_tag":"Set","elementType":"I64","elements":[]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-23791218"},{"_tag":"I64","value":"-32"}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":"-Infinity"}]},{"_tag":"Set","elementType":"Bool","elements":[]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-4936092710727"},{"_tag":"I64","value":"5"}]},{"_tag":"Set","elementType":"Double","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-806769.4008258417}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-13468},{"_tag":"I16","value":47}]},{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"2d64766f77692d66667469687871766c6575676f6f6372c3bc736a696e7a706420c3a97a6d775f6974f09f8c8df09f8c8d7571787666772d656d6d6973647a646b7a6166767073776d2e66772e6c696e657a706877756e67666566782e6f71767262c3bc2f76757779747920f09f8c8d73c3bc2e5f7776756bf09f8c8d776b707563c3bc71697275e697a5715f676ee697a56d7a"},{"_tag":"Binary","value":"7169686777622d6978"}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-125884.6128305624},{"_tag":"Double","value":-1.8715606978227384e-29}]},{"_tag":"Set","elementType":"Binary","elements":[]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"685147360437969"},{"_tag":"I64","value":"-741852656664147216"}]}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"5939"}]},{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-128},{"_tag":"I8","value":2},{"_tag":"I8","value":-2},{"_tag":"I8","value":50},{"_tag":"I8","value":-22},{"_tag":"I8","value":-64},{"_tag":"I8","value":49},{"_tag":"I8","value":-1},{"_tag":"I8","value":-1},{"_tag":"I8","value":-128},{"_tag":"I8","value":127},{"_tag":"I8","value":-4},{"_tag":"I8","value":-54},{"_tag":"I8","value":-6},{"_tag":"I8","value":57},{"_tag":"I8","value":-55},{"_tag":"I8","value":1},{"_tag":"I8","value":13},{"_tag":"I8","value":-15},{"_tag":"I8","value":4},{"_tag":"I8","value":0},{"_tag":"I8","value":1},{"_tag":"I8","value":0},{"_tag":"I8","value":-5},{"_tag":"I8","value":127},{"_tag":"I8","value":-41},{"_tag":"I8","value":-44},{"_tag":"I8","value":-115},{"_tag":"I8","value":38},{"_tag":"I8","value":-4}]},{"_tag":"Set","elementType":"Double","elements":[]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":1103354}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-347163.6667304501}]},{"_tag":"Set","elementType":"Double","elements":[]},{"_tag":"Set","elementType":"Double","elements":[]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-1992546868536719"},{"_tag":"I64","value":"-28"},{"_tag":"I64","value":"-4290877311871"},{"_tag":"I64","value":"-29"},{"_tag":"I64","value":"-24"}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"438668725589094"},{"_tag":"I64","value":"-1135528594"},{"_tag":"I64","value":"-53"},{"_tag":"I64","value":"-57624430482"},{"_tag":"I64","value":"27"},{"_tag":"I64","value":"-14646"},{"_tag":"I64","value":"3594027"},{"_tag":"I64","value":"36"},{"_tag":"I64","value":"-12"},{"_tag":"I64","value":"-97433691581596"},{"_tag":"I64","value":"166149248299216"},{"_tag":"I64","value":"1271286348153"},{"_tag":"I64","value":"3715410760"},{"_tag":"I64","value":"174411"}]},{"_tag":"Set","elementType":"Map","elements":[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"Map","keyType":"Binary","valueType":"Double","entries":[[{"_tag":"Binary","value":"75e52e5a414738f0a16bb12058137c0a"},{"_tag":"Double","value":314.0}],[{"_tag":"Binary","value":"6d6c666ce697a5756d716d6ff09f8c8d746674"},{"_tag":"Double","value":-718516.0100142225}],[{"_tag":"Binary","value":"2ef3ef3810"},{"_tag":"Double","value":95383.43421510188}]]}]},{"_tag":"Set","elementType":"Set","elements":[{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"477db9e3ecdcdea3d38a91bb44efefbe"},{"_tag":"Uuid","value":"b616232a02b2e7eb538f96b145cc6873"}]},{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":true}]},{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-1},{"_tag":"I8","value":3}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-431.0}]},{"_tag":"Set","elementType":"Bool","elements":[]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"46"}]},{"_tag":"Set","elementType":"I16","elements":[]},{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"d3ae6e44f2dfa9af4a58a514f1b15b4c"}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-49}]},{"_tag":"Set","elementType":"Binary","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-877.0}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"35"}]},{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":false}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-260142}]}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":28},{"_tag":"I32","value":-113002},{"_tag":"I32","value":-46},{"_tag":"I32","value":-3450682}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":165.0},{"_tag":"Double","value":-850019.1681439443},{"_tag":"Double","value":-5.502567453469574e-130},{"_tag":"Double","value":836.0},{"_tag":"Double","value":399376.1356118282}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-4.40049044001022e+112},{"_tag":"Double","value":5669.648760814569},{"_tag":"Double","value":531.0},{"_tag":"Double","value":-792570.3652043612},{"_tag":"Double","value":-298424.7264261646},{"_tag":"Double","value":780093.1760323783},{"_tag":"Double","value":-397970.5750144677},{"_tag":"Double","value":464.0},{"_tag":"Double","value":-780246.608121047},{"_tag":"Double","value":-2.8611722136487722e+259},{"_tag":"Double","value":-457125.0237101967},{"_tag":"Double","value":9.310164013054245e-56},{"_tag":"Double","value":435301.5027740041},{"_tag":"Double","value":-9273.39891526138},{"_tag":"Double","value":-68324.15767884243}]},{"_tag":"Set","elementType":"I64","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":939.0},{"_tag":"Double","value":436444.1417126134},{"_tag":"Double","value":-327138.7833760235},{"_tag":"Double","value":350.0},{"_tag":"Double","value":-4.85635151516202e+116},{"_tag":"Double","value":980.0}]},{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-1},{"_tag":"I8","value":1},{"_tag":"I8","value":127},{"_tag":"I8","value":-24},{"_tag":"I8","value":-1}]},{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"-27355694"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"-280017229477638965"},{"_tag":"I64","value":"-50"},{"_tag":"I64","value":"-49"}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-3},{"_tag":"I32","value":-4222},{"_tag":"I32","value":-1221695},{"_tag":"I32","value":-202433},{"_tag":"I32","value":-910796},{"_tag":"I32","value":23}]},{"_tag":"Set","elementType":"Set","elements":[]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"228666359764950"},{"_tag":"I64","value":"-47"},{"_tag":"I64","value":"-21704"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"-24"},{"_tag":"I64","value":"-4"},{"_tag":"I64","value":"29"},{"_tag":"I64","value":"1381855430189927"},{"_tag":"I64","value":"-5"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"989697930131"},{"_tag":"I64","value":"-540864607094515"},{"_tag":"I64","value":"7895327"},{"_tag":"I64","value":"-11980492"}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-224}]}]}}]},"bytes":"0d007a54dd1d13ba5f38f3897cd70ff60497aafa1ee5e207cd02ec9faf89082ecb1c6789cc01f0ccf8a60159a6c2960d0193ef85de01a07e00263939f70f0000000000e0894000000000008059c0fe0f1d0c5d76e0a0ab39e9676b742932b0b5dc52f1fdf7c0f08f32f5aee5fd40f29766620ec11425a0b94b637bad2c41c4ae7c33c2ef1141f0faaf9fa484f9c066ffa5b9280f19c13cb57e55c574d00f0000000000807bc00000000000f885407a531eb7f37e2bc126a733cafee0b902f40f01fd0907017fda05ba0226b9820135680237d101ad07510102020102fa1e0417000000000000000016fffeecbacae8cfab07281279637866687a5f6b2e64c3bc20772e77676b00143d1d4502a26bdd22ceceb7d37d67aa828e9c1da507254a89499c2f0f5223368f6169c9170000000000c07640011d1573738e07b226eed8db85d6b241b721253ef8d6f0ca010d0d27000000000000f87f2052b0310d392e410527075999326b465fac0000000000c08d400311020626e399d8163f17000000000000f0ff01268dddebe0a89f020a0717730b39cde29e28c124b7d2015e2894012d64766f77692d66667469687871766c6575676f6f6372c3bc736a696e7a706420c3a97a6d775f6974f09f8c8df09f8c8d7571787666772d656d6d6973647a646b7a6166767073776d2e66772e6c696e657a706877756e67666566782e6f71767262c3bc2f76757779747920f09f8c8d73c3bc2e5f7776756bf09f8c8d776b707563c3bc71697275e697a5715f676ee697a56d7a097169686777622d697827786b27cec9bbfec0a99c0ecc8fb9f7b90826a29bcdd0e9c8b7029fd4abc2f1c3cbcb1416e65cf31e8002fe32eac031ffff807ffccafa39c9010df104000100fb7fd7d48d26fc0715f4d78601171a63bbaa6e3015c10707569d96e184bf8d8a0737fdadbec3e1f901392fe6ccd180d9f0bdc701a3aaf6ba0869a39ef8aaad0336ebe401d6dcb6034817b7b29bc9b1a72ca0c3e5ae94c74bf2e591e9ff4990ada5d71b96a5152b0003871075e52e5a414738f0a16bb12058137c0a0000000000a07340136d6c666ce697a5756d716d6ff09f8c8d7466748c95200568ed25c1052ef3ef3810e0888bf27649f740ea2d477db9e3ecdcdea3d38a91bb44efefbeb616232a02b2e7eb538f96b145cc687321010123ff03170000000000f07ac001165c041dd3ae6e44f2dfa9af4a58a514f1b15b4c156108170000000000688bc01646110215dbe01f4538d3e50d5bf39ca503570000000000a064408cf61656c6f029c131cbaea42f6918a50000000000208a40bcd3dd8a40601841f70f7d17f623474c52d780513015a625b6400000000000988040e710fcbaf42f28c1b042dce7e23612c178ea205a7ace2741b697d04c4a4a18c10000000000007d4052a45b37adcf27c14f078e3f27c6cdf55e7c471894e61bc169fd418a444382344430d70296911a41c0c0a70fb31cc2c0f83fda8542aef0c006670000000000588d40801c1d9170a31a4106532d228bf713c10000000000e075406713697278a628d80000000000a08e4053ff017fe8fff11e01010201020102010102010101020201010202010102010201020101020156dba88b1a01e9cc87e0f88be9e20763616505fb41fd90950181db1897976f2e0ae6acdfaa9291fe675d8fd302012f073ace8de1a0c9b2f4040901a6cebfe9cd39e59bb8acbbfaf501bee4c30797bbb60b15bf0300"},{"id":"random/157","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Struct","fields":[{"id":-26702,"value":{"_tag":"I64","value":"-5493957"}},{"id":-26574,"value":{"_tag":"I8","value":1}},{"id":-26562,"value":{"_tag":"I64","value":"1342"}},{"id":-26560,"value":{"_tag":"Double","value":388.0}},{"id":24699,"value":{"_tag":"I8","value":0}}]}},{"id":200,"value":{"_tag":"Double","value":-3.8447456544675803e+33}},{"id":208,"value":{"_tag":"Binary","value":"beefe9"}}]},"bytes":"bc069ba10389d39e05039b9f0301c6fc1427000000000040784003f681030000079003b5563267f0b1e7c68803beefe900"},{"id":"random/158","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-3,"value":{"_tag":"Bool","value":false}},{"id":-2,"value":{"_tag":"I8","value":7}}]},"bytes":"0205130700"},{"id":"random/159","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I32","value":-25}},{"id":3,"value":{"_tag":"I32","value":0}},{"id":15,"value":{"_tag":"Map","keyType":"I8","valueType":"I8","entries":[[{"_tag":"I8","value":-1},{"_tag":"I8","value":-19}],[{"_tag":"I8","value":-41},{"_tag":"I8","value":38}],[{"_tag":"I8","value":-32},{"_tag":"I8","value":-58}],[{"_tag":"I8","value":-3},{"_tag":"I8","value":7}]]}}]},"bytes":"25311500cb0433ffedd726e0c6fd0700"},{"id":"random/160","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":59}},{"id":13,"value":{"_tag":"I32","value":-723}},{"id":288,"value":{"_tag":"I32","value":3}},{"id":289,"value":{"_tag":"Uuid","value":"c250a608391e211a70eceb14cbe29d2e"}}]},"bytes":"3476a5a50b05c004061dc250a608391e211a70eceb14cbe29d2e00"},{"id":"random/161","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I16","value":23}},{"id":2,"value":{"_tag":"I16","value":89}},{"id":14,"value":{"_tag":"Double","value":-715995.9899583382}},{"id":18,"value":{"_tag":"Bool","value":false}},{"id":19,"value":{"_tag":"I8","value":-6}}]},"bytes":"942e0404b201c7bed1dbfab7d925c14213fa00"},{"id":"random/162","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Uuid","value":"db391460979e5993e742f9cc9ab1ea48"}}]},"bytes":"7ddb391460979e5993e742f9cc9ab1ea4800"},{"id":"random/163","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Binary","value":"8bb217c999ff3bdf0e0588b2d0458948949877"}},{"id":9,"value":{"_tag":"I64","value":"-54"}},{"id":15,"value":{"_tag":"Double","value":0.0}},{"id":23,"value":{"_tag":"Double","value":"NaN"}}]},"bytes":"58138bb217c999ff3bdf0e0588b2d0458948949877466b67000000000000000087000000000000f87f00"},{"id":"random/164","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I8","value":59}},{"id":17,"value":{"_tag":"I8","value":8}}]},"bytes":"833b930800"},{"id":"random/165","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Bool","value":true}},{"id":257,"value":{"_tag":"I64","value":"-5"}}]},"bytes":"c10682040900"},{"id":"random/166","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Binary","value":"c1bf1ea803b0e0b18dae4bf1520566f7d8"}},{"id":20,"value":{"_tag":"I32","value":219}},{"id":31,"value":{"_tag":"I16","value":32}},{"id":32,"value":{"_tag":"I16","value":305}},{"id":42,"value":{"_tag":"Binary","value":"657668727967c3a9c3a9766563672e61645f7a662e682e5f7562717a6e7073c3a97a785f756c73616a2d6df09f8c8d616266c3a9796b6a777469727575767262c3a92e2d7477c3bc65c3a9746762717a2069c3a97073772e6b7a7af09f8c8d7972e697a5666f61666b696674677577e697a56b7a7420786f7a662065e697a57a6af09f8c8d6c616561c3a969636de697a52f70666c2d70686866"}},{"id":241,"value":{"_tag":"Binary","value":""}},{"id":244,"value":{"_tag":"Uuid","value":"1c4d0c99067035753a167da9ae36f0aa"}},{"id":249,"value":{"_tag":"Bool","value":false}}]},"bytes":"9811c1bf1ea803b0e0b18dae4bf1520566f7d8b5b603b44014e204a89a01657668727967c3a9c3a9766563672e61645f7a662e682e5f7562717a6e7073c3a97a785f756c73616a2d6df09f8c8d616266c3a9796b6a777469727575767262c3a92e2d7477c3bc65c3a9746762717a2069c3a97073772e6b7a7af09f8c8d7972e697a5666f61666b696674677577e697a56b7a7420786f7a662065e697a57a6af09f8c8d6c616561c3a969636de697a52f70666c2d7068686608e203003d1c4d0c99067035753a167da9ae36f0aa5200"},{"id":"random/167","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"I16","value":-7}},{"id":26,"value":{"_tag":"Uuid","value":"a91860dda5e422edf03462a18eb423f2"}},{"id":5,"value":{"_tag":"Uuid","value":"6be942d2997b3c960db338ac9277db2d"}},{"id":-1,"value":{"_tag":"I32","value":-58}},{"id":281,"value":{"_tag":"I32","value":-2}},{"id":28844,"value":{"_tag":"Bool","value":true}},{"id":28865,"value":{"_tag":"Bool","value":false}},{"id":-21292,"value":{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":2},{"_tag":"I8","value":10},{"_tag":"I8","value":25},{"_tag":"I8","value":-7},{"_tag":"I8","value":-37}]}}]},"bytes":"b40dfda91860dda5e422edf03462a18eb423f20d0a6be942d2997b3c960db338ac9277db2d05017305b2040301d8c2030282c30309d7cc0253020a19f9db00"},{"id":"random/168","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-3,"value":{"_tag":"Double","value":8.60303117398961e-292}},{"id":6,"value":{"_tag":"Uuid","value":"e3910deb3adc583d8897e522c767d6b4"}},{"id":11,"value":{"_tag":"Double","value":-339838.3133877105}},{"id":247,"value":{"_tag":"Uuid","value":"f783fa46520f4418e2435f9ffdce3371"}},{"id":249,"value":{"_tag":"Bool","value":false}}]},"bytes":"0705bb92f63b972b81039de3910deb3adc583d8897e522c767d6b4573eb5e840f9bd14c10dee03f783fa46520f4418e2435f9ffdce33712200"},{"id":"random/169","type":"Struct","value":{"_tag":"Struct","fields":[{"id":242,"value":{"_tag":"Double","value":-361628.054738467}},{"id":255,"value":{"_tag":"I32","value":12887140}},{"id":9734,"value":{"_tag":"I32","value":-5}},{"id":10006,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I32","value":5083692}},{"id":5,"value":{"_tag":"I64","value":"5"}},{"id":132,"value":{"_tag":"Uuid","value":"4d092980020b71e24554427034f47d6c"}},{"id":218,"value":{"_tag":"I16","value":0}},{"id":374,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"Binary","value":"7278f887b0793025254b32"}},{"id":121,"value":{"_tag":"I64","value":"-1183995912178623"}},{"id":129,"value":{"_tag":"Uuid","value":"5f652f9a44357fdd1781fee7bb7ceda8"}}]},{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"Double","value":"Infinity"}},{"id":29,"value":{"_tag":"Binary","value":"7974e697a57871"}}]},{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I16","value":-2}},{"id":5,"value":{"_tag":"Binary","value":"7366f09f8c8d7269616e6c7462616373f09f8c8d676d76657a71722ef09f8c8d796877696b2064206c2e64695f657968c3bc7ae697a570c3a97573646b70716120202f74717778676d72756b2dc3bc6920686fc3bc612e6ee697a55f5f6467c3a92f706d787a65662079206f756cc3bc2e2f706c6b79f09f8c8d6a6664637871686470737168c3a9e697a5675f737174e697a5706e6770736dc3bc"}}]},{"_tag":"Struct","fields":[]},{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Double","value":-179932.47060910007}},{"id":13487,"value":{"_tag":"Uuid","value":"b736e43f0b979a212fa2d66698e193f7"}},{"id":13499,"value":{"_tag":"Binary","value":"6d7765f09f8c8d726a71"}}]}]}},{"id":376,"value":{"_tag":"Bool","value":false}},{"id":387,"value":{"_tag":"Double","value":"-Infinity"}}]}},{"id":10021,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":10035,"value":{"_tag":"Binary","value":"09bf20f3f2d9a5c8c7562e0a952edd0bdc6438"}}]},"bytes":"07e403565c0d38701216c1d5c891a50c058c9801090cac9c0135d8c8ec04260a0d88024d092980020b71e24554427034f47d6c04b4030009ec055c880b7278f887b0793025254b3206f201fdae96fcd0b59a048d5f652f9a44357fdd1781fee7bb7ceda800e7000000000000f07ff8077974e697a57871008403080a9b017366f09f8c8d7269616e6c7462616373f09f8c8d676d76657a71722ef09f8c8d796877696b2064206c2e64695f657968c3bc7ae697a570c3a97573646b70716120202f74717778676d72756b2dc3bc6920686fc3bc612e6ee697a55f5f6467c3a92f706d787a65662079206f756cc3bc2e2f706c6b79f09f8c8d6a6664637871686470737168c3a9e697a5675f737174e697a5706e6770736dc3bc00009730b4cec3e3f605c10dded201b736e43f0b979a212fa2d66698e193f7c80a6d7765f09f8c8d726a710022b7000000000000f0ff00fb00e81309bf20f3f2d9a5c8c7562e0a952edd0bdc643800"},{"id":"random/170","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Bool","value":false}},{"id":19,"value":{"_tag":"Uuid","value":"21d4086343464453fa3d8d1fe9209015"}},{"id":33,"value":{"_tag":"Binary","value":"02ac26"}},{"id":36,"value":{"_tag":"I16","value":-118}},{"id":215,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false}]}},{"id":-18983,"value":{"_tag":"I32","value":-53}},{"id":-18976,"value":{"_tag":"Uuid","value":"f768ad2ef47cea0425aca641f7eefa09"}},{"id":-18965,"value":{"_tag":"Uuid","value":"8aa3d656cdda78f46e60c1d60f49bc56"}}]},"bytes":"92ad21d4086343464453fa3d8d1fe9209015e80302ac2634eb0109ae036101020101020205cda802697df768ad2ef47cea0425aca641f7eefa09bd8aa3d656cdda78f46e60c1d60f49bc5600"},{"id":"random/171","type":"Struct","value":{"_tag":"Struct","fields":[{"id":45,"value":{"_tag":"Double","value":-4.004842129001973e-275}},{"id":55,"value":{"_tag":"Double","value":-203325.62038598815}},{"id":59,"value":{"_tag":"I64","value":"-40"}},{"id":73,"value":{"_tag":"I16","value":19}},{"id":283,"value":{"_tag":"Uuid","value":"5de40b04c58aeb62da411c98a7928498"}},{"id":478,"value":{"_tag":"Map","keyType":"I16","valueType":"I16","entries":[[{"_tag":"I16","value":10},{"_tag":"I16","value":-3}],[{"_tag":"I16","value":-33},{"_tag":"I16","value":713}],[{"_tag":"I16","value":-11784},{"_tag":"I16","value":0}],[{"_tag":"I16","value":0},{"_tag":"I16","value":-111}],[{"_tag":"I16","value":47},{"_tag":"I16","value":-32768}]]}},{"id":485,"value":{"_tag":"Bool","value":true}}]},"bytes":"075af5c299d7622ff686a7d0ed8cf6ecd108c1464fe4260db6045de40b04c58aeb62da411c98a79284980bbc070544140541920b8fb8010000dd015effff037100"},{"id":"random/172","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"17"}},{"id":-2,"value":{"_tag":"Uuid","value":"3f70a95ad894d4f1886f73da5c55a094"}}]},"bytes":"36220d033f70a95ad894d4f1886f73da5c55a09400"},{"id":"random/173","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/174","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"Double","value":-968590.5298175063}},{"id":-21521,"value":{"_tag":"I8","value":-42}},{"id":-21508,"value":{"_tag":"Binary","value":"5f6465e697a57a626c6c796a742ff09f8c8d797976746f70f09f8c8dc3bc6671e697a5666c67c3bc77e697a56c7a6a2e6a6a6c6764632d6b646a206f64c3a978732e2ef09f8c8d775f6573652ef09f8c8d696f206c672d76736d696165612e616d6e2e64f09f8c8d686e646d7671736e70c3a9657475206b696462f09f8c8d2f727a6f2d676e746e6d20765f646f7169c3a97370202f736e7467736a75"}},{"id":-21503,"value":{"_tag":"Binary","value":"6e5f746b796de697a567f09f8c8d7572"}},{"id":-21502,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"0348ac4a52b965f5e3de6b2b68cbb292"},{"_tag":"Uuid","value":"0327f4b088a5ce392bac058eb119c4a2"}]}},{"id":1,"value":{"_tag":"I64","value":"77927319108786234"}}]},"bytes":"877d3d440f1d8f2dc103a1d002d6d89d015f6465e697a57a626c6c796a742ff09f8c8d797976746f70f09f8c8dc3bc6671e697a5666c67c3bc77e697a56c7a6a2e6a6a6c6764632d6b646a206f64c3a978732e2ef09f8c8d775f6573652ef09f8c8d696f206c672d76736d696165612e616d6e2e64f09f8c8d686e646d7671736e70c3a9657475206b696462f09f8c8d2f727a6f2d676e746e6d20765f646f7169c3a97370202f736e7467736a7558106e5f746b796de697a567f09f8c8d7572192d0348ac4a52b965f5e3de6b2b68cbb2920327f4b088a5ce392bac058eb119c4a20602f4e0b8dcfb9eed940200"},{"id":"random/175","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Double","value":"-Infinity"}},{"id":292,"value":{"_tag":"Uuid","value":"66a8392e84fc71f6087d629379737daa"}},{"id":385,"value":{"_tag":"Double","value":920933.9476508114}},{"id":390,"value":{"_tag":"I32","value":-11}},{"id":395,"value":{"_tag":"I32","value":75}},{"id":398,"value":{"_tag":"Uuid","value":"d8c75b2d78852bc616fd477329735084"}},{"id":-13980,"value":{"_tag":"Double","value":"Infinity"}},{"id":-13967,"value":{"_tag":"I8","value":-47}}]},"bytes":"67000000000000f0ff0dc80466a8392e84fc71f6087d629379737daa078206b67c32e5cb1a2c4155155596013dd8c75b2d78852bc616fd47732973508407b7da01000000000000f07fd3d100"},{"id":"random/176","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":7}},{"id":279,"value":{"_tag":"I64","value":"701087283"}}]},"bytes":"130706ae04e6f8cd9c0500"},{"id":"random/177","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Double","value":14043.414786439622}}]},"bytes":"b780d8b817b56dcb4000"},{"id":"random/178","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"85398c392bdcf6ace2c6673191fe89731f08"}},{"id":9,"value":{"_tag":"Bool","value":false}},{"id":18142,"value":{"_tag":"Binary","value":""}},{"id":18148,"value":{"_tag":"Binary","value":"697971c3a9662e5f6a6e2f"}},{"id":18276,"value":{"_tag":"I16","value":1720}},{"id":18282,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"28"},{"_tag":"Binary","value":"1ad534780d5ade1f16c871f33af5"},{"_tag":"Binary","value":"5c7551fd3d4d51b01bf92048440f"},{"_tag":"Binary","value":"6182786ff2"},{"_tag":"Binary","value":"777966e697a5"},{"_tag":"Binary","value":"64746578f09f8c8d757065"},{"_tag":"Binary","value":"059a"},{"_tag":"Binary","value":"70"},{"_tag":"Binary","value":"5f612d"},{"_tag":"Binary","value":"a711d995612c9498b17eb53e470a238ea3308b"},{"_tag":"Binary","value":"2077662f67"},{"_tag":"Binary","value":"8b75b35dc1dd"},{"_tag":"Binary","value":"c73e4105ae5cbd32393950ec4e8d09125d9a"},{"_tag":"Binary","value":"6f696f7669f09f8c8d6679c3bcf09f8c8d"},{"_tag":"Binary","value":"71c3a96c716d6f76e697a5622e786261767278742f2f7962682e636f6262656a6a73712e66766e6d67616f6178697678636e775fe697a56a79666975776d2e626170655ff09f8c8d665f7772c3a96562e697a56672686b6f7a7070666272647172612063f09f8c8d62642d7179c3a9786e2f74616974677967635f6668e697a56163686e66782f73746a206261676bc3bc"}]},{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}]}}]},"bytes":"181285398c392bdcf6ace2c6673191fe89731f088208bc9b0200680b697971c3a9662e5f6a6e2f04c89d02f01a6929f80f01280e1ad534780d5ade1f16c871f33af50e5c7551fd3d4d51b01bf92048440f056182786ff206777966e697a50b64746578f09f8c8d75706502059a0170035f612d13a711d995612c9498b17eb53e470a238ea3308b052077662f67068b75b35dc1dd12c73e4105ae5cbd32393950ec4e8d09125d9a116f696f7669f09f8c8d6679c3bcf09f8c8d910171c3a96c716d6f76e697a5622e786261767278742f2f7962682e636f6262656a6a73712e66766e6d67616f6178697678636e775fe697a56a79666975776d2e626170655ff09f8c8d665f7772c3a96562e697a56672686b6f7a7070666272647172612063f09f8c8d62642d7179c3a9786e2f74616974677967635f6668e697a56163686e66782f73746a206261676bc3bc51020202020100"},{"id":"random/179","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I32","value":-63}},{"id":-3,"value":{"_tag":"I32","value":2386}}]},"bytes":"957d0505a42500"},{"id":"random/180","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7388,"value":{"_tag":"I64","value":"82073287428689840"}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"Uuid","value":"1437f20ce0cd250da93ec921d2931d89"}},{"id":18,"value":{"_tag":"Binary","value":"657a"}},{"id":21,"value":{"_tag":"Double","value":-758453.9758385094}},{"id":196,"value":{"_tag":"Bool","value":false}}]},"bytes":"06b873e0bee2eb8acecaa3020504003d1437f20ce0cd250da93ec921d2931d89d802657a37e81aa1f36b2527c102880300"},{"id":"random/181","type":"Struct","value":{"_tag":"Struct","fields":[{"id":211,"value":{"_tag":"Uuid","value":"73e95304d39b6915b0f7d10dee0c4f7c"}},{"id":219,"value":{"_tag":"I64","value":"46"}},{"id":370,"value":{"_tag":"Bool","value":true}},{"id":-4,"value":{"_tag":"Binary","value":"841268"}},{"id":4,"value":{"_tag":"I8","value":21}},{"id":14,"value":{"_tag":"I8","value":-56}},{"id":144,"value":{"_tag":"I64","value":"-6"}},{"id":297,"value":{"_tag":"I8","value":1}}]},"bytes":"0da60373e95304d39b6915b0f7d10dee0c4f7c865c01e4050807038412688315a3c806a0020b03d2040100"},{"id":"random/182","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I8","value":-6}}]},"bytes":"33fa00"},{"id":"random/183","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/184","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Uuid","value":"570da9d66756dc21a09f284703a4c1e0"}},{"id":24,"value":{"_tag":"I16","value":-2}},{"id":26,"value":{"_tag":"I16","value":386}},{"id":29,"value":{"_tag":"Bool","value":false}}]}},{"id":6,"value":{"_tag":"Double","value":-353473.6495467725}},{"id":8,"value":{"_tag":"Binary","value":"5f766df09f8c8d67796972"}},{"id":-1,"value":{"_tag":"Double","value":669134.8062210064}},{"id":13,"value":{"_tag":"Double","value":904.0}}]},"bytes":"1ccd570da9d66756dc21a09f284703a4c1e0c40324840632005704ca2299069315c1280b5f766df09f8c8d677969720701f0ffc89c9d6b2441e70000000000408c4000"},{"id":"random/185","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Map","keyType":"I16","valueType":"Bool","entries":[[{"_tag":"I16","value":-57},{"_tag":"Bool","value":false}]]}},{"id":12,"value":{"_tag":"I8","value":32}},{"id":236,"value":{"_tag":"I16","value":-27}}]},"bytes":"2b01417102a32004d8033500"},{"id":"random/186","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/187","type":"Struct","value":{"_tag":"Struct","fields":[{"id":242,"value":{"_tag":"Double","value":-745498.1018863805}},{"id":424,"value":{"_tag":"Binary","value":"6a6375"}}]},"bytes":"07e403a0732a3434c026c108d006036a637500"},{"id":"random/188","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/189","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Bool","value":false}},{"id":21,"value":{"_tag":"I16","value":9789}},{"id":181,"value":{"_tag":"I32","value":1}},{"id":9222,"value":{"_tag":"I32","value":-4766942}}]},"bytes":"c294fa980105ea0202058c9001bbf3c50400"},{"id":"random/190","type":"Struct","value":{"_tag":"Struct","fields":[{"id":152,"value":{"_tag":"Binary","value":"99"}},{"id":194,"value":{"_tag":"I32","value":-805}},{"id":3,"value":{"_tag":"I32","value":-28379}},{"id":253,"value":{"_tag":"I32","value":-62}},{"id":261,"value":{"_tag":"Bool","value":false}},{"id":-2,"value":{"_tag":"Double","value":-2.9723028684042826e-184}},{"id":12870,"value":{"_tag":"I16","value":-33}},{"id":13051,"value":{"_tag":"I8","value":-54}}]},"bytes":"08b0020199058403c90c0506b5bb0305fa037b82070311f85d0b1835d499048cc9014103f6cb01ca00"},{"id":"random/191","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I64","value":"60"}},{"id":230,"value":{"_tag":"Bool","value":true}},{"id":-3,"value":{"_tag":"I16","value":-24}}]},"bytes":"167801cc0304052f00"},{"id":"random/192","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Map","keyType":"I8","valueType":"Bool","entries":[[{"_tag":"I8","value":16},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-59},{"_tag":"Bool","value":false}],[{"_tag":"I8","value":-18},{"_tag":"Bool","value":true}]]}},{"id":7,"value":{"_tag":"I8","value":-2}},{"id":14,"value":{"_tag":"I16","value":-231}}]},"bytes":"3b03311002c502ee0143fe74cd0300"},{"id":"random/193","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Double","value":213.0}},{"id":11,"value":{"_tag":"I8","value":-64}}]},"bytes":"a70000000000a06a4013c000"},{"id":"random/194","type":"Struct","value":{"_tag":"Struct","fields":[{"id":179,"value":{"_tag":"Double","value":612.0}},{"id":18891,"value":{"_tag":"Double","value":802295.7192933345}}]},"bytes":"07e60200000000002083400796a70248374770ef7b284100"},{"id":"random/195","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Bool","value":true}},{"id":9,"value":{"_tag":"Bool","value":true}},{"id":3,"value":{"_tag":"Bool","value":true}},{"id":2,"value":{"_tag":"Struct","fields":[{"id":58,"value":{"_tag":"Bool","value":true}},{"id":61,"value":{"_tag":"Double","value":989.0}},{"id":74,"value":{"_tag":"Uuid","value":"0b8ebdb26f2ae7cf274bbd22b5cd39b9"}},{"id":5,"value":{"_tag":"I32","value":20}},{"id":234,"value":{"_tag":"I32","value":-90}},{"id":267,"value":{"_tag":"Double","value":-870999.4470261727}}]}},{"id":12,"value":{"_tag":"I16","value":-23806}},{"id":74,"value":{"_tag":"I64","value":"-7"}}]},"bytes":"316101060c040174370000000000e88e40dd0b8ebdb26f2ae7cf274bbd22b5cd39b9050a2805d403b301079604509de0e4ae942ac100a4fbf3020694010d00"},{"id":"random/196","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"I64","value":"183022398310386"}},{"id":9,"value":{"_tag":"Binary","value":"6e2f67"}},{"id":14341,"value":{"_tag":"I32","value":-2}},{"id":14347,"value":{"_tag":"Uuid","value":"25cea12717f7e61ac520cff9b641b489"}},{"id":14350,"value":{"_tag":"I64","value":"16053"}},{"id":14612,"value":{"_tag":"Binary","value":""}},{"id":14625,"value":{"_tag":"Double","value":97803501054.21117}},{"id":30664,"value":{"_tag":"I8","value":-9}}]},"bytes":"0607e49fcc90a79d53d8036e2f67058ae001036d25cea12717f7e61ac520cff9b641b48936eafa0108a8e40100d70f36fefd8ac536420390df03f700"},{"id":"random/197","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Binary","value":"888f7e3aa3436ce80d41d3"}},{"id":11,"value":{"_tag":"I64","value":"6163"}},{"id":276,"value":{"_tag":"Binary","value":"62e697a52f672e73"}},{"id":367,"value":{"_tag":"I32","value":249767229}}]},"bytes":"280b888f7e3aa3436ce80d41d396a66008a8040862e697a52f672e7305de05fa9499ee0100"},{"id":"random/198","type":"Struct","value":{"_tag":"Struct","fields":[{"id":72,"value":{"_tag":"I16","value":15}},{"id":291,"value":{"_tag":"I16","value":-35}}]},"bytes":"0490011e04c6044500"},{"id":"random/199","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Map","keyType":"I8","valueType":"Double","entries":[[{"_tag":"I8","value":1},{"_tag":"Double","value":528.0}],[{"_tag":"I8","value":0},{"_tag":"Double","value":769.0}],[{"_tag":"I8","value":2},{"_tag":"Double","value":166150.56532268738}]]}},{"id":16,"value":{"_tag":"Binary","value":"6171202d777067"}},{"id":18,"value":{"_tag":"I8","value":34}},{"id":25,"value":{"_tag":"I64","value":"33"}},{"id":-30925,"value":{"_tag":"Bool","value":true}},{"id":-30860,"value":{"_tag":"Uuid","value":"8bdacdcf7c552bb21baa1cb20b8f34e0"}}]},"bytes":"9b033701000000000080804000000000000008884002b0e6c7853448044178076171202d777067232276420199e3030d97e2038bdacdcf7c552bb21baa1cb20b8f34e000"},{"id":"random/200","type":"Struct","value":{"_tag":"Struct","fields":[{"id":195,"value":{"_tag":"Bool","value":true}},{"id":209,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true}]}},{"id":220,"value":{"_tag":"I32","value":-27}},{"id":221,"value":{"_tag":"Double","value":-838931.8870946185}},{"id":222,"value":{"_tag":"Bool","value":true}}]},"bytes":"018603ea510101010101b535170e4431c6279a29c11100"},{"id":"random/201","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I16","value":5}},{"id":9,"value":{"_tag":"Double","value":-595.0}},{"id":23,"value":{"_tag":"I32","value":2042}},{"id":5121,"value":{"_tag":"Uuid","value":"b66e94671bb0eb7cd05242dce5c6c937"}},{"id":25735,"value":{"_tag":"Double","value":339791.9734279441}},{"id":25742,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"530963194310756"},{"_tag":"I64","value":"48"},{"_tag":"I64","value":"-90040077097263"},{"_tag":"I64","value":"26"},{"_tag":"I64","value":"60"}]}},{"id":25748,"value":{"_tag":"I32","value":0}}]},"bytes":"540a4700000000009882c0e5f41f0d8250b66e94671bb0eb7cd05242dce5c6c937078e9203844bcae43fbd14417a56c8c1af8990baf10160dde4fcec82f9283478650000"},{"id":"random/202","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Bool","value":false}},{"id":55,"value":{"_tag":"Map","keyType":"List","valueType":"Uuid","entries":[[{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"2d6bc3a9662077616f70616571c3a92f77"},{"_tag":"Binary","value":"6d76696b6c657ac3a96e6d6f"}]},{"_tag":"Uuid","value":"3d7112194ea1d88dab4d9156e252c56d"}],[{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"15122465221715"},{"_tag":"I64","value":"32"}]},{"_tag":"Uuid","value":"5d50b093c4eadda59f0f1e6e5e7a139a"}],[{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":1},{"_tag":"I8","value":-10}]},{"_tag":"Uuid","value":"2b935525a38872af35e47fa469d00b95"}]]}},{"id":69,"value":{"_tag":"I32","value":-2}},{"id":72,"value":{"_tag":"I8","value":1}},{"id":-6820,"value":{"_tag":"Uuid","value":"ccb3779099eb2b06a34326bda74fb129"}},{"id":0,"value":{"_tag":"Uuid","value":"1000c5093368ee3f78eb743308195c36"}}]},"bytes":"b20b6e039d28112d6bc3a9662077616f70616571c3a92f770c6d76696b6c657ac3a96e6d6f3d7112194ea1d88dab4d9156e252c56d26a6b1d7929ff006405d50b093c4eadda59f0f1e6e5e7a139a2301f62b935525a38872af35e47fa469d00b95e50333010dc76accb3779099eb2b06a34326bda74fb1290d001000c5093368ee3f78eb743308195c3600"},{"id":"random/203","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"Bool","value":false}},{"id":6,"value":{"_tag":"I16","value":-47}},{"id":5,"value":{"_tag":"I16","value":-14}}]},"bytes":"22445d040a1b00"},{"id":"random/204","type":"Struct","value":{"_tag":"Struct","fields":[{"id":290,"value":{"_tag":"I16","value":0}}]},"bytes":"04c4040000"},{"id":"random/205","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Uuid","value":"a1d03ba485ecb07ccc2590e0628aaac0"}},{"id":-2,"value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I32","value":-2}},{"id":196,"value":{"_tag":"Binary","value":"7a6a62737261"}},{"id":206,"value":{"_tag":"Double","value":0.0}},{"id":210,"value":{"_tag":"Double","value":-258111.53512359783}},{"id":215,"value":{"_tag":"Uuid","value":"e3053236e32cea9ff4d50c16a587bb8c"}},{"id":-1,"value":{"_tag":"List","elementType":"I16","elements":[{"_tag":"I16","value":31},{"_tag":"I16","value":32767}]}}]}}]},"bytes":"5da1d03ba485ecb07ccc2590e0628aaac00c039503088803067a6a62737261a700000000000000004780e1ee47fc810fc15de3053236e32cea9ff4d50c16a587bb8c0901243efeff030000"},{"id":"random/206","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"57"}},{"id":7,"value":{"_tag":"Double","value":700944.8469645118}},{"id":3,"value":{"_tag":"I16","value":38}},{"id":224,"value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I8","value":15}},{"id":13,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":124708428},{"_tag":"I32","value":-1733699},{"_tag":"I32","value":-17371},{"_tag":"I32","value":-18},{"_tag":"I32","value":309450},{"_tag":"I32","value":30}]}},{"id":28,"value":{"_tag":"I8","value":26}},{"id":72,"value":{"_tag":"I64","value":"6513019"}}]}}]},"bytes":"3672471e55a5b12164254104064c0cc003530f8a659899f77685d1d301b58f022394e3253cf31a069001f6859b060000"},{"id":"random/207","type":"Struct","value":{"_tag":"Struct","fields":[{"id":148,"value":{"_tag":"Binary","value":"62"}},{"id":20070,"value":{"_tag":"List","elementType":"I16","elements":[]}},{"id":20076,"value":{"_tag":"I64","value":"-21"}},{"id":20091,"value":{"_tag":"Uuid","value":"e4ca8bbf87bf3ef50e33a5db032566fc"}},{"id":20102,"value":{"_tag":"I8","value":48}},{"id":2,"value":{"_tag":"I8","value":-2}},{"id":5,"value":{"_tag":"Double","value":892.0}}]},"bytes":"08a802016209ccb902046629fde4ca8bbf87bf3ef50e33a5db032566fcb3300304fe370000000000e08b4000"},{"id":"random/208","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-92},{"_tag":"I16","value":-16},{"_tag":"I16","value":0},{"_tag":"I16","value":-3956},{"_tag":"I16","value":-13},{"_tag":"I16","value":-51},{"_tag":"I16","value":0},{"_tag":"I16","value":-50},{"_tag":"I16","value":-640},{"_tag":"I16","value":-63},{"_tag":"I16","value":198},{"_tag":"I16","value":599},{"_tag":"I16","value":3},{"_tag":"I16","value":8},{"_tag":"I16","value":11},{"_tag":"I16","value":21}]}},{"id":9,"value":{"_tag":"I8","value":-5}},{"id":3293,"value":{"_tag":"Double","value":9.579081941189292e+57}}]},"bytes":"6af410b7011f00e73d19650063ff097d8c03ae090610162a33fb07ba33490cae82a46af84b00"},{"id":"random/209","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Binary","value":"7a6f6e"}},{"id":235,"value":{"_tag":"Double","value":-86.0}},{"id":337,"value":{"_tag":"I64","value":"-9528607"}},{"id":3,"value":{"_tag":"I64","value":"-17170966625"}},{"id":17,"value":{"_tag":"I8","value":26}},{"id":19,"value":{"_tag":"Binary","value":"3673bdf4cea1c07d53e00c01fa83f6d5b3"}}]},"bytes":"f8037a6f6e07d60300000000008055c006a205bd948b090606c1a1c1f77fe31a28113673bdf4cea1c07d53e00c01fa83f6d5b300"},{"id":"random/210","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/211","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I16","value":0}},{"id":16,"value":{"_tag":"Bool","value":true}},{"id":75,"value":{"_tag":"I16","value":11}},{"id":79,"value":{"_tag":"I8","value":-3}},{"id":82,"value":{"_tag":"Struct","fields":[{"id":23,"value":{"_tag":"I32","value":34}},{"id":38,"value":{"_tag":"Set","elementType":"Bool","elements":[]}},{"id":219,"value":{"_tag":"Bool","value":false}},{"id":223,"value":{"_tag":"Bool","value":true}},{"id":327,"value":{"_tag":"Binary","value":"ed277b394d9e6d38417b"}},{"id":336,"value":{"_tag":"Double","value":-236.0}},{"id":-8214,"value":{"_tag":"Bool","value":true}},{"id":-8205,"value":{"_tag":"I32","value":-2147483648}}]}}]},"bytes":"e400210496011643fd3c052e44fa0102b60341088e050aed277b394d9e6d38417b970000000000806dc001ab800195ffffffff0f0000"},{"id":"random/212","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Binary","value":"313b7e761bd1e72e720a2b32f3"}},{"id":21,"value":{"_tag":"Bool","value":true}},{"id":290,"value":{"_tag":"I16","value":-329}},{"id":296,"value":{"_tag":"Binary","value":"6d637763"}}]},"bytes":"780d313b7e761bd1e72e720a2b32f3e104c404910568046d63776300"},{"id":"random/213","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Binary","value":"66e697a563762e782d68776df09f8c8d61616e2f666e667866616e782e6262686476656e726a677869e697a5f09f8c8d7366646173e697a5672e6a76716c6c7563e697a56120f09f8c8d2e61636e78717a74736c2d7769652f6a6866626e7375687166666762796f2f6865766d2d772d695f6d6d667572c3a97873742d77c3bc76746d636477746969c3bc6f6a73e697a5742e6261"}},{"id":209,"value":{"_tag":"I16","value":-150}},{"id":214,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"Uuid","value":"f0d41b19765ee84df41b2f0177d08ed5"}},{"id":-13945,"value":{"_tag":"I8","value":-2}},{"id":-13938,"value":{"_tag":"I8","value":22}},{"id":-13725,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"44af7a43c3be511b13"}]}},{"id":-13723,"value":{"_tag":"I32","value":5}},{"id":-13722,"value":{"_tag":"Uuid","value":"0d03fffd0df44a181ea552606f1f2a5f"}},{"id":-13665,"value":{"_tag":"I32","value":9}}]},{"_tag":"Struct","fields":[]}]}},{"id":220,"value":{"_tag":"Uuid","value":"ce25512863b9ae2eaa2694ddf7d83edd"}},{"id":232,"value":{"_tag":"I8","value":-2}}]},"bytes":"38950166e697a563762e782d68776df09f8c8d61616e2f666e667866616e782e6262686476656e726a677869e697a5f09f8c8d7366646173e697a5672e6a76716c6c7563e697a56120f09f8c8d2e61636e78717a74736c2d7769652f6a6866626e7375687166666762796f2f6865766d2d772d695f6d6d667572c3a97873742d77c3bc76746d636477746969c3bc6f6a73e697a5742e626104a203ab02592cddf0d41b19765ee84df41b2f0177d08ed503f1d901fe73160ab9d601180944af7a43c3be511b13250a1d0d03fffd0df44a181ea552606f1f2a5f05c1d5011200006dce25512863b9ae2eaa2694ddf7d83eddc3fe00"},{"id":"random/214","type":"Struct","value":{"_tag":"Struct","fields":[{"id":24164,"value":{"_tag":"I8","value":1}},{"id":24338,"value":{"_tag":"I16","value":-2}},{"id":24346,"value":{"_tag":"Double","value":-776233.399993409}},{"id":-4,"value":{"_tag":"I64","value":"3273"}},{"id":-5,"value":{"_tag":"I16","value":1877}}]},"bytes":"03c8f9020104a4fc020387a5efcbcc52b027c1060792330409aa1d00"},{"id":"random/215","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1577,"value":{"_tag":"I64","value":"-15019880"}},{"id":-1442,"value":{"_tag":"I8","value":-20}},{"id":-1250,"value":{"_tag":"Double","value":0.0}},{"id":-1001,"value":{"_tag":"Bool","value":true}},{"id":4,"value":{"_tag":"I64","value":"9223372036854775807"}},{"id":-4,"value":{"_tag":"I16","value":-24}},{"id":7,"value":{"_tag":"Set","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Uuid","value":"ac9c963e9bec69121f0a4c8d03b8ca85"}},{"id":298,"value":{"_tag":"Binary","value":"68706b2063f09f8c8d696e662fc3bc70f09f8c8d67686767726c7265685f5f7779776c2067206f2f6e5f6d78696361777765706d78746b642ec3bc70f09f8c8d6e72657570737a7777656c66796c7475c3bc6366e697a57a686e6962646f65c3a9736e6b71f09f8c8d7977c3a9637a6f702e77646576f09f8c8d6463657a6af09f8c8d645f6a74c3bc756176202073652f6373722e2073662f74e697a572646e622071c3bc7376e697a57566676b676f6a2d672e6b62e697a56cf09f8c8d6e20746e2e7771682f737177747075675f7af09f8c8d636e2e682e2d737869f09f8c8d756a715f6a636878e697a56f79716b2e2e78c3bc"}},{"id":306,"value":{"_tag":"I8","value":-18}},{"id":-5,"value":{"_tag":"I32","value":8844743}},{"id":6,"value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"Uuid","value":"8f5eb387d5f09215803b3c835e6a8db5"}}]}},{"id":1838,"value":{"_tag":"I16","value":-2494}}]},{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"I16","value":-25}},{"id":28,"value":{"_tag":"Uuid","value":"ff68edb2d0643b7c87bbc45917aba1da"}},{"id":34,"value":{"_tag":"Double","value":1.361088501157034e+134}},{"id":5,"value":{"_tag":"Double","value":-5.790539697003349e+138}},{"id":113,"value":{"_tag":"Bool","value":false}},{"id":121,"value":{"_tag":"Uuid","value":"309f460ba0dc3ae9c47b1985d2680a0f"}}]},{"_tag":"Struct","fields":[{"id":298,"value":{"_tag":"Binary","value":"55b623e798f56d8f278c161a141c75a2e91e61"}},{"id":1,"value":{"_tag":"I32","value":-63}},{"id":160,"value":{"_tag":"Bool","value":false}}]},{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Bool","value":false}},{"id":66,"value":{"_tag":"Binary","value":"a732a66861b9add7a0e336327bd0b6"}},{"id":77,"value":{"_tag":"Binary","value":"d02df0"}},{"id":84,"value":{"_tag":"Binary","value":"7879646570206d2e72206461c3bc2df09f8c8d76"}},{"id":92,"value":{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":43},{"_tag":"I16","value":-10}]}},{"id":367,"value":{"_tag":"Binary","value":"77617872676575766d2ef09f8c8d5f636e7a642e6d7a2d68796a696a6f627377682d75612d73652d716b625f70686ce697a569646e6262656df09f8c8d6c2de697a56172e697a5726b7a652fc3bc6f7a6e706371746567776176776d6a6b77c3a92f767669736af09f8c8d6d20636c676bf09f8c8d796cf09f8c8d7367652d2d786c73f09f8c8dc3bc706a6568766b667561f09f8c8d6d2e6a2f61756a762063666c6d6d72777a722e6178205f76696b646f7179207a7a6e697768206b6f6970c3a9666a662d78796c76655f75626578646d6c64767a616577666ff09f8c8d2e6f796673726863677a69"}},{"id":6417,"value":{"_tag":"I64","value":"-18"}},{"id":6430,"value":{"_tag":"Binary","value":"7972662d20706d6cc3a9c3bc2f5f20667a68c3a96f6274666468626a76786b666f73c3a9696bf09f8c8d7a75c3a97164206dc3bc206363c3bc79612e7261645f676bc3a97666626b74636f727575c3a9f09f8c8d2f6a6c2e2f2e63636b2f677a5f5f6674762d7369652d76726a74736b6ce697a56dc3bc7a78e697a573e697a57020622073726b72752e6b78767968726ac3a9636c736a"}}]},{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I16","value":4335}},{"id":16,"value":{"_tag":"I64","value":"-68167304068571"}}]}]}},{"id":10,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}}]},"bytes":"06d118cfbda90e03c316ec07c313000000000000000001d10f0608feffffffffffffffff0104072fba5cadac9c963e9bec69121f0a4c8d03b8ca8508d404f50168706b2063f09f8c8d696e662fc3bc70f09f8c8d67686767726c7265685f5f7779776c2067206f2f6e5f6d78696361777765706d78746b642ec3bc70f09f8c8d6e72657570737a7777656c66796c7475c3bc6366e697a57a686e6962646f65c3a9736e6b71f09f8c8d7977c3a9637a6f702e77646576f09f8c8d6463657a6af09f8c8d645f6a74c3bc756176202073652f6373722e2073662f74e697a572646e622071c3bc7376e697a57566676b676f6a2d672e6b62e697a56cf09f8c8d6e20746e2e7771682f737177747075675f7af09f8c8d636e2e682e2d737869f09f8c8d756a715f6a636878e697a56f79716b2e2e78c3bc83ee05098ed7b708bced8f5eb387d5f09215803b3c835e6a8db50004dc1cfb2600d431fdff68edb2d0643b7c87bbc45917aba1da67892be6a02ef8c75b070ab5adab0bbf1ebfdc02e2018d309f460ba0dc3ae9c47b1985d2680a0f0008d4041355b623e798f56d8f278c161a141c75a2e91e6105027d02c00200720884010fa732a66861b9add7a0e336327bd0b6b803d02df078147879646570206d2e72206461c3bc2df09f8c8d768a24561308de05ea0177617872676575766d2ef09f8c8d5f636e7a642e6d7a2d68796a696a6f627377682d75612d73652d716b625f70686ce697a569646e6262656df09f8c8d6c2de697a56172e697a5726b7a652fc3bc6f7a6e706371746567776176776d6a6b77c3a92f767669736af09f8c8d6d20636c676bf09f8c8d796cf09f8c8d7367652d2d786c73f09f8c8dc3bc706a6568766b667561f09f8c8d6d2e6a2f61756a762063666c6d6d72777a722e6178205f76696b646f7179207a7a6e697768206b6f6970c3a9666a662d78796c76655f75626578646d6c64767a616577666ff09f8c8d2e6f796673726863677a6906a26423d897017972662d20706d6cc3a9c3bc2f5f20667a68c3a96f6274666468626a76786b666f73c3a9696bf09f8c8d7a75c3a97164206dc3bc206363c3bc79612e7261645f676bc3a97666626b74636f727575c3a9f09f8c8d2f6a6c2e2f2e63636b2f677a5f5f6674762d7369652d76726a74736b6ce697a56dc3bc7a78e697a573e697a57020622073726b72752e6b78767968726ac3a9636c736a0054de43b6b5f78dffedff1e003b0000"},{"id":"random/216","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"Bool","value":true}},{"id":12,"value":{"_tag":"Uuid","value":"b1641fbc07639dcb84580d953ceb12eb"}},{"id":17,"value":{"_tag":"Binary","value":"a5bc63f431820ff9958908683b8d1e"}},{"id":19,"value":{"_tag":"I64","value":"-15190819858452565"}}]},"bytes":"913db1641fbc07639dcb84580d953ceb12eb580fa5bc63f431820ff9958908683b8d1e26a9c194d88bfefb3500"},{"id":"random/217","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"I32","value":386}},{"id":18,"value":{"_tag":"Bool","value":true}},{"id":3,"value":{"_tag":"Binary","value":"865120d2d5fac17d167d48116416210d970509"}},{"id":136,"value":{"_tag":"I64","value":"2602073617371409120"}}]},"bytes":"458406e1080613865120d2d5fac17d167d48116416210d970509069002c0abb5ade58ab69c4800"},{"id":"random/218","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"I32","value":11}}]},"bytes":"f51600"},{"id":"random/219","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Bool","value":false}},{"id":8,"value":{"_tag":"Uuid","value":"c7841b87204d8bb16d9201f0626ec992"}},{"id":200,"value":{"_tag":"Double","value":109563.64377445448}},{"id":363,"value":{"_tag":"I16","value":51}},{"id":19441,"value":{"_tag":"I32","value":52}}]},"bytes":"424dc7841b87204d8bb16d9201f0626ec9920790034071e64cbabffa4004d6056605e2af026800"},{"id":"random/220","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Binary","value":"647671e697a56e65f09f8c8d2d6273c3bc7a6d6d616ce697a56663"}},{"id":95,"value":{"_tag":"Binary","value":"66715fc3bc636b79627571626c79762f702e"}},{"id":102,"value":{"_tag":"I16","value":-31}},{"id":8145,"value":{"_tag":"Double","value":-58932.267224866024}},{"id":8150,"value":{"_tag":"Double","value":928820.4109685028}},{"id":8156,"value":{"_tag":"I16","value":-21}}]},"bytes":"581b647671e697a56e65f09f8c8d2d6273c3bc7a6d6d616ce697a5666308be011266715fc3bc636b79627571626c79762f702e743d07a27f88291b8d88c6ecc057ae766ad268582c41642900"},{"id":"random/221","type":"Struct","value":{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"Bool","value":true}},{"id":-14772,"value":{"_tag":"Uuid","value":"d7052682536eac388e7418e185ba9e53"}},{"id":-14514,"value":{"_tag":"Uuid","value":"a6f7db17db6bd9bb45fb9f5212006c99"}},{"id":-14445,"value":{"_tag":"I64","value":"-89144"}}]},"bytes":"d10de7e601d7052682536eac388e7418e185ba9e530de3e201a6f7db17db6bd9bb45fb9f5212006c9906d9e101eff00a00"},{"id":"random/222","type":"Struct","value":{"_tag":"Struct","fields":[{"id":248,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":260,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"68c5d1c8d2a8fd854e24414a522e1c8b"},{"_tag":"Uuid","value":"5e94567a124b6614e1daeb099d96b255"},{"_tag":"Uuid","value":"62ae714a2e3d1191d49b565bcedc9e03"},{"_tag":"Uuid","value":"2186251c8ca302bdd4721d7d37bd87d3"},{"_tag":"Uuid","value":"34044b59e24247aa1e3f3e6f9e5342d7"},{"_tag":"Uuid","value":"1a9636bd493cd6b3f3192f8ddb09e9f2"},{"_tag":"Uuid","value":"63452a39c5325d06c655c7db4f0931ae"},{"_tag":"Uuid","value":"b2ffe65aa3bc9ddb8c39b165c99aa7ab"},{"_tag":"Uuid","value":"01161d60ba5ce5d6b4123ceb6b0f7ae9"},{"_tag":"Uuid","value":"40eb7206a113208230becaf1e226a7f7"},{"_tag":"Uuid","value":"a89426aa3666306679b21eca66ed4a0f"},{"_tag":"Uuid","value":"9b1f265ceea9dccbd7241818b94f7b89"},{"_tag":"Uuid","value":"ff126b3fd68055684ec2dfe93c02d7b4"},{"_tag":"Uuid","value":"83aa9236e18912ecbddaa33b118810eb"},{"_tag":"Uuid","value":"4cfee128cbf33881abae2519de42dd0e"},{"_tag":"Uuid","value":"fade086f8c8972214c8349d2d6daa724"},{"_tag":"Uuid","value":"77b558cdfc010eec6e05eaa967f5bc44"},{"_tag":"Uuid","value":"be77942f92df52239e82d70e36c232c3"},{"_tag":"Uuid","value":"ce81be34b3134120dfc57dcabf8da267"},{"_tag":"Uuid","value":"3f564966543e629527addba9862335f4"},{"_tag":"Uuid","value":"b2fac26b75812c6b65ab7f6f8b67380f"},{"_tag":"Uuid","value":"c891b75c4a7bd0f18f2080581ad6c3c1"},{"_tag":"Uuid","value":"3220d3ce845eb299ae9612549ca86ed8"},{"_tag":"Uuid","value":"228fd8904e3c4685ebd674acced3352f"},{"_tag":"Uuid","value":"1111cdbdd9241c9c1b789ab2b7016bd9"},{"_tag":"Uuid","value":"b97ed03ead03405ee19ed8d80839144b"},{"_tag":"Uuid","value":"cba94b22fa706c327bef05fd840e2ef2"},{"_tag":"Uuid","value":"f774464d55f42b60e0b8ae37157300d6"},{"_tag":"Uuid","value":"d72a8ce10c8e70e9f3e77fa425241543"},{"_tag":"Uuid","value":"870e1ae9f205e9d1d3665b2901b5f641"}]}},{"id":272,"value":{"_tag":"Struct","fields":[]}},{"id":285,"value":{"_tag":"Double","value":-713559.2466903226}}]},"bytes":"0bf00300c9fd1e68c5d1c8d2a8fd854e24414a522e1c8b5e94567a124b6614e1daeb099d96b25562ae714a2e3d1191d49b565bcedc9e032186251c8ca302bdd4721d7d37bd87d334044b59e24247aa1e3f3e6f9e5342d71a9636bd493cd6b3f3192f8ddb09e9f263452a39c5325d06c655c7db4f0931aeb2ffe65aa3bc9ddb8c39b165c99aa7ab01161d60ba5ce5d6b4123ceb6b0f7ae940eb7206a113208230becaf1e226a7f7a89426aa3666306679b21eca66ed4a0f9b1f265ceea9dccbd7241818b94f7b89ff126b3fd68055684ec2dfe93c02d7b483aa9236e18912ecbddaa33b118810eb4cfee128cbf33881abae2519de42dd0efade086f8c8972214c8349d2d6daa72477b558cdfc010eec6e05eaa967f5bc44be77942f92df52239e82d70e36c232c3ce81be34b3134120dfc57dcabf8da2673f564966543e629527addba9862335f4b2fac26b75812c6b65ab7f6f8b67380fc891b75c4a7bd0f18f2080581ad6c3c13220d3ce845eb299ae9612549ca86ed8228fd8904e3c4685ebd674acced3352f1111cdbdd9241c9c1b789ab2b7016bd9b97ed03ead03405ee19ed8d80839144bcba94b22fa706c327bef05fd840e2ef2f774464d55f42b60e0b8ae37157300d6d72a8ce10c8e70e9f3e77fa425241543870e1ae9f205e9d1d3665b2901b5f641cc00d7a8314e7eaec625c100"},{"id":"random/223","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I64","value":"-55"}}]},"bytes":"566d00"},{"id":"random/224","type":"Struct","value":{"_tag":"Struct","fields":[{"id":196,"value":{"_tag":"Bool","value":false}},{"id":206,"value":{"_tag":"Uuid","value":"803f82a74cf4d122a5bc443c93418b40"}},{"id":209,"value":{"_tag":"I32","value":-35}},{"id":3,"value":{"_tag":"I8","value":0}},{"id":4,"value":{"_tag":"I16","value":32767}},{"id":7,"value":{"_tag":"I64","value":"0"}},{"id":-5,"value":{"_tag":"I8","value":-56}}]},"bytes":"028803ad803f82a74cf4d122a5bc443c93418b40354503060014feff0336000309c800"},{"id":"random/225","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"I32","value":0}}]},"bytes":"450000"},{"id":"random/226","type":"Struct","value":{"_tag":"Struct","fields":[{"id":27538,"value":{"_tag":"I32","value":-10}},{"id":27540,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"796b2e69732f702e7475"},{"_tag":"Binary","value":"e697a5756ff09f8c8d756761c3bc6b69626662632ef09f8c8d7a6a626662c3bc7a726767636b76792f626e67f09f8c8d6b797562672f6d657665756f65c3a971686e6e2f70696978e697a56b71706c5f787875767877676e63616a20f09f8c8dc3a96f2061746961706b75676b677063e697a52e70c3bc7976762f676f716874627866727261e697a577687874755f726978c3bce697a5f09f8c8d6769f09f8c8d"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"776272c3a96577777a"},{"_tag":"Binary","value":"727561786766207978786f68796a752f7174762f"},{"_tag":"Binary","value":"c3a967666c6a6779c3bc2e6e76646970"}]}},{"id":27552,"value":{"_tag":"Uuid","value":"be1f9f0f79a420039c24fb6eab1bbbf1"}},{"id":0,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-19098558"},{"_tag":"I64","value":"-118863735894409071"},{"_tag":"I64","value":"1"},{"_tag":"I64","value":"21"},{"_tag":"I64","value":"-26"},{"_tag":"I64","value":"-56"}]}},{"id":1,"value":{"_tag":"Uuid","value":"2fe239d53f4161003783c9523c7b3d58"}},{"id":-1,"value":{"_tag":"Binary","value":"6270f09f8c8d"}}]},"bytes":"05a4ae031329680a796b2e69732f702e7475a101e697a5756ff09f8c8d756761c3bc6b69626662632ef09f8c8d7a6a626662c3bc7a726767636b76792f626e67f09f8c8d6b797562672f6d657665756f65c3a971686e6e2f70696978e697a56b71706c5f787875767877676e63616a20f09f8c8dc3a96f2061746961706b75676b677063e697a52e70c3bc7976762f676f716874627866727261e697a577687874755f726978c3bce697a5f09f8c8d6769f09f8c8d0009776272c3a96577777a14727561786766207978786f68796a752f7174762f10c3a967666c6a6779c3bc2e6e76646970cdbe1f9f0f79a420039c24fb6eab1bbbf1090066fbae9b12ddfdcbaa84fca4a603022a336f1d2fe239d53f4161003783c9523c7b3d580801066270f09f8c8d00"},{"id":"random/227","type":"Struct","value":{"_tag":"Struct","fields":[{"id":44,"value":{"_tag":"Binary","value":"24da463b66b19233364a518d"}},{"id":57,"value":{"_tag":"I64","value":"40"}},{"id":67,"value":{"_tag":"I16","value":222}},{"id":-2,"value":{"_tag":"I64","value":"-1592725211"}},{"id":297,"value":{"_tag":"I64","value":"-10521"}},{"id":428,"value":{"_tag":"I64","value":"0"}}]},"bytes":"08580c24da463b66b19233364a518dd650a4bc030603b5bbf8ee0b06d204b1a40106d8060000"},{"id":"random/228","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/229","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Binary","value":"77667066767176626b616b6875f09f8c8d77786767"}},{"id":-10045,"value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Binary","value":"a6f52ed3bda7e877f9b7c5a87a0725deb1f8cc3fee6fc5bd96814853ef68d57c18e830a0baab3b1aef4aa1cf10f9b6c1906b5d5d12f4e181d8aabf293bbb0b2be7d237fc9825d19057e0a5e1f88bce7d8d9dcf90df0cd22bc9bec26025f8b8b5b121f622d4230dfb076a86206860dd57c985c9c803ed4b5f45c3ea46c554f4"}}]}},{"id":-10042,"value":{"_tag":"I64","value":"-1511356334"}}]},"bytes":"481577667066767176626b616b6875f09f8c8d777867670cf99c01f87fa6f52ed3bda7e877f9b7c5a87a0725deb1f8cc3fee6fc5bd96814853ef68d57c18e830a0baab3b1aef4aa1cf10f9b6c1906b5d5d12f4e181d8aabf293bbb0b2be7d237fc9825d19057e0a5e1f88bce7d8d9dcf90df0cd22bc9bec26025f8b8b5b121f622d4230dfb076a86206860dd57c985c9c803ed4b5f45c3ea46c554f40036dbdeaba10b00"},{"id":"random/230","type":"Struct","value":{"_tag":"Struct","fields":[{"id":62,"value":{"_tag":"Binary","value":"717862782e6c63726962666b71786c"}},{"id":67,"value":{"_tag":"Double","value":-980.0}},{"id":74,"value":{"_tag":"Uuid","value":"3958cca7fe9ab965154cd80cca368c77"}},{"id":88,"value":{"_tag":"Bool","value":false}},{"id":98,"value":{"_tag":"Struct","fields":[{"id":-28267,"value":{"_tag":"I64","value":"56"}},{"id":0,"value":{"_tag":"Uuid","value":"5cd52fd55c24c9e3f2b03e6248e1e695"}}]}},{"id":110,"value":{"_tag":"Double","value":-1.6891144884183598e+91}},{"id":132,"value":{"_tag":"I16","value":4}},{"id":146,"value":{"_tag":"I64","value":"-64"}}]},"bytes":"087c0f717862782e6c63726962666b71786c570000000000a08ec07d3958cca7fe9ab965154cd80cca368c77e2ac06d5b903700d005cd52fd55c24c9e3f2b03e6248e1e69500c759bd56c38395e0d204880208e67f00"},{"id":"random/231","type":"Struct","value":{"_tag":"Struct","fields":[{"id":92,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"626a8a71c12dea1f1d1b18fe7271e1d0"}]}},{"id":103,"value":{"_tag":"I8","value":4}}]},"bytes":"0ab8011d626a8a71c12dea1f1d1b18fe7271e1d0b30400"},{"id":"random/232","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-2,"value":{"_tag":"I8","value":3}}]},"bytes":"03030300"},{"id":"random/233","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/234","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Bool","value":false}},{"id":23,"value":{"_tag":"I8","value":-57}},{"id":-4,"value":{"_tag":"I32","value":42}},{"id":8,"value":{"_tag":"I8","value":-1}},{"id":17,"value":{"_tag":"Uuid","value":"b704bfc8a7e028d267e10fa65102f566"}},{"id":24,"value":{"_tag":"Binary","value":"5f2e2073762f71f09f8c8d7067f09f8c8d6f207667"}},{"id":28,"value":{"_tag":"Bool","value":true}}]},"bytes":"32032ec7050754c3ff9db704bfc8a7e028d267e10fa65102f56678155f2e2073762f71f09f8c8d7067f09f8c8d6f2076674100"},{"id":"random/235","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Map","keyType":"Double","valueType":"I8","entries":[[{"_tag":"Double","value":-802.0},{"_tag":"I8","value":-128}],[{"_tag":"Double","value":-785689.9785907138},{"_tag":"I8","value":1}],[{"_tag":"Double","value":502233.09097354976},{"_tag":"I8","value":-9}],[{"_tag":"Double","value":663.0},{"_tag":"I8","value":-3}],[{"_tag":"Double","value":-243224.69910060836},{"_tag":"I8","value":-72}],[{"_tag":"Double","value":987913.0068029854},{"_tag":"I8","value":55}],[{"_tag":"Double","value":-836.0},{"_tag":"I8","value":1}],[{"_tag":"Double","value":-1.8002021314860538e-228},{"_tag":"I8","value":-31}],[{"_tag":"Double","value":1.9423271399513855e+252},{"_tag":"I8","value":-59}],[{"_tag":"Double","value":-587.0},{"_tag":"I8","value":48}],[{"_tag":"Double","value":-277810.43949948426},{"_tag":"I8","value":15}],[{"_tag":"Double","value":-62.0},{"_tag":"I8","value":0}],[{"_tag":"Double","value":-50.0},{"_tag":"I8","value":-7}],[{"_tag":"Double","value":608.0},{"_tag":"I8","value":-45}],[{"_tag":"Double","value":-26425.509325389517},{"_tag":"I8","value":34}],[{"_tag":"Double","value":600.0},{"_tag":"I8","value":15}]]}},{"id":2,"value":{"_tag":"Uuid","value":"69f78f44b26953eeb0b8e3706149469d"}},{"id":11,"value":{"_tag":"Struct","fields":[]}},{"id":13,"value":{"_tag":"Bool","value":true}},{"id":28,"value":{"_tag":"Bool","value":true}},{"id":-4,"value":{"_tag":"I32","value":231}}]},"bytes":"7b107300000000001089c08090d709f533fa27c101942b285d64a71e41f70000000000b88440fd4c0fc297c5b00dc1b850ae7b0312262e41370000000000208ac0012b449458b1d5a590e1e133afdc8ff45074c500000000005882c0301e270cc2c9f410c10f0000000000004fc00000000000000049c0f90000000000008340d3c084c99860ced9c0220000000000c082400f0d0469f78f44b26953eeb0b8e3706149469d9c0021f10507ce0300"},{"id":"random/236","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"Binary","value":"5f"}},{"id":183,"value":{"_tag":"Map","keyType":"Map","valueType":"List","entries":[[{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"ad4616b1aa49567dad1018f766d714d8"},{"_tag":"Uuid","value":"fa9d1216c2bab04c82c705f45db1aa20"},{"_tag":"Uuid","value":"c42dee5776b6eb0d5d7c374be9f172e9"},{"_tag":"Uuid","value":"58c8b05898d7a9d386ceb6ee6558188b"},{"_tag":"Uuid","value":"37176777e74395eddc84336b81a99027"}]}],[{"_tag":"Map","keyType":"I32","valueType":"I64","entries":[[{"_tag":"I32","value":62},{"_tag":"I64","value":"-20"}],[{"_tag":"I32","value":84},{"_tag":"I64","value":"106179444695739"}]]},{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"-2299119770503063766"},{"_tag":"I64","value":"-7847420"},{"_tag":"I64","value":"1926051508673144"}]}],[{"_tag":"Map","keyType":"Binary","valueType":"Binary","entries":[[{"_tag":"Binary","value":"6d7a646a796f2d6d766d70c3a9646675706574c3bc70677a70c3bc73796c777a70755f5f666b6e64c3bc71717073e697a56a6a206871767362712ee697a5656b6f6769c3bc6f677a645fc3a96f6a736b736166642f792e71c3a96c71656b7368692076707a79e697a5747279736871746c67687879646a6c7371206677737af09f8c8d7476766a77767a7a71736e70"},{"_tag":"Binary","value":"6e6d707a6b6f67f09f8c8dc3bc626268696e757176c3a96561f09f8c8d766a6f65742f2dc3a9646af09f8c8d7a6c6a6af09f8c8d6ef09f8c8d76732d772f6d67716c71642074c3a9c3bc7920f09f8c8d6d7a6d676fc3a971f09f8c8d7864646372776d772f6d656f6c78c3a9f09f8c8d6269e697a562c3a96d7175755f6c20717a6b2e6869c3a9716d2f667274c3a96a7171725f67652063677661702f792dc3bcc3a9e697a5712e"}],[{"_tag":"Binary","value":"45628aa75c36a34b2e23f964668435a7e4be60b6"},{"_tag":"Binary","value":"736c2062f09f8c8d726cf09f8c8d6f796de697a5756b6a"}],[{"_tag":"Binary","value":"647461e697a57474637176712d68766865c3bc642f6c2d636b6ce697a5c3bc732d7972626372f09f8c8d7a745f6d6a6f2f63632f69c3bc6d78c3a9796e2f705f202dc3bc5fc3bc2d7a636be697a5736a2076c3a92df09f8c8d6a2d77725fc3bc2e642f70652f687778712e6b766b68687a7877676a792f6f5f6f696976766ef09f8c8d72f09f8c8d6e6b6f6f2e726173c3a9f09f8c8d785ff09f8c8d6b206165e697a5"},{"_tag":"Binary","value":"20662d5f6b2d61642e622dc3bc6671776e6365647763702fc3a969706f6866772e5f7a762d646b6776636e2d616e2d65716e6c756c7077c3bc696376706a71687970202e2e787a7161732e707377c3bc7379e697a56b637567766875206567796d766172f09f8c8d7366696679e697a5667363772f697770c3a970656375735fc3bc2d2e62786ff09f8c8de697a57564656b"}],[{"_tag":"Binary","value":"70712d62"},{"_tag":"Binary","value":"d55196a5afae473afad36e"}],[{"_tag":"Binary","value":"696b2d6b69c3bc64716474736820625f2d73"},{"_tag":"Binary","value":"6e6b6c696967e697a578"}]]},{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"70656a722e626320685f6862"}]}]]}},{"id":195,"value":{"_tag":"I64","value":"-179527674259"}},{"id":201,"value":{"_tag":"Set","elementType":"Set","elements":[{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"fb024585aa78fbb5d1df"},{"_tag":"Binary","value":"f09f8c8d67675f"},{"_tag":"Binary","value":"b3ff26e05cf84fb6b353"},{"_tag":"Binary","value":"f1"},{"_tag":"Binary","value":"707520"}]},{"_tag":"Set","elementType":"I8","elements":[]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":-12272.152007369907},{"_tag":"Double","value":-5.429098181216496e-268},{"_tag":"Double","value":-1.13916719160083e-70}]},{"_tag":"Set","elementType":"Map","elements":[]},{"_tag":"Set","elementType":"I64","elements":[]}]}},{"id":498,"value":{"_tag":"I16","value":42}},{"id":-2,"value":{"_tag":"Set","elementType":"Set","elements":[]}}]},"bytes":"0800015f0bee0203b9005dad4616b1aa49567dad1018f766d714d8fa9d1216c2bab04c82c705f45db1aa20c42dee5776b6eb0d5d7c374be9f172e958c8b05898d7a9d386ceb6ee6558188b37176777e74395eddc84336b81a9902702567c27a801f68aabc1baa430560101aba3cd8e80d08ee83ff7f7bd07f0e9f2fff9eeeb0605888f016d7a646a796f2d6d766d70c3a9646675706574c3bc70677a70c3bc73796c777a70755f5f666b6e64c3bc71717073e697a56a6a206871767362712ee697a5656b6f6769c3bc6f677a645fc3a96f6a736b736166642f792e71c3a96c71656b7368692076707a79e697a5747279736871746c67687879646a6c7371206677737af09f8c8d7476766a77767a7a71736e70a8016e6d707a6b6f67f09f8c8dc3bc626268696e757176c3a96561f09f8c8d766a6f65742f2dc3a9646af09f8c8d7a6c6a6af09f8c8d6ef09f8c8d76732d772f6d67716c71642074c3a9c3bc7920f09f8c8d6d7a6d676fc3a971f09f8c8d7864646372776d772f6d656f6c78c3a9f09f8c8d6269e697a562c3a96d7175755f6c20717a6b2e6869c3a9716d2f667274c3a96a7171725f67652063677661702f792dc3bcc3a9e697a5712e1445628aa75c36a34b2e23f964668435a7e4be60b617736c2062f09f8c8d726cf09f8c8d6f796de697a5756b6aa301647461e697a57474637176712d68766865c3bc642f6c2d636b6ce697a5c3bc732d7972626372f09f8c8d7a745f6d6a6f2f63632f69c3bc6d78c3a9796e2f705f202dc3bc5fc3bc2d7a636be697a5736a2076c3a92df09f8c8d6a2d77725fc3bc2e642f70652f687778712e6b766b68687a7877676a792f6f5f6f696976766ef09f8c8d72f09f8c8d6e6b6f6f2e726173c3a9f09f8c8d785ff09f8c8d6b206165e697a5920120662d5f6b2d61642e622dc3bc6671776e6365647763702fc3a969706f6866772e5f7a762d646b6776636e2d616e2d65716e6c756c7077c3bc696376706a71687970202e2e787a7161732e707377c3bc7379e697a56b637567766875206567796d766172f09f8c8d7366696679e697a5667363772f697770c3a970656375735fc3bc2d2e62786ff09f8c8de697a57564656b0470712d620bd55196a5afae473afad36e12696b2d6b69c3bc64716474736820625f2d730a6e6b6c696967e697a578180c70656a722e626320685f6862c6a5a6f7cab90a6a5a580afb024585aa78fbb5d1df07f09f8c8d67675f0ab3ff26e05cf84fb6b35301f1037075200337403dfa7413f8c7c07e73cf830fed71887eb023ddbf2869b10b0604e407540a030a00"},{"id":"random/237","type":"Struct","value":{"_tag":"Struct","fields":[{"id":250,"value":{"_tag":"Uuid","value":"6700da466ae3d56bbbfd5c5365e549d4"}},{"id":-26304,"value":{"_tag":"Uuid","value":"306083e276546c17ca182f900f7556c2"}},{"id":-26131,"value":{"_tag":"I32","value":2147483647}},{"id":-26125,"value":{"_tag":"I32","value":-60}},{"id":-24129,"value":{"_tag":"I16","value":0}}]},"bytes":"0df4036700da466ae3d56bbbfd5c5365e549d40dff9a03306083e276546c17ca182f900f7556c205a59803feffffff0f65770481f9020000"},{"id":"random/238","type":"Struct","value":{"_tag":"Struct","fields":[{"id":30,"value":{"_tag":"I32","value":38}},{"id":31,"value":{"_tag":"I16","value":-1}},{"id":140,"value":{"_tag":"Bool","value":true}}]},"bytes":"053c4c140101980200"},{"id":"random/239","type":"Struct","value":{"_tag":"Struct","fields":[{"id":213,"value":{"_tag":"Double","value":853190.9032760744}},{"id":436,"value":{"_tag":"Binary","value":"c0b6495e5ff05afe8f4d"}},{"id":447,"value":{"_tag":"Double","value":-738991.9357896219}},{"id":456,"value":{"_tag":"I8","value":-14}},{"id":666,"value":{"_tag":"I8","value":-1}}]},"bytes":"07aa039e337ace8d092a4108e8060ac0b6495e5ff05afe8f4db73cd11fdf5f8d26c193f203b40aff00"},{"id":"random/240","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12537,"value":{"_tag":"Double","value":5e-324}},{"id":12582,"value":{"_tag":"I16","value":26}},{"id":12588,"value":{"_tag":"Map","keyType":"Uuid","valueType":"Set","entries":[[{"_tag":"Uuid","value":"e41f49d7670636364198c194c0732d17"},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-3},{"_tag":"I16","value":32767},{"_tag":"I16","value":17},{"_tag":"I16","value":-455},{"_tag":"I16","value":58},{"_tag":"I16","value":-150},{"_tag":"I16","value":-15198},{"_tag":"I16","value":58},{"_tag":"I16","value":64},{"_tag":"I16","value":0},{"_tag":"I16","value":-1},{"_tag":"I16","value":58},{"_tag":"I16","value":2062},{"_tag":"I16","value":-4061}]}],[{"_tag":"Uuid","value":"781cc001af87f68fa18a46c7c9ade876"},{"_tag":"Set","elementType":"Map","elements":[{"_tag":"Map","keyType":"I16","valueType":"Uuid","entries":[[{"_tag":"I16","value":38},{"_tag":"Uuid","value":"e21cb09fcf30f75c4ac5124a174ed661"}],[{"_tag":"I16","value":-34},{"_tag":"Uuid","value":"1d8a11ab4d9ca573e5ab145aeb3d94e6"}],[{"_tag":"I16","value":46},{"_tag":"Uuid","value":"030ad13ab1e3acf0326cf7a4bdcd9064"}]]},{"_tag":"Map","keyType":"I8","valueType":"I16","entries":[[{"_tag":"I8","value":1},{"_tag":"I16","value":-329}],[{"_tag":"I8","value":0},{"_tag":"I16","value":688}],[{"_tag":"I8","value":1},{"_tag":"I16","value":-12300}],[{"_tag":"I8","value":-13},{"_tag":"I16","value":32767}],[{"_tag":"I8","value":0},{"_tag":"I16","value":-80}]]},{"_tag":"Map","keyType":"I16","valueType":"Double","entries":[[{"_tag":"I16","value":1597},{"_tag":"Double","value":2.887371864553667e+196}],[{"_tag":"I16","value":28},{"_tag":"Double","value":0.08609397831019376}]]},{"_tag":"Map","keyType":"Bool","valueType":"I32","entries":[[{"_tag":"Bool","value":true},{"_tag":"I32","value":25624094}]]},{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"Map","keyType":"I8","valueType":"Uuid","entries":[[{"_tag":"I8","value":-2},{"_tag":"Uuid","value":"361b7cabe509fdc18f7069cb4687896f"}],[{"_tag":"I8","value":-19},{"_tag":"Uuid","value":"11b1310d4319b10d6507d4bc3dd79fc0"}],[{"_tag":"I8","value":0},{"_tag":"Uuid","value":"5ad68aaae05bfe79a6897205d41d47c1"}]]},{"_tag":"Map","keyType":"I16","valueType":"Double","entries":[[{"_tag":"I16","value":-2},{"_tag":"Double","value":753105.0697204219}],[{"_tag":"I16","value":-48},{"_tag":"Double","value":391951.9044972472}]]},{"_tag":"Map","keyType":"Double","valueType":"Double","entries":[[{"_tag":"Double","value":-804728.1840654408},{"_tag":"Double","value":94092.03474531975}]]},{"_tag":"Map","keyType":"I64","valueType":"Bool","entries":[[{"_tag":"I64","value":"58"},{"_tag":"Bool","value":false}],[{"_tag":"I64","value":"-52"},{"_tag":"Bool","value":true}],[{"_tag":"I64","value":"2249913800104395"},{"_tag":"Bool","value":true}]]},{"_tag":"Map","keyType":"Uuid","valueType":"I8","entries":[[{"_tag":"Uuid","value":"87b76ad0c10b1d58a4284e4b61eef11e"},{"_tag":"I8","value":11}]]},{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]},{"_tag":"Map","keyType":"I16","valueType":"Uuid","entries":[[{"_tag":"I16","value":-20},{"_tag":"Uuid","value":"8ed497cb0092f3091d0da0d437a29f2f"}],[{"_tag":"I16","value":51},{"_tag":"Uuid","value":"8873b9fa6de019b745b6262c7113b113"}],[{"_tag":"I16","value":26},{"_tag":"Uuid","value":"dba1fdcb9277955451d11f8a9e67fb3e"}],[{"_tag":"I16","value":-14},{"_tag":"Uuid","value":"d123f96e88a31dc2a0dfca50bbd49d09"}],[{"_tag":"I16","value":20},{"_tag":"Uuid","value":"d32a5db45ec2c3ec622d9b218fc86cbd"}]]},{"_tag":"Map","keyType":"I64","valueType":"Bool","entries":[[{"_tag":"I64","value":"19"},{"_tag":"Bool","value":true}]]},{"_tag":"Map","keyType":"Bool","valueType":"Double","entries":[[{"_tag":"Bool","value":false},{"_tag":"Double","value":705.0}],[{"_tag":"Bool","value":true},{"_tag":"Double","value":"Infinity"}]]},{"_tag":"Map","keyType":"I16","valueType":"Uuid","entries":[[{"_tag":"I16","value":19},{"_tag":"Uuid","value":"70a54c31502299087a6d5816dce933bf"}],[{"_tag":"I16","value":-39},{"_tag":"Uuid","value":"d88af2b8c7a2d8ed192ef424000ca897"}],[{"_tag":"I16","value":0},{"_tag":"Uuid","value":"66fc0633e927425846d5dd82c2612a33"}],[{"_tag":"I16","value":161},{"_tag":"Uuid","value":"457f43a4532d514d516ff98846030489"}]]},{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}]}],[{"_tag":"Uuid","value":"3667697533aa273ff3d6986ba73cde69"},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-1},{"_tag":"I32","value":42},{"_tag":"I32","value":-956099871},{"_tag":"I32","value":-179430689},{"_tag":"I32","value":-47266032},{"_tag":"I32","value":-44},{"_tag":"I32","value":60},{"_tag":"I32","value":17652979},{"_tag":"I32","value":27},{"_tag":"I32","value":-2147483648},{"_tag":"I32","value":5339265},{"_tag":"I32","value":-42},{"_tag":"I32","value":-21024945},{"_tag":"I32","value":-40},{"_tag":"I32","value":3259053},{"_tag":"I32","value":5},{"_tag":"I32","value":-62},{"_tag":"I32","value":-61021},{"_tag":"I32","value":188},{"_tag":"I32","value":-1},{"_tag":"I32","value":-482902556},{"_tag":"I32","value":-1},{"_tag":"I32","value":13093309},{"_tag":"I32","value":-3931236},{"_tag":"I32","value":59645},{"_tag":"I32","value":7285},{"_tag":"I32","value":-51},{"_tag":"I32","value":-1},{"_tag":"I32","value":-1274179024},{"_tag":"I32","value":1}]}]]}},{"id":12597,"value":{"_tag":"I8","value":-4}},{"id":12603,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false}]}}]},"bytes":"07f2c301010000000000000004ccc401346b03dae41f49d7670636364198c194c0732d17e405feff03228d0774ab02bbed017480010001749c20b93f781cc001af87f68fa18a46c7c9ade876fb10034d4ce21cb09fcf30f75c4ac5124a174ed661431d8a11ab4d9ca573e5ab145aeb3d94e65c030ad13ab1e3acf0326cf7a4bdcd9064053401910500e00a0197c001f3feff03009f010247fa18fc10d69291b8b86838838d3945410ab63f011501bcf8b71800033dfe361b7cabe509fdc18f7069cb4687896fed11b1310d4319b10d6507d4bc3dd79fc0005ad68aaae05bfe79a6897205d41d47c10247032865b223a2fb26415fc086349e3fec1741017751d33d5ef08e28c1c01b518ec0f8f64003617402670196e7a48d9c92ff070101d387b76ad0c10b1d58a4284e4b61eef11e0b00054d278ed497cb0092f3091d0da0d437a29f2f668873b9fa6de019b745b6262c7113b11334dba1fdcb9277955451d11f8a9e67fb3e1bd123f96e88a31dc2a0dfca50bbd49d0928d32a5db45ec2c3ec622d9b218fc86cbd01612601021702000000000008864001000000000000f07f044d2670a54c31502299087a6d5816dce933bf4dd88af2b8c7a2d8ed192ef424000ca8970066fc0633e927425846d5dd82c2612a33c202457f43a4532d514d516ff98846030489003667697533aa273ff3d6986ba73cde69f51e0154bdb4e78f07c1948fab01dfe3892d5778e6f3ea1036ffffffff0f82e28b0553e1c286144fdaea8d030a7bb9b907f80201b788c4cc0301faa6be0cc7f1df03faa307ea7165019fb793bf090293fc6951010102010200"},{"id":"random/241","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"5e2b0fa1b1f3540ce0fd3eb7a76f89c0"}]}},{"id":-4,"value":{"_tag":"Bool","value":true}},{"id":-10032,"value":{"_tag":"I64","value":"-109763"}},{"id":31143,"value":{"_tag":"I32","value":-118}}]},"bytes":"a91d5e2b0fa1b1f3540ce0fd3eb7a76f89c0010706df9c0185b30d05cee603eb0100"},{"id":"random/242","type":"Struct","value":{"_tag":"Struct","fields":[{"id":72,"value":{"_tag":"Bool","value":true}},{"id":81,"value":{"_tag":"Double","value":-786.0}},{"id":154,"value":{"_tag":"Double","value":360569.71112599014}},{"id":161,"value":{"_tag":"Uuid","value":"8ffae65f4e2dd147b68eee93f1f88f0d"}},{"id":-3,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true},{"_tag":"Bool","value":true}]}}]},"bytes":"0190019700000000009088c007b4025c6931d8e60116417d8ffae65f4e2dd147b68eee93f1f88f0d0a05f1100201020202010101010101020201010100"},{"id":"random/243","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"I16","value":2}},{"id":22,"value":{"_tag":"I16","value":-14}},{"id":161,"value":{"_tag":"Bool","value":true}},{"id":171,"value":{"_tag":"Bool","value":false}},{"id":179,"value":{"_tag":"I32","value":-3}}]},"bytes":"7404f41b01c202a2850500"},{"id":"random/244","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Uuid","value":"30d44fd3ced386facfcd1029efb8ddbb"}},{"id":29,"value":{"_tag":"Bool","value":false}},{"id":35,"value":{"_tag":"I16","value":-3697}},{"id":42,"value":{"_tag":"Map","keyType":"Binary","valueType":"I16","entries":[[{"_tag":"Binary","value":"d2fffb1acc076e7d"},{"_tag":"I16","value":-17}],[{"_tag":"Binary","value":"6569c3bc707a7871f09f8c8d2f69"},{"_tag":"I16","value":-5}],[{"_tag":"Binary","value":""},{"_tag":"I16","value":-46}],[{"_tag":"Binary","value":"723cd7e9"},{"_tag":"I16","value":-62}],[{"_tag":"Binary","value":"622f6d7761646b6f6c666a6469"},{"_tag":"I16","value":-1}]]}}]},"bytes":"fd30d44fd3ced386facfcd1029efb8ddbbe264e1397b058408d2fffb1acc076e7d210e6569c3bc707a7871f09f8c8d2f6909005b04723cd7e97b0d622f6d7761646b6f6c666a64690100"},{"id":"random/245","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-27794,"value":{"_tag":"I8","value":0}},{"id":-27779,"value":{"_tag":"Map","keyType":"Uuid","valueType":"I32","entries":[[{"_tag":"Uuid","value":"ed88d2dec5a72b6d255aa2bbe06263ef"},{"_tag":"I32","value":-7677}]]}},{"id":-27775,"value":{"_tag":"I16","value":29}}]},"bytes":"03a3b20300fb01d5ed88d2dec5a72b6d255aa2bbe06263eff977443a00"},{"id":"random/246","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/247","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Uuid","value":"2c1b52b1cc5caf4ac399e4b3f25339dc"}}]},"bytes":"cd2c1b52b1cc5caf4ac399e4b3f25339dc00"},{"id":"random/248","type":"Struct","value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"2355"},{"_tag":"I64","value":"-1"},{"_tag":"I64","value":"27"},{"_tag":"I64","value":"-580172990"},{"_tag":"I64","value":"192173436906"},{"_tag":"I64","value":"16"}]}}]}},{"id":21,"value":{"_tag":"I8","value":1}},{"id":34,"value":{"_tag":"I16","value":-923}}]},"bytes":"cc7a66e6240136fbf2a5a904d4bfefe6970b20009301d4b50e00"},{"id":"random/249","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Uuid","value":"0bee3a846cbe31a3809e4cec777555d0"}},{"id":3179,"value":{"_tag":"I16","value":53}},{"id":3187,"value":{"_tag":"Bool","value":false}},{"id":3255,"value":{"_tag":"I8","value":-19}},{"id":3260,"value":{"_tag":"Double","value":"-0"}}]},"bytes":"fd0bee3a846cbe31a3809e4cec777555d004d6316a8203ee32ed57000000000000008000"},{"id":"random/250","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/251","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Double","value":-185037.2130638964}},{"id":8,"value":{"_tag":"I16","value":-3037}},{"id":11,"value":{"_tag":"I16","value":5}},{"id":15,"value":{"_tag":"Bool","value":true}},{"id":300,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"4"},{"_tag":"I64","value":"55122906956845"},{"_tag":"I64","value":"-3582464459519774707"}]}},{"id":17278,"value":{"_tag":"Bool","value":true}}]},"bytes":"6718d85ab4699606c124b92f340a4109d8043608da90ced4c98819e5bf9aa2bf9cbcb76301fc8d0200"},{"id":"random/252","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I64","value":"-44"}},{"id":14,"value":{"_tag":"Double","value":-729.0}},{"id":17,"value":{"_tag":"Double","value":986.0}},{"id":165,"value":{"_tag":"Uuid","value":"1af3c194aafdce4286ae48abe1aaf876"}},{"id":174,"value":{"_tag":"I32","value":-18285}},{"id":-21448,"value":{"_tag":"Uuid","value":"8d5769569d1357304df2230cdc255b81"}}]},"bytes":"9657570000000000c886c0370000000000d08e400dca021af3c194aafdce4286ae48abe1aaf87695d99d020d8fcf028d5769569d1357304df2230cdc255b8100"},{"id":"random/253","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"Bool","value":true}},{"id":16,"value":{"_tag":"Uuid","value":"05a2d3fada5b0cf79d6afe61446b8547"}},{"id":25,"value":{"_tag":"I16","value":1}}]},"bytes":"f11d05a2d3fada5b0cf79d6afe61446b8547940200"},{"id":"random/254","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I16","value":-1230}}]},"bytes":"949b1300"},{"id":"random/255","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-13396,"value":{"_tag":"I32","value":-69620}},{"id":-13298,"value":{"_tag":"I32","value":-33247478}},{"id":-13288,"value":{"_tag":"Double","value":799.0}}]},"bytes":"05a7d101e7bf0805e3cf01ebc3da1fa70000000000f8884000"},{"id":"random/256","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I32","value":34}},{"id":4,"value":{"_tag":"I64","value":"14377669819503"}},{"id":5,"value":{"_tag":"Binary","value":"5f782f6af09f8c8d77f09f8c8d"}},{"id":144,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"8b4d66ec69940872c85589d533f7593bad4bd6c54d9dd43dec5e134e33ecd36a20a8bdf86e5e99fe8a66c8aa5d93667d9b919a381905c0de86acaf8a4183e18709304bbec2189c89d4ff9c87cc6acfe00833c8c449ab98b43cfa88b36eeecbf4c4a1df6ad866dbe55adbf8683f7ac42452229771d585c049a30105347e637c0d"},{"_tag":"Binary","value":"0732af02e57a71191e76c90de7a277c0"},{"_tag":"Binary","value":"d18ce00015959584"},{"_tag":"Binary","value":"652a6f2e3e1718b38f9b33a6"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"b2dc89f0e35f4882c494e7"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"c3bc6568726363e697a52020"},{"_tag":"Binary","value":"0bb6ab033541889f6a8b714a8be6cc0c281ae8"},{"_tag":"Binary","value":"a3902816fc82b95176497768dc2154db290d564f9538d4e14480263fe4b70af7451b61ffb4a8c4a81ea6ac49924f9621c5a327b073ec862ad8c2cab08baf00d7376bd47b99c24d4f33024c751c3b3b6eab77e8fc4861fcef47c6cd4a555ebdbfa4a43d90c862b038693568a406b51185b561fadf282dead9672dcb819c102ee3"},{"_tag":"Binary","value":"7978672065c3a9792d2d72"},{"_tag":"Binary","value":"57fc2c0360a36ddbb9afc428191538"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"6a5f7672202d7a206c71c3a97a726c75"},{"_tag":"Binary","value":"655f73c3bc757a20206a73746ae697a56f66"}]}},{"id":147,"value":{"_tag":"I16","value":-1}},{"id":368,"value":{"_tag":"Binary","value":"4fcbf8605ddb"}}]},"bytes":"254426def1b7fef1c406180d5f782f6af09f8c8d77f09f8c8d0aa002f80f80018b4d66ec69940872c85589d533f7593bad4bd6c54d9dd43dec5e134e33ecd36a20a8bdf86e5e99fe8a66c8aa5d93667d9b919a381905c0de86acaf8a4183e18709304bbec2189c89d4ff9c87cc6acfe00833c8c449ab98b43cfa88b36eeecbf4c4a1df6ad866dbe55adbf8683f7ac42452229771d585c049a30105347e637c0d100732af02e57a71191e76c90de7a277c008d18ce000159595840c652a6f2e3e1718b38f9b33a6000bb2dc89f0e35f4882c494e7000cc3bc6568726363e697a52020130bb6ab033541889f6a8b714a8be6cc0c281ae88001a3902816fc82b95176497768dc2154db290d564f9538d4e14480263fe4b70af7451b61ffb4a8c4a81ea6ac49924f9621c5a327b073ec862ad8c2cab08baf00d7376bd47b99c24d4f33024c751c3b3b6eab77e8fc4861fcef47c6cd4a555ebdbfa4a43d90c862b038693568a406b51185b561fadf282dead9672dcb819c102ee30b7978672065c3a9792d2d720f57fc2c0360a36ddbb9afc42819153800106a5f7672202d7a206c71c3a97a726c7512655f73c3bc757a20206a73746ae697a56f66340108e005064fcbf8605ddb00"},{"id":"random/257","type":"Struct","value":{"_tag":"Struct","fields":[{"id":17719,"value":{"_tag":"Map","keyType":"Struct","valueType":"I64","entries":[[{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"Bool","value":false}}]},{"_tag":"I64","value":"32866065"}]]}},{"id":-30999,"value":{"_tag":"I32","value":753}},{"id":-30885,"value":{"_tag":"I16","value":-1}},{"id":-30870,"value":{"_tag":"I8","value":2}},{"id":-30863,"value":{"_tag":"I8","value":10}},{"id":-1151,"value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Binary","value":"747a6c682f2f2071776a2e7368656c716b796e707978c3bc76712d69726b2fc3bcc3a972627975737273722e6575757469786a6864612d71762f6a6320736474746ff09f8c8d6863766b722e7320c3bc6c78716b61206de697a567626bf09f8c8d792d2ff09f8c8d6e7167f09f8c8d672f65766c2d6b2d667a726e2f66777069c3a9f09f8c8d696c68202d67797373642d205f657777756fc3a9f09f8c8d745f7579756b762f71e697a56a75c3bc6c7a67c3a96cc3a9797263757a6264e697a5626365745f78c3a9676d2f7062626168622064206b7869c3bc6c6f66e697a5f09f8c8d632ef09f8c8dc3bc792e67f09f8c8d2f6f627964"}},{"id":-5,"value":{"_tag":"I8","value":0}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":-983,"value":{"_tag":"Double","value":940.0}}]},"bytes":"0bee940201c6d200a2fcab1f05ade403e20b04c9e20301f302730a0cfd11c8f701747a6c682f2f2071776a2e7368656c716b796e707978c3bc76712d69726b2fc3bcc3a972627975737273722e6575757469786a6864612d71762f6a6320736474746ff09f8c8d6863766b722e7320c3bc6c78716b61206de697a567626bf09f8c8d792d2ff09f8c8d6e7167f09f8c8d672f65766c2d6b2d667a726e2f66777069c3a9f09f8c8d696c68202d67797373642d205f657777756fc3a9f09f8c8d745f7579756b762f71e697a56a75c3bc6c7a67c3a96cc3a9797263757a6264e697a5626365745f78c3a9676d2f7062626168622064206b7869c3bc6c6f66e697a5f09f8c8d632ef09f8c8dc3bc792e67f09f8c8d2f6f627964030900710007ad0f0000000000608d4000"},{"id":"random/258","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Uuid","value":"458ab34a34e536d34063e3c4e324a477"}}]},"bytes":"6d458ab34a34e536d34063e3c4e324a47700"},{"id":"random/259","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"Double","value":805.0}},{"id":9,"value":{"_tag":"I8","value":127}},{"id":3,"value":{"_tag":"Binary","value":"74642bd1829d66e088ae"}}]},"bytes":"07070000000000288940d37f08060a74642bd1829d66e088ae00"},{"id":"random/260","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"Double","value":-616.0}},{"id":2,"value":{"_tag":"I32","value":-1}},{"id":68,"value":{"_tag":"I64","value":"9"}},{"id":76,"value":{"_tag":"I16","value":-5013}},{"id":-16705,"value":{"_tag":"Uuid","value":"e14e23d2280e68c004a8655d605125e2"}},{"id":-16695,"value":{"_tag":"Double","value":-217550.9387380439}},{"id":-16666,"value":{"_tag":"Bool","value":true}},{"id":-16655,"value":{"_tag":"I8","value":-1}}]},"bytes":"070700000000004083c065010688011284a94e0d818502e14e23d2280e68c004a8655d605125e2a770178982778e0ac101b38402b3ff00"},{"id":"random/261","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/262","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/263","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"I16","value":23}},{"id":13,"value":{"_tag":"I64","value":"-11220572"}},{"id":256,"value":{"_tag":"Bool","value":false}}]},"bytes":"642e76b7d9d90a02800400"},{"id":"random/264","type":"Struct","value":{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"Uuid","value":"d4c7e889646c45c1afed0007faa2be33"}},{"id":18,"value":{"_tag":"I64","value":"768032477497"}},{"id":23,"value":{"_tag":"I16","value":-58}}]},"bytes":"ddd4c7e889646c45c1afed0007faa2be3356f2c4eca4da2c547300"},{"id":"random/265","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/266","type":"Struct","value":{"_tag":"Struct","fields":[{"id":208,"value":{"_tag":"I32","value":2}},{"id":-2,"value":{"_tag":"Binary","value":"586008ee5e102f75fa553eebe15b9fe6d16be8c6103dc5f19119d0e15ef72a43e35a96d465f13e0359ccbe85ba178392237b26e78b47e816a13b2d0a77436afab8c72c55ea3d1700051dd0b8cd642042316a046b3050e0469b9712ffbe022a39615af806c0256b8ac8d5baf58f5891e8000402fd604b6e9ae583e9cfb9e0db"}},{"id":10,"value":{"_tag":"Binary","value":"07"}},{"id":21,"value":{"_tag":"I32","value":-21}}]},"bytes":"05a0030408037f586008ee5e102f75fa553eebe15b9fe6d16be8c6103dc5f19119d0e15ef72a43e35a96d465f13e0359ccbe85ba178392237b26e78b47e816a13b2d0a77436afab8c72c55ea3d1700051dd0b8cd642042316a046b3050e0469b9712ffbe022a39615af806c0256b8ac8d5baf58f5891e8000402fd604b6e9ae583e9cfb9e0dbc80107b52900"},{"id":"random/267","type":"Struct","value":{"_tag":"Struct","fields":[{"id":13,"value":{"_tag":"I8","value":-7}},{"id":23,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"18afcf4243a42954dc94c2cc3930a888"},{"_tag":"Uuid","value":"c9fff42afa47d66a0bd8c9e7452bcb75"},{"_tag":"Uuid","value":"d99a746b3f24cc39ef60b547eff9f84e"},{"_tag":"Uuid","value":"0925292e1de197fcc75fbee648c3e046"},{"_tag":"Uuid","value":"25451a968ee85cbe1a6da76439838d93"}]}}]},"bytes":"d3f9aa5d18afcf4243a42954dc94c2cc3930a888c9fff42afa47d66a0bd8c9e7452bcb75d99a746b3f24cc39ef60b547eff9f84e0925292e1de197fcc75fbee648c3e04625451a968ee85cbe1a6da76439838d9300"},{"id":"random/268","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15989,"value":{"_tag":"I16","value":119}}]},"bytes":"04eaf901ee0100"},{"id":"random/269","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Bool","value":true}},{"id":14,"value":{"_tag":"Binary","value":"637870"}},{"id":25,"value":{"_tag":"Bool","value":false}},{"id":187,"value":{"_tag":"Bool","value":false}},{"id":391,"value":{"_tag":"I64","value":"0"}},{"id":2,"value":{"_tag":"Uuid","value":"1ee2b5a090ab227ecefb43a5a998e68d"}},{"id":11,"value":{"_tag":"I32","value":-69}}]},"bytes":"b13803637870b202f602068e06000d041ee2b5a090ab227ecefb43a5a998e68d95890100"},{"id":"random/270","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-29510,"value":{"_tag":"I32","value":-62}},{"id":-29496,"value":{"_tag":"Binary","value":"616e612de697a5c3bcf09f8c8d6365c3a96262612d6b76786e2fe697a5"}},{"id":-29492,"value":{"_tag":"Double","value":"NaN"}},{"id":0,"value":{"_tag":"I64","value":"1"}},{"id":10,"value":{"_tag":"Bool","value":true}}]},"bytes":"058bcd037be81d616e612de697a5c3bcf09f8c8d6365c3a96262612d6b76786e2fe697a547000000000000f87f060002a100"},{"id":"random/271","type":"Struct","value":{"_tag":"Struct","fields":[{"id":102,"value":{"_tag":"Uuid","value":"3295c403a3acd9d50714ace37a2aaf13"}},{"id":369,"value":{"_tag":"Set","elementType":"Set","elements":[{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":2147483647},{"_tag":"I32","value":-53},{"_tag":"I32","value":63},{"_tag":"I32","value":-20},{"_tag":"I32","value":51},{"_tag":"I32","value":4750}]}]}},{"id":5176,"value":{"_tag":"Uuid","value":"e472e18a92db317e561c7a4ae9878ebf"}},{"id":5185,"value":{"_tag":"Map","keyType":"I8","valueType":"I64","entries":[[{"_tag":"I8","value":0},{"_tag":"I64","value":"-41061651610610900"}],[{"_tag":"I8","value":-13},{"_tag":"I64","value":"-114794"}],[{"_tag":"I8","value":-1},{"_tag":"I64","value":"-1"}]]}},{"id":5187,"value":{"_tag":"I16","value":-9322}},{"id":5410,"value":{"_tag":"I16","value":50}}]},"bytes":"0dcc013295c403a3acd9d50714ace37a2aaf130ae2051a65feffffff0f697e27669c4a0df050e472e18a92db317e561c7a4ae9878ebf9b033600a7b3a1e2d8d6f09101f3d3810eff0124d3910104c4546400"},{"id":"random/272","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"I32","value":53}},{"id":228,"value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"I16","value":-21}},{"id":9,"value":{"_tag":"Bool","value":false}},{"id":-2,"value":{"_tag":"I64","value":"-42447337395"}},{"id":5,"value":{"_tag":"I32","value":116}}]}},{"id":237,"value":{"_tag":"I16","value":98}},{"id":242,"value":{"_tag":"I32","value":-26}},{"id":249,"value":{"_tag":"I32","value":46}},{"id":254,"value":{"_tag":"Uuid","value":"cc11e6df58962c1d5eef8bfe6335b166"}},{"id":259,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"fef39c0ff6efcc920fe73eecfa3176f7"},{"_tag":"Uuid","value":"eadb96f70ba63b3e0a8cea5cb151cb99"}]}}]},"bytes":"b56a0cc8037429220603e5def7a0bc0275e8010094c4015533755c5dcc11e6df58962c1d5eef8bfe6335b1665a2dfef39c0ff6efcc920fe73eecfa3176f7eadb96f70ba63b3e0a8cea5cb151cb9900"},{"id":"random/273","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I16","value":58}},{"id":61,"value":{"_tag":"Double","value":-195.0}},{"id":32574,"value":{"_tag":"Struct","fields":[{"id":12,"value":{"_tag":"Binary","value":"ee17909e"}},{"id":27,"value":{"_tag":"I32","value":-5}}]}},{"id":32767,"value":{"_tag":"Map","keyType":"I32","valueType":"Binary","entries":[[{"_tag":"I32","value":28},{"_tag":"Binary","value":"722ec3bc"}]]}},{"id":32767,"value":{"_tag":"Bool","value":true}},{"id":-11194,"value":{"_tag":"Uuid","value":"6746fbefa9bbe595352781d5fadc4ece"}}]},"bytes":"1474077a00000000006068c00cfcfc03c804ee17909ef509000bfeff0301583804722ec3bc01feff030df3ae016746fbefa9bbe595352781d5fadc4ece00"},{"id":"random/274","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"I32","value":156530}},{"id":-29261,"value":{"_tag":"Uuid","value":"c4760bc3373c07058afcb841271074e5"}},{"id":0,"value":{"_tag":"I16","value":-3565}}]},"bytes":"a5e48d130d99c903c4760bc3373c07058afcb841271074e50400d93700"},{"id":"random/275","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":514}}]},"bytes":"34840800"},{"id":"random/276","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"Binary","value":"6c6a7971"}}]},"bytes":"e8046c6a797100"},{"id":"random/277","type":"Struct","value":{"_tag":"Struct","fields":[{"id":19925,"value":{"_tag":"I8","value":34}},{"id":-1,"value":{"_tag":"Bool","value":false}},{"id":13,"value":{"_tag":"Bool","value":true}},{"id":38,"value":{"_tag":"I16","value":-1264}},{"id":47,"value":{"_tag":"Set","elementType":"Uuid","elements":[]}}]},"bytes":"03aab702220201e1044cdf139a0d00"},{"id":"random/278","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I16","value":-2297}},{"id":16,"value":{"_tag":"I8","value":6}},{"id":2,"value":{"_tag":"I16","value":27}},{"id":15,"value":{"_tag":"I64","value":"-20"}},{"id":22,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"e697a52e2ee697a5702f632e2e63"},{"_tag":"Binary","value":"f3"},{"_tag":"Binary","value":"6026"},{"_tag":"Binary","value":"6f6974776963f09f8c8d6466676e2f76796620697178e697a5f09f8c8de697a578e697a56877682e666f787464706ec3a973746962776a687670c3a97067655f6965717a2f6272c3bc6464726171f09f8c8d65635f74667a5f7af09f8c8d6d7a206579742e2d71626d6e6572776c2e71776a626a71736f71706975c3a96970797a6f6be697a5776a7371685f737263747669697761617a652d7769e697a56d6d5f736a646f755f6c2d6964e697a56c796f5f612d70686c6dc3a92073786b70c3a96676632d775f2e726f6220746e706d68636b716e6f2f636c716c6e6b6d616462757370785f"},{"_tag":"Binary","value":"18def721ce490caaef63318d4b"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"3c663a116a4132ff24cc6afb"},{"_tag":"Binary","value":"1d"},{"_tag":"Binary","value":"1bea24a6df875caf1db2ed"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"566d"},{"_tag":"Binary","value":"792e6e2ec3a9786664"},{"_tag":"Binary","value":"10d9be4b8e9f3329ac601b83f375d48445c536fa2ef6090100e9efee322c6d8abd6ef934a91f1597ab4d47892bbc782e149224a7d86a0d13ead806c08cb1837c710f3833732c34b184f862ca3984c7d1017343408bf68b31e6037de728a3821061b76465cf8181975064174569cffcba0b725def043fa2ad0b4b9991c39f742afab4545370bebff7f2b09c6f59a607ccc464c8d8f29476164248907a56775145715fd0fa29837552554d40f6d9b8be8e63670bc4887f4c418f2863e4039b0c6fd168962ffc89646b"},{"_tag":"Binary","value":"796d6372622e7620627879"},{"_tag":"Binary","value":"736f64"}]}},{"id":260,"value":{"_tag":"I32","value":-4}},{"id":456,"value":{"_tag":"I8","value":0}}]},"bytes":"94f1237306040436d62779f80f0ee697a52e2ee697a5702f632e2e6301f3026026e6016f6974776963f09f8c8d6466676e2f76796620697178e697a5f09f8c8de697a578e697a56877682e666f787464706ec3a973746962776a687670c3a97067655f6965717a2f6272c3bc6464726171f09f8c8d65635f74667a5f7af09f8c8d6d7a206579742e2d71626d6e6572776c2e71776a626a71736f71706975c3a96970797a6f6be697a5776a7371685f737263747669697761617a652d7769e697a56d6d5f736a646f755f6c2d6964e697a56c796f5f612d70686c6dc3a92073786b70c3a96676632d775f2e726f6220746e706d68636b716e6f2f636c716c6e6b6d616462757370785f0d18def721ce490caaef63318d4b000c3c663a116a4132ff24cc6afb011d0b1bea24a6df875caf1db2ed0002566d09792e6e2ec3a9786664c80110d9be4b8e9f3329ac601b83f375d48445c536fa2ef6090100e9efee322c6d8abd6ef934a91f1597ab4d47892bbc782e149224a7d86a0d13ead806c08cb1837c710f3833732c34b184f862ca3984c7d1017343408bf68b31e6037de728a3821061b76465cf8181975064174569cffcba0b725def043fa2ad0b4b9991c39f742afab4545370bebff7f2b09c6f59a607ccc464c8d8f29476164248907a56775145715fd0fa29837552554d40f6d9b8be8e63670bc4887f4c418f2863e4039b0c6fd168962ffc89646b0b796d6372622e762062787903736f64058804070390070000"},{"id":"random/279","type":"Struct","value":{"_tag":"Struct","fields":[{"id":151,"value":{"_tag":"I16","value":-51}},{"id":152,"value":{"_tag":"I32","value":-437904}}]},"bytes":"04ae0265159fba3500"},{"id":"random/280","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/281","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/282","type":"Struct","value":{"_tag":"Struct","fields":[{"id":150,"value":{"_tag":"I8","value":-1}},{"id":426,"value":{"_tag":"Bool","value":true}},{"id":431,"value":{"_tag":"I16","value":30}},{"id":444,"value":{"_tag":"I32","value":263800}},{"id":459,"value":{"_tag":"I8","value":-34}},{"id":472,"value":{"_tag":"Bool","value":true}},{"id":-7157,"value":{"_tag":"Double","value":-413028.2065547373}},{"id":-25298,"value":{"_tag":"Map","keyType":"I64","valueType":"Bool","entries":[[{"_tag":"I64","value":"19"},{"_tag":"Bool","value":true}],[{"_tag":"I64","value":"-1"},{"_tag":"Bool","value":true}],[{"_tag":"I64","value":"50"},{"_tag":"Bool","value":false}],[{"_tag":"I64","value":"-621"},{"_tag":"Bool","value":false}]]}}]},"bytes":"03ac02ff01d406543cd5f09920f3ded107e96fc61583d3903519c10ba38b030461260101016402d9090200"},{"id":"random/283","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-4,"value":{"_tag":"I64","value":"292721"}},{"id":8,"value":{"_tag":"Double","value":-605871.3666418185}},{"id":2,"value":{"_tag":"Bool","value":true}},{"id":14,"value":{"_tag":"I8","value":-1}},{"id":274,"value":{"_tag":"I64","value":"1"}},{"id":275,"value":{"_tag":"I8","value":0}},{"id":-4,"value":{"_tag":"I32","value":58}}]},"bytes":"0607e2dd23c7f879b8bb5e7d22c10104c3ff06a40402130005077400"},{"id":"random/284","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Double","value":-438580.8007887597}},{"id":288,"value":{"_tag":"I64","value":"-41"}},{"id":465,"value":{"_tag":"Double","value":590866.1419729535}},{"id":4,"value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"Bool","value":true}},{"id":14,"value":{"_tag":"Map","keyType":"Binary","valueType":"I8","entries":[[{"_tag":"Binary","value":"c9c2cb809394e1368bbdf605049f9ef7cb"},{"_tag":"I8","value":-1}],[{"_tag":"Binary","value":"a1"},{"_tag":"I8","value":1}],[{"_tag":"Binary","value":"6f63666e6c747867"},{"_tag":"I8","value":-36}]]}},{"id":27,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"fee9acf31c96ce40da75"}},{"id":85,"value":{"_tag":"I64","value":"-12"}},{"id":156,"value":{"_tag":"I32","value":-15}},{"id":4,"value":{"_tag":"Double","value":3.0158664289042023e-149}},{"id":36,"value":{"_tag":"I16","value":-46}},{"id":45,"value":{"_tag":"Binary","value":"626378e697a520e697a57069716d"}},{"id":53,"value":{"_tag":"I64","value":"673629632"}}]}},{"id":30,"value":{"_tag":"Uuid","value":"c6544f516607ed928cd83fa6fc76c33e"}},{"id":274,"value":{"_tag":"I16","value":58}},{"id":306,"value":{"_tag":"I8","value":-57}},{"id":319,"value":{"_tag":"I32","value":-17}}]}},{"id":5,"value":{"_tag":"I16","value":-219}},{"id":1,"value":{"_tag":"Uuid","value":"df0022dcec97a4a8426e01d15f865b6c"}},{"id":-28771,"value":{"_tag":"Bool","value":true}}]},"bytes":"17f8f70134d3c41ac106c0045107a207d0adb048240822410c08717b038311c9c2cb809394e1368bbdf605049f9ef7cbff01a101086f63666e6c747867dcdc180afee9acf31c96ce40da7506aa011705b8021d0708b091165026ae182104485b980e626378e697a520e697a57069716d868097b68205003dc6544f516607ed928cd83fa6fc76c33e04a4047403e404c7d5210014b5030d02df0022dcec97a4a8426e01d15f865b6c01c5c10300"},{"id":"random/285","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I16","value":8}},{"id":299,"value":{"_tag":"I64","value":"43804281748513120"}},{"id":506,"value":{"_tag":"Map","keyType":"I32","valueType":"I64","entries":[[{"_tag":"I32","value":1399},{"_tag":"I64","value":"43"}],[{"_tag":"I32","value":2147483647},{"_tag":"I64","value":"32745"}]]}},{"id":-28813,"value":{"_tag":"Double","value":474.0}},{"id":-28721,"value":{"_tag":"I64","value":"57"}},{"id":-28478,"value":{"_tag":"Double","value":44111.66167533293}},{"id":31977,"value":{"_tag":"Bool","value":false}},{"id":31984,"value":{"_tag":"Uuid","value":"2a3d18aa46eb29724188166ffeaf3980"}}]},"bytes":"241006d604c08597d4e3f0cf9b010bf4070256ee1556feffffff0fd2ff030799c2030000000000a07d4006e1c0037207fbbc0370bf712cf589e54002d2f3037d2a3d18aa46eb29724188166ffeaf398000"},{"id":"random/286","type":"Struct","value":{"_tag":"Struct","fields":[{"id":11,"value":{"_tag":"Bool","value":false}},{"id":24,"value":{"_tag":"Binary","value":"6c"}},{"id":34,"value":{"_tag":"Binary","value":"c3bc736a6f676664f09f8c8d74c3bc772067c3bc656778f09f8c8d2f6177657179c3a970622e7662672f616d6e2fc3a96c6c6e6968685f79682d61772d2d756d78c3a96876772e7120e697a5726e752f2e7a616e686971726b7468f09f8c8d617a66c3a964c3a96e626de697a56a61687072656165742e6f6f7063676b6d69c3bc6c62f09f8c8d73686f7973656d65776cc3bc6dc3bc775f6b2e"}},{"id":43,"value":{"_tag":"Struct","fields":[{"id":-2,"value":{"_tag":"I64","value":"869522"}},{"id":8,"value":{"_tag":"Bool","value":true}},{"id":28177,"value":{"_tag":"Bool","value":true}}]}}]},"bytes":"b2d8016ca89a01c3bc736a6f676664f09f8c8d74c3bc772067c3bc656778f09f8c8d2f6177657179c3a970622e7662672f616d6e2fc3a96c6c6e6968685f79682d61772d2d756d78c3a96876772e7120e697a5726e752f2e7a616e686971726b7468f09f8c8d617a66c3a964c3a96e626de697a56a61687072656165742e6f6f7063676b6d69c3bc6c62f09f8c8d73686f7973656d65776cc3bc6dc3bc775f6b2e9c0603a4926aa101a2b8030000"},{"id":"random/287","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"Double","value":200533.15863065305}},{"id":13,"value":{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":-24.0},{"_tag":"Double","value":-33186.76069079852},{"_tag":"Double","value":4.2823435886387345e-248},{"_tag":"Double","value":"Infinity"},{"_tag":"Double","value":-782.0},{"_tag":"Double","value":-690992.3127693953}]}},{"id":25,"value":{"_tag":"I8","value":14}}]},"bytes":"87d825e044a97a0841596700000000000038c0c03a94575834e0c02b949541aa29930c000000000000f07f00000000007088c0684f23a0601625c1c30e00"},{"id":"random/288","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I8","value":5}},{"id":15,"value":{"_tag":"I8","value":-4}},{"id":24,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"7a6670e697a5c3bc7475c3a9686a666b696274776be697a578"}]}},{"id":30,"value":{"_tag":"Struct","fields":[{"id":17972,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"e3ef4d215625ece32ae8ceaf63647a47"},{"_tag":"Uuid","value":"9e2a300ec0f0aed8af783eb0a0827b48"},{"_tag":"Uuid","value":"bac1e77ebc86e99c021ea2efd9f3d3b1"},{"_tag":"Uuid","value":"05a91347134b5127e41c97154b0312c5"},{"_tag":"Uuid","value":"b4c72dcb7c26a90c3777c8cbb3c003d0"},{"_tag":"Uuid","value":"55bca5de0cd8739c2ca139d5fa335f10"},{"_tag":"Uuid","value":"8f89de6b7cba1253f6a7266b02c3a8d1"},{"_tag":"Uuid","value":"7c90405420587c4348175013b9e98d55"},{"_tag":"Uuid","value":"af9ed125f5f464da3247b73b21e37e55"},{"_tag":"Uuid","value":"64146770f0e2fbaf1167f1c206af6814"},{"_tag":"Uuid","value":"9d1d3c6572a4a59ee76b872d4b467bc2"},{"_tag":"Uuid","value":"f063ae8f398a0f141f3f3cf0ec93881a"},{"_tag":"Uuid","value":"a7d716fb9ed985b8b7b8d23bf09213e6"},{"_tag":"Uuid","value":"a4936aa7d5f2ffe639c1025fc6632efb"},{"_tag":"Uuid","value":"924f167b0b78f9dd7cf73626c9078612"},{"_tag":"Uuid","value":"b21d08805aebc2d6ebe3aa69870dd542"},{"_tag":"Uuid","value":"04e21c0fba3040a837097dcdf7db847d"},{"_tag":"Uuid","value":"b17f92206a792d89605a510e1e8d0068"},{"_tag":"Uuid","value":"3a753bcdfca1ba6669fdf9d0647afe90"},{"_tag":"Uuid","value":"851586ae113cf19f74774a835b6d7f6d"},{"_tag":"Uuid","value":"a2cffa8d7e4094d95b2161d5cf2f107f"},{"_tag":"Uuid","value":"e169cca1f58ac824cfb97df7429359dd"},{"_tag":"Uuid","value":"41336f31149986a1771c395f4e6a0ca0"},{"_tag":"Uuid","value":"b3dec806b5ce1d93855b7ee1078fabac"},{"_tag":"Uuid","value":"30a19b890d3f2037f76f0928c0d2f073"},{"_tag":"Uuid","value":"d0fc982418ed0749b234e0922f5f32e9"},{"_tag":"Uuid","value":"acf623701fd2adfd01578bc102b6641e"},{"_tag":"Uuid","value":"58b272fc044123f0bf50b07c80318b23"},{"_tag":"Uuid","value":"e684f67adb830bda159c585b10d83e7c"},{"_tag":"Uuid","value":"53861e979122a972f1319f52e2af5f1d"}]}},{"id":17980,"value":{"_tag":"I16","value":5}},{"id":18220,"value":{"_tag":"Set","elementType":"Set","elements":[{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-2},{"_tag":"I8","value":5}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":1229}]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-1047533},{"_tag":"I32","value":-4900498}]},{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"106"}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":3},{"_tag":"I16","value":23}]},{"_tag":"Set","elementType":"I8","elements":[]},{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":-54},{"_tag":"I32","value":70510}]},{"_tag":"Set","elementType":"I8","elements":[]},{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":974698.8653169367}]},{"_tag":"Set","elementType":"Double","elements":[{"_tag":"Double","value":934270.2477328258}]},{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-2}]},{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":-46},{"_tag":"I8","value":-20}]},{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":8},{"_tag":"I16","value":-1}]}]}},{"id":-12432,"value":{"_tag":"Double","value":22641.759094976354}},{"id":-12418,"value":{"_tag":"I64","value":"30807909"}}]}},{"id":41,"value":{"_tag":"Bool","value":false}},{"id":55,"value":{"_tag":"Binary","value":"6a7574625f7462732fc3bc647972752d67647877c3a961f09f8c8dc3bc746f2fe697a565766ee697a571207864777374766b64c3a965667379736363716820672d716a765f672079716274787977756f6569772f6c2d2ec3a92d74757472787020717a786bf09f8c8d7269c3a97a2f6f7873776967766f68636f6d6564677774672e657367762f66672dc3bc72722df09f8c8d2f"}}]},"bytes":"2305d3fc9a18197a6670e697a5c3bc7475c3a9686a666b696274776be697a5786c0ae89802fd1ee3ef4d215625ece32ae8ceaf63647a479e2a300ec0f0aed8af783eb0a0827b48bac1e77ebc86e99c021ea2efd9f3d3b105a91347134b5127e41c97154b0312c5b4c72dcb7c26a90c3777c8cbb3c003d055bca5de0cd8739c2ca139d5fa335f108f89de6b7cba1253f6a7266b02c3a8d17c90405420587c4348175013b9e98d55af9ed125f5f464da3247b73b21e37e5564146770f0e2fbaf1167f1c206af68149d1d3c6572a4a59ee76b872d4b467bc2f063ae8f398a0f141f3f3cf0ec93881aa7d716fb9ed985b8b7b8d23bf09213e6a4936aa7d5f2ffe639c1025fc6632efb924f167b0b78f9dd7cf73626c9078612b21d08805aebc2d6ebe3aa69870dd54204e21c0fba3040a837097dcdf7db847db17f92206a792d89605a510e1e8d00683a753bcdfca1ba6669fdf9d0647afe90851586ae113cf19f74774a835b6d7f6da2cffa8d7e4094d95b2161d5cf2f107fe169cca1f58ac824cfb97df7429359dd41336f31149986a1771c395f4e6a0ca0b3dec806b5ce1d93855b7ee1078fabac30a19b890d3f2037f76f0928c0d2f073d0fc982418ed0749b234e0922f5f32e9acf623701fd2adfd01578bc102b6641e58b272fc044123f0bf50b07c80318b23e684f67adb830bda159c585b10d83e7c53861e979122a972f1319f52e2af5f1d840a0ad89c02ea23fe05149a1325d9ef7fa39ad60416d40124062e03256bdccd08032102011750d20abbd5be2d411742d6d67efc822c4113fe23d2ec241001079fc20180180395701cd640e6caddb01d00b2e894016a7574625f7462732fc3bc647972752d67647877c3a961f09f8c8dc3bc746f2fe697a565766ee697a571207864777374766b64c3a965667379736363716820672d716a765f672079716274787977756f6569772f6c2d2ec3a92d74757472787020717a786bf09f8c8d7269c3a97a2f6f7873776967766f68636f6d6564677774672e657367762f66672dc3bc72722df09f8c8d2f00"},{"id":"random/289","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/290","type":"Struct","value":{"_tag":"Struct","fields":[{"id":6,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"39d70f29cd8684a64aee22df4577bf"},{"_tag":"Binary","value":"d68dbab0f9"},{"_tag":"Binary","value":"f09f8c8d736a2f796178f09f8c8d736f71716264"}]}},{"id":-2,"value":{"_tag":"Bool","value":false}}]},"bytes":"6a380f39d70f29cd8684a64aee22df4577bf05d68dbab0f914f09f8c8d736a2f796178f09f8c8d736f71716264020300"},{"id":"random/291","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}},{"id":0,"value":{"_tag":"I8","value":-12}},{"id":14,"value":{"_tag":"I32","value":-6030293}},{"id":59,"value":{"_tag":"Double","value":-933.0}},{"id":63,"value":{"_tag":"Binary","value":"742d746c6f76756472"}},{"id":318,"value":{"_tag":"Binary","value":"67706965c3bc20c3a96a7766"}},{"id":325,"value":{"_tag":"Bool","value":true}}]},"bytes":"0b010013f4e5a98fe00507760000000000288dc04809742d746c6f7675647208fc040c67706965c3bc20c3a96a77667100"},{"id":"random/292","type":"Struct","value":{"_tag":"Struct","fields":[{"id":294,"value":{"_tag":"I32","value":-3983477}},{"id":488,"value":{"_tag":"Uuid","value":"4239bf5b7b310cd0b1c3eb8da7b2fab0"}},{"id":489,"value":{"_tag":"I32","value":2083209860}},{"id":-7560,"value":{"_tag":"I16","value":4}},{"id":-7551,"value":{"_tag":"Bool","value":false}},{"id":-7538,"value":{"_tag":"Set","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"30c5adfa30d11a2bb6c5511618f69b38"},{"_tag":"Uuid","value":"139eca9b215153e50eb72a12028b43fa"},{"_tag":"Uuid","value":"3f276c719dd63bb8a27d4b6b841a06ea"},{"_tag":"Uuid","value":"1e4a53579677bec4eed92fdc5fc0da9d"},{"_tag":"Uuid","value":"c668912dcb0eebd2243288c9fd69cf45"},{"_tag":"Uuid","value":"bed5ae44ee6c5e283db26c6116638b89"},{"_tag":"Uuid","value":"49046b9b279ef0578de7aafd0d1e45ec"},{"_tag":"Uuid","value":"a04accd79343bff8751bda98e8f5165f"},{"_tag":"Uuid","value":"25fdc9b0ceeb4830d25cb27bfc5327a5"},{"_tag":"Uuid","value":"8e7f5b7cfd292ff5804c6a50a9926af5"},{"_tag":"Uuid","value":"fd0380815557b58ceaf09a4a8788e858"},{"_tag":"Uuid","value":"9abefb36a03d2049806f6855db891260"},{"_tag":"Uuid","value":"e6541632374738ada338ebf59c3234b2"},{"_tag":"Uuid","value":"fc5424765be69800013f9b34070107b5"},{"_tag":"Uuid","value":"b292ca1e13ed02c110d7e40acf3773eb"},{"_tag":"Uuid","value":"510e6b2e1abeb1fb5d11d66accbe09e1"}]}}]},"bytes":"05cc04e9a1e6030dd0074239bf5b7b310cd0b1c3eb8da7b2fab015888adac20f048f760892dafd1030c5adfa30d11a2bb6c5511618f69b38139eca9b215153e50eb72a12028b43fa3f276c719dd63bb8a27d4b6b841a06ea1e4a53579677bec4eed92fdc5fc0da9dc668912dcb0eebd2243288c9fd69cf45bed5ae44ee6c5e283db26c6116638b8949046b9b279ef0578de7aafd0d1e45eca04accd79343bff8751bda98e8f5165f25fdc9b0ceeb4830d25cb27bfc5327a58e7f5b7cfd292ff5804c6a50a9926af5fd0380815557b58ceaf09a4a8788e8589abefb36a03d2049806f6855db891260e6541632374738ada338ebf59c3234b2fc5424765be69800013f9b34070107b5b292ca1e13ed02c110d7e40acf3773eb510e6b2e1abeb1fb5d11d66accbe09e100"},{"id":"random/293","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I32","value":-575246}}]},"bytes":"359b9c4600"},{"id":"random/294","type":"Struct","value":{"_tag":"Struct","fields":[{"id":7,"value":{"_tag":"I8","value":-44}},{"id":-27919,"value":{"_tag":"I64","value":"51"}},{"id":-27916,"value":{"_tag":"I64","value":"10973209584"}},{"id":-27912,"value":{"_tag":"Uuid","value":"236f0c6c965f6bb91089764be643b189"}}]},"bytes":"73d4069db4036636e08fefe0514d236f0c6c965f6bb91089764be643b18900"},{"id":"random/295","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"random/296","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-20153,"value":{"_tag":"I32","value":-80608}},{"id":-20072,"value":{"_tag":"Double","value":498.0}},{"id":9901,"value":{"_tag":"Uuid","value":"ff0db252b20f9a90119a1d4e64a5819c"}}]},"bytes":"05f1ba02bfeb0907cfb9020000000000207f400dda9a01ff0db252b20f9a90119a1d4e64a5819c00"},{"id":"random/297","type":"Struct","value":{"_tag":"Struct","fields":[{"id":175,"value":{"_tag":"Binary","value":"7669676a7772206e"}},{"id":187,"value":{"_tag":"I32","value":233491}},{"id":195,"value":{"_tag":"Binary","value":"928bbe423afea5d2c22d972a1409785296316e"}},{"id":-8745,"value":{"_tag":"I32","value":-79320757}},{"id":12693,"value":{"_tag":"Double","value":371128.788136672}},{"id":12705,"value":{"_tag":"I64","value":"-1"}}]},"bytes":"08de02087669676a7772206ec5a6c01c8813928bbe423afea5d2c22d972a1409785296316e05d18801e9dad24b07aac601bc4c0d27e3a61641c60100"},{"id":"random/298","type":"Struct","value":{"_tag":"Struct","fields":[{"id":61,"value":{"_tag":"I64","value":"44"}},{"id":327,"value":{"_tag":"I64","value":"10796937520517"}},{"id":336,"value":{"_tag":"I16","value":25}},{"id":349,"value":{"_tag":"I16","value":55}}]},"bytes":"067a58068e058a9681babbf4049432d46e00"},{"id":"random/299","type":"Struct","value":{"_tag":"Struct","fields":[{"id":9,"value":{"_tag":"I16","value":-706}},{"id":12,"value":{"_tag":"Binary","value":"414f49050ed41688919828c6ab2917fd05f4"}},{"id":23,"value":{"_tag":"Bool","value":false}}]},"bytes":"94830b3812414f49050ed41688919828c6ab2917fd05f4b200"}],"parquet":[{"id":"data_index_bloom_encoding_stats.parquet/footer","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Binary","value":"64617461"}},{"id":5,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"537472696e67"}},{"id":6,"value":{"_tag":"I32","value":0}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]}]}},{"id":3,"value":{"_tag":"I64","value":"14"}},{"id":4,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"4"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":4},{"_tag":"I32","value":3},{"_tag":"I32","value":0}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"537472696e67"}]}},{"id":4,"value":{"_tag":"I32","value":2}},{"id":5,"value":{"_tag":"I64","value":"14"}},{"id":6,"value":{"_tag":"I64","value":"163"}},{"id":7,"value":{"_tag":"I64","value":"152"}},{"id":9,"value":{"_tag":"I64","value":"4"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"746f646179"}},{"id":6,"value":{"_tag":"Binary","value":"48656c6c6f"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":1}}]}]}},{"id":14,"value":{"_tag":"I64","value":"192"}}]}},{"id":4,"value":{"_tag":"I64","value":"181"}},{"id":5,"value":{"_tag":"I32","value":11}},{"id":6,"value":{"_tag":"I64","value":"156"}},{"id":7,"value":{"_tag":"I32","value":25}}]}]}},{"id":2,"value":{"_tag":"I64","value":"163"}},{"id":3,"value":{"_tag":"I64","value":"14"}},{"id":5,"value":{"_tag":"I64","value":"4"}},{"id":6,"value":{"_tag":"I64","value":"152"}},{"id":7,"value":{"_tag":"I16","value":0}}]}]}},{"id":5,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"706172717565742e6176726f2e736368656d61"}},{"id":2,"value":{"_tag":"Binary","value":"7b2274797065223a227265636f7264222c226e616d65223a2264617461222c226669656c6473223a5b7b226e616d65223a22537472696e67222c2274797065223a5b226e756c6c222c22737472696e67225d2c22646f63223a225479706520696e6665727265642066726f6d202748656c6c6f27222c2264656661756c74223a6e756c6c7d5d7d"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"7772697465722e6d6f64656c2e6e616d65"}},{"id":2,"value":{"_tag":"Binary","value":"6176726f"}}]}]}},{"id":6,"value":{"_tag":"Binary","value":"706172717565742d6d722076657273696f6e20312e31332e302d534e415053484f5420286275696c64203733393864396235323237333363363639643439376332353439356339656661316338363039393429"}},{"id":7,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}]}}]},"bytes":"1502192c480464617461150200150c25021806537472696e6725004c1c000000161c191c191c26081c150c1935080600191806537472696e671504161c16c60216b00226083c36002805746f646179180548656c6c6f00191c150015001502001680030016ea02151616b80215320016c602161c260816b002140000192c1813706172717565742e6176726f2e736368656d611887017b2274797065223a227265636f7264222c226e616d65223a2264617461222c226669656c6473223a5b7b226e616d65223a22537472696e67222c2274797065223a5b226e756c6c222c22737472696e67225d2c22646f63223a225479706520696e6665727265642066726f6d202748656c6c6f27222c2264656661756c74223a6e756c6c7d5d7d0018117772697465722e6d6f64656c2e6e616d6518046176726f001853706172717565742d6d722076657273696f6e20312e31332e302d534e415053484f5420286275696c64203733393864396235323237333363363639643439376332353439356339656661316338363039393429191c1c000000"},{"id":"data_index_bloom_encoding_stats.parquet/page-header","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":138}},{"id":3,"value":{"_tag":"I32","value":127}},{"id":4,"value":{"_tag":"I32","value":-779741599}},{"id":5,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":14}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":3}},{"id":4,"value":{"_tag":"I32","value":4}}]}}]},"bytes":"150015940215fe0115bda6cfe7051c151c1500150615080000"},{"id":"data_index_bloom_encoding_stats.parquet/bloom-filter-header","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1024}},{"id":2,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}},{"id":4,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]},"bytes":"1580101c1c00001c1c00001c1c000000"},{"id":"data_index_bloom_encoding_stats.parquet/offset-index","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I64","value":"4"}},{"id":2,"value":{"_tag":"I32","value":152}},{"id":3,"value":{"_tag":"I64","value":"0"}}]}]}}]},"bytes":"191c160815b00216000000"},{"id":"data_index_bloom_encoding_stats.parquet/column-index","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false}]}},{"id":2,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"48656c6c6f"}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"746f646179"}]}},{"id":4,"value":{"_tag":"I32","value":1}},{"id":5,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"0"}]}}]},"bytes":"19110219180548656c6c6f191805746f646179150219160000"},{"id":"alltypes_tiny_pages.parquet/footer","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Binary","value":"686976655f736368656d61"}},{"id":5,"value":{"_tag":"I32","value":13}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"6964"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"626f6f6c5f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"74696e79696e745f636f6c"}},{"id":6,"value":{"_tag":"I32","value":15}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":8}},{"id":2,"value":{"_tag":"Bool","value":true}}]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"736d616c6c696e745f636f6c"}},{"id":6,"value":{"_tag":"I32","value":16}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":16}},{"id":2,"value":{"_tag":"Bool","value":true}}]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"696e745f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"626967696e745f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":4}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"666c6f61745f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":5}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"646f75626c655f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"646174655f737472696e675f636f6c"}},{"id":6,"value":{"_tag":"I32","value":0}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"737472696e675f636f6c"}},{"id":6,"value":{"_tag":"I32","value":0}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":3}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"74696d657374616d705f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"79656172"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"6d6f6e7468"}}]}]}},{"id":3,"value":{"_tag":"I64","value":"7300"}},{"id":4,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"4"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":3},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"6964"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"37325"}},{"id":7,"value":{"_tag":"I64","value":"37325"}},{"id":9,"value":{"_tag":"I64","value":"4"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"831c0000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"831c0000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"394311"}},{"id":5,"value":{"_tag":"I32","value":3503}},{"id":6,"value":{"_tag":"I64","value":"323583"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"37329"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":3},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"626f6f6c5f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"3022"}},{"id":7,"value":{"_tag":"I64","value":"3022"}},{"id":9,"value":{"_tag":"I64","value":"37329"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"01"}},{"id":2,"value":{"_tag":"Binary","value":"00"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"01"}},{"id":6,"value":{"_tag":"Binary","value":"00"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":82}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"397814"}},{"id":5,"value":{"_tag":"I32","value":823}},{"id":6,"value":{"_tag":"I64","value":"327502"}},{"id":7,"value":{"_tag":"I32","value":507}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"40351"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"74696e79696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"40351"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"09000000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"09000000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"398637"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"328009"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"52745"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"736d616c6c696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"52745"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"09000000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"09000000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"401888"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"331928"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"65139"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"65139"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"09000000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"09000000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"405139"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"335847"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"77533"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"626967696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"17515"}},{"id":7,"value":{"_tag":"I64","value":"17515"}},{"id":9,"value":{"_tag":"I64","value":"77533"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"5a00000000000000"}},{"id":2,"value":{"_tag":"Binary","value":"0000000000000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"5a00000000000000"}},{"id":6,"value":{"_tag":"Binary","value":"0000000000000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":528}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"408390"}},{"id":5,"value":{"_tag":"I32","value":5280}},{"id":6,"value":{"_tag":"I64","value":"339766"}},{"id":7,"value":{"_tag":"I32","value":10579}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"95048"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":4}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"666c6f61745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"95048"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"66661e41"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"66661e41"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"413670"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"350345"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"107442"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":5}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"646f75626c655f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"17515"}},{"id":7,"value":{"_tag":"I64","value":"17515"}},{"id":9,"value":{"_tag":"I64","value":"107442"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"9999999999b95640"}},{"id":2,"value":{"_tag":"Binary","value":"0000000000000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"9999999999b95640"}},{"id":6,"value":{"_tag":"Binary","value":"0000000000000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":528}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"416921"}},{"id":5,"value":{"_tag":"I32","value":5280}},{"id":6,"value":{"_tag":"I64","value":"354264"}},{"id":7,"value":{"_tag":"I32","value":10579}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"124957"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"646174655f737472696e675f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"42118"}},{"id":7,"value":{"_tag":"I64","value":"42118"}},{"id":9,"value":{"_tag":"I64","value":"124957"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"31322f33312f3130"}},{"id":6,"value":{"_tag":"Binary","value":"30312f30312f3039"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":974}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"422201"}},{"id":5,"value":{"_tag":"I32","value":9735}},{"id":6,"value":{"_tag":"I64","value":"364843"}},{"id":7,"value":{"_tag":"I32","value":19499}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"167075"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"737472696e675f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"13083"}},{"id":7,"value":{"_tag":"I64","value":"13083"}},{"id":9,"value":{"_tag":"I64","value":"167075"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"39"}},{"id":6,"value":{"_tag":"Binary","value":"30"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":352}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"431936"}},{"id":5,"value":{"_tag":"I32","value":3521}},{"id":6,"value":{"_tag":"I64","value":"384342"}},{"id":7,"value":{"_tag":"I32","value":2131}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"180158"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":3}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"74696d657374616d705f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"126532"}},{"id":7,"value":{"_tag":"I64","value":"126532"}},{"id":9,"value":{"_tag":"I64","value":"180158"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1055}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"435457"}},{"id":5,"value":{"_tag":"I32","value":10545}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"306690"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"79656172"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"8311"}},{"id":7,"value":{"_tag":"I64","value":"8311"}},{"id":9,"value":{"_tag":"I64","value":"306690"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"da070000"}},{"id":2,"value":{"_tag":"Binary","value":"d9070000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"da070000"}},{"id":6,"value":{"_tag":"Binary","value":"d9070000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"446002"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"386473"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"315001"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"6d6f6e7468"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"8582"}},{"id":7,"value":{"_tag":"I64","value":"8582"}},{"id":9,"value":{"_tag":"I64","value":"315001"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"0c000000"}},{"id":2,"value":{"_tag":"Binary","value":"01000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"0c000000"}},{"id":6,"value":{"_tag":"Binary","value":"01000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"449253"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"390392"}},{"id":7,"value":{"_tag":"I32","value":3919}}]}]}},{"id":2,"value":{"_tag":"I64","value":"323579"}},{"id":3,"value":{"_tag":"I64","value":"7300"}}]}]}},{"id":5,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"7772697465722e6d6f64656c2e6e616d65"}},{"id":2,"value":{"_tag":"Binary","value":"322e312e312d636468362e782d534e415053484f54"}}]}]}},{"id":6,"value":{"_tag":"Binary","value":"706172717565742d6d722076657273696f6e20312e31322e302d534e415053484f5420286275696c64203639303161323034303834386336623337666136316634623061373632343634343566333936646229"}},{"id":7,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}]}}]},"bytes":"150219ec480b686976655f736368656d61151a00150225021802696400150025021808626f6f6c5f636f6c0015022502180b74696e79696e745f636f6c251e4cac13081100000015022502180c736d616c6c696e745f636f6c25204cac131011000000150225021807696e745f636f6c0015042502180a626967696e745f636f6c00150825021809666c6f61745f636f6c00150a2502180a646f75626c655f636f6c00150c2502180f646174655f737472696e675f636f6c25004c1c000000150c2502180a737472696e675f636f6c25004c1c00000015062502180d74696d657374616d705f636f6c0015022502180479656172001502250218056d6f6e746800168872191c19dc26081c1502193500060819180269641500168872169ac704169ac70426083c1804831c000018040000000016002804831c000018040000000000191c15001500158a050000168e913015de3616febf27159e3d0026a2c7041c15001935000608191808626f6f6c5f636f6c1500168872169c2f169c2f26a2c7043c180101180100160028010118010000191c1500150015a401000016ecc73015ee0c169cfd2715f6070026bef6041c1502193506040819180b74696e79696e745f636f6c150016887216d4c10116d4c10126bef6043c180409000000180400000000160028040900000018040000000000192c1504150415020015001504158a05000016dad43015e63216928528159e3d002692b8061c1502193506040819180c736d616c6c696e745f636f6c150016887216d4c10116d4c1012692b8063c180409000000180400000000160028040900000018040000000000192c1504150415020015001504158a05000016c0873115e63216b0c228159e3d0026e6f9071c15021935060408191807696e745f636f6c150016887216d4c10116d4c10126e6f9073c180409000000180400000000160028040900000018040000000000192c1504150415020015001504158a05000016a6ba3115e63216ceff28159e3d0026babb091c1504193506040819180a626967696e745f636f6c150016887216d6910216d6910226babb093c18085a0000000000000018080000000000000000160028085a000000000000001808000000000000000000192c150415041502001500150415a0080000168ced3115c05216ecbc2915a6a501002690cd0b1c15081935060408191809666c6f61745f636f6c150016887216d4c10116d4c1012690cd0b3c180466661e411804000000001600280466661e4118040000000000192c1504150415020015001504158a05000016ccbf3215e6321692e22a159e3d0026e48e0d1c150a193506040819180a646f75626c655f636f6c150016887216d6910216d6910226e48e0d3c18089999999999b9564018080000000000000000160028089999999999b956401808000000000000000000192c150415041502001500150415a008000016b2f23215c05216b09f2b15a6a5010026baa00f1c150c193506040819180f646174655f737472696e675f636f6c1500168872168c9205168c920526baa00f3c3600280831322f33312f3130180830312f30312f303900192c1504150415020015001504159c0f000016f2c433158e980116d6c42c15d6b0020026c6b2141c150c193506040819180a737472696e675f636f6c150016887216b6cc0116b6cc0126c6b2143c360028013918013000192c150415041502001500150415c00500001680dd3415823716acf52e15a6210026fcfe151c1506193506040819180d74696d657374616d705f636f6c15001688721688b90f1688b90f26fcfe153c360000192c150415041502001500150415be1000001682943515e2a401002684b8251c1502193506040819180479656172150016887216ee810116ee81012684b8253c1804da0700001804d907000016002804da0700001804d907000000192c1504150415020015001504158a05000016e4b83615e63216d2962f159e3d0026f2b9261c150219350604081918056d6f6e74681500168872168c8601168c860126f2b9263c18040c000000180401000000160028040c00000018040100000000192c1504150415020015001504158a05000016caeb3615e63216f0d32f159e3d0016f6bf2716887200191c18117772697465722e6d6f64656c2e6e616d651815322e312e312d636468362e782d534e415053484f54001853706172717565742d6d722076657273696f6e20312e31322e302d534e415053484f5420286275696c6420363930316132303430383438633662333766613631663462306137363234363434356633393664622919dc1c00001c00001c00001c00001c00001c00001c00001c00001c00001c00001c00001c00001c000000"},{"id":"alltypes_tiny_pages.parquet/page-header","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":90}},{"id":3,"value":{"_tag":"I32","value":90}},{"id":5,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":21}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":3}},{"id":4,"value":{"_tag":"I32","value":4}}]}}]},"bytes":"150015b40115b4012c152a1500150615080000"}]} +{"cases":[{"id":"struct/empty","type":"Struct","value":{"_tag":"Struct","fields":[]},"bytes":"00"},{"id":"bool/fields","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}},{"id":2,"value":{"_tag":"Bool","value":false}},{"id":3,"value":{"_tag":"Bool","value":true}}]},"bytes":"11121100"},{"id":"bool/long-form","type":"Struct","value":{"_tag":"Struct","fields":[{"id":20,"value":{"_tag":"Bool","value":true}},{"id":-1,"value":{"_tag":"Bool","value":false}}]},"bytes":"0128020100"},{"id":"field/delta-1","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"I32","value":2}}]},"bytes":"1502150400"},{"id":"field/delta-15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":15,"value":{"_tag":"I32","value":1}},{"id":30,"value":{"_tag":"I32","value":2}}]},"bytes":"f502f50400"},{"id":"field/delta-16","type":"Struct","value":{"_tag":"Struct","fields":[{"id":16,"value":{"_tag":"I32","value":1}},{"id":32,"value":{"_tag":"I32","value":2}}]},"bytes":"05200205400400"},{"id":"field/id-0","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"I32","value":1}}]},"bytes":"05000200"},{"id":"field/decreasing","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":2}},{"id":4,"value":{"_tag":"I32","value":3}}]},"bytes":"5502050604150600"},{"id":"field/repeated","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"I32","value":1}},{"id":5,"value":{"_tag":"I32","value":2}}]},"bytes":"5502050a0400"},{"id":"field/negative","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-1,"value":{"_tag":"I32","value":1}},{"id":-32768,"value":{"_tag":"I32","value":2}},{"id":1,"value":{"_tag":"I32","value":3}}]},"bytes":"05010205ffff030405020600"},{"id":"field/max","type":"Struct","value":{"_tag":"Struct","fields":[{"id":32767,"value":{"_tag":"I32","value":1}}]},"bytes":"05feff030200"},{"id":"field/nested-resets","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":1}},{"id":2,"value":{"_tag":"I8","value":2}}]}},{"id":6,"value":{"_tag":"I8","value":3}}]},"bytes":"5c1301130200130300"},{"id":"i8/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":-128}},{"id":2,"value":{"_tag":"I8","value":127}},{"id":3,"value":{"_tag":"I8","value":0}},{"id":4,"value":{"_tag":"I8","value":-1}}]},"bytes":"1380137f130013ff00"},{"id":"i16/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I16","value":-32768}},{"id":2,"value":{"_tag":"I16","value":32767}},{"id":3,"value":{"_tag":"I16","value":0}},{"id":4,"value":{"_tag":"I16","value":-1}},{"id":5,"value":{"_tag":"I16","value":64}},{"id":6,"value":{"_tag":"I16","value":-65}}]},"bytes":"14ffff0314feff031400140114800114810100"},{"id":"i32/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":-2147483648}},{"id":2,"value":{"_tag":"I32","value":2147483647}},{"id":3,"value":{"_tag":"I32","value":0}},{"id":4,"value":{"_tag":"I32","value":-1}},{"id":5,"value":{"_tag":"I32","value":63}},{"id":6,"value":{"_tag":"I32","value":64}},{"id":7,"value":{"_tag":"I32","value":8191}},{"id":8,"value":{"_tag":"I32","value":8192}},{"id":9,"value":{"_tag":"I32","value":-8193}}]},"bytes":"15ffffffff0f15feffffff0f15001501157e15800115fe7f158080011581800100"},{"id":"i64/bounds","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I64","value":"0"}},{"id":2,"value":{"_tag":"I64","value":"-1"}},{"id":3,"value":{"_tag":"I64","value":"1"}},{"id":4,"value":{"_tag":"I64","value":"2147483648"}},{"id":5,"value":{"_tag":"I64","value":"-2147483649"}},{"id":6,"value":{"_tag":"I64","value":"9007199254740991"}},{"id":7,"value":{"_tag":"I64","value":"-9007199254740991"}},{"id":8,"value":{"_tag":"I64","value":"9007199254740992"}},{"id":9,"value":{"_tag":"I64","value":"-9007199254740992"}},{"id":10,"value":{"_tag":"I64","value":"9007199254740993"}},{"id":11,"value":{"_tag":"I64","value":"-9007199254740993"}},{"id":12,"value":{"_tag":"I64","value":"4611686018427387904"}},{"id":13,"value":{"_tag":"I64","value":"9223372036854775807"}},{"id":14,"value":{"_tag":"I64","value":"-9223372036854775808"}},{"id":15,"value":{"_tag":"I64","value":"72057594037927935"}},{"id":16,"value":{"_tag":"I64","value":"562949953421312"}}]},"bytes":"16001601160216808080801016818080801016feffffffffffff1f16fdffffffffffff1f16808080808080802016ffffffffffffff1f168280808080808020168180808080808020168080808080808080800116feffffffffffffffff0116ffffffffffffffffff0116feffffffffffffff0116808080808080800200"},{"id":"double/values","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Double","value":0.0}},{"id":2,"value":{"_tag":"Double","value":"-0"}},{"id":3,"value":{"_tag":"Double","value":1.5}},{"id":4,"value":{"_tag":"Double","value":"NaN"}},{"id":5,"value":{"_tag":"Double","value":"Infinity"}},{"id":6,"value":{"_tag":"Double","value":"-Infinity"}},{"id":7,"value":{"_tag":"Double","value":5e-324}},{"id":8,"value":{"_tag":"Double","value":1.7976931348623157e+308}},{"id":9,"value":{"_tag":"Double","value":3.141592653589793}},{"id":10,"value":{"_tag":"Double","value":-2.5}}]},"bytes":"17000000000000000017000000000000008017000000000000f83f17000000000000f87f17000000000000f07f17000000000000f0ff17010000000000000017ffffffffffffef7f17182d4454fb2109401700000000000004c000"},{"id":"binary/lengths","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":""}},{"id":2,"value":{"_tag":"Binary","value":"00"}},{"id":3,"value":{"_tag":"Binary","value":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f"}}]},"bytes":"1800180100188001000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f00"},{"id":"binary/utf8","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"68c3a96c6c6f20f09f8c8d"}},{"id":2,"value":{"_tag":"Binary","value":"6173636969"}}]},"bytes":"180b68c3a96c6c6f20f09f8c8d1805617363696900"},{"id":"uuid","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Uuid","value":"000102030405060708090a0b0c0d0e0f"}},{"id":2,"value":{"_tag":"Uuid","value":"00000000000000000000000000000000"}}]},"bytes":"1d000102030405060708090a0b0c0d0e0f1d0000000000000000000000000000000000"},{"id":"list/empty","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I32","elements":[]}},{"id":2,"value":{"_tag":"List","elementType":"Struct","elements":[]}},{"id":3,"value":{"_tag":"List","elementType":"Bool","elements":[]}}]},"bytes":"1905190c190100"},{"id":"list/size-14","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":1},{"_tag":"I32","value":2},{"_tag":"I32","value":3},{"_tag":"I32","value":4},{"_tag":"I32","value":5},{"_tag":"I32","value":6},{"_tag":"I32","value":7},{"_tag":"I32","value":8},{"_tag":"I32","value":9},{"_tag":"I32","value":10},{"_tag":"I32","value":11},{"_tag":"I32","value":12},{"_tag":"I32","value":13}]}}]},"bytes":"19e500020406080a0c0e10121416181a00"},{"id":"list/size-15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":1},{"_tag":"I32","value":2},{"_tag":"I32","value":3},{"_tag":"I32","value":4},{"_tag":"I32","value":5},{"_tag":"I32","value":6},{"_tag":"I32","value":7},{"_tag":"I32","value":8},{"_tag":"I32","value":9},{"_tag":"I32","value":10},{"_tag":"I32","value":11},{"_tag":"I32","value":12},{"_tag":"I32","value":13},{"_tag":"I32","value":14}]}}]},"bytes":"19f50f00020406080a0c0e10121416181a1c00"},{"id":"list/bool","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]}}]},"bytes":"193101020100"},{"id":"list/i8","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I8","elements":[{"_tag":"I8","value":-1},{"_tag":"I8","value":0},{"_tag":"I8","value":127}]}}]},"bytes":"1933ff007f00"},{"id":"list/i64","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"-9223372036854775808"},{"_tag":"I64","value":"0"},{"_tag":"I64","value":"9223372036854775807"}]}}]},"bytes":"1936ffffffffffffffffff0100feffffffffffffffff0100"},{"id":"list/double","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":1.0},{"_tag":"Double","value":"NaN"}]}}]},"bytes":"1927000000000000f03f000000000000f87f00"},{"id":"list/binary","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"61"},{"_tag":"Binary","value":""},{"_tag":"Binary","value":"6263"}]}}]},"bytes":"193801610002626300"},{"id":"list/uuid","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"000102030405060708090a0b0c0d0e0f"},{"_tag":"Uuid","value":"101112131415161718191a1b1c1d1e1f"}]}}]},"bytes":"192d000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f00"},{"id":"list/struct","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}}]},{"_tag":"Struct","fields":[]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I32","value":7}}]}]}}]},"bytes":"193c110000250e0000"},{"id":"list/list","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":1}]},{"_tag":"List","elementType":"I32","elements":[]},{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false}]}]}}]},"bytes":"1939150205110200"},{"id":"list/map","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Map","elements":[{"_tag":"Map","keyType":"I32","valueType":"I32","entries":[[{"_tag":"I32","value":1},{"_tag":"I32","value":2}]]},{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}]}}]},"bytes":"192b015502040000"},{"id":"set/values","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":1}]}},{"id":2,"value":{"_tag":"Set","elementType":"Binary","elements":[]}},{"id":3,"value":{"_tag":"Set","elementType":"Bool","elements":[{"_tag":"Bool","value":true}]}}]},"bytes":"1a2506021a081a110100"},{"id":"set/size-15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Set","elementType":"I8","elements":[{"_tag":"I8","value":0},{"_tag":"I8","value":1},{"_tag":"I8","value":2},{"_tag":"I8","value":3},{"_tag":"I8","value":4},{"_tag":"I8","value":5},{"_tag":"I8","value":6},{"_tag":"I8","value":7},{"_tag":"I8","value":8},{"_tag":"I8","value":9},{"_tag":"I8","value":10},{"_tag":"I8","value":11},{"_tag":"I8","value":12},{"_tag":"I8","value":13},{"_tag":"I8","value":14}]}}]},"bytes":"1af30f000102030405060708090a0b0c0d0e00"},{"id":"map/empty","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Stop","valueType":"Stop","entries":[]}}]},"bytes":"1b0000"},{"id":"map/one","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Binary","valueType":"I64","entries":[[{"_tag":"Binary","value":"6b"},{"_tag":"I64","value":"-5"}]]}}]},"bytes":"1b0186016b0900"},{"id":"map/bools","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Bool","valueType":"Bool","entries":[[{"_tag":"Bool","value":true},{"_tag":"Bool","value":false}],[{"_tag":"Bool","value":false},{"_tag":"Bool","value":true}]]}}]},"bytes":"1b02110102020100"},{"id":"map/i32-struct","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"I32","valueType":"Struct","entries":[[{"_tag":"I32","value":1},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Bool","value":true}}]}],[{"_tag":"I32","value":2},{"_tag":"Struct","fields":[]}]]}}]},"bytes":"1b025c021100040000"},{"id":"map/nested","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"I32","valueType":"List","entries":[[{"_tag":"I32","value":1},{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":2},{"_tag":"I32","value":3}]}]]}}]},"bytes":"1b01590225040600"},{"id":"map/uuid","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Map","keyType":"Uuid","valueType":"Uuid","entries":[[{"_tag":"Uuid","value":"000102030405060708090a0b0c0d0e0f"},{"_tag":"Uuid","value":"00000000000000000000000000000000"}]]}}]},"bytes":"1b01dd000102030405060708090a0b0c0d0e0f0000000000000000000000000000000000"},{"id":"struct/deep-5","type":"Struct","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":5}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]}},{"id":2,"value":{"_tag":"Bool","value":false}}]}},{"id":2,"value":{"_tag":"Bool","value":true}}]},"bytes":"1c1c1c1c1c150a0011001200110012001100"},{"id":"top/list","type":"List","value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":1},{"_tag":"I32","value":-1}]},"bytes":"250201"},{"id":"top/i64","type":"I64","value":{"_tag":"I64","value":"-1099511627776"},"bytes":"ffffffffff3f"},{"id":"top/binary","type":"Binary","value":{"_tag":"Binary","value":"746f70"},"bytes":"03746f70"},{"id":"top/map","type":"Map","value":{"_tag":"Map","keyType":"Binary","valueType":"Binary","entries":[[{"_tag":"Binary","value":"61"},{"_tag":"Binary","value":"62"}]]},"bytes":"018801610162"},{"id":"top/set","type":"Set","value":{"_tag":"Set","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":1}}]}]},"bytes":"1c130100"},{"id":"top/double","type":"Double","value":{"_tag":"Double","value":0.1},"bytes":"9a9999999999b93f"},{"id":"random/0","type":"Struct","value":{"_tag":"Struct","fields":[{"id":0,"value":{"_tag":"I64","value":"-29"}},{"id":14,"value":{"_tag":"I16","value":-59}},{"id":23,"value":{"_tag":"Double","value":1.4115744041294328e+88}}]},"bytes":"060039e475971730f51829623c5200"},{"id":"random/1","type":"Struct","value":{"_tag":"Struct","fields":[{"id":14,"value":{"_tag":"I16","value":-3320}},{"id":-4,"value":{"_tag":"Set","elementType":"I16","elements":[{"_tag":"I16","value":-56},{"_tag":"I16","value":61},{"_tag":"I16","value":1},{"_tag":"I16","value":263},{"_tag":"I16","value":2},{"_tag":"I16","value":60},{"_tag":"I16","value":-19},{"_tag":"I16","value":-8},{"_tag":"I16","value":13},{"_tag":"I16","value":39},{"_tag":"I16","value":2},{"_tag":"I16","value":32767},{"_tag":"I16","value":514},{"_tag":"I16","value":9}]}}]},"bytes":"e4ef330a07e46f7a028e040478250f1a4e04feff0384081200"},{"id":"random/2","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-29451,"value":{"_tag":"List","elementType":"Uuid","elements":[{"_tag":"Uuid","value":"cec1ac0bf57cef30a5082be06f325c79"},{"_tag":"Uuid","value":"092897dae3c87bf715c8600adb36d5f5"},{"_tag":"Uuid","value":"a32829ea541c8063a7ab79c1abf328d3"},{"_tag":"Uuid","value":"45db9230ad34c2b629c5e8a9cf30b96a"},{"_tag":"Uuid","value":"6d8da58bcda9ddd0875eeb9381d9a522"}]}}]},"bytes":"0995cc035dcec1ac0bf57cef30a5082be06f325c79092897dae3c87bf715c8600adb36d5f5a32829ea541c8063a7ab79c1abf328d345db9230ad34c2b629c5e8a9cf30b96a6d8da58bcda9ddd0875eeb9381d9a52200"},{"id":"random/3","type":"Struct","value":{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I8","value":4}},{"id":7,"value":{"_tag":"I8","value":-4}},{"id":8,"value":{"_tag":"I32","value":-8197731}},{"id":-3,"value":{"_tag":"I16","value":11}},{"id":90,"value":{"_tag":"Map","keyType":"Double","valueType":"I16","entries":[[{"_tag":"Double","value":-26380.583127614926},{"_tag":"I16","value":13}],[{"_tag":"Double","value":-239.0},{"_tag":"I16","value":-12}],[{"_tag":"Double","value":-5.969692745028776e-185},{"_tag":"I16","value":-19965}],[{"_tag":"Double","value":-258737.13185530005},{"_tag":"I16","value":-81}]]}},{"id":96,"value":{"_tag":"Binary","value":"677768"}},{"id":108,"value":{"_tag":"I32","value":0}},{"id":112,"value":{"_tag":"Struct","fields":[{"id":-8192,"value":{"_tag":"I64","value":"-34"}},{"id":-8181,"value":{"_tag":"I64","value":"56"}},{"id":-8167,"value":{"_tag":"I32","value":-1047514659}},{"id":-8166,"value":{"_tag":"Uuid","value":"e26c925eab7f1c6a0c0fd61ec2749a09"}},{"id":8426,"value":{"_tag":"I64","value":"-71"}}]}}]},"bytes":"230453fc15c5d9e8070405160bb4010474e07cf65125c3d9c01a0000000000e06dc01761e38ba6f03bb099f9b702cc260a0e89950fc1a1016803677768c5004c06ff7f43b670e5c5b8fee6071de26c925eab7f1c6a0c0fd61ec2749a0906d483018d010000"},{"id":"random/4","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":-855}},{"id":16,"value":{"_tag":"Uuid","value":"a958bb6c386785bcad834ad8fcbf2e67"}},{"id":92,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"6916868375171133"}]}},{"id":102,"value":{"_tag":"Map","keyType":"I16","valueType":"I32","entries":[[{"_tag":"I16","value":10},{"_tag":"I32","value":-44}],[{"_tag":"I16","value":-41},{"_tag":"I32","value":17077}],[{"_tag":"I16","value":-15},{"_tag":"I32","value":74}],[{"_tag":"I16","value":-45},{"_tag":"I32","value":23}],[{"_tag":"I16","value":-32768},{"_tag":"I32","value":499905923}],[{"_tag":"I16","value":8790},{"_tag":"I32","value":32}],[{"_tag":"I16","value":7121},{"_tag":"I32","value":-1}],[{"_tag":"I16","value":-18},{"_tag":"I32","value":-4866846}],[{"_tag":"I16","value":44},{"_tag":"I32","value":-64635}],[{"_tag":"I16","value":-36},{"_tag":"I32","value":-49}],[{"_tag":"I16","value":27},{"_tag":"I32","value":-9}],[{"_tag":"I16","value":-18},{"_tag":"I32","value":-26465}],[{"_tag":"I16","value":-8181},{"_tag":"I32","value":15}],[{"_tag":"I16","value":1943},{"_tag":"I32","value":-2524193}],[{"_tag":"I16","value":49},{"_tag":"I32","value":-138126707}],[{"_tag":"I16","value":-4340},{"_tag":"I32","value":0}]]}},{"id":110,"value":{"_tag":"I8","value":3}},{"id":119,"value":{"_tag":"Bool","value":true}},{"id":131,"value":{"_tag":"List","elementType":"Uuid","elements":[]}},{"id":132,"value":{"_tag":"Uuid","value":"31c6135e3ddd116a08793a1decc0fc25"}}]},"bytes":"34ad0ddda958bb6c386785bcad834ad8fcbf2e6709b80116fac0aea7ddb6c918ab1045145751ea8a021d9401592effff0386d6dfdc03ac890140a26f0123bb8cd20458f5f1074761361123c19d03e97f1eae1ec190b40262e595dd8301e74300830391c90d1d31c6135e3ddd116a08793a1decc0fc2500"},{"id":"random/5","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Double","value":651.0}},{"id":12,"value":{"_tag":"Bool","value":false}},{"id":21,"value":{"_tag":"Bool","value":true}},{"id":33,"value":{"_tag":"Uuid","value":"b61b7133dd5c9e8d585c922bfa35c6fe"}},{"id":247,"value":{"_tag":"List","elementType":"List","elements":[{"_tag":"List","elementType":"I16","elements":[]}]}},{"id":257,"value":{"_tag":"Map","keyType":"Bool","valueType":"Double","entries":[[{"_tag":"Bool","value":true},{"_tag":"Double","value":"-Infinity"}]]}}]},"bytes":"3700000000005884409291cdb61b7133dd5c9e8d585c922bfa35c6fe09ee031904ab011701000000000000f0ff00"},{"id":"random/6","type":"Struct","value":{"_tag":"Struct","fields":[{"id":206,"value":{"_tag":"Double","value":658410.8958999449}},{"id":219,"value":{"_tag":"I16","value":-32}},{"id":225,"value":{"_tag":"I16","value":-54}}]},"bytes":"079c03c865b3cad5172441d43f646b00"},{"id":"random/7","type":"Struct","value":{"_tag":"Struct","fields":[{"id":129,"value":{"_tag":"I8","value":-2}},{"id":1,"value":{"_tag":"I32","value":-80551}},{"id":13,"value":{"_tag":"Double","value":744831.811722348}},{"id":18,"value":{"_tag":"Bool","value":true}},{"id":24,"value":{"_tag":"I32","value":3287}}]},"bytes":"038202fe0502cdea09c754129a9fffba26415165ae3300"},{"id":"random/8","type":"Struct","value":{"_tag":"Struct","fields":[{"id":114,"value":{"_tag":"I8","value":13}},{"id":129,"value":{"_tag":"Double","value":-73.0}},{"id":139,"value":{"_tag":"I8","value":0}},{"id":7,"value":{"_tag":"Binary","value":"7a"}},{"id":19,"value":{"_tag":"Double","value":724235.1509789564}}]},"bytes":"03e4010df700000000004052c0a300080e017ac7201d4d4d161a264100"},{"id":"random/9","type":"Struct","value":{"_tag":"Struct","fields":[{"id":231,"value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Bool","value":true}},{"id":12,"value":{"_tag":"Uuid","value":"612360392bfdc4be16e62bd8bf1ec79e"}},{"id":17,"value":{"_tag":"I64","value":"-1623860795"}},{"id":23,"value":{"_tag":"Binary","value":"5450a788777788558a96"}},{"id":29,"value":{"_tag":"Binary","value":"782062"}},{"id":173,"value":{"_tag":"I8","value":10}}]}},{"id":437,"value":{"_tag":"Bool","value":true}},{"id":-4,"value":{"_tag":"I64","value":"-1"}},{"id":11,"value":{"_tag":"Bool","value":false}},{"id":14,"value":{"_tag":"Bool","value":false}},{"id":19,"value":{"_tag":"Binary","value":"6d6f78e697a52d792f2e2d752f"}},{"id":-5481,"value":{"_tag":"Binary","value":"74717a"}},{"id":-5479,"value":{"_tag":"Bool","value":false}}]},"bytes":"0cce03517d612360392bfdc4be16e62bd8bf1ec79e56f598d18c0c680a5450a788777788558a96680378206203da020a0001ea06060701f232580d6d6f78e697a52d792f2e2d752f08d1550374717a2200"},{"id":"random/10","type":"Struct","value":{"_tag":"Struct","fields":[{"id":43,"value":{"_tag":"Double","value":-825.0}},{"id":-10189,"value":{"_tag":"Binary","value":"cd39bc"}},{"id":-10027,"value":{"_tag":"List","elementType":"Double","elements":[{"_tag":"Double","value":5.384133272083983e+208},{"_tag":"Double","value":-313926.1086485876},{"_tag":"Double","value":-765168.6597299525},{"_tag":"Double","value":640.0},{"_tag":"Double","value":"-Infinity"}]}},{"id":-9729,"value":{"_tag":"Uuid","value":"62b7ae846d41664daf7c1ed29c606fd5"}},{"id":-9725,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":true},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false},{"_tag":"Bool","value":false}]}}]},"bytes":"07560000000000c889c008999f0103cd39bc09d59c0157270e047978f6446b4a93416f182913c1d41fc851e15927c10000000000008440000000000000f0ff0d81980162b7ae846d41664daf7c1ed29c606fd5496102020102020200"},{"id":"random/11","type":"Struct","value":{"_tag":"Struct","fields":[{"id":5,"value":{"_tag":"Binary","value":"6a7661642f6b79702dc3bc6f64707377"}},{"id":-2,"value":{"_tag":"Bool","value":true}},{"id":6,"value":{"_tag":"I8","value":-17}},{"id":31203,"value":{"_tag":"Double","value":527667.6184021656}},{"id":7810,"value":{"_tag":"Bool","value":false}},{"id":7876,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"Binary","value":"332d"}},{"id":16,"value":{"_tag":"Set","elementType":"Binary","elements":[{"_tag":"Binary","value":"62"},{"_tag":"Binary","value":"636f6f2e6163617a796c2ef09f8c8d747ac3a9712f6ac3a9"}]}},{"id":31,"value":{"_tag":"Uuid","value":"b8b2a7a5ce3bebd218f91766697e64a1"}},{"id":290,"value":{"_tag":"Uuid","value":"f0dc62129edd1e72f7dd4306cf593cb5"}},{"id":326,"value":{"_tag":"I32","value":42}}]}}]},"bytes":"58106a7661642f6b79702dc3bc6f64707377010383ef07c6e7036a359f3c671a204102847a0c887b3802332dda28016218636f6f2e6163617a796c2ef09f8c8d747ac3a9712f6ac3a9fdb8b2a7a5ce3bebd218f91766697e64a10dc404f0dc62129edd1e72f7dd4306cf593cb5058c05540000"},{"id":"random/12","type":"Struct","value":{"_tag":"Struct","fields":[{"id":8,"value":{"_tag":"I32","value":40}}]},"bytes":"855000"},{"id":"random/13","type":"Struct","value":{"_tag":"Struct","fields":[{"id":279,"value":{"_tag":"Bool","value":true}},{"id":281,"value":{"_tag":"I16","value":81}}]},"bytes":"01ae0424a20100"},{"id":"random/14","type":"Struct","value":{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"696c69"}]}},{"id":10,"value":{"_tag":"I32","value":-63}},{"id":19,"value":{"_tag":"I8","value":6}},{"id":33,"value":{"_tag":"Binary","value":"7a6c646a76"}}]},"bytes":"491803696c69657d9306e8057a6c646a7600"},{"id":"random/15","type":"Struct","value":{"_tag":"Struct","fields":[{"id":-11701,"value":{"_tag":"Double","value":-6.001937545422097e+177}},{"id":-11698,"value":{"_tag":"Set","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":-1},{"_tag":"I32","value":0},{"_tag":"I32","value":-5}]}},{"id":10974,"value":{"_tag":"I8","value":-12}},{"id":10983,"value":{"_tag":"I8","value":-2}},{"id":10994,"value":{"_tag":"Double","value":421711.43568380247}},{"id":0,"value":{"_tag":"Set","elementType":"I64","elements":[{"_tag":"I64","value":"29"}]}}]},"bytes":"07e9b60162de986ab8b2d7e43a450001000903bcab01f493feb70ce523be3dbd19410a00163a00"},{"id":"random/16","type":"Struct","value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Double","value":78889.97803132748}}]},"bytes":"a7602d04a69f42f34000"},{"id":"random/17","type":"Struct","value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I16","value":26}},{"id":16,"value":{"_tag":"I16","value":6}}]},"bytes":"3434d40c00"},{"id":"random/18","type":"Struct","value":{"_tag":"Struct","fields":[{"id":272,"value":{"_tag":"Binary","value":"6b66762d6870636e"}},{"id":287,"value":{"_tag":"I16","value":-33}},{"id":579,"value":{"_tag":"I8","value":-41}},{"id":16972,"value":{"_tag":"I32","value":16}},{"id":-17242,"value":{"_tag":"I8","value":-22}},{"id":-17229,"value":{"_tag":"Bool","value":false}},{"id":27585,"value":{"_tag":"I64","value":"56"}},{"id":-5705,"value":{"_tag":"I16","value":22846}}]},"bytes":"08a004086b66762d6870636ef441038609d7059889022003b38d02ead20682af0370049159fce40200"},{"id":"random/19","type":"Struct","value":{"_tag":"Struct","fields":[{"id":206,"value":{"_tag":"Double","value":499840.97379794903}},{"id":-23604,"value":{"_tag":"Bool","value":false}}]},"bytes":"079c03204a2be503821e4102e7f00200"}],"parquet":[{"id":"data_index_bloom_encoding_stats.parquet/footer","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Binary","value":"64617461"}},{"id":5,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"537472696e67"}},{"id":6,"value":{"_tag":"I32","value":0}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]}]}},{"id":3,"value":{"_tag":"I64","value":"14"}},{"id":4,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"4"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":4},{"_tag":"I32","value":3},{"_tag":"I32","value":0}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"537472696e67"}]}},{"id":4,"value":{"_tag":"I32","value":2}},{"id":5,"value":{"_tag":"I64","value":"14"}},{"id":6,"value":{"_tag":"I64","value":"163"}},{"id":7,"value":{"_tag":"I64","value":"152"}},{"id":9,"value":{"_tag":"I64","value":"4"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"746f646179"}},{"id":6,"value":{"_tag":"Binary","value":"48656c6c6f"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":1}}]}]}},{"id":14,"value":{"_tag":"I64","value":"192"}}]}},{"id":4,"value":{"_tag":"I64","value":"181"}},{"id":5,"value":{"_tag":"I32","value":11}},{"id":6,"value":{"_tag":"I64","value":"156"}},{"id":7,"value":{"_tag":"I32","value":25}}]}]}},{"id":2,"value":{"_tag":"I64","value":"163"}},{"id":3,"value":{"_tag":"I64","value":"14"}},{"id":5,"value":{"_tag":"I64","value":"4"}},{"id":6,"value":{"_tag":"I64","value":"152"}},{"id":7,"value":{"_tag":"I16","value":0}}]}]}},{"id":5,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"706172717565742e6176726f2e736368656d61"}},{"id":2,"value":{"_tag":"Binary","value":"7b2274797065223a227265636f7264222c226e616d65223a2264617461222c226669656c6473223a5b7b226e616d65223a22537472696e67222c2274797065223a5b226e756c6c222c22737472696e67225d2c22646f63223a225479706520696e6665727265642066726f6d202748656c6c6f27222c2264656661756c74223a6e756c6c7d5d7d"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"7772697465722e6d6f64656c2e6e616d65"}},{"id":2,"value":{"_tag":"Binary","value":"6176726f"}}]}]}},{"id":6,"value":{"_tag":"Binary","value":"706172717565742d6d722076657273696f6e20312e31332e302d534e415053484f5420286275696c64203733393864396235323237333363363639643439376332353439356339656661316338363039393429"}},{"id":7,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}]}}]},"bytes":"1502192c480464617461150200150c25021806537472696e6725004c1c000000161c191c191c26081c150c1935080600191806537472696e671504161c16c60216b00226083c36002805746f646179180548656c6c6f00191c150015001502001680030016ea02151616b80215320016c602161c260816b002140000192c1813706172717565742e6176726f2e736368656d611887017b2274797065223a227265636f7264222c226e616d65223a2264617461222c226669656c6473223a5b7b226e616d65223a22537472696e67222c2274797065223a5b226e756c6c222c22737472696e67225d2c22646f63223a225479706520696e6665727265642066726f6d202748656c6c6f27222c2264656661756c74223a6e756c6c7d5d7d0018117772697465722e6d6f64656c2e6e616d6518046176726f001853706172717565742d6d722076657273696f6e20312e31332e302d534e415053484f5420286275696c64203733393864396235323237333363363639643439376332353439356339656661316338363039393429191c1c000000"},{"id":"data_index_bloom_encoding_stats.parquet/page-header","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":138}},{"id":3,"value":{"_tag":"I32","value":127}},{"id":4,"value":{"_tag":"I32","value":-779741599}},{"id":5,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":14}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":3}},{"id":4,"value":{"_tag":"I32","value":4}}]}}]},"bytes":"150015940215fe0115bda6cfe7051c151c1500150615080000"},{"id":"data_index_bloom_encoding_stats.parquet/bloom-filter-header","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1024}},{"id":2,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}},{"id":4,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]},"bytes":"1580101c1c00001c1c00001c1c000000"},{"id":"data_index_bloom_encoding_stats.parquet/offset-index","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I64","value":"4"}},{"id":2,"value":{"_tag":"I32","value":152}},{"id":3,"value":{"_tag":"I64","value":"0"}}]}]}}]},"bytes":"191c160815b00216000000"},{"id":"data_index_bloom_encoding_stats.parquet/column-index","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Bool","elements":[{"_tag":"Bool","value":false}]}},{"id":2,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"48656c6c6f"}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"746f646179"}]}},{"id":4,"value":{"_tag":"I32","value":1}},{"id":5,"value":{"_tag":"List","elementType":"I64","elements":[{"_tag":"I64","value":"0"}]}}]},"bytes":"19110219180548656c6c6f191805746f646179150219160000"},{"id":"alltypes_tiny_pages.parquet/footer","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":4,"value":{"_tag":"Binary","value":"686976655f736368656d61"}},{"id":5,"value":{"_tag":"I32","value":13}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"6964"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"626f6f6c5f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"74696e79696e745f636f6c"}},{"id":6,"value":{"_tag":"I32","value":15}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":8}},{"id":2,"value":{"_tag":"Bool","value":true}}]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"736d616c6c696e745f636f6c"}},{"id":6,"value":{"_tag":"I32","value":16}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I8","value":16}},{"id":2,"value":{"_tag":"Bool","value":true}}]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"696e745f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"626967696e745f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":4}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"666c6f61745f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":5}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"646f75626c655f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"646174655f737472696e675f636f6c"}},{"id":6,"value":{"_tag":"I32","value":0}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"737472696e675f636f6c"}},{"id":6,"value":{"_tag":"I32","value":0}},{"id":10,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":3}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"74696d657374616d705f636f6c"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"79656172"}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":3,"value":{"_tag":"I32","value":1}},{"id":4,"value":{"_tag":"Binary","value":"6d6f6e7468"}}]}]}},{"id":3,"value":{"_tag":"I64","value":"7300"}},{"id":4,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"4"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":3},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"6964"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"37325"}},{"id":7,"value":{"_tag":"I64","value":"37325"}},{"id":9,"value":{"_tag":"I64","value":"4"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"831c0000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"831c0000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"394311"}},{"id":5,"value":{"_tag":"I32","value":3503}},{"id":6,"value":{"_tag":"I64","value":"323583"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"37329"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":0},{"_tag":"I32","value":3},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"626f6f6c5f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"3022"}},{"id":7,"value":{"_tag":"I64","value":"3022"}},{"id":9,"value":{"_tag":"I64","value":"37329"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"01"}},{"id":2,"value":{"_tag":"Binary","value":"00"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"01"}},{"id":6,"value":{"_tag":"Binary","value":"00"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":82}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"397814"}},{"id":5,"value":{"_tag":"I32","value":823}},{"id":6,"value":{"_tag":"I64","value":"327502"}},{"id":7,"value":{"_tag":"I32","value":507}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"40351"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"74696e79696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"40351"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"09000000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"09000000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"398637"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"328009"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"52745"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"736d616c6c696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"52745"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"09000000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"09000000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"401888"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"331928"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"65139"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"65139"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"09000000"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"09000000"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"405139"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"335847"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"77533"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"626967696e745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"17515"}},{"id":7,"value":{"_tag":"I64","value":"17515"}},{"id":9,"value":{"_tag":"I64","value":"77533"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"5a00000000000000"}},{"id":2,"value":{"_tag":"Binary","value":"0000000000000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"5a00000000000000"}},{"id":6,"value":{"_tag":"Binary","value":"0000000000000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":528}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"408390"}},{"id":5,"value":{"_tag":"I32","value":5280}},{"id":6,"value":{"_tag":"I64","value":"339766"}},{"id":7,"value":{"_tag":"I32","value":10579}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"95048"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":4}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"666c6f61745f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"12394"}},{"id":7,"value":{"_tag":"I64","value":"12394"}},{"id":9,"value":{"_tag":"I64","value":"95048"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"66661e41"}},{"id":2,"value":{"_tag":"Binary","value":"00000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"66661e41"}},{"id":6,"value":{"_tag":"Binary","value":"00000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"413670"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"350345"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"107442"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":5}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"646f75626c655f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"17515"}},{"id":7,"value":{"_tag":"I64","value":"17515"}},{"id":9,"value":{"_tag":"I64","value":"107442"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"9999999999b95640"}},{"id":2,"value":{"_tag":"Binary","value":"0000000000000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"9999999999b95640"}},{"id":6,"value":{"_tag":"Binary","value":"0000000000000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":528}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"416921"}},{"id":5,"value":{"_tag":"I32","value":5280}},{"id":6,"value":{"_tag":"I64","value":"354264"}},{"id":7,"value":{"_tag":"I32","value":10579}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"124957"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"646174655f737472696e675f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"42118"}},{"id":7,"value":{"_tag":"I64","value":"42118"}},{"id":9,"value":{"_tag":"I64","value":"124957"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"31322f33312f3130"}},{"id":6,"value":{"_tag":"Binary","value":"30312f30312f3039"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":974}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"422201"}},{"id":5,"value":{"_tag":"I32","value":9735}},{"id":6,"value":{"_tag":"I64","value":"364843"}},{"id":7,"value":{"_tag":"I32","value":19499}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"167075"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":6}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"737472696e675f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"13083"}},{"id":7,"value":{"_tag":"I64","value":"13083"}},{"id":9,"value":{"_tag":"I64","value":"167075"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"39"}},{"id":6,"value":{"_tag":"Binary","value":"30"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":352}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"431936"}},{"id":5,"value":{"_tag":"I32","value":3521}},{"id":6,"value":{"_tag":"I64","value":"384342"}},{"id":7,"value":{"_tag":"I32","value":2131}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"180158"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":3}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"74696d657374616d705f636f6c"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"126532"}},{"id":7,"value":{"_tag":"I64","value":"126532"}},{"id":9,"value":{"_tag":"I64","value":"180158"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":3,"value":{"_tag":"I64","value":"0"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1055}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"435457"}},{"id":5,"value":{"_tag":"I32","value":10545}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"306690"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"79656172"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"8311"}},{"id":7,"value":{"_tag":"I64","value":"8311"}},{"id":9,"value":{"_tag":"I64","value":"306690"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"da070000"}},{"id":2,"value":{"_tag":"Binary","value":"d9070000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"da070000"}},{"id":6,"value":{"_tag":"Binary","value":"d9070000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"446002"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"386473"}},{"id":7,"value":{"_tag":"I32","value":3919}}]},{"_tag":"Struct","fields":[{"id":2,"value":{"_tag":"I64","value":"315001"}},{"id":3,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":1}},{"id":2,"value":{"_tag":"List","elementType":"I32","elements":[{"_tag":"I32","value":3},{"_tag":"I32","value":2},{"_tag":"I32","value":4}]}},{"id":3,"value":{"_tag":"List","elementType":"Binary","elements":[{"_tag":"Binary","value":"6d6f6e7468"}]}},{"id":4,"value":{"_tag":"I32","value":0}},{"id":5,"value":{"_tag":"I64","value":"7300"}},{"id":6,"value":{"_tag":"I64","value":"8582"}},{"id":7,"value":{"_tag":"I64","value":"8582"}},{"id":9,"value":{"_tag":"I64","value":"315001"}},{"id":12,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"0c000000"}},{"id":2,"value":{"_tag":"Binary","value":"01000000"}},{"id":3,"value":{"_tag":"I64","value":"0"}},{"id":5,"value":{"_tag":"Binary","value":"0c000000"}},{"id":6,"value":{"_tag":"Binary","value":"01000000"}}]}},{"id":13,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":2}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":1}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":2}},{"id":3,"value":{"_tag":"I32","value":325}}]}]}}]}},{"id":4,"value":{"_tag":"I64","value":"449253"}},{"id":5,"value":{"_tag":"I32","value":3251}},{"id":6,"value":{"_tag":"I64","value":"390392"}},{"id":7,"value":{"_tag":"I32","value":3919}}]}]}},{"id":2,"value":{"_tag":"I64","value":"323579"}},{"id":3,"value":{"_tag":"I64","value":"7300"}}]}]}},{"id":5,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Binary","value":"7772697465722e6d6f64656c2e6e616d65"}},{"id":2,"value":{"_tag":"Binary","value":"322e312e312d636468362e782d534e415053484f54"}}]}]}},{"id":6,"value":{"_tag":"Binary","value":"706172717565742d6d722076657273696f6e20312e31322e302d534e415053484f5420286275696c64203639303161323034303834386336623337666136316634623061373632343634343566333936646229"}},{"id":7,"value":{"_tag":"List","elementType":"Struct","elements":[{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]},{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"Struct","fields":[]}}]}]}}]},"bytes":"150219ec480b686976655f736368656d61151a00150225021802696400150025021808626f6f6c5f636f6c0015022502180b74696e79696e745f636f6c251e4cac13081100000015022502180c736d616c6c696e745f636f6c25204cac131011000000150225021807696e745f636f6c0015042502180a626967696e745f636f6c00150825021809666c6f61745f636f6c00150a2502180a646f75626c655f636f6c00150c2502180f646174655f737472696e675f636f6c25004c1c000000150c2502180a737472696e675f636f6c25004c1c00000015062502180d74696d657374616d705f636f6c0015022502180479656172001502250218056d6f6e746800168872191c19dc26081c1502193500060819180269641500168872169ac704169ac70426083c1804831c000018040000000016002804831c000018040000000000191c15001500158a050000168e913015de3616febf27159e3d0026a2c7041c15001935000608191808626f6f6c5f636f6c1500168872169c2f169c2f26a2c7043c180101180100160028010118010000191c1500150015a401000016ecc73015ee0c169cfd2715f6070026bef6041c1502193506040819180b74696e79696e745f636f6c150016887216d4c10116d4c10126bef6043c180409000000180400000000160028040900000018040000000000192c1504150415020015001504158a05000016dad43015e63216928528159e3d002692b8061c1502193506040819180c736d616c6c696e745f636f6c150016887216d4c10116d4c1012692b8063c180409000000180400000000160028040900000018040000000000192c1504150415020015001504158a05000016c0873115e63216b0c228159e3d0026e6f9071c15021935060408191807696e745f636f6c150016887216d4c10116d4c10126e6f9073c180409000000180400000000160028040900000018040000000000192c1504150415020015001504158a05000016a6ba3115e63216ceff28159e3d0026babb091c1504193506040819180a626967696e745f636f6c150016887216d6910216d6910226babb093c18085a0000000000000018080000000000000000160028085a000000000000001808000000000000000000192c150415041502001500150415a0080000168ced3115c05216ecbc2915a6a501002690cd0b1c15081935060408191809666c6f61745f636f6c150016887216d4c10116d4c1012690cd0b3c180466661e411804000000001600280466661e4118040000000000192c1504150415020015001504158a05000016ccbf3215e6321692e22a159e3d0026e48e0d1c150a193506040819180a646f75626c655f636f6c150016887216d6910216d6910226e48e0d3c18089999999999b9564018080000000000000000160028089999999999b956401808000000000000000000192c150415041502001500150415a008000016b2f23215c05216b09f2b15a6a5010026baa00f1c150c193506040819180f646174655f737472696e675f636f6c1500168872168c9205168c920526baa00f3c3600280831322f33312f3130180830312f30312f303900192c1504150415020015001504159c0f000016f2c433158e980116d6c42c15d6b0020026c6b2141c150c193506040819180a737472696e675f636f6c150016887216b6cc0116b6cc0126c6b2143c360028013918013000192c150415041502001500150415c00500001680dd3415823716acf52e15a6210026fcfe151c1506193506040819180d74696d657374616d705f636f6c15001688721688b90f1688b90f26fcfe153c360000192c150415041502001500150415be1000001682943515e2a401002684b8251c1502193506040819180479656172150016887216ee810116ee81012684b8253c1804da0700001804d907000016002804da0700001804d907000000192c1504150415020015001504158a05000016e4b83615e63216d2962f159e3d0026f2b9261c150219350604081918056d6f6e74681500168872168c8601168c860126f2b9263c18040c000000180401000000160028040c00000018040100000000192c1504150415020015001504158a05000016caeb3615e63216f0d32f159e3d0016f6bf2716887200191c18117772697465722e6d6f64656c2e6e616d651815322e312e312d636468362e782d534e415053484f54001853706172717565742d6d722076657273696f6e20312e31322e302d534e415053484f5420286275696c6420363930316132303430383438633662333766613631663462306137363234363434356633393664622919dc1c00001c00001c00001c00001c00001c00001c00001c00001c00001c00001c00001c00001c000000"},{"id":"alltypes_tiny_pages.parquet/page-header","value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":0}},{"id":2,"value":{"_tag":"I32","value":90}},{"id":3,"value":{"_tag":"I32","value":90}},{"id":5,"value":{"_tag":"Struct","fields":[{"id":1,"value":{"_tag":"I32","value":21}},{"id":2,"value":{"_tag":"I32","value":0}},{"id":3,"value":{"_tag":"I32","value":3}},{"id":4,"value":{"_tag":"I32","value":4}}]}}]},"bytes":"150015b40115b4012c152a1500150615080000"}]} From 3166345a019d79fc85a627a13d8a85a5bf8e6bba Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:35:14 +0000 Subject: [PATCH 05/32] Validate Thrift reader limits and writer initialSize --- packages/effect/src/encoding/Thrift.ts | 23 ++++++++++++----- packages/effect/test/encoding/Thrift.test.ts | 27 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 932574645c9..540a06d977f 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -134,6 +134,14 @@ const fail = (kind: ThriftError["kind"], message: string): never => { throw new ThriftError({ kind, message }) } +const integerOption = (name: string, value: number | undefined, min: number, fallback: number): number => { + if (value === undefined) return fallback + if (!Number.isInteger(value) || value < min) { + fail("Usage", `${name} must be an integer of at least ${min}, got ${value}`) + } + return value +} + // ----------------------------------------------------------------------------- // Types // ----------------------------------------------------------------------------- @@ -256,12 +264,13 @@ export interface Options { /** * How deep structs and containers may nest, counting each struct and each * container entered by {@link Reader.readStructBegin}, - * {@link Reader.skip} or {@link Reader.readValue}. Defaults to 64. + * {@link Reader.skip} or {@link Reader.readValue}. A positive integer; + * defaults to 64. */ readonly maxDepth?: number | undefined - /** The longest binary or string value, in bytes. No limit by default. */ + /** The longest binary or string value, in bytes: a non-negative integer. No limit by default. */ readonly maxBinaryLength?: number | undefined - /** The largest list, set or map. No limit by default. */ + /** The largest list, set or map: a non-negative integer. No limit by default. */ readonly maxContainerSize?: number | undefined } @@ -326,9 +335,9 @@ export class Reader { this.bytes = bytes this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) this.end = bytes.length - this.maxDepth = options?.maxDepth ?? 64 - this.maxBinaryLength = options?.maxBinaryLength ?? Infinity - this.maxContainerSize = options?.maxContainerSize ?? Infinity + this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 64) + this.maxBinaryLength = integerOption("maxBinaryLength", options?.maxBinaryLength, 0, Infinity) + this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity) } /** Where the next read starts in {@link bytes}. Set it to read from elsewhere. */ @@ -799,7 +808,7 @@ export class Writer { private boolFieldId = 0 constructor(options?: { readonly initialSize?: number | undefined }) { - this.bytes = new Uint8Array(Math.max(16, options?.initialSize ?? 1024)) + this.bytes = new Uint8Array(Math.max(16, integerOption("initialSize", options?.initialSize, 0, 1024))) this.view = new DataView(this.bytes.buffer) } diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index f70525c00ae..f6b435edc91 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -322,6 +322,26 @@ describe("Thrift", () => { assert.isAbove(failures, 0) }) + it("rejects limits that aren't integers in range", () => { + // NaN used to disable a limit and a fractional one to round it. + for ( + const options of [ + { maxDepth: NaN }, + { maxDepth: 0 }, + { maxDepth: 1.5 }, + { maxBinaryLength: -1 }, + { maxBinaryLength: NaN }, + { maxContainerSize: Infinity } + ] + ) { + assert.strictEqual( + thrown(() => Thrift.makeReader(new Uint8Array(0), options)), + "Usage", + Object.entries(options).join() + ) + } + }) + it("decode rethrows errors that aren't ThriftErrors", () => { assert.throws(() => Thrift.decode(Uint8Array.of(0), () => { @@ -368,6 +388,13 @@ describe("Thrift", () => { it(`rejects ${name}`, () => assert.strictEqual(thrown(() => f(Thrift.makeWriter())), "Usage")) } + it("rejects an initialSize that isn't a non-negative integer", () => { + // NaN used to make the buffer grow forever. + for (const initialSize of [NaN, Infinity, -1, 1.5]) { + assert.strictEqual(thrown(() => Thrift.makeWriter({ initialSize })), "Usage", String(initialSize)) + } + }) + it("grows from a small buffer and returns a copy", () => { const bytes = parquet("alltypes_tiny_pages.parquet/footer") const w = Thrift.makeWriter({ initialSize: 1 }) From d7ba781a5758aadfec594e93511a14ff48068034 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:35:29 +0000 Subject: [PATCH 06/32] Keep a leading byte order mark in Thrift strings --- packages/effect/src/encoding/Thrift.ts | 3 ++- packages/effect/test/encoding/Thrift.test.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 540a06d977f..c6980868a24 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -274,7 +274,8 @@ export interface Options { readonly maxContainerSize?: number | undefined } -const utf8 = new TextDecoder("utf-8", { fatal: true }) +// Keeps a leading U+FEFF, which is part of the string. +const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) const MAX_SAFE = Number.MAX_SAFE_INTEGER const BIGINT_1 = BigInt(1) const BIGINT_7 = BigInt(7) diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index f6b435edc91..51481888a9e 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -209,6 +209,10 @@ describe("Thrift", () => { assert.strictEqual(roundTrip((w) => w.writeString(s), (r) => r.readString()), expected) }) + it("keeps a leading byte order mark in a string", () => { + assert.strictEqual(roundTrip((w) => w.writeString("\uFEFFa"), (r) => r.readString()), "\uFEFFa") + }) + it("reads i64 as a number only when it is a safe integer", () => { const max = BigInt(Number.MAX_SAFE_INTEGER) const asNumber = (n: bigint) => Thrift.decode(write((w) => w.writeI64(n)), (r) => r.readI64Number()) From cad41c387baec9e40408332c3b836a5c1b44008c Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:36:07 +0000 Subject: [PATCH 07/32] Reject out-of-range Thrift i16 values and field ids --- packages/effect/src/encoding/Thrift.ts | 24 ++++++++++++++------ packages/effect/test/encoding/Thrift.test.ts | 4 ++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index c6980868a24..7258bc5a9be 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -93,8 +93,10 @@ const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" * * - `Truncated`: the input ends inside a value, or a length or container * size is larger than the bytes that remain. - * - `Malformed`: an unknown type code, a varint that is too long, or a value - * of type `Stop`. + * - `Malformed`: an unknown type code, a varint longer than its type + * allows (5 bytes and 32 bits for `i32` and sizes, 16 bits for `i16` and + * field ids, 10 bytes and 64 bits for `i64`), a field id over 32767, or a + * non-empty container of `Stop`. * - `InvalidUtf8`: a string is not valid UTF-8. * - `Range`: an `i64` read with {@link Reader.readI64Number} is not a safe * integer. @@ -398,6 +400,13 @@ export class Reader { return this.truncated(what) } + /** A zigzag varint of at most 16 bits. */ + private varint16(what: string): number { + const n = this.varint32(what) + if (n > 0xffff) fail("Malformed", `${what} before byte ${this.pos} is longer than 16 bits`) + return (n >>> 1) ^ -(n & 1) + } + /** * A zigzag varint of at most 64 bits as a number: returns the magnitude * `|v|` for v >= 0 or `|v| - 1` for v < 0 (exact up to 2^53) and sets @@ -475,10 +484,10 @@ export class Reader { const delta = header >> 4 let id: number if (delta === 0) { - const n = this.varint32("a field id") - id = (((n >>> 1) ^ -(n & 1)) << 16) >> 16 + id = this.varint16("a field id") } else { id = this.lastFieldId + delta + if (id > 32767) fail("Malformed", `the field header at byte ${this.pos - 1} has id ${id}, over 32767`) } this.lastFieldId = id this.fieldId = id @@ -542,8 +551,7 @@ export class Reader { /** Reads an `i16`. */ readI16(): number { - const n = this.varint32("an i16") - return (((n >>> 1) ^ -(n & 1)) << 16) >> 16 + return this.varint16("an i16") } /** Reads an `i32`. */ @@ -628,8 +636,10 @@ export class Reader { this.advance(1, "an i8") return case Type.I16: + this.varint16("an i16") + return case Type.I32: - this.varint32("an integer") + this.varint32("an i32") return case Type.I64: this.varint64("an i64") diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 51481888a9e..0c58721dd01 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -260,6 +260,10 @@ describe("Thrift", () => { ["an i32 varint over 32 bits", "ffffffff1f", (r) => r.readI32(), "Malformed"], ["an i64 varint over 10 bytes", "ffffffffffffffffff8101", (r) => r.readI64(), "Malformed"], ["an i64 varint over 64 bits", "ffffffffffffffffff02", (r) => r.readI64(), "Malformed"], + // Other readers wrap these to 16 bits or keep them, so fields would alias. + ["an i16 over 16 bits", "808004", (r) => r.readI16(), "Malformed"], + ["a field id over 16 bits", "0580800400", struct, "Malformed"], + ["a field id delta past 32767", "05feff0300f50000", struct, "Malformed"], ["an unknown field type", "1e", struct, "Malformed"], ["an unknown list element type", "1e", (r) => r.readListBegin(), "Malformed"], ["a non-empty list of Stop", "10", (r) => r.readListBegin(), "Malformed"], From 638455ee52d069c0e562cdf79bfa153ca017b81b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:36:29 +0000 Subject: [PATCH 08/32] Rethrow Thrift usage errors from decode --- packages/effect/src/encoding/Thrift.ts | 16 +++++++++++----- packages/effect/test/encoding/Thrift.test.ts | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 7258bc5a9be..e88962d9749 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -103,8 +103,9 @@ const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" * - `Limit`: the input nests deeper than {@link Options.maxDepth}, or a * length or size exceeds {@link Options.maxBinaryLength} or * {@link Options.maxContainerSize}. - * - `Usage`: the reader or writer was called out of order, or a value is - * out of range for its type. + * - `Usage`: the reader or writer was called out of order or with an + * invalid argument or option, or a written value is out of range for its + * type. {@link decode} rethrows these caller errors. * * @stability experimental * @category errors @@ -685,7 +686,7 @@ export class Reader { return } default: - fail("Malformed", `can't skip a value of type ${describe(type)} at byte ${this.pos}`) + fail("Usage", `can't skip a value of type ${describe(type)}`) } } @@ -746,7 +747,7 @@ export class Reader { return { _tag: "Map", keyType: typeNames[key], valueType: typeNames[value], entries } } default: - return fail("Malformed", `can't read a value of type ${describe(type)} at byte ${this.pos}`) + return fail("Usage", `can't read a value of type ${describe(type)}`) } } } @@ -764,6 +765,11 @@ export const makeReader = (bytes: Uint8Array, options?: Options): Reader => new * Reads `bytes` with `read`, which gets a {@link Reader} at the start of * them. Malformed input becomes a failed `Result` instead of an exception. * + * **Gotchas** + * + * `Usage` errors and errors other than {@link ThriftError} are rethrown: + * they are bugs in `read` or invalid options, not bad input. + * * @stability experimental * @category reading * @since 4.1.5 @@ -776,7 +782,7 @@ export const decode = ( try { return Result.succeed(read(new Reader(bytes, options))) } catch (cause) { - if (isThriftError(cause)) return Result.fail(cause) + if (isThriftError(cause) && cause.kind !== "Usage") return Result.fail(cause) throw cause } } diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 0c58721dd01..aaf2dbc090c 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -356,6 +356,20 @@ describe("Thrift", () => { throw new Error("boom") }), "boom") }) + + it("decode rethrows Usage errors, which are the caller's", () => { + const misuse: ReadonlyArray<(r: Thrift.Reader) => unknown> = [ + (r) => r.readStructEnd(), + (r) => r.position = -1, + (r) => r.position = 2, + (r) => r.position = 0.5, + (r) => r.skip(Thrift.Type.Stop), + (r) => r.readValue(7 as Thrift.Type) + ] + for (const f of misuse) { + assert.strictEqual(thrown(() => Thrift.decode(Uint8Array.of(0), f)), "Usage", String(f)) + } + }) }) describe("Writer", () => { From 96843ae83614028762f85fad94b90469fbf25898 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:36:55 +0000 Subject: [PATCH 09/32] Forget a pending Thrift bool field when the reader moves --- packages/effect/src/encoding/Thrift.ts | 1 + packages/effect/test/encoding/Thrift.test.ts | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index e88962d9749..637854ade0a 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -353,6 +353,7 @@ export class Reader { fail("Usage", `position ${position} is outside the ${this.end}-byte input`) } this.pos = position + this.bool = 0 } private truncated(what: string): never { diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index aaf2dbc090c..0c3c88bac6b 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -233,6 +233,15 @@ describe("Thrift", () => { assert.strictEqual(r.position, 5) }) + it("forgets a bool field header when the position moves", () => { + const r = Thrift.makeReader(Uint8Array.of(0x11, 0x02)) + r.readStructBegin() + assert.strictEqual(r.readFieldBegin(), Thrift.Type.Bool) + r.position = 1 + assert.isFalse(r.readBool()) + assert.strictEqual(r.position, 2) + }) + it("reads binary as a view of the input", () => { const bytes = Uint8Array.of(2, 7, 8) const view = Thrift.makeReader(bytes).readBinary() From de2c3687d9a5f83d7bf933ecfb3cabfa0775793c Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:37:13 +0000 Subject: [PATCH 10/32] Reject Thrift values of no type in writeValue --- packages/effect/src/encoding/Thrift.ts | 2 ++ packages/effect/test/encoding/Thrift.test.ts | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 637854ade0a..27408fe09e2 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -1133,6 +1133,8 @@ export class Writer { } return } + default: + fail("Usage", `${String((value as { readonly _tag: unknown })._tag)} is not a Thrift value`) } } diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 0c3c88bac6b..2a0ed20f05c 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -413,7 +413,9 @@ describe("Thrift", () => { [ "a list element of another type", (w) => w.writeValue({ _tag: "List", elementType: "I32", elements: [{ _tag: "I8", value: 1 }] }) - ] + ], + // Used to write nothing. + ["a value of no Thrift type", (w) => w.writeValue({ _tag: "Text", value: "a" } as unknown as Thrift.Value)] ] for (const [name, f] of usage) { it(`rejects ${name}`, () => assert.strictEqual(thrown(() => f(Thrift.makeWriter())), "Usage")) From f7821953c9e11583b4cc84435946b41a4d2df4b5 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:37:46 +0000 Subject: [PATCH 11/32] Share Thrift header code and check limits before the input size --- packages/effect/src/encoding/Thrift.ts | 78 ++++++++++---------- packages/effect/test/encoding/Thrift.test.ts | 2 +- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 27408fe09e2..90107016a2f 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -437,13 +437,33 @@ export class Reader { return half } + /** Checks a container size; every element takes at least `bytesPerElement` bytes. */ private checkSize(size: number, bytesPerElement: number, what: string): void { + if (size > this.maxContainerSize) { + fail("Limit", `${what} of ${size} elements exceeds the limit of ${this.maxContainerSize}`) + } if (size * bytesPerElement > this.end - this.pos) { fail("Truncated", `${what} of ${size} elements at byte ${this.pos} is larger than the remaining input`) } - if (size > this.maxContainerSize) { - fail("Limit", `${what} of ${size} elements exceeds the limit of ${this.maxContainerSize}`) + } + + /** Reads a binary length and moves past the bytes; returns where they start. */ + private binary(what: string): number { + const length = this.varint32(`${what} length`) + if (length > this.maxBinaryLength) { + fail("Limit", `${what} of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) } + return this.advance(length, `${what} of ${length} bytes`) + } + + /** Reads a list or set header. */ + private listHeader(what: string): number { + const header = this.byte(`${what} header`) + let size = header >> 4 + if (size === 15) size = this.varint32(`${what} size`) + this.elementType = this.elementOf(header & 0x0f, size, what) + this.checkSize(size, 1, what) + return size } private enter(): void { @@ -503,22 +523,12 @@ export class Reader { /** Reads a list header: returns the list's size and sets {@link elementType}. */ readListBegin(): number { - const header = this.byte("a list header") - let size = header >> 4 - if (size === 15) size = this.varint32("a list size") - this.elementType = this.elementOf(header & 0x0f, size, "a list") - this.checkSize(size, 1, "a list") - return size + return this.listHeader("a list") } /** Reads a set header: returns the set's size and sets {@link elementType}. */ readSetBegin(): number { - const header = this.byte("a set header") - let size = header >> 4 - if (size === 15) size = this.varint32("a set size") - this.elementType = this.elementOf(header & 0x0f, size, "a set") - this.checkSize(size, 1, "a set") - return size + return this.listHeader("a set") } /** Reads a map header: returns the map's size and sets {@link keyType} and {@link valueType}. */ @@ -596,23 +606,14 @@ export class Reader { /** Reads a `binary` value as a view of {@link bytes}, not a copy. */ readBinary(): Uint8Array { - const length = this.varint32("a binary length") - if (length > this.maxBinaryLength) { - fail("Limit", `a binary value of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) - } - const at = this.advance(length, `a binary value of ${length} bytes`) - return this.bytes.subarray(at, at + length) + return this.bytes.subarray(this.binary("a binary value"), this.pos) } /** Reads a `string`: a `binary` value decoded as UTF-8. */ readString(): string { - const length = this.varint32("a string length") - if (length > this.maxBinaryLength) { - fail("Limit", `a string of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) - } - const at = this.advance(length, `a string of ${length} bytes`) + const at = this.binary("a string") try { - return utf8.decode(this.bytes.subarray(at, at + length)) + return utf8.decode(this.bytes.subarray(at, this.pos)) } catch { return fail("InvalidUtf8", `the string at byte ${at} is not valid UTF-8`) } @@ -649,14 +650,9 @@ export class Reader { case Type.Double: this.advance(8, "a double") return - case Type.Binary: { - const length = this.varint32("a binary length") - if (length > this.maxBinaryLength) { - fail("Limit", `a binary value of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) - } - this.advance(length, `a binary value of ${length} bytes`) + case Type.Binary: + this.binary("a binary value") return - } case Type.Uuid: this.advance(16, "a uuid") return @@ -938,11 +934,10 @@ export class Writer { this.writeByte(0) } - /** Writes a list header; write `size` elements of `elementType` next. */ - writeListBegin(elementType: Type, size: number): void { - this.checkNoBool("writeListBegin") - this.checkSize(size, "list size") - const nibble = this.elementNibble(elementType, size, "list element type") + private listHeader(elementType: Type, size: number, what: string): void { + this.checkNoBool(`write${what}Begin`) + this.checkSize(size, `${what} size`) + const nibble = this.elementNibble(elementType, size, `${what} element type`) if (size < 15) { this.writeByte((size << 4) | nibble) } else { @@ -951,9 +946,14 @@ export class Writer { } } + /** Writes a list header; write `size` elements of `elementType` next. */ + writeListBegin(elementType: Type, size: number): void { + this.listHeader(elementType, size, "List") + } + /** Writes a set header; write `size` elements of `elementType` next. */ writeSetBegin(elementType: Type, size: number): void { - this.writeListBegin(elementType, size) + this.listHeader(elementType, size, "Set") } /** Writes a map header; write `size` key-value pairs next. */ diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 2a0ed20f05c..487e1e3821d 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -284,7 +284,7 @@ describe("Thrift", () => { ["a skipped binary over maxBinaryLength", "1803616263", (r) => r.skip(Thrift.Type.Struct), "Limit", { maxBinaryLength: 2 }], - ["a list over maxContainerSize", "35010203", (r) => r.readListBegin(), "Limit", { maxContainerSize: 2 }], + ["a list over maxContainerSize", "35", (r) => r.readListBegin(), "Limit", { maxContainerSize: 2 }], ["a struct over maxDepth", "1c1c000000", struct, "Limit", { maxDepth: 2 }], ["lists over maxDepth", "1919191500", (r) => r.skip(Thrift.Type.List), "Limit", { maxDepth: 2 }] ] From 81999f30a52bbac483e0c39aa3bbb209ad743b98 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:37:46 +0000 Subject: [PATCH 12/32] Rename Thrift Options to ReaderOptions --- packages/effect/src/encoding/Thrift.ts | 16 ++++++++-------- packages/effect/test/encoding/Thrift.test.ts | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 90107016a2f..09818a81c24 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -65,7 +65,7 @@ * {@link ThriftError} (or makes {@link decode} fail) instead of reading past * the bytes. A length or container size larger than the bytes that remain * fails before anything is allocated, and nesting is limited by - * {@link Options.maxDepth}. Binary values are views into the input, not + * {@link ReaderOptions.maxDepth}. Binary values are views into the input, not * copies. Strings must be valid UTF-8. * * `i64` values read as `bigint` ({@link Reader.readI64}) or, when they are @@ -100,9 +100,9 @@ const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" * - `InvalidUtf8`: a string is not valid UTF-8. * - `Range`: an `i64` read with {@link Reader.readI64Number} is not a safe * integer. - * - `Limit`: the input nests deeper than {@link Options.maxDepth}, or a - * length or size exceeds {@link Options.maxBinaryLength} or - * {@link Options.maxContainerSize}. + * - `Limit`: the input nests deeper than {@link ReaderOptions.maxDepth}, or a + * length or size exceeds {@link ReaderOptions.maxBinaryLength} or + * {@link ReaderOptions.maxContainerSize}. * - `Usage`: the reader or writer was called out of order or with an * invalid argument or option, or a written value is out of range for its * type. {@link decode} rethrows these caller errors. @@ -263,7 +263,7 @@ export interface Field { * @category models * @since 4.1.5 */ -export interface Options { +export interface ReaderOptions { /** * How deep structs and containers may nest, counting each struct and each * container entered by {@link Reader.readStructBegin}, @@ -335,7 +335,7 @@ export class Reader { /** Set by {@link varint64}: whether the zigzag value is negative. */ private negative = false - constructor(bytes: Uint8Array, options?: Options) { + constructor(bytes: Uint8Array, options?: ReaderOptions) { this.bytes = bytes this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) this.end = bytes.length @@ -756,7 +756,7 @@ export class Reader { * @category reading * @since 4.1.5 */ -export const makeReader = (bytes: Uint8Array, options?: Options): Reader => new Reader(bytes, options) +export const makeReader = (bytes: Uint8Array, options?: ReaderOptions): Reader => new Reader(bytes, options) /** * Reads `bytes` with `read`, which gets a {@link Reader} at the start of @@ -774,7 +774,7 @@ export const makeReader = (bytes: Uint8Array, options?: Options): Reader => new export const decode = ( bytes: Uint8Array, read: (reader: Reader) => A, - options?: Options + options?: ReaderOptions ): Result.Result => { try { return Result.succeed(read(new Reader(bytes, options))) diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 487e1e3821d..9dd7c5f3437 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -79,7 +79,7 @@ const toJson = (v: Thrift.Value): Json => { } } -const readAll = (bytes: Uint8Array, type: Thrift.TypeName, options?: Thrift.Options) => +const readAll = (bytes: Uint8Array, type: Thrift.TypeName, options?: Thrift.ReaderOptions) => Thrift.decode(bytes, (r) => { const value = r.readValue(Thrift.Type[type]) assert.strictEqual(r.position, bytes.length) @@ -261,7 +261,7 @@ describe("Thrift", () => { describe("hostile input", () => { const struct = (r: Thrift.Reader) => r.readValue(Thrift.Type.Struct) const table: ReadonlyArray< - readonly [string, string, (r: Thrift.Reader) => unknown, Thrift.ThriftError["kind"], Thrift.Options?] + readonly [string, string, (r: Thrift.Reader) => unknown, Thrift.ThriftError["kind"], Thrift.ReaderOptions?] > = [ ["empty input", "", struct, "Truncated"], ["a struct without its stop", "1502", struct, "Truncated"], From 9c8ed80e295db8c98d194f81658096819612d038 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:39:14 +0000 Subject: [PATCH 13/32] Document Thrift view lifetimes, option validation and stack depth --- packages/effect/src/encoding/Thrift.ts | 44 +++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 09818a81c24..b2e6c38726f 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -64,9 +64,13 @@ * Every read is bounds-checked: malformed or hostile input throws a * {@link ThriftError} (or makes {@link decode} fail) instead of reading past * the bytes. A length or container size larger than the bytes that remain - * fails before anything is allocated, and nesting is limited by - * {@link ReaderOptions.maxDepth}. Binary values are views into the input, not - * copies. Strings must be valid UTF-8. + * fails before anything is allocated, varints are capped at the width of + * their type, and nesting is limited by {@link ReaderOptions.maxDepth}. + * Strings must be valid UTF-8. + * + * Binary and `uuid` values, also inside a {@link Value}, are views into the + * input, not copies: they change when the input does and keep its buffer + * alive. Copy one with `slice()` to keep it on its own. * * `i64` values read as `bigint` ({@link Reader.readI64}) or, when they are * known to be safe integers such as offsets and counts, as `number` @@ -257,7 +261,14 @@ export interface Field { // ----------------------------------------------------------------------------- /** - * Limits for reading untrusted input. + * Limits for reading untrusted input. A value that isn't an integer in range + * throws a {@link ThriftError} of kind `Usage`. + * + * **Gotchas** + * + * {@link Reader.skip} and {@link Reader.readValue} recurse, so a `maxDepth` + * in the thousands can overflow the stack, which throws a `RangeError` + * instead of a {@link ThriftError}. * * @stability experimental * @category models @@ -304,6 +315,12 @@ const I64_MAX = (BigInt(1) << BIGINT_63) - BIGINT_1 * after a `Bool` field header returns it without reading a byte. Inside a * container, each `bool` is a byte. * + * **Gotchas** + * + * {@link readBinary}, {@link readUuid} and the binary values of + * {@link readValue} are views of {@link bytes}: copy them before mutating or + * reusing the input. + * * @stability experimental * @category reading * @since 4.1.5 @@ -344,7 +361,11 @@ export class Reader { this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity) } - /** Where the next read starts in {@link bytes}. Set it to read from elsewhere. */ + /** + * Where the next read starts in {@link bytes}. Set it to read from + * elsewhere; a position outside the input is a `Usage` error, so check + * offsets read from the input first. + */ get position(): number { return this.pos } @@ -589,7 +610,8 @@ export class Reader { /** * Reads an `i64` as a `number`. Fails with kind `Range` when the value is - * not a safe integer. + * not a safe integer, leaving {@link position} at the value so that + * {@link readI64} can read it. */ readI64Number(): number { const start = this.pos @@ -750,7 +772,7 @@ export class Reader { } /** - * A {@link Reader} at the start of `bytes`. + * Creates a {@link Reader} at the start of `bytes`, which it reads in place. * * @stability experimental * @category reading @@ -805,8 +827,8 @@ const encoder = new TextEncoder() * {@link writeMapBegin}) followed by exactly that many elements (for a map, * key then value). * - * Integers outside their type's range, and calls out of order, throw a - * {@link ThriftError} of kind `Usage`. + * Integers outside their type's range, values of no {@link Type}, and calls + * out of order throw a {@link ThriftError} of kind `Usage`. * * @stability experimental * @category writing @@ -1147,8 +1169,8 @@ export class Writer { } /** - * A new {@link Writer}. `initialSize` is the first buffer size; it grows as - * needed. + * Creates a {@link Writer}. `initialSize` is the first buffer size in bytes, + * a non-negative integer (default 1024); the buffer grows as needed. * * @stability experimental * @category writing From ac8909eb38fa0b4aec4acbded060590937005124 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:40:25 +0000 Subject: [PATCH 14/32] Shorten the Thrift changeset --- .changeset/encoding-thrift.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/encoding-thrift.md b/.changeset/encoding-thrift.md index 8c98c74101d..f5e20a9605a 100644 --- a/.changeset/encoding-thrift.md +++ b/.changeset/encoding-thrift.md @@ -2,4 +2,4 @@ "effect": patch --- -Add `effect/encoding/Thrift` for reading and writing the Apache Thrift compact protocol without an IDL. The reader is a bounds-checked cursor that reads binary values without copying, and `decode` returns a `Result`. The writer produces the same bytes as the official Thrift library. +Add the experimental `effect/encoding/Thrift` module, which reads and writes the Apache Thrift compact protocol without an IDL. From 9a3b07623e130fafeb5e1cf84503c3e488f798ac Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:58:55 +0000 Subject: [PATCH 15/32] Keep BOMs in the Thrift string round-trip oracle --- packages/effect/test/encoding/Thrift.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index 9dd7c5f3437..e82ceb3975f 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -205,7 +205,7 @@ describe("Thrift", () => { it.prop("string round-trips through UTF-8", [Schema.String], ([s]) => { // Lone surrogates have no UTF-8 encoding and become U+FFFD. - const expected = new TextDecoder().decode(new TextEncoder().encode(s)) + const expected = new TextDecoder("utf-8", { ignoreBOM: true }).decode(new TextEncoder().encode(s)) assert.strictEqual(roundTrip((w) => w.writeString(s), (r) => r.readString()), expected) }) From d43d38b9002a50eae2876f0295bc4b095d494738 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:58:55 +0000 Subject: [PATCH 16/32] Cap Thrift maxDepth at 1000 so recursive reads stay within the stack --- packages/effect/src/encoding/Thrift.ts | 25 ++++++++++---------- packages/effect/test/encoding/Thrift.test.ts | 4 +++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index b2e6c38726f..0a94eab059d 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -141,10 +141,17 @@ const fail = (kind: ThriftError["kind"], message: string): never => { throw new ThriftError({ kind, message }) } -const integerOption = (name: string, value: number | undefined, min: number, fallback: number): number => { +const integerOption = ( + name: string, + value: number | undefined, + min: number, + fallback: number, + max = Infinity +): number => { if (value === undefined) return fallback - if (!Number.isInteger(value) || value < min) { - fail("Usage", `${name} must be an integer of at least ${min}, got ${value}`) + if (!Number.isInteger(value) || value < min || value > max) { + const range = max === Infinity ? `of at least ${min}` : `from ${min} to ${max}` + fail("Usage", `${name} must be an integer ${range}, got ${value}`) } return value } @@ -264,12 +271,6 @@ export interface Field { * Limits for reading untrusted input. A value that isn't an integer in range * throws a {@link ThriftError} of kind `Usage`. * - * **Gotchas** - * - * {@link Reader.skip} and {@link Reader.readValue} recurse, so a `maxDepth` - * in the thousands can overflow the stack, which throws a `RangeError` - * instead of a {@link ThriftError}. - * * @stability experimental * @category models * @since 4.1.5 @@ -278,8 +279,8 @@ export interface ReaderOptions { /** * How deep structs and containers may nest, counting each struct and each * container entered by {@link Reader.readStructBegin}, - * {@link Reader.skip} or {@link Reader.readValue}. A positive integer; - * defaults to 64. + * {@link Reader.skip} or {@link Reader.readValue}. An integer from 1 to + * 1000, which keeps their recursion within the stack; defaults to 64. */ readonly maxDepth?: number | undefined /** The longest binary or string value, in bytes: a non-negative integer. No limit by default. */ @@ -356,7 +357,7 @@ export class Reader { this.bytes = bytes this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) this.end = bytes.length - this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 64) + this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 64, 1000) this.maxBinaryLength = integerOption("maxBinaryLength", options?.maxBinaryLength, 0, Infinity) this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity) } diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index e82ceb3975f..c21922cc5c5 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -292,7 +292,7 @@ describe("Thrift", () => { it(`fails on ${name}`, () => assert.strictEqual(kindOf(Thrift.decode(fromHex(hex), read, options)), kind)) } - it("limits nesting to 64 by default", () => { + it("limits nesting to 64 by default and 1000 at most", () => { const nest = (depth: number): Thrift.Value => { let v: Thrift.Value = { _tag: "Struct", fields: [] } for (let i = 1; i < depth; i++) v = { _tag: "Struct", fields: [{ id: 1, value: v }] } @@ -302,6 +302,7 @@ describe("Thrift", () => { assert.isTrue(Result.isSuccess(readAll(bytesOf(64), "Struct"))) assert.strictEqual(kindOf(readAll(bytesOf(65), "Struct")), "Limit") assert.strictEqual(kindOf(Thrift.decode(bytesOf(65), (r) => r.skip(Thrift.Type.Struct))), "Limit") + assert.strictEqual(kindOf(readAll(bytesOf(1001), "Struct", { maxDepth: 1000 })), "Limit") }) it("fails every truncated gold case with Truncated", () => { @@ -346,6 +347,7 @@ describe("Thrift", () => { { maxDepth: NaN }, { maxDepth: 0 }, { maxDepth: 1.5 }, + { maxDepth: 1001 }, { maxBinaryLength: -1 }, { maxBinaryLength: NaN }, { maxContainerSize: Infinity } From 997f399afa898f9c9e43508f380b85ddee854e9a Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:16:19 +0000 Subject: [PATCH 17/32] Simplify Thrift internals Share one integer range check, decode strings with internal/utf8, drop the INVALID sentinel, the Reader's struct counter and the i64 bound constants, and build Writer container labels only on failure. Section banners go and the Reader and Writer docs no longer repeat their members. --- packages/effect/src/encoding/Thrift.ts | 199 +++++++++---------------- 1 file changed, 72 insertions(+), 127 deletions(-) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts index 0a94eab059d..f424e5cf906 100644 --- a/packages/effect/src/encoding/Thrift.ts +++ b/packages/effect/src/encoding/Thrift.ts @@ -11,7 +11,7 @@ * - This module works at the protocol level, as generated Thrift code does. * A {@link Reader} is a cursor over the bytes: read a field header, look at * its id and type, read the value or {@link Reader.skip} it. A - * {@link Writer} writes the same calls back. + * {@link Writer} mirrors those calls. * - The method names and their meaning follow Thrift's `TProtocol`, so code * written against another Thrift implementation reads the same: a struct * is `readStructBegin`, then `readFieldBegin` until it returns @@ -83,13 +83,15 @@ * @since 4.1.5 */ import * as Data from "../Data.ts" +import * as Utf8 from "../internal/utf8.ts" import * as Predicate from "../Predicate.ts" import * as Result from "../Result.ts" const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" /** - * Error raised for malformed input or a misused {@link Writer}. + * Error raised for malformed input or misuse of a {@link Reader} or + * {@link Writer}. * * **Details** * @@ -97,10 +99,9 @@ const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" * * - `Truncated`: the input ends inside a value, or a length or container * size is larger than the bytes that remain. - * - `Malformed`: an unknown type code, a varint longer than its type - * allows (5 bytes and 32 bits for `i32` and sizes, 16 bits for `i16` and - * field ids, 10 bytes and 64 bits for `i64`), a field id over 32767, or a - * non-empty container of `Stop`. + * - `Malformed`: an unknown type code, a varint wider than its type (16 + * bits for `i16` and field ids, 32 for `i32` and sizes, 64 for `i64`), a + * field id over 32767, or a non-empty container of `Stop`. * - `InvalidUtf8`: a string is not valid UTF-8. * - `Range`: an `i64` read with {@link Reader.readI64Number} is not a safe * integer. @@ -141,24 +142,15 @@ const fail = (kind: ThriftError["kind"], message: string): never => { throw new ThriftError({ kind, message }) } -const integerOption = ( - name: string, - value: number | undefined, - min: number, - fallback: number, - max = Infinity -): number => { - if (value === undefined) return fallback +const checkInteger = (what: string, value: number, min: number, max: number): number => { if (!Number.isInteger(value) || value < min || value > max) { - const range = max === Infinity ? `of at least ${min}` : `from ${min} to ${max}` - fail("Usage", `${name} must be an integer ${range}, got ${value}`) + fail("Usage", `${what} ${value} is not an integer from ${min} to ${max}`) } return value } -// ----------------------------------------------------------------------------- -// Types -// ----------------------------------------------------------------------------- +const integerOption = (name: string, value: number | undefined, min: number, max: number, fallback: number): number => + value === undefined ? fallback : checkInteger(name, value, min, max) /** * The type of a field or container element. The codes are Thrift's `TType` @@ -202,27 +194,20 @@ export type Type = typeof Type[keyof typeof Type] */ export type TypeName = keyof typeof Type -const INVALID = 255 +/** Compact type nibble → TType. Nibbles 1 and 2 are a bool field's true and false; 14 and 15 are unassigned. */ +const fromCompact: ReadonlyArray = [0, 2, 2, 3, 6, 8, 10, 4, 11, 15, 14, 13, 12, 16] -/** Compact type nibble → TType. */ -const fromCompact = [0, 2, 2, 3, 6, 8, 10, 4, 11, 15, 14, 13, 12, 16, INVALID, INVALID] - -/** TType → compact type nibble (bool as an element; fields encode the value). */ -const toCompact: Array = new Array(17).fill(INVALID) -toCompact[0] = 0 -for (let nibble = 1; nibble < 14; nibble++) { - if (nibble !== 2) toCompact[fromCompact[nibble]] = nibble -} +/** TType → compact type nibble, with `Bool` as 1. */ +const toCompact: Array = [] +fromCompact.forEach((type, nibble) => { + if (nibble !== 2) toCompact[type] = nibble +}) const typeNames: Array = [] for (const name of Object.keys(Type) as Array) typeNames[Type[name]] = name const describe = (type: number): string => typeNames[type] ?? String(type) -// ----------------------------------------------------------------------------- -// Values -// ----------------------------------------------------------------------------- - /** * Any Thrift value, tagged with the name of its {@link Type}: what * {@link Reader.readValue} returns and {@link Writer.writeValue} writes. @@ -263,10 +248,6 @@ export interface Field { readonly value: Value } -// ----------------------------------------------------------------------------- -// Reading -// ----------------------------------------------------------------------------- - /** * Limits for reading untrusted input. A value that isn't an integer in range * throws a {@link ThriftError} of kind `Usage`. @@ -289,38 +270,27 @@ export interface ReaderOptions { readonly maxContainerSize?: number | undefined } -// Keeps a leading U+FEFF, which is part of the string. -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) const MAX_SAFE = Number.MAX_SAFE_INTEGER const BIGINT_1 = BigInt(1) const BIGINT_7 = BigInt(7) const BIGINT_63 = BigInt(63) const BIGINT_0x7f = BigInt(0x7f) -const I64_MIN = -(BigInt(1) << BIGINT_63) -const I64_MAX = (BigInt(1) << BIGINT_63) - BIGINT_1 + +const zigzag = (n: number): number => ((n << 1) ^ (n >> 31)) >>> 0 +const unzigzag = (n: number): number => (n >>> 1) ^ -(n & 1) /** * A cursor over compact-protocol bytes. Each `read*` method reads one value - * at {@link position} and moves past it; reads throw a {@link ThriftError} - * on malformed input. - * - * **Details** - * - * After {@link readFieldBegin}, {@link fieldId} holds the field's id; after - * {@link readListBegin} or {@link readSetBegin}, {@link elementType} holds - * the element type; after {@link readMapBegin}, {@link keyType} and - * {@link valueType}. Copy the property into a local before reading a nested - * value, which overwrites it. - * - * A `bool` field keeps its value in the field header: {@link readBool} right - * after a `Bool` field header returns it without reading a byte. Inside a - * container, each `bool` is a byte. + * at {@link position} and moves past it, or throws a {@link ThriftError} on + * malformed input. * * **Gotchas** * - * {@link readBinary}, {@link readUuid} and the binary values of - * {@link readValue} are views of {@link bytes}: copy them before mutating or - * reusing the input. + * - A nested read overwrites {@link fieldId}, {@link elementType}, + * {@link keyType} and {@link valueType}: copy one into a local first. + * - {@link readBinary}, {@link readUuid} and the binary values of + * {@link readValue} are views of {@link bytes}: copy them before mutating + * or reusing the input. * * @stability experimental * @category reading @@ -346,7 +316,6 @@ export class Reader { private readonly maxContainerSize: number private lastFieldId = 0 private readonly fieldIds: Array = [] - private structs = 0 private depth = 0 /** The compact type of a bool field header not yet read: 1 true, 2 false, 0 none. */ private bool = 0 @@ -357,9 +326,9 @@ export class Reader { this.bytes = bytes this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) this.end = bytes.length - this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 64, 1000) - this.maxBinaryLength = integerOption("maxBinaryLength", options?.maxBinaryLength, 0, Infinity) - this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity) + this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 1000, 64) + this.maxBinaryLength = integerOption("maxBinaryLength", options?.maxBinaryLength, 0, Infinity, Infinity) + this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity, Infinity) } /** @@ -371,10 +340,7 @@ export class Reader { return this.pos } set position(position: number) { - if (!Number.isInteger(position) || position < 0 || position > this.end) { - fail("Usage", `position ${position} is outside the ${this.end}-byte input`) - } - this.pos = position + this.pos = checkInteger("position", position, 0, this.end) this.bool = 0 } @@ -428,13 +394,12 @@ export class Reader { private varint16(what: string): number { const n = this.varint32(what) if (n > 0xffff) fail("Malformed", `${what} before byte ${this.pos} is longer than 16 bits`) - return (n >>> 1) ^ -(n & 1) + return unzigzag(n) } /** - * A zigzag varint of at most 64 bits as a number: returns the magnitude - * `|v|` for v >= 0 or `|v| - 1` for v < 0 (exact up to 2^53) and sets - * {@link negative}. + * A zigzag varint of at most 64 bits as a number: returns `v` for v >= 0 or + * `-v - 1` for v < 0 (exact below 2^53) and sets {@link negative}. */ private varint64(what: string): number { const bytes = this.bytes @@ -449,8 +414,8 @@ export class Reader { while (b >= 0x80) { if (pos >= end) return this.truncated(what) b = bytes[pos++] - if (pos - start === 10) { - if (b > 0x01) return fail("Malformed", `${what} at byte ${start} is longer than 64 bits`) + if (pos - start === 10 && b > 0x01) { + return fail("Malformed", `${what} at byte ${start} is longer than 64 bits`) } half += (b & 0x7f) * scale scale *= 128 @@ -478,7 +443,6 @@ export class Reader { return this.advance(length, `${what} of ${length} bytes`) } - /** Reads a list or set header. */ private listHeader(what: string): number { const header = this.byte(`${what} header`) let size = header >> 4 @@ -496,24 +460,24 @@ export class Reader { private elementOf(nibble: number, size: number, what: string): Type { const type = fromCompact[nibble] - if (type === INVALID || (type === Type.Stop && size > 0)) { + if (type === undefined || (type === Type.Stop && size > 0)) { return fail("Malformed", `${what} at byte ${this.pos} has element type code ${nibble}`) } - return type as Type + return type } /** Starts reading a struct. Read its fields with {@link readFieldBegin}. */ readStructBegin(): void { this.enter() - this.fieldIds[this.structs++] = this.lastFieldId + this.fieldIds.push(this.lastFieldId) this.lastFieldId = 0 } /** Finishes the struct started with {@link readStructBegin}, after its `Stop`. */ readStructEnd(): void { - if (this.structs === 0) fail("Usage", "readStructEnd without readStructBegin") + if (this.fieldIds.length === 0) fail("Usage", "readStructEnd without readStructBegin") this.depth-- - this.lastFieldId = this.fieldIds[--this.structs] + this.lastFieldId = this.fieldIds.pop()! } /** @@ -536,11 +500,11 @@ export class Reader { this.lastFieldId = id this.fieldId = id const type = fromCompact[nibble] - if (type === INVALID) { + if (type === undefined) { return fail("Malformed", `the field header at byte ${this.pos - 1} has type code ${nibble}`) } if (nibble <= 2) this.bool = nibble - return type as Type + return type } /** Reads a list header: returns the list's size and sets {@link elementType}. */ @@ -590,8 +554,7 @@ export class Reader { /** Reads an `i32`. */ readI32(): number { - const n = this.varint32("an i32") - return (n >>> 1) ^ -(n & 1) + return unzigzag(this.varint32("an i32")) } /** Reads an `i64` as a `bigint`. */ @@ -635,11 +598,7 @@ export class Reader { /** Reads a `string`: a `binary` value decoded as UTF-8. */ readString(): string { const at = this.binary("a string") - try { - return utf8.decode(this.bytes.subarray(at, this.pos)) - } catch { - return fail("InvalidUtf8", `the string at byte ${at} is not valid UTF-8`) - } + return Utf8.decode(this.bytes, at, this.pos) ?? fail("InvalidUtf8", `the string at byte ${at} is not valid UTF-8`) } /** Reads a `uuid`: 16 bytes, as a view of {@link bytes}. */ @@ -655,8 +614,7 @@ export class Reader { skip(type: Type): void { switch (type) { case Type.Bool: - if (this.bool !== 0) this.bool = 0 - else this.advance(1, "a bool") + this.readBool() return case Type.I8: this.advance(1, "an i8") @@ -807,10 +765,6 @@ export const decode = ( } } -// ----------------------------------------------------------------------------- -// Writing -// ----------------------------------------------------------------------------- - const encoder = new TextEncoder() /** @@ -821,8 +775,7 @@ const encoder = new TextEncoder() * * Write a struct as {@link writeStructBegin}, then for each field * {@link writeFieldBegin} and its value, then {@link writeFieldStop} and - * {@link writeStructEnd}. A `bool` field's value goes into its header: - * `writeFieldBegin(Type.Bool, id)` writes nothing until {@link writeBool}. + * {@link writeStructEnd}. * * Containers are a header ({@link writeListBegin}, {@link writeSetBegin}, * {@link writeMapBegin}) followed by exactly that many elements (for a map, @@ -845,7 +798,7 @@ export class Writer { private boolFieldId = 0 constructor(options?: { readonly initialSize?: number | undefined }) { - this.bytes = new Uint8Array(Math.max(16, integerOption("initialSize", options?.initialSize, 0, 1024))) + this.bytes = new Uint8Array(Math.max(16, integerOption("initialSize", options?.initialSize, 0, Infinity, 1024))) this.view = new DataView(this.bytes.buffer) } @@ -886,22 +839,20 @@ export class Writer { this.position = pos } - private integer(value: number, min: number, max: number, what: string): void { - this.checkNoBool(what) - if (!Number.isInteger(value) || value < min || value > max) { - fail("Usage", `${what} ${value} is not an integer from ${min} to ${max}`) - } + private raw(value: Uint8Array): void { + this.reserve(value.length) + this.bytes.set(value, this.position) + this.position += value.length } - private checkSize(size: number, what: string): void { - if (!Number.isInteger(size) || size < 0 || size > 0xffffffff) { - fail("Usage", `${what} ${size} is not an integer from 0 to ${0xffffffff}`) - } + private integer(value: number, min: number, max: number, what: string): void { + this.checkNoBool(what) + checkInteger(what, value, min, max) } private elementNibble(type: number, size: number, what: string): number { const nibble = toCompact[type] - if (nibble === undefined || nibble === INVALID || (type === Type.Stop && size > 0)) { + if (nibble === undefined || (type === Type.Stop && size > 0)) { return fail("Usage", `${describe(type)} is not a valid ${what}`) } return nibble @@ -913,7 +864,7 @@ export class Writer { this.writeByte((delta << 4) | nibble) } else { this.writeByte(nibble) - this.varint32(((id << 1) ^ (id >> 31)) >>> 0) + this.varint32(zigzag(id)) } this.lastFieldId = id } @@ -936,16 +887,14 @@ export class Writer { writeFieldBegin(type: Type, id: number): void { this.checkNoBool("writeFieldBegin") if (this.fieldIds.length === 0) fail("Usage", "writeFieldBegin outside writeStructBegin/writeStructEnd") - if (!Number.isInteger(id) || id < -32768 || id > 32767) { - fail("Usage", `field id ${id} is not an integer from -32768 to 32767`) - } + checkInteger("field id", id, -32768, 32767) if (type === Type.Bool) { this.boolPending = true this.boolFieldId = id return } const nibble = toCompact[type] - if (nibble === undefined || nibble === INVALID || type === Type.Stop) { + if (nibble === undefined || type === Type.Stop) { fail("Usage", `${describe(type)} is not a field type`) } this.fieldHeader(nibble, id) @@ -957,10 +906,10 @@ export class Writer { this.writeByte(0) } - private listHeader(elementType: Type, size: number, what: string): void { - this.checkNoBool(`write${what}Begin`) - this.checkSize(size, `${what} size`) - const nibble = this.elementNibble(elementType, size, `${what} element type`) + private listHeader(elementType: Type, size: number, method: string): void { + this.checkNoBool(method) + checkInteger("container size", size, 0, 0xffffffff) + const nibble = this.elementNibble(elementType, size, "element type") if (size < 15) { this.writeByte((size << 4) | nibble) } else { @@ -971,18 +920,18 @@ export class Writer { /** Writes a list header; write `size` elements of `elementType` next. */ writeListBegin(elementType: Type, size: number): void { - this.listHeader(elementType, size, "List") + this.listHeader(elementType, size, "writeListBegin") } /** Writes a set header; write `size` elements of `elementType` next. */ writeSetBegin(elementType: Type, size: number): void { - this.listHeader(elementType, size, "Set") + this.listHeader(elementType, size, "writeSetBegin") } /** Writes a map header; write `size` key-value pairs next. */ writeMapBegin(keyType: Type, valueType: Type, size: number): void { this.checkNoBool("writeMapBegin") - this.checkSize(size, "map size") + checkInteger("container size", size, 0, 0xffffffff) const key = this.elementNibble(keyType, size, "map key type") const value = this.elementNibble(valueType, size, "map value type") if (size === 0) { @@ -1012,13 +961,13 @@ export class Writer { /** Writes an `i16`. */ writeI16(value: number): void { this.integer(value, -32768, 32767, "i16") - this.varint32(((value << 1) ^ (value >> 31)) >>> 0) + this.varint32(zigzag(value)) } /** Writes an `i32`. */ writeI32(value: number): void { this.integer(value, -2147483648, 2147483647, "i32") - this.varint32(((value << 1) ^ (value >> 31)) >>> 0) + this.varint32(zigzag(value)) } /** Writes an `i64` given as a safe-integer `number`. */ @@ -1044,8 +993,8 @@ export class Writer { /** Writes an `i64`. */ writeI64(value: bigint): void { this.checkNoBool("writeI64") - if (value < I64_MIN || value > I64_MAX) fail("Usage", `i64 ${value} is out of range`) if (value >= -MAX_SAFE && value <= MAX_SAFE) return this.writeI64Number(Number(value)) + if (BigInt.asIntN(64, value) !== value) fail("Usage", `i64 ${value} is out of range`) let u = BigInt.asUintN(64, (value << BIGINT_1) ^ (value >> BIGINT_63)) this.reserve(10) while (u > BIGINT_0x7f) { @@ -1067,9 +1016,7 @@ export class Writer { writeBinary(value: Uint8Array): void { this.checkNoBool("writeBinary") this.varint32(value.length) - this.reserve(value.length) - this.bytes.set(value, this.position) - this.position += value.length + this.raw(value) } /** Writes a `string` as UTF-8 `binary`. Lone surrogates, which UTF-8 can't encode, become U+FFFD. */ @@ -1100,9 +1047,7 @@ export class Writer { writeUuid(value: Uint8Array): void { this.checkNoBool("writeUuid") if (value.length !== 16) fail("Usage", `a uuid is 16 bytes, got ${value.length}`) - this.reserve(16) - this.bytes.set(value, this.position) - this.position += 16 + this.raw(value) } /** Writes a {@link Value}: a field's value after its header, a container element, or a whole value. */ From 81b9a192eb13662d6a4bdfeb5639ed06ba457756 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:16:19 +0000 Subject: [PATCH 18/32] Simplify Thrift tests and fixture generator Compare gold cases as Thrift.Value instead of converting back to JSON, use effect/encoding/Hex, and remove an identity helper and duplicated instructions from generate.py. --- packages/effect/test/encoding/Thrift.test.ts | 38 ++++--------------- .../effect/test/fixtures/thrift/generate.py | 22 +++-------- 2 files changed, 12 insertions(+), 48 deletions(-) diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts index c21922cc5c5..a50da6b98bb 100644 --- a/packages/effect/test/encoding/Thrift.test.ts +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -1,5 +1,6 @@ import { assert, describe, it } from "@effect/vitest" import { Result, Schema } from "effect" +import * as Hex from "effect/encoding/Hex" import * as Thrift from "effect/encoding/Thrift" import { readFileSync } from "node:fs" @@ -31,8 +32,7 @@ const gold: { readonly cases: ReadonlyArray; readonly parquet: ReadonlyArr const corpus: ReadonlyArray = [...gold.cases, ...gold.parquet.map((c) => ({ ...c, type: "Struct" as const }))] const parquet = (id: string) => fromHex(gold.parquet.find((c) => c.id === id)!.bytes) -const fromHex = (hex: string) => Uint8Array.from(hex.match(/../g) ?? [], (b) => parseInt(b, 16)) -const toHex = (bytes: Uint8Array) => Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("") +const fromHex = (hex: string) => Result.getOrThrow(Hex.decode(hex)) const fromJson = (v: Json): Thrift.Value => { switch (v._tag) { @@ -55,30 +55,6 @@ const fromJson = (v: Json): Thrift.Value => { } } -const toJson = (v: Thrift.Value): Json => { - switch (v._tag) { - case "I64": - return { _tag: "I64", value: String(v.value) } - case "Double": - return { - _tag: "Double", - value: Object.is(v.value, -0) ? "-0" : Number.isFinite(v.value) ? v.value : String(v.value) - } - case "Binary": - case "Uuid": - return { _tag: v._tag, value: toHex(v.value) } - case "Struct": - return { _tag: "Struct", fields: v.fields.map((f) => ({ id: f.id, value: toJson(f.value) })) } - case "List": - case "Set": - return { ...v, elements: v.elements.map(toJson) } - case "Map": - return { ...v, entries: v.entries.map(([k, x]) => [toJson(k), toJson(x)] as const) } - default: - return v - } -} - const readAll = (bytes: Uint8Array, type: Thrift.TypeName, options?: Thrift.ReaderOptions) => Thrift.decode(bytes, (r) => { const value = r.readValue(Thrift.Type[type]) @@ -110,11 +86,12 @@ describe("Thrift", () => { it("reads, skips and writes every gold case as the official library does", () => { for (const c of corpus) { const bytes = fromHex(c.bytes) - assert.deepStrictEqual(toJson(Result.getOrThrow(readAll(bytes, c.type))), c.value, c.id) + const value = fromJson(c.value) + assert.deepStrictEqual(Result.getOrThrow(readAll(bytes, c.type)), value, c.id) const r = Thrift.makeReader(bytes) r.skip(Thrift.Type[c.type]) assert.strictEqual(r.position, bytes.length, c.id) - assert.strictEqual(toHex(write((w) => w.writeValue(fromJson(c.value)))), c.bytes, c.id) + assert.strictEqual(Hex.encode(write((w) => w.writeValue(value))), c.bytes, c.id) } }) @@ -341,7 +318,7 @@ describe("Thrift", () => { }) it("rejects limits that aren't integers in range", () => { - // NaN used to disable a limit and a fractional one to round it. + // NaN compares false with everything, so it would switch a limit off. for ( const options of [ { maxDepth: NaN }, @@ -416,7 +393,6 @@ describe("Thrift", () => { "a list element of another type", (w) => w.writeValue({ _tag: "List", elementType: "I32", elements: [{ _tag: "I8", value: 1 }] }) ], - // Used to write nothing. ["a value of no Thrift type", (w) => w.writeValue({ _tag: "Text", value: "a" } as unknown as Thrift.Value)] ] for (const [name, f] of usage) { @@ -424,7 +400,7 @@ describe("Thrift", () => { } it("rejects an initialSize that isn't a non-negative integer", () => { - // NaN used to make the buffer grow forever. + // A NaN size would never satisfy the loop that grows the buffer. for (const initialSize of [NaN, Infinity, -1, 1.5]) { assert.strictEqual(thrown(() => Thrift.makeWriter({ initialSize })), "Usage", String(initialSize)) } diff --git a/packages/effect/test/fixtures/thrift/generate.py b/packages/effect/test/fixtures/thrift/generate.py index 1478066dbc1..b0ce460fb88 100644 --- a/packages/effect/test/fixtures/thrift/generate.py +++ b/packages/effect/test/fixtures/thrift/generate.py @@ -1,11 +1,7 @@ """ -Gold corpus for effect/encoding/Thrift: values written by the official Apache -Thrift Python library's TCompactProtocol (thrift 0.25.0), and the bytes -it produces. The TypeScript test reads every case back into a Thrift.Value, -expects the same tree, and writes the tree back, expecting the same bytes. - - PARQUET_TESTING=/path/to/apache/parquet-testing \ - uv run --with thrift==0.25.0 python test/fixtures/thrift/generate.py +Generates gold.json, the gold corpus for effect/encoding/Thrift: values and +the bytes the official Apache Thrift Python library's TCompactProtocol writes +for them. README.md describes the corpus and how to regenerate it. Values are JSON trees shaped like Thrift.Value: {"_tag": "Bool", "value": true} @@ -18,11 +14,7 @@ {"_tag": "Map", "keyType": "I32", "valueType": "Bool", "entries": [[k, v]]} A case is {"id", "type", "value", "bytes"}: `type` is the Type name of the top-level value. Every case is also read back here with the Python reader to -check the generator. - -The "parquet" cases are Thrift structs cut from parquet-testing files (a -footer, page headers, a bloom filter header, column and offset indexes), -with the tree the Python reader reads from them. +check the generator. The "parquet" cases are structs and carry no `type`. """ import json @@ -72,10 +64,6 @@ def double_json(d): return d -def double_py(v): - return float(v) if isinstance(v, str) else float(v) - - def write(p, v): tag = v["_tag"] if tag == "Bool": @@ -89,7 +77,7 @@ def write(p, v): elif tag == "I64": p.writeI64(int(v["value"])) elif tag == "Double": - p.writeDouble(double_py(v["value"])) + p.writeDouble(float(v["value"])) elif tag == "Binary": p.writeBinary(bytes.fromhex(v["value"])) elif tag == "Uuid": From 0c97830d2bade03af7484b3bee3618f277f7eead Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 20:05:23 +0000 Subject: [PATCH 19/32] Add effect/encoding/Parquet reader --- .changeset/encoding-parquet.md | 5 + dprint.json | 1 + packages/effect/src/encoding/Parquet.ts | 1817 +++++++++++++++++ packages/effect/src/encoding/index.ts | 6 + .../effect/src/internal/parquet/encodings.ts | 552 +++++ packages/effect/test/encoding/Parquet.test.ts | 1175 +++++++++++ .../test/fixtures/parquet/CONFORMANCE.md | 95 + .../effect/test/fixtures/parquet/README.md | 31 + .../test/fixtures/parquet/dictionary.parquet | Bin 0 -> 3193 bytes .../test/fixtures/parquet/empty.parquet | Bin 0 -> 495 bytes .../test/fixtures/parquet/encodings.parquet | Bin 0 -> 11604 bytes .../fixtures/parquet/encodings_v2.parquet | Bin 0 -> 11632 bytes .../effect/test/fixtures/parquet/generate.py | 319 +++ .../effect/test/fixtures/parquet/gold.json | 1 + .../effect/test/fixtures/parquet/gzip.parquet | Bin 0 -> 3056 bytes .../test/fixtures/parquet/int96.parquet | Bin 0 -> 1014 bytes .../test/fixtures/parquet/nested.parquet | Bin 0 -> 9726 bytes .../test/fixtures/parquet/nested_v2.parquet | Bin 0 -> 9569 bytes .../test/fixtures/parquet/plain.parquet | Bin 0 -> 8857 bytes 19 files changed, 4002 insertions(+) create mode 100644 .changeset/encoding-parquet.md create mode 100644 packages/effect/src/encoding/Parquet.ts create mode 100644 packages/effect/src/internal/parquet/encodings.ts create mode 100644 packages/effect/test/encoding/Parquet.test.ts create mode 100644 packages/effect/test/fixtures/parquet/CONFORMANCE.md create mode 100644 packages/effect/test/fixtures/parquet/README.md create mode 100644 packages/effect/test/fixtures/parquet/dictionary.parquet create mode 100644 packages/effect/test/fixtures/parquet/empty.parquet create mode 100644 packages/effect/test/fixtures/parquet/encodings.parquet create mode 100644 packages/effect/test/fixtures/parquet/encodings_v2.parquet create mode 100644 packages/effect/test/fixtures/parquet/generate.py create mode 100644 packages/effect/test/fixtures/parquet/gold.json create mode 100644 packages/effect/test/fixtures/parquet/gzip.parquet create mode 100644 packages/effect/test/fixtures/parquet/int96.parquet create mode 100644 packages/effect/test/fixtures/parquet/nested.parquet create mode 100644 packages/effect/test/fixtures/parquet/nested_v2.parquet create mode 100644 packages/effect/test/fixtures/parquet/plain.parquet diff --git a/.changeset/encoding-parquet.md b/.changeset/encoding-parquet.md new file mode 100644 index 00000000000..3f50b37ccae --- /dev/null +++ b/.changeset/encoding-parquet.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add the experimental `effect/encoding/Parquet` module, which reads Apache Parquet metadata and column chunks into Arrow-style buffers. diff --git a/dprint.json b/dprint.json index d8e26de0455..4a1e34baa16 100644 --- a/dprint.json +++ b/dprint.json @@ -23,6 +23,7 @@ "packages/effect/src/StandardSchema.ts", "packages/tools/graphql-generator/test/generated/**/*.graphql.ts", "packages/effect/test/fixtures/thrift/gold.json", + "packages/effect/test/fixtures/parquet/gold.json", "!scratchpad/**/*", "**/.*", "!.agents", diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts new file mode 100644 index 00000000000..3b92ca35a98 --- /dev/null +++ b/packages/effect/src/encoding/Parquet.ts @@ -0,0 +1,1817 @@ +/** + * Reads Apache Parquet files: the footer metadata and the values of one column + * chunk at a time, in buffers laid out the way Apache Arrow lays out arrays. + * + * **Mental model** + * + * - A Parquet file holds row groups; each row group holds one column chunk per + * leaf column of the schema. {@link readMetadata} reads the footer, which + * describes the schema, the row groups and where every column chunk is. + * - {@link readColumn} reads one column chunk and returns a {@link Column}: one + * {@link Layer} per schema node on the column's path, with Arrow-style + * validity bitmaps and list offsets, and the leaf's {@link Values} with a slot + * for every null. + * - Both read through a {@link Source}, so the file can be in memory + * ({@link fromBytes}), on disk or behind HTTP range requests. + * - Compressed pages go through a {@link Decompress} function supplied per + * codec in {@link readColumn}'s options. This module has no codecs of its own. + * + * **Example** (Reading a column) + * + * ```ts + * import { Effect } from "effect" + * import { Parquet } from "effect/encoding" + * import { readFile } from "node:fs/promises" + * + * const program = Effect.gen(function*() { + * const source = Parquet.fromBytes(new Uint8Array(yield* Effect.promise(() => readFile("data.parquet")))) + * const metadata = yield* Parquet.readMetadata(source) + * const column = metadata.columns.findIndex((c) => c.path.join(".") === "price") + * const { values, layers } = yield* Parquet.readColumn(source, metadata, { rowGroup: 0, column }) + * // layers[0].validity marks the non-null rows, values.values holds them + * return { values, validity: layers[0].validity } + * }) + * ``` + * + * **Details** + * + * The reader follows + * [apache/parquet-format 2.14.0](https://github.com/apache/parquet-format/tree/apache-parquet-format-2.14.0): + * + * - Physical types: all eight. Values stay in their physical type; logical + * types and converted types are decoded into the schema but not applied + * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). + * - Encodings: `PLAIN`, `PLAIN_DICTIONARY` and `RLE_DICTIONARY` (values + * are looked up in the dictionary), `RLE` (booleans), `DELTA_BINARY_PACKED`, + * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; + * levels in `RLE` or the deprecated `BIT_PACKED`. + * - Pages: data pages v1 and v2 and dictionary pages; index pages and page + * types added after 2.14.0 are skipped. + * - Nesting: any depth of optional, required and repeated fields, assembled from + * repetition and definition levels into {@link Layer}s. + * + * Not supported, failing with a {@link ParquetError} of kind `Unsupported`: + * encrypted files and columns, the `ALP` encoding, column chunks in other + * files (`file_path`), and compressed pages without a matching + * {@link Decompress}. Not read: page indexes, bloom filters, page checksums, + * size and geospatial statistics. + * + * Input is untrusted: every offset and length is checked against the bytes + * that are there, and the sizes a file claims are capped by options + * (`maxFooterLength`, `maxDepth`, `maxPageSize`, `maxValues`, + * `maxBinaryLength`) before anything that large is allocated. + * + * Writing Parquet is not supported. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Data from "../Data.ts" +import * as Effect from "../Effect.ts" +import * as Encodings from "../internal/parquet/encodings.ts" +import * as Predicate from "../Predicate.ts" +import * as Thrift from "./Thrift.ts" + +const ParquetErrorTypeId = "~effect/encoding/Parquet/ParquetError" + +/** + * Error raised when a Parquet file can't be read. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `Malformed`: the file breaks the Parquet format, for example a bad magic + * number, invalid Thrift metadata, an offset outside the file, or pages that + * don't add up to their column chunk. + * - `Unsupported`: the file uses a feature this module doesn't read, or a + * compression codec with no {@link Decompress} in the options. + * - `Limit`: the file needs more than an option such as `maxPageSize` allows. + * - `Codec`: a {@link Decompress} function failed; its error is the `cause`. + * - `Usage`: an option or argument is invalid. These are bugs in the caller, + * so they are defects, not failures. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class ParquetError extends Data.TaggedError("ParquetError")<{ + readonly kind: "Malformed" | "Unsupported" | "Limit" | "Codec" | "Usage" + readonly message: string + readonly cause?: unknown +}> { + /** + * Marks this value as a Parquet error for runtime guards. + * + * @stability experimental + * @since 4.1.5 + */ + readonly [ParquetErrorTypeId] = ParquetErrorTypeId +} + +/** + * Checks whether a value is a {@link ParquetError}. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isParquetError = (u: unknown): u is ParquetError => Predicate.hasProperty(u, ParquetErrorTypeId) + +/** + * A Parquet physical type: how values are stored. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type PhysicalType = + | "BOOLEAN" + | "INT32" + | "INT64" + | "INT96" + | "FLOAT" + | "DOUBLE" + | "BYTE_ARRAY" + | "FIXED_LEN_BYTE_ARRAY" + +/** + * Whether a schema field is required, optional (may be null) or repeated (a + * list of zero or more values). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Repetition = "REQUIRED" | "OPTIONAL" | "REPEATED" + +/** + * A Parquet encoding. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Encoding = + | "PLAIN" + | "PLAIN_DICTIONARY" + | "RLE" + | "BIT_PACKED" + | "DELTA_BINARY_PACKED" + | "DELTA_LENGTH_BYTE_ARRAY" + | "DELTA_BYTE_ARRAY" + | "RLE_DICTIONARY" + | "BYTE_STREAM_SPLIT" + | "ALP" + +/** + * A Parquet compression codec. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type CompressionCodec = "UNCOMPRESSED" | "SNAPPY" | "GZIP" | "LZO" | "BROTLI" | "LZ4" | "ZSTD" | "LZ4_RAW" + +/** + * A legacy type annotation, superseded by {@link LogicalType}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type ConvertedType = + | "UTF8" + | "MAP" + | "MAP_KEY_VALUE" + | "LIST" + | "ENUM" + | "DECIMAL" + | "DATE" + | "TIME_MILLIS" + | "TIME_MICROS" + | "TIMESTAMP_MILLIS" + | "TIMESTAMP_MICROS" + | "UINT_8" + | "UINT_16" + | "UINT_32" + | "UINT_64" + | "INT_8" + | "INT_16" + | "INT_32" + | "INT_64" + | "JSON" + | "BSON" + | "INTERVAL" + +/** + * A type annotation that says how to interpret a field's physical values. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type LogicalType = + | { + readonly type: + | "STRING" + | "MAP" + | "LIST" + | "ENUM" + | "DATE" + | "UNKNOWN" + | "JSON" + | "BSON" + | "UUID" + | "FLOAT16" + | "FILE" + } + | { readonly type: "DECIMAL"; readonly scale: number; readonly precision: number } + | { + readonly type: "TIME" | "TIMESTAMP" + readonly isAdjustedToUTC: boolean + readonly unit: "MILLIS" | "MICROS" | "NANOS" + } + | { readonly type: "INTEGER"; readonly bitWidth: number; readonly isSigned: boolean } + | { readonly type: "VARIANT"; readonly specificationVersion: number | undefined } + | { readonly type: "GEOMETRY"; readonly crs: string | undefined } + | { + readonly type: "GEOGRAPHY" + readonly crs: string | undefined + readonly algorithm: "SPHERICAL" | "VINCENTY" | "THOMAS" | "ANDOYER" | "KARNEY" | number | undefined + } + +/** + * The sort order of a column's `minValue` and `maxValue` statistics. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type ColumnOrder = "TYPE_ORDER" | "IEEE_754_TOTAL_ORDER" | "INT96_TIMESTAMP_ORDER" + +/** + * A node of the schema tree: a group with `children`, or a leaf with a + * `physicalType`. + * + * **Details** + * + * Enum-valued fields hold the spec's name, or the numeric code of a value this + * module doesn't know. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface SchemaNode { + readonly name: string + /** `undefined` only for the root. */ + readonly repetition: Repetition | undefined + /** `undefined` for groups. */ + readonly physicalType: PhysicalType | undefined + /** The byte width of `FIXED_LEN_BYTE_ARRAY` values. */ + readonly typeLength: number | undefined + readonly convertedType: ConvertedType | number | undefined + /** `undefined` when absent or of a kind this module doesn't know. */ + readonly logicalType: LogicalType | undefined + readonly scale: number | undefined + readonly precision: number | undefined + readonly fieldId: number | undefined + /** `undefined` for leaves. */ + readonly children: ReadonlyArray | undefined +} + +/** + * A leaf column of the schema, in the order of the column chunks of every row + * group. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ColumnDescriptor { + /** The names of {@link nodes}. */ + readonly path: ReadonlyArray + /** The schema nodes from a child of the root down to the leaf. */ + readonly nodes: ReadonlyArray + readonly physicalType: PhysicalType + readonly typeLength: number | undefined + /** The number of optional and repeated nodes on the path. */ + readonly maxDefinitionLevel: number + /** The number of repeated nodes on the path. */ + readonly maxRepetitionLevel: number + /** `undefined` when the file has no column orders, or one this module doesn't know. */ + readonly columnOrder: ColumnOrder | undefined +} + +/** + * Key/value metadata of a file or column chunk. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface KeyValue { + readonly key: string + readonly value: string | undefined +} + +/** + * Statistics of a column chunk, as written. + * + * **Details** + * + * A count that is `undefined` is unknown, not zero. Min and max values are + * the plain-encoded bytes of a value, ordered by the column's + * {@link ColumnDescriptor.columnOrder}; `min` and `max` are the deprecated + * fields, ordered by signed comparison. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Statistics { + readonly nullCount: number | undefined + readonly distinctCount: number | undefined + readonly nanCount: number | undefined + readonly minValue: Uint8Array | undefined + readonly maxValue: Uint8Array | undefined + readonly isMinValueExact: boolean | undefined + readonly isMaxValueExact: boolean | undefined + readonly min: Uint8Array | undefined + readonly max: Uint8Array | undefined +} + +/** + * Where a column chunk's pages are and how they are written. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ColumnMetaData { + readonly physicalType: PhysicalType + readonly encodings: ReadonlyArray + readonly pathInSchema: ReadonlyArray + readonly codec: CompressionCodec | number + /** The number of values, nulls included. */ + readonly numValues: number + readonly totalUncompressedSize: number + readonly totalCompressedSize: number + readonly keyValueMetadata: ReadonlyArray + readonly dataPageOffset: number + readonly indexPageOffset: number | undefined + readonly dictionaryPageOffset: number | undefined + readonly statistics: Statistics | undefined +} + +/** + * A column chunk of a row group. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ColumnChunk { + /** Set when the chunk is in another file, which this module doesn't read. */ + readonly filePath: string | undefined + /** Whether the chunk is encrypted, which this module doesn't read. */ + readonly encrypted: boolean + /** `undefined` only when the chunk is encrypted. */ + readonly metaData: ColumnMetaData | undefined +} + +/** + * A row group: one column chunk per leaf column. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface RowGroup { + readonly numRows: number + readonly totalByteSize: number + /** In the order of {@link FileMetaData.columns}. */ + readonly columns: ReadonlyArray +} + +/** + * The metadata in a Parquet file's footer. + * + * **Gotchas** + * + * Binary statistics are views of the footer bytes read from the + * {@link Source}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface FileMetaData { + readonly version: number + readonly numRows: number + readonly createdBy: string | undefined + readonly keyValueMetadata: ReadonlyArray + /** The root of the schema tree. */ + readonly schema: SchemaNode + /** The leaf columns, in schema order. */ + readonly columns: ReadonlyArray + readonly rowGroups: ReadonlyArray +} + +/** + * Random access to the bytes of a Parquet file. + * + * **Details** + * + * `read` returns the `length` bytes at `offset`; the reader asks only for + * ranges inside `size`. Returning fewer bytes fails with `Malformed`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Source { + readonly size: number + readonly read: (offset: number, length: number) => Effect.Effect +} + +/** + * Creates a {@link Source} over bytes in memory. + * + * **Gotchas** + * + * Reads return views of `bytes`, so do not mutate them while reading. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromBytes = (bytes: Uint8Array): Source => ({ + size: bytes.length, + read: (offset, length) => Effect.succeed(bytes.subarray(offset, offset + length)) +}) + +/** + * Decompresses one page: the bytes as written, and the size the page header + * says they decompress to. + * + * **Details** + * + * A result of another size fails with `Malformed`; a failure becomes a + * {@link ParquetError} of kind `Codec` with the error as its `cause`. For + * `GZIP`, pages may hold several gzip members. `LZ4` is the deprecated + * Hadoop-framed codec; `LZ4_RAW` is the plain LZ4 block format. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Decompress = (data: Uint8Array, uncompressedSize: number) => Effect.Effect + +/** + * The values of a column chunk's leaf, one slot per slot of the leaf's + * {@link Layer}, in Arrow's layout. + * + * **Details** + * + * - `BOOLEAN` values are a bitmap, least significant bit first. + * - `INT96` values are 12 bytes each, `FIXED_LEN_BYTE_ARRAY` values `width` + * bytes each. + * - `BYTE_ARRAY` value `i` is `values.subarray(offsets[i], offsets[i + 1])`. + * - A null slot holds zero, `false`, zero bytes or an empty byte array. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Values = + | { readonly type: "BOOLEAN"; readonly values: Uint8Array } + | { readonly type: "INT32"; readonly values: Int32Array } + | { readonly type: "INT64"; readonly values: BigInt64Array } + | { readonly type: "INT96"; readonly values: Uint8Array } + | { readonly type: "FLOAT"; readonly values: Float32Array } + | { readonly type: "DOUBLE"; readonly values: Float64Array } + | { readonly type: "BYTE_ARRAY"; readonly offsets: Int32Array; readonly values: Uint8Array } + | { readonly type: "FIXED_LEN_BYTE_ARRAY"; readonly width: number; readonly values: Uint8Array } + +/** + * One schema node of a column's path, as an Arrow array without its values. + * + * **Details** + * + * A node that isn't repeated has one slot per slot of its parent (per row for + * a child of the root). A repeated node has one slot per element, and its + * `offsets` give, for slot `i` of the parent, the elements from + * `offsets[i]` to `offsets[i + 1]`. + * + * An optional node has a `validity` bitmap, least significant bit first: a + * set bit is a non-null slot. A slot whose ancestor is null is null too. A + * repeated node has no validity; an empty and a null list are told apart by + * the optional node above it. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Layer { + readonly node: SchemaNode + readonly length: number + /** For optional nodes. */ + readonly validity: Uint8Array | undefined + readonly nullCount: number + /** For repeated nodes: `length` of the parent plus one. */ + readonly offsets: Int32Array | undefined +} + +/** + * A column chunk's values. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Column { + readonly descriptor: ColumnDescriptor + readonly numRows: number + /** One per node of {@link ColumnDescriptor.nodes}. */ + readonly layers: ReadonlyArray + /** One slot per slot of the last layer. */ + readonly values: Values +} + +// --------------------------------------------------------------------------- +// Errors and options +// --------------------------------------------------------------------------- + +const { Failure, fail } = Encodings + +const usage = (message: string): never => { + throw new ParquetError({ kind: "Usage", message }) +} + +const limit = (name: string, value: number | undefined, fallback: number, max: number): number => { + if (value === undefined) return fallback + if (!Number.isSafeInteger(value) || value < 1 || value > max) { + usage(`${name} must be an integer from 1 to ${max}, got ${value}`) + } + return value +} + +const MAX_INT32 = 0x7fffffff + +/** Runs synchronous decoding, turning its failures into {@link ParquetError}s. */ +const attempt = (f: () => A): Effect.Effect => + Effect.suspend(() => { + try { + return Effect.succeed(f()) + } catch (cause) { + if (cause instanceof Failure) return Effect.fail(new ParquetError({ kind: cause.kind, message: cause.message })) + if (Thrift.isThriftError(cause) && cause.kind !== "Usage") { + return Effect.fail( + new ParquetError({ + kind: cause.kind === "Limit" ? "Limit" : "Malformed", + message: `invalid Thrift metadata: ${cause.message}` + }) + ) + } + return Effect.die(cause) + } + }) + +const error = (kind: ParquetError["kind"], message: string) => new ParquetError({ kind, message }) + +/** Prefixes an error's message with where in the file it happened. */ +const locate = (where: string) => (e: ParquetError): ParquetError => + new ParquetError({ + kind: e.kind, + message: `${where}: ${e.message}`, + ...(e.cause === undefined ? {} : { cause: e.cause }) + }) + +const readRange = ( + source: Source, + offset: number, + length: number +): Effect.Effect => + Effect.flatMap(source.read(offset, length), (bytes) => + bytes.length === length + ? Effect.succeed(bytes) + : Effect.fail( + error("Malformed", `the source returned ${bytes.length} bytes at offset ${offset} instead of ${length}`) + )) + +// --------------------------------------------------------------------------- +// Thrift metadata +// --------------------------------------------------------------------------- + +const T = Thrift.Type + +const physicalTypes: ReadonlyArray = [ + "BOOLEAN", + "INT32", + "INT64", + "INT96", + "FLOAT", + "DOUBLE", + "BYTE_ARRAY", + "FIXED_LEN_BYTE_ARRAY" +] +const repetitions: ReadonlyArray = ["REQUIRED", "OPTIONAL", "REPEATED"] +const encodings: ReadonlyArray = [ + "PLAIN", + undefined, + "PLAIN_DICTIONARY", + "RLE", + "BIT_PACKED", + "DELTA_BINARY_PACKED", + "DELTA_LENGTH_BYTE_ARRAY", + "DELTA_BYTE_ARRAY", + "RLE_DICTIONARY", + "BYTE_STREAM_SPLIT", + "ALP" +] +const codecs: ReadonlyArray = [ + "UNCOMPRESSED", + "SNAPPY", + "GZIP", + "LZO", + "BROTLI", + "LZ4", + "ZSTD", + "LZ4_RAW" +] +const convertedTypes: ReadonlyArray = [ + "UTF8", + "MAP", + "MAP_KEY_VALUE", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME_MILLIS", + "TIME_MICROS", + "TIMESTAMP_MILLIS", + "TIMESTAMP_MICROS", + "UINT_8", + "UINT_16", + "UINT_32", + "UINT_64", + "INT_8", + "INT_16", + "INT_32", + "INT_64", + "JSON", + "BSON", + "INTERVAL" +] +const edgeAlgorithms = ["SPHERICAL", "VINCENTY", "THOMAS", "ANDOYER", "KARNEY"] as const +const timeUnits = [undefined, "MILLIS", "MICROS", "NANOS"] as const +const columnOrders = [undefined, "TYPE_ORDER", "IEEE_754_TOTAL_ORDER", "INT96_TIMESTAMP_ORDER"] as const + +const name = (names: ReadonlyArray, code: number): A | number => names[code] ?? code + +type Reader = Thrift.Reader + +const struct = (r: Reader, field: (id: number, type: Thrift.Type) => void): void => { + r.readStructBegin() + for (let type = r.readFieldBegin(); type !== T.Stop; type = r.readFieldBegin()) field(r.fieldId, type) + r.readStructEnd() +} + +// Generated Thrift code skips a field of an unexpected type, as these do. +const skip = (r: Reader, type: Thrift.Type): undefined => { + r.skip(type) + return undefined +} +const i32 = (r: Reader, type: Thrift.Type) => type === T.I32 ? r.readI32() : skip(r, type) +const i8 = (r: Reader, type: Thrift.Type) => type === T.I8 ? r.readI8() : skip(r, type) +const i64 = (r: Reader, type: Thrift.Type) => type === T.I64 ? r.readI64Number() : skip(r, type) +const bool = (r: Reader, type: Thrift.Type) => type === T.Bool ? r.readBool() : skip(r, type) +const string = (r: Reader, type: Thrift.Type) => type === T.Binary ? r.readString() : skip(r, type) +const binary = (r: Reader, type: Thrift.Type) => type === T.Binary ? r.readBinary() : skip(r, type) +const list = (r: Reader, type: Thrift.Type, element: Thrift.Type, read: () => A): Array | undefined => { + if (type !== T.List) return skip(r, type) + const out: Array = [] + for (let n = r.readListBegin(), elementType = r.elementType; n > 0; n--) { + if (elementType !== element) return fail("Malformed", "a metadata list has elements of the wrong type") + out.push(read()) + } + return out +} + +const required = (value: A | undefined, what: string): A => + value === undefined ? fail("Malformed", `${what} is missing`) : value + +const keyValue = (r: Reader): KeyValue => { + let key: string | undefined + let value: string | undefined + struct(r, (id, type) => { + if (id === 1) key = string(r, type) + else if (id === 2) value = string(r, type) + else r.skip(type) + }) + return { key: required(key, "KeyValue.key"), value } +} + +const statistics = (r: Reader): Statistics => { + const s = { + nullCount: undefined as number | undefined, + distinctCount: undefined as number | undefined, + nanCount: undefined as number | undefined, + minValue: undefined as Uint8Array | undefined, + maxValue: undefined as Uint8Array | undefined, + isMinValueExact: undefined as boolean | undefined, + isMaxValueExact: undefined as boolean | undefined, + min: undefined as Uint8Array | undefined, + max: undefined as Uint8Array | undefined + } + struct(r, (id, type) => { + switch (id) { + case 1: + s.max = binary(r, type) + break + case 2: + s.min = binary(r, type) + break + case 3: + s.nullCount = i64(r, type) + break + case 4: + s.distinctCount = i64(r, type) + break + case 5: + s.maxValue = binary(r, type) + break + case 6: + s.minValue = binary(r, type) + break + case 7: + s.isMaxValueExact = bool(r, type) + break + case 8: + s.isMinValueExact = bool(r, type) + break + case 9: + s.nanCount = i64(r, type) + break + default: + r.skip(type) + } + }) + return s +} + +const simpleLogicalTypes = { + 1: "STRING", + 2: "MAP", + 3: "LIST", + 4: "ENUM", + 6: "DATE", + 11: "UNKNOWN", + 12: "JSON", + 13: "BSON", + 14: "UUID", + 15: "FLOAT16", + 19: "FILE" +} as const + +/** Reads a struct's fields into an array indexed by field id; unread field types are skipped. */ +const fields = (r: Reader, read: (id: number, type: Thrift.Type) => unknown): Array => { + const out: Array = [] + struct(r, (id, type) => { + out[id] = read(id, type) + }) + return out +} + +const timeUnit = (r: Reader): "MILLIS" | "MICROS" | "NANOS" | undefined => { + let unit: "MILLIS" | "MICROS" | "NANOS" | undefined + struct(r, (id, type) => { + r.skip(type) + if (type === T.Struct) unit = timeUnits[id as 1 | 2 | 3] + }) + return unit +} + +const logicalType = (r: Reader): LogicalType | undefined => { + let out: LogicalType | undefined + struct(r, (id, type) => { + if (type !== T.Struct) return r.skip(type) + const simple = simpleLogicalTypes[id as keyof typeof simpleLogicalTypes] + if (simple !== undefined) { + r.skip(type) + out = { type: simple } + return + } + switch (id) { + case 5: { + const f = fields(r, (_, t) => i32(r, t)) + out = { + type: "DECIMAL", + scale: required(f[1] as number | undefined, "DecimalType.scale"), + precision: required(f[2] as number | undefined, "DecimalType.precision") + } + return + } + case 7: + case 8: { + const f = fields(r, (fid, t) => fid === 1 ? bool(r, t) : fid === 2 && t === T.Struct ? timeUnit(r) : skip(r, t)) + const unit = f[2] as "MILLIS" | "MICROS" | "NANOS" | undefined + const isAdjustedToUTC = required(f[1] as boolean | undefined, "TimeType.isAdjustedToUTC") + out = unit === undefined ? undefined : { type: id === 7 ? "TIME" : "TIMESTAMP", isAdjustedToUTC, unit } + return + } + case 10: { + const f = fields(r, (fid, t) => fid === 1 ? i8(r, t) : fid === 2 ? bool(r, t) : skip(r, t)) + out = { + type: "INTEGER", + bitWidth: required(f[1] as number | undefined, "IntType.bitWidth"), + isSigned: required(f[2] as boolean | undefined, "IntType.isSigned") + } + return + } + case 16: { + const f = fields(r, (fid, t) => fid === 1 ? i8(r, t) : skip(r, t)) + out = { type: "VARIANT", specificationVersion: f[1] as number | undefined } + return + } + case 17: { + const f = fields(r, (fid, t) => fid === 1 ? string(r, t) : skip(r, t)) + out = { type: "GEOMETRY", crs: f[1] as string | undefined } + return + } + case 18: { + const f = fields(r, (fid, t) => fid === 1 ? string(r, t) : fid === 2 ? i32(r, t) : skip(r, t)) + const algorithm = f[2] as number | undefined + out = { + type: "GEOGRAPHY", + crs: f[1] as string | undefined, + algorithm: algorithm === undefined ? undefined : name(edgeAlgorithms, algorithm) + } + return + } + default: + r.skip(type) + out = undefined + } + }) + return out +} + +interface Element { + readonly type: number | undefined + readonly typeLength: number | undefined + readonly repetition: number | undefined + readonly name: string + readonly numChildren: number | undefined + readonly convertedType: number | undefined + readonly scale: number | undefined + readonly precision: number | undefined + readonly fieldId: number | undefined + readonly logicalType: LogicalType | undefined +} + +const schemaElement = (r: Reader): Element => { + const f = fields(r, (id, type) => { + switch (id) { + case 4: + return string(r, type) + case 10: + return type === T.Struct ? logicalType(r) : skip(r, type) + case 1: + case 2: + case 3: + case 5: + case 6: + case 7: + case 8: + case 9: + return i32(r, type) + default: + return skip(r, type) + } + }) + return { + type: f[1] as number | undefined, + typeLength: f[2] as number | undefined, + repetition: f[3] as number | undefined, + name: required(f[4] as string | undefined, "SchemaElement.name"), + numChildren: f[5] as number | undefined, + convertedType: f[6] as number | undefined, + scale: f[7] as number | undefined, + precision: f[8] as number | undefined, + fieldId: f[9] as number | undefined, + logicalType: f[10] as LogicalType | undefined + } +} + +const columnMetaData = (r: Reader): ColumnMetaData => { + const f = fields(r, (id, type) => { + switch (id) { + case 1: + case 4: + return i32(r, type) + case 2: + return list(r, type, T.I32, () => name(encodings, r.readI32())) + case 3: + return list(r, type, T.Binary, () => r.readString()) + case 5: + case 6: + case 7: + case 9: + case 10: + case 11: + return i64(r, type) + case 8: + return list(r, type, T.Struct, () => keyValue(r)) + case 12: + return type === T.Struct ? statistics(r) : skip(r, type) + default: + return skip(r, type) + } + }) + const type = required(f[1] as number | undefined, "ColumnMetaData.type") + return { + physicalType: physicalTypes[type] ?? fail("Unsupported", `physical type ${type} is not supported`), + encodings: required(f[2] as Array | undefined, "ColumnMetaData.encodings"), + pathInSchema: required(f[3] as Array | undefined, "ColumnMetaData.path_in_schema"), + codec: name(codecs, required(f[4] as number | undefined, "ColumnMetaData.codec")), + numValues: required(f[5] as number | undefined, "ColumnMetaData.num_values"), + totalUncompressedSize: required(f[6] as number | undefined, "ColumnMetaData.total_uncompressed_size"), + totalCompressedSize: required(f[7] as number | undefined, "ColumnMetaData.total_compressed_size"), + keyValueMetadata: (f[8] as Array | undefined) ?? [], + dataPageOffset: required(f[9] as number | undefined, "ColumnMetaData.data_page_offset"), + indexPageOffset: f[10] as number | undefined, + dictionaryPageOffset: f[11] as number | undefined, + statistics: f[12] as Statistics | undefined + } +} + +const columnChunk = (r: Reader): ColumnChunk => { + const f = fields(r, (id, type) => { + switch (id) { + case 1: + return string(r, type) + case 3: + return type === T.Struct ? columnMetaData(r) : skip(r, type) + case 8: + case 9: + r.skip(type) + return true + default: + return skip(r, type) + } + }) + const encrypted = f[8] === true || f[9] === true + const metaData = f[3] as ColumnMetaData | undefined + if (!encrypted && metaData === undefined) fail("Malformed", "ColumnChunk.meta_data is missing") + return { filePath: f[1] as string | undefined, encrypted, metaData } +} + +const rowGroup = (r: Reader): RowGroup => { + const f = fields(r, (id, type) => { + switch (id) { + case 1: + return list(r, type, T.Struct, () => columnChunk(r)) + case 2: + case 3: + return i64(r, type) + default: + return skip(r, type) + } + }) + return { + columns: required(f[1] as Array | undefined, "RowGroup.columns"), + totalByteSize: required(f[2] as number | undefined, "RowGroup.total_byte_size"), + numRows: required(f[3] as number | undefined, "RowGroup.num_rows") + } +} + +const columnOrder = (r: Reader): ColumnOrder | undefined => { + let order: ColumnOrder | undefined + struct(r, (id, type) => { + r.skip(type) + if (type === T.Struct) order = columnOrders[id as 1 | 2 | 3] + }) + return order +} + +interface Schema { + readonly root: SchemaNode + readonly columns: Array +} + +/** Rebuilds the schema tree from its depth-first list of elements. */ +const schemaTree = ( + elements: ReadonlyArray, + orders: ReadonlyArray | undefined, + maxDepth: number +): Schema => { + const columns: Array = [] + let index = 0 + const node = (path: ReadonlyArray, definition: number, repetitionLevel: number): SchemaNode => { + if (index >= elements.length) fail("Malformed", "the schema ends inside a group") + const e = elements[index++] + const isRoot = path.length === 0 && index === 1 + let repetition: Repetition | undefined + if (!isRoot) { + if (path.length + 1 > maxDepth) fail("Limit", `the schema nests deeper than maxDepth (${maxDepth})`) + const code = required(e.repetition, `the repetition of schema element "${e.name}"`) + repetition = repetitions[code] ?? fail("Malformed", `schema element "${e.name}" has repetition ${code}`) + if (repetition !== "REQUIRED") definition++ + if (repetition === "REPEATED") repetitionLevel++ + } + const base = { + name: e.name, + repetition, + typeLength: e.typeLength, + convertedType: e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType), + logicalType: e.logicalType, + scale: e.scale, + precision: e.precision, + fieldId: e.fieldId + } + if (e.type === undefined) { + const count = required(e.numChildren, `the type or children of schema element "${e.name}"`) + if (count < 0) fail("Malformed", `schema element "${e.name}" has ${count} children`) + const children: Array = [] + const self: SchemaNode = { ...base, physicalType: undefined, children } + const nodes = isRoot ? path : [...path, self] + for (let i = 0; i < count; i++) children.push(node(nodes, definition, repetitionLevel)) + return self + } + if (isRoot) fail("Malformed", "the schema root is not a group") + if (e.numChildren !== undefined && e.numChildren !== 0) { + fail("Malformed", `schema element "${e.name}" has both a type and children`) + } + const physicalType = physicalTypes[e.type] ?? fail("Unsupported", `physical type ${e.type} is not supported`) + if (physicalType === "FIXED_LEN_BYTE_ARRAY" && !(e.typeLength !== undefined && e.typeLength > 0)) { + fail("Malformed", `FIXED_LEN_BYTE_ARRAY column "${e.name}" has no positive type_length`) + } + const self: SchemaNode = { ...base, physicalType, children: undefined } + const nodes = [...path, self] + columns.push({ + path: nodes.map((n) => n.name), + nodes, + physicalType, + typeLength: physicalType === "FIXED_LEN_BYTE_ARRAY" ? e.typeLength : undefined, + maxDefinitionLevel: definition, + maxRepetitionLevel: repetitionLevel, + columnOrder: orders?.[columns.length] + }) + return self + } + const root = node([], 0, 0) + if (index !== elements.length) fail("Malformed", `the schema has ${elements.length - index} elements after its root`) + if (orders !== undefined && orders.length !== columns.length) { + fail("Malformed", `the file has ${orders.length} column orders for ${columns.length} columns`) + } + return { root, columns } +} + +const fileMetaData = (footer: Uint8Array, maxDepth: number): FileMetaData => { + const r = Thrift.makeReader(footer) + const f = fields(r, (id, type) => { + switch (id) { + case 1: + return i32(r, type) + case 2: + return list(r, type, T.Struct, () => schemaElement(r)) + case 3: + return i64(r, type) + case 4: + return list(r, type, T.Struct, () => rowGroup(r)) + case 5: + return list(r, type, T.Struct, () => keyValue(r)) + case 6: + return string(r, type) + case 7: + return list(r, type, T.Struct, () => columnOrder(r)) + default: + return skip(r, type) + } + }) + const elements = required(f[2] as Array | undefined, "FileMetaData.schema") + if (elements.length === 0) fail("Malformed", "FileMetaData.schema is empty") + const { columns, root } = schemaTree(elements, f[7] as Array | undefined, maxDepth) + const rowGroups = required(f[4] as Array | undefined, "FileMetaData.row_groups") + for (const group of rowGroups) { + if (group.columns.length !== columns.length) { + fail("Malformed", `a row group has ${group.columns.length} column chunks for ${columns.length} columns`) + } + group.columns.forEach((chunk, i) => { + const meta = chunk.metaData + const column = columns[i] + if (meta === undefined) return + if (meta.physicalType !== column.physicalType) { + fail( + "Malformed", + `column chunk ${i} is ${meta.physicalType}, but column "${column.path.join(".")}" is ${column.physicalType}` + ) + } + if (meta.pathInSchema.join("\u0000") !== column.path.join("\u0000")) { + fail("Malformed", `column chunk ${i} is for "${meta.pathInSchema.join(".")}", not "${column.path.join(".")}"`) + } + }) + } + return { + version: required(f[1] as number | undefined, "FileMetaData.version"), + numRows: required(f[3] as number | undefined, "FileMetaData.num_rows"), + createdBy: f[6] as string | undefined, + keyValueMetadata: (f[5] as Array | undefined) ?? [], + schema: root, + columns, + rowGroups + } +} + +const MAGIC = 0x31524150 // "PAR1" little-endian +const MAGIC_ENCRYPTED = 0x45524150 // "PARE" + +const u32 = (bytes: Uint8Array, at: number): number => + (bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16) | (bytes[at + 3] << 24)) >>> 0 + +/** + * Reads the metadata in the footer of a Parquet file. + * + * **Details** + * + * This reads the last 8 bytes, the first 4 and the footer. `maxFooterLength` + * (default 64 MiB) caps the footer's size and `maxDepth` (default 64, at most + * 255) how deep the schema may nest. + * + * @stability experimental + * @category decoding + * @since 4.1.5 + */ +export const readMetadata = ( + source: Source, + options?: { + readonly maxFooterLength?: number | undefined + readonly maxDepth?: number | undefined + } +): Effect.Effect => + Effect.gen(function*() { + const maxFooterLength = limit("maxFooterLength", options?.maxFooterLength, 64 * 1024 * 1024, 0xffffffff) + const maxDepth = limit("maxDepth", options?.maxDepth, 64, 255) + const size = source.size + if (!Number.isSafeInteger(size) || size < 0) usage(`the source size ${size} is not a non-negative integer`) + if (size < 12) return yield* error("Malformed", `a file of ${size} bytes is too short to be Parquet`) + const tail = yield* readRange(source, size - 8, 8) + const magic = u32(tail, 4) + if (magic === MAGIC_ENCRYPTED) { + return yield* error("Unsupported", "files with an encrypted footer are not supported") + } + if (magic !== MAGIC) return yield* error("Malformed", "the file does not end with the magic number PAR1") + if (u32(yield* readRange(source, 0, 4), 0) !== MAGIC) { + return yield* error("Malformed", "the file does not start with the magic number PAR1") + } + const length = u32(tail, 0) + if (length > size - 12) return yield* error("Malformed", `the footer length ${length} is larger than the file`) + if (length > maxFooterLength) { + return yield* error("Limit", `the footer of ${length} bytes exceeds maxFooterLength (${maxFooterLength})`) + } + const footer = yield* readRange(source, size - 8 - length, length) + return yield* attempt(() => fileMetaData(footer, maxDepth)) + }) + +// --------------------------------------------------------------------------- +// Pages +// --------------------------------------------------------------------------- + +const PLAIN = 0 +const PLAIN_DICTIONARY = 2 +const RLE = 3 +const BIT_PACKED = 4 +const DELTA_BINARY_PACKED = 5 +const DELTA_LENGTH_BYTE_ARRAY = 6 +const DELTA_BYTE_ARRAY = 7 +const RLE_DICTIONARY = 8 +const BYTE_STREAM_SPLIT = 9 + +const DATA_PAGE = 0 +const DICTIONARY_PAGE = 2 +const DATA_PAGE_V2 = 3 + +interface PageHeader { + readonly type: number + readonly uncompressedSize: number + readonly compressedSize: number + readonly numValues: number + readonly encoding: number + readonly definitionEncoding: number + readonly repetitionEncoding: number + readonly numNulls: number + readonly numRows: number + readonly definitionLength: number + readonly repetitionLength: number + readonly compressed: boolean +} + +const pageHeader = (r: Reader): PageHeader => { + const ints = (id: number, type: Thrift.Type) => id === 7 ? bool(r, type) : i32(r, type) + const f = fields( + r, + (id, type) => id <= 3 ? i32(r, type) : id >= 5 && id <= 8 && type === T.Struct ? fields(r, ints) : skip(r, type) + ) + const type = required(f[1] as number | undefined, "PageHeader.type") + const uncompressedSize = required(f[2] as number | undefined, "PageHeader.uncompressed_page_size") + const compressedSize = required(f[3] as number | undefined, "PageHeader.compressed_page_size") + if (uncompressedSize < 0 || compressedSize < 0) fail("Malformed", "a page header has a negative size") + const page = (id: number, what: string) => required(f[id] as Array | undefined, what) + const header = { type, uncompressedSize, compressedSize } + const count = (n: number | boolean | undefined, what: string): number => { + const value = required(n as number | undefined, what) + if (value < 0) fail("Malformed", `${what} is negative`) + return value + } + switch (type) { + case DATA_PAGE: { + const p = page(5, "PageHeader.data_page_header") + return { + ...header, + numValues: count(p[1], "DataPageHeader.num_values"), + encoding: required(p[2] as number | undefined, "DataPageHeader.encoding"), + definitionEncoding: required(p[3] as number | undefined, "DataPageHeader.definition_level_encoding"), + repetitionEncoding: required(p[4] as number | undefined, "DataPageHeader.repetition_level_encoding"), + numNulls: 0, + numRows: 0, + definitionLength: 0, + repetitionLength: 0, + compressed: true + } + } + case DATA_PAGE_V2: { + const p = page(8, "PageHeader.data_page_header_v2") + return { + ...header, + numValues: count(p[1], "DataPageHeaderV2.num_values"), + numNulls: count(p[2], "DataPageHeaderV2.num_nulls"), + numRows: count(p[3], "DataPageHeaderV2.num_rows"), + encoding: required(p[4] as number | undefined, "DataPageHeaderV2.encoding"), + definitionLength: count(p[5], "DataPageHeaderV2.definition_levels_byte_length"), + repetitionLength: count(p[6], "DataPageHeaderV2.repetition_levels_byte_length"), + compressed: p[7] !== false, + definitionEncoding: RLE, + repetitionEncoding: RLE + } + } + case DICTIONARY_PAGE: { + const p = page(7, "PageHeader.dictionary_page_header") + return { + ...header, + numValues: count(p[1], "DictionaryPageHeader.num_values"), + encoding: required(p[2] as number | undefined, "DictionaryPageHeader.encoding"), + definitionEncoding: RLE, + repetitionEncoding: RLE, + numNulls: 0, + numRows: 0, + definitionLength: 0, + repetitionLength: 0, + compressed: true + } + } + default: + return { + ...header, + numValues: 0, + encoding: PLAIN, + definitionEncoding: RLE, + repetitionEncoding: RLE, + numNulls: 0, + numRows: 0, + definitionLength: 0, + repetitionLength: 0, + compressed: true + } + } +} + +const kindOf = (column: ColumnDescriptor): Encodings.Kind => { + switch (column.physicalType) { + case "BOOLEAN": + return { tag: "bool" } + case "INT32": + return { tag: "i32", width: 4 } + case "FLOAT": + return { tag: "f32", width: 4 } + case "INT64": + return { tag: "i64", width: 8 } + case "DOUBLE": + return { tag: "f64", width: 8 } + case "INT96": + return { tag: "fixed", width: 12 } + case "FIXED_LEN_BYTE_ARRAY": + return { tag: "fixed", width: column.typeLength! } + case "BYTE_ARRAY": + return { tag: "bytes" } + } +} + +const encodingName = (code: number): string => String(name(encodings, code)) + +/** Decodes the pages of one column chunk and assembles its {@link Column}. */ +class ChunkDecoder { + readonly kind: Encodings.Kind + readonly maxDefinition: number + readonly maxRepetition: number + private dictionary: Encodings.Dense | undefined + private seenData = false + private readonly definitions: Array = [] + private readonly repetitions: Array = [] + private readonly dense: Array = [] + /** Levels decoded so far: values, nulls included. */ + count = 0 + + readonly column: ColumnDescriptor + readonly numValues: number + private readonly budget: Encodings.Budget + + constructor(column: ColumnDescriptor, numValues: number, budget: Encodings.Budget) { + this.column = column + this.numValues = numValues + this.budget = budget + this.kind = kindOf(column) + this.maxDefinition = column.maxDefinitionLevel + this.maxRepetition = column.maxRepetitionLevel + } + + dictionaryPage(header: PageHeader, data: Uint8Array): void { + if (this.dictionary !== undefined || this.seenData) { + fail("Malformed", "a dictionary page must be the first page of a column chunk, and the only one") + } + if (header.encoding !== PLAIN && header.encoding !== PLAIN_DICTIONARY) { + fail("Malformed", `a dictionary page is encoded with ${encodingName(header.encoding)}, not PLAIN`) + } + this.dictionary = Encodings.plain(this.kind, data, 0, data.length, header.numValues, this.budget) + } + + private start(n: number): void { + if (n > this.numValues - this.count) fail("Malformed", "the pages hold more values than their column chunk") + this.seenData = true + } + + /** A v1 data page: repetition levels, definition levels and values, all after decompression. */ + dataPage(header: PageHeader, page: Uint8Array): void { + const n = header.numValues + this.start(n) + let pos = 0 + const levels = (encoding: number, max: number, what: string): Uint8Array | undefined => { + if (max === 0) return undefined + const out = new Uint8Array(n) + const width = Encodings.bitWidth(max) + if (encoding === RLE) { + const end = Encodings.lengthPrefix(page, pos, page.length, what) + Encodings.hybrid(page, pos + 4, end, width, n, out, what) + pos = end + } else if (encoding === BIT_PACKED) { + pos = Encodings.bitPacked(page, pos, page.length, width, n, out, what) + } else { + fail("Unsupported", `${what} encoded with ${encodingName(encoding)} are not supported`) + } + return out + } + const repetition = levels(header.repetitionEncoding, this.maxRepetition, "repetition levels") + const definition = levels(header.definitionEncoding, this.maxDefinition, "definition levels") + this.levels(n, repetition, definition) + this.values(header.encoding, page, pos, this.nonNull(n, definition)) + } + + /** A v2 data page: the levels as written and the values after decompression. */ + dataPageV2(header: PageHeader, levels: Uint8Array, values: Uint8Array): void { + const n = header.numValues + this.start(n) + const decode = (start: number, end: number, max: number, what: string) => { + if (max === 0) return undefined + const out = new Uint8Array(n) + Encodings.hybrid(levels, start, end, Encodings.bitWidth(max), n, out, what) + return out + } + const repetitionEnd = header.repetitionLength + const repetition = decode(0, repetitionEnd, this.maxRepetition, "repetition levels") + const definition = decode(repetitionEnd, levels.length, this.maxDefinition, "definition levels") + this.levels(n, repetition, definition) + const nonNull = this.nonNull(n, definition) + if (n - nonNull !== header.numNulls) { + fail("Malformed", `a v2 data page has ${n - nonNull} nulls, but its header says ${header.numNulls}`) + } + if (repetition !== undefined && n > 0 && repetition[0] !== 0) { + fail("Malformed", "a v2 data page does not start at a row boundary") + } + let rows = n + if (repetition !== undefined) { + rows = 0 + for (let i = 0; i < n; i++) if (repetition[i] === 0) rows++ + } + if (rows !== header.numRows) { + fail("Malformed", `a v2 data page has ${rows} rows, but its header says ${header.numRows}`) + } + this.values(header.encoding, values, 0, nonNull) + } + + private levels(n: number, repetition: Uint8Array | undefined, definition: Uint8Array | undefined): void { + const check = (levels: Uint8Array | undefined, max: number, what: string) => { + if (levels === undefined) return + for (let i = 0; i < n; i++) { + if (levels[i] > max) fail("Malformed", `${what} ${levels[i]} is above the column's maximum of ${max}`) + } + } + check(repetition, this.maxRepetition, "repetition level") + check(definition, this.maxDefinition, "definition level") + if (repetition !== undefined) this.repetitions.push(repetition) + if (definition !== undefined) this.definitions.push(definition) + this.count += n + } + + private nonNull(n: number, definition: Uint8Array | undefined): number { + if (definition === undefined) return n + let count = 0 + for (let i = 0; i < n; i++) if (definition[i] === this.maxDefinition) count++ + return count + } + + private values(encoding: number, bytes: Uint8Array, pos: number, n: number): void { + const kind = this.kind + const end = bytes.length + const type = this.column.physicalType + const invalid = (): never => fail("Malformed", `${encodingName(encoding)} is not an encoding for ${type} values`) + let dense: Encodings.Dense + switch (encoding) { + case PLAIN: + dense = Encodings.plain(kind, bytes, pos, end, n, this.budget) + break + case PLAIN_DICTIONARY: + case RLE_DICTIONARY: { + if (this.dictionary === undefined) fail("Malformed", "a dictionary-encoded page has no dictionary page") + if (n === 0) return + if (pos >= end) fail("Malformed", "dictionary indices are truncated") + const width = bytes[pos] + if (width > 32) fail("Malformed", `dictionary indices have a bit width of ${width}, more than 32`) + const indices = new Int32Array(n) + Encodings.hybrid(bytes, pos + 1, end, width, n, indices, "dictionary indices") + dense = Encodings.gather(kind, this.dictionary!, indices, this.budget) + break + } + case RLE: { + if (kind.tag !== "bool") return invalid() + const out = new Uint8Array(n) + const stop = Encodings.lengthPrefix(bytes, pos, end, "RLE BOOLEAN values") + Encodings.hybrid(bytes, pos + 4, stop, 1, n, out, "RLE BOOLEAN values") + dense = out + break + } + case DELTA_BINARY_PACKED: + if (kind.tag === "i32") dense = Encodings.deltaBinaryPacked(bytes, pos, end, n, false)[0] + else if (kind.tag === "i64") dense = Encodings.deltaBinaryPacked(bytes, pos, end, n, true)[0] + else return invalid() + break + case DELTA_LENGTH_BYTE_ARRAY: + if (kind.tag !== "bytes") return invalid() + dense = Encodings.deltaLengthByteArray(bytes, pos, end, n, this.budget)[0] + break + case DELTA_BYTE_ARRAY: + if (kind.tag === "bytes") dense = Encodings.deltaByteArray(bytes, pos, end, n, this.budget) + else if (type === "FIXED_LEN_BYTE_ARRAY") { + dense = Encodings.toFixed( + Encodings.deltaByteArray(bytes, pos, end, n, this.budget), + kind.tag === "fixed" ? kind.width : 0, + "DELTA_BYTE_ARRAY" + ) + } else return invalid() + break + case BYTE_STREAM_SPLIT: + if (kind.tag === "bool" || kind.tag === "bytes" || type === "INT96") return invalid() + dense = Encodings.byteStreamSplit(kind, bytes, pos, end, n) + break + case BIT_PACKED: + return invalid() + default: + return fail("Unsupported", `the ${encodingName(encoding)} encoding is not supported`) + } + this.dense.push(dense) + } + + finish(numRows: number): Column { + if (this.count !== this.numValues) { + fail("Malformed", `the column chunk ends after ${this.count} of its ${this.numValues} values`) + } + const join = (parts: Array) => + parts.length === 1 ? parts[0] : parts.length === 0 ? undefined : concatBytes(parts, this.count) + const dense = this.dense.length === 0 ? Encodings.empty(this.kind) : Encodings.concat(this.kind, this.dense) + return assemble(this.column, this.kind, numRows, this.count, join(this.repetitions), join(this.definitions), dense) + } +} + +const concatBytes = (parts: ReadonlyArray, total: number): Uint8Array => { + const out = new Uint8Array(total) + let at = 0 + for (const part of parts) { + out.set(part, at) + at += part.length + } + return out +} + +/** + * Turns repetition and definition levels into one {@link Layer} per node of the + * column's path, and spreads the dense values over the leaf's slots. + * + * Each repeated node starts a new "space" of slots (its elements); the other + * nodes share the space of their nearest repeated ancestor, or the rows. A + * level entry (r, d) adds a slot to every space s >= r whose repeated node it + * defines, so to spaces r, r + 1, ... while d reaches the space's definition + * level. + */ +const assemble = ( + column: ColumnDescriptor, + kind: Encodings.Kind, + numRows: number, + n: number, + repetition: Uint8Array | undefined, + definition: Uint8Array | undefined, + dense: Encodings.Dense +): Column => { + const nodes = column.nodes + const maxRepetition = column.maxRepetitionLevel + const maxDefinition = column.maxDefinitionLevel + // spaceDefinition[s]: the definition level at which space s has a slot + const spaceDefinition = [0] + const nodeDefinition: Array = [] + const nodeSpace: Array = [] + const optionalIn: Array> = [[]] + const repeatedOf: Array = [-1] + let d = 0 + for (let i = 0; i < nodes.length; i++) { + const repetitionType = nodes[i].repetition + if (repetitionType !== "REQUIRED") d++ + if (repetitionType === "REPEATED") { + spaceDefinition.push(d) + optionalIn.push([]) + repeatedOf.push(i) + } + nodeDefinition.push(d) + nodeSpace.push(spaceDefinition.length - 1) + if (repetitionType === "OPTIONAL") optionalIn[spaceDefinition.length - 1].push(i) + } + + const counts = new Array(maxRepetition + 1).fill(0) + let open = -1 + for (let e = 0; e < n; e++) { + const r = repetition === undefined ? 0 : repetition[e] + const level = definition === undefined ? maxDefinition : definition[e] + if (r > 0 && (open < r || level < spaceDefinition[r])) { + fail("Malformed", `level ${e} repeats a list that has no elements (repetition ${r}, definition ${level})`) + } + let s = r + while (s <= maxRepetition && level >= spaceDefinition[s]) counts[s++]++ + open = s - 1 + } + if (counts[0] !== numRows) { + fail("Malformed", `the column chunk holds ${counts[0]} rows, but its row group has ${numRows}`) + } + + const validity = nodes.map((node, i) => + node.repetition === "OPTIONAL" ? new Uint8Array((counts[nodeSpace[i]] + 7) >> 3) : undefined + ) + const valid = new Array(nodes.length).fill(0) + const offsets = nodes.map((node, i) => + node.repetition === "REPEATED" ? new Int32Array(counts[nodeSpace[i] - 1] + 1) : undefined + ) + const slots = counts[maxRepetition] + const present = new Uint8Array(slots) + const next = new Array(maxRepetition + 1).fill(0) + for (let e = 0; e < n; e++) { + const r = repetition === undefined ? 0 : repetition[e] + const level = definition === undefined ? maxDefinition : definition[e] + for (let s = r; s <= maxRepetition && level >= spaceDefinition[s]; s++) { + const slot = next[s]++ + if (s < maxRepetition) offsets[repeatedOf[s + 1]]![slot] = next[s + 1] + for (const i of optionalIn[s]) { + if (level >= nodeDefinition[i]) { + validity[i]![slot >> 3] |= 1 << (slot & 7) + valid[i]++ + } + } + if (s === maxRepetition && level === maxDefinition) present[slot] = 1 + } + } + for (let s = 1; s <= maxRepetition; s++) offsets[repeatedOf[s]]![counts[s - 1]] = counts[s] + + return { + descriptor: column, + numRows, + layers: nodes.map((node, i) => { + const length = counts[nodeSpace[i]] + return { + node, + length, + validity: validity[i], + nullCount: validity[i] === undefined ? 0 : length - valid[i], + offsets: offsets[i] + } + }), + values: spread(column, kind, dense, present, slots) + } +} + +/** Gives every leaf slot a value, leaving null slots empty. */ +const spread = ( + column: ColumnDescriptor, + kind: Encodings.Kind, + dense: Encodings.Dense, + present: Uint8Array, + slots: number +): Values => { + const full = Encodings.lengthOf(kind, dense) === slots + switch (kind.tag) { + case "bool": { + const bits = dense as Uint8Array + const out = new Uint8Array((slots + 7) >> 3) + for (let j = 0, k = 0; j < slots; j++) { + if (present[j] === 1 && bits[k++] === 1) out[j >> 3] |= 1 << (j & 7) + } + return { type: "BOOLEAN", values: out } + } + case "bytes": { + const values = dense as Encodings.Binary + let offsets = values.offsets + if (!full) { + offsets = new Int32Array(slots + 1) + for (let j = 0, k = 0; j < slots; j++) { + offsets[j + 1] = offsets[j] + (present[j] === 1 ? values.offsets[k + 1] - values.offsets[k++] : 0) + } + } + return { type: "BYTE_ARRAY", offsets, values: values.data } + } + case "fixed": { + const width = kind.width + let values = dense as Uint8Array + if (!full) { + values = new Uint8Array(slots * width) + const from = dense as Uint8Array + for (let j = 0, k = 0; j < slots; j++) { + if (present[j] === 1) values.set(from.subarray(k * width, ++k * width), j * width) + } + } + return column.physicalType === "INT96" + ? { type: "INT96", values } + : { type: "FIXED_LEN_BYTE_ARRAY", width, values } + } + default: { + let values = dense as Int32Array + if (!full) { + const from = values + values = new (from.constructor as Int32ArrayConstructor)(slots) + for (let j = 0, k = 0; j < slots; j++) if (present[j] === 1) values[j] = from[k++] + } + return { type: column.physicalType, values } as Values + } + } +} + +const decompress = ( + codec: CompressionCodec | number, + decompressors: { readonly [codec: string]: Decompress | undefined } | undefined, + data: Uint8Array, + size: number +): Effect.Effect => { + const checked = (out: Uint8Array) => + out.length === size + ? Effect.succeed(out) + : Effect.fail(error("Malformed", `a page decompressed to ${out.length} bytes, but its header says ${size}`)) + if (codec === "UNCOMPRESSED") return checked(data) + const f = typeof codec === "string" ? decompressors?.[codec] : undefined + if (f === undefined) { + return Effect.fail(error("Unsupported", `${codec} pages need a Decompress for ${codec} in the codecs option`)) + } + return Effect.flatMap( + Effect.mapError( + f(data, size), + (cause) => new ParquetError({ kind: "Codec", message: `${codec} decompression failed`, cause }) + ), + checked + ) +} + +/** + * Reads the values of one column chunk: column `column` of + * `metadata.columns` in row group `rowGroup`. + * + * **Details** + * + * Pages compressed with a codec other than `UNCOMPRESSED` need a + * {@link Decompress} for it in `codecs`. Dictionary-encoded values are looked + * up in the dictionary, so the result never refers to it. + * + * Limits for untrusted files: `maxPageSize` (default 256 MiB) caps each page's + * compressed and uncompressed size, `maxValues` (default 2^26) the values of + * the column chunk, nulls included, and `maxBinaryLength` (default 1 GiB) the + * total bytes of its `BYTE_ARRAY` values. Each is at most 2^31 - 1. + * + * **Gotchas** + * + * `rowGroup` and `column` must be indexes into `metadata`: anything else is + * a defect, not a failure. + * + * @stability experimental + * @category decoding + * @since 4.1.5 + */ +export const readColumn = ( + source: Source, + metadata: FileMetaData, + options: { + readonly rowGroup: number + readonly column: number + readonly codecs?: { readonly [K in Exclude]?: Decompress | undefined } | undefined + readonly maxPageSize?: number | undefined + readonly maxValues?: number | undefined + readonly maxBinaryLength?: number | undefined + } +): Effect.Effect => + Effect.gen(function*() { + const maxPageSize = limit("maxPageSize", options.maxPageSize, 256 * 1024 * 1024, MAX_INT32) + const maxValues = limit("maxValues", options.maxValues, 2 ** 26, MAX_INT32) + const maxBinaryLength = limit("maxBinaryLength", options.maxBinaryLength, 1024 * 1024 * 1024, MAX_INT32) + const group = Number.isInteger(options.rowGroup) ? metadata.rowGroups[options.rowGroup] : undefined + const column = Number.isInteger(options.column) ? metadata.columns[options.column] : undefined + if (group === undefined) return usage(`the file has no row group ${options.rowGroup}`) + if (column === undefined) return usage(`the file has no column ${options.column}`) + const chunk = group.columns[options.column] + const where = `column "${column.path.join(".")}" of row group ${options.rowGroup}` + if (chunk.filePath !== undefined) { + return yield* error("Unsupported", `${where} is in another file, ${chunk.filePath}`) + } + if (chunk.encrypted || chunk.metaData === undefined) { + return yield* error("Unsupported", `${where} is encrypted`) + } + const meta = chunk.metaData + if (meta.numValues > maxValues) { + return yield* error("Limit", `${where} has ${meta.numValues} values, more than maxValues (${maxValues})`) + } + const decoder = new ChunkDecoder(column, meta.numValues, { left: maxBinaryLength }) + // A chunk without values needs no pages; parquet-cpp writes data_page_offset 0 for one. + if (meta.numValues === 0) return yield* Effect.mapError(attempt(() => decoder.finish(group.numRows)), locate(where)) + let start = meta.dataPageOffset + const dictionary = meta.dictionaryPageOffset + if (dictionary !== undefined && dictionary > 0 && dictionary < start) start = dictionary + const length = meta.totalCompressedSize + if (start < 4 || length < 0 || length > source.size - start) { + return yield* error("Malformed", `${where} has pages outside the file`) + } + const bytes = yield* readRange(source, start, length) + const thrift = Thrift.makeReader(bytes) + let pos = 0 + while (decoder.count < meta.numValues) { + if (pos >= bytes.length) { + return yield* error("Malformed", `${where} ends after ${decoder.count} of its ${meta.numValues} values`) + } + const headerAt = pos + const header = yield* Effect.mapError( + attempt(() => { + thrift.position = pos + return pageHeader(thrift) + }), + locate(`${where}, page header at byte ${start + headerAt}`) + ) + const body = thrift.position + if (header.compressedSize > bytes.length - body) { + return yield* error("Malformed", `a page of ${where} is larger than its column chunk`) + } + if (header.compressedSize > maxPageSize || header.uncompressedSize > maxPageSize) { + return yield* error("Limit", `a page of ${where} is larger than maxPageSize (${maxPageSize})`) + } + pos = body + header.compressedSize + const page = bytes.subarray(body, pos) + const decompressed = (data: Uint8Array, size: number, codec = meta.codec) => + decompress(codec, options.codecs, data, size) + const decode = Effect.gen(function*() { + switch (header.type) { + case DICTIONARY_PAGE: { + const data = yield* decompressed(page, header.uncompressedSize) + return yield* attempt(() => decoder.dictionaryPage(header, data)) + } + case DATA_PAGE: { + const data = yield* decompressed(page, header.uncompressedSize) + return yield* attempt(() => decoder.dataPage(header, data)) + } + case DATA_PAGE_V2: { + const levels = header.repetitionLength + header.definitionLength + if (levels > page.length || levels > header.uncompressedSize) { + return yield* error("Malformed", "the levels of a v2 data page are larger than the page") + } + const data = yield* decompressed( + page.subarray(levels), + header.uncompressedSize - levels, + header.compressed ? meta.codec : "UNCOMPRESSED" + ) + return yield* attempt(() => decoder.dataPageV2(header, page.subarray(0, levels), data)) + } + // Index pages and page types added later are skipped. + } + }) + yield* Effect.mapError(decode, locate(`${where}, page at byte ${start + headerAt}`)) + } + return yield* Effect.mapError(attempt(() => decoder.finish(group.numRows)), locate(where)) + }) diff --git a/packages/effect/src/encoding/index.ts b/packages/effect/src/encoding/index.ts index 8247d26b903..d61740e8e97 100644 --- a/packages/effect/src/encoding/index.ts +++ b/packages/effect/src/encoding/index.ts @@ -65,6 +65,12 @@ export * as Jsonc from "./Jsonc.ts" */ export * as Ndjson from "./Ndjson.ts" +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Parquet from "./Parquet.ts" + /** * @stability unstable * @since 4.0.0 diff --git a/packages/effect/src/internal/parquet/encodings.ts b/packages/effect/src/internal/parquet/encodings.ts new file mode 100644 index 00000000000..9d49a026238 --- /dev/null +++ b/packages/effect/src/internal/parquet/encodings.ts @@ -0,0 +1,552 @@ +/** + * Value and level decoders for the Parquet encodings + * (https://github.com/apache/parquet-format/blob/apache-parquet-format-2.14.0/Encodings.md). + * + * Every decoder reads from `bytes[pos, end)` and throws a {@link Failure} instead of + * reading past `end`. Values come out dense: one entry per non-null value. + * + * @internal + */ + +/** @internal */ +export class Failure { + readonly kind: "Malformed" | "Unsupported" | "Limit" + readonly message: string + constructor(kind: "Malformed" | "Unsupported" | "Limit", message: string) { + this.kind = kind + this.message = message + } +} + +/** @internal */ +export const fail = (kind: Failure["kind"], message: string): never => { + throw new Failure(kind, message) +} + +/** + * How a physical type's values are held while decoding: `bool` one byte (0 or 1) per + * value, `fixed` `width` bytes per value (INT96 and FIXED_LEN_BYTE_ARRAY), `bytes` + * offsets into concatenated data (BYTE_ARRAY). + * + * @internal + */ +export type Kind = + | { readonly tag: "bool" } + | { readonly tag: "i32" | "f32"; readonly width: 4 } + | { readonly tag: "i64" | "f64"; readonly width: 8 } + | { readonly tag: "fixed"; readonly width: number } + | { readonly tag: "bytes" } + +/** @internal */ +export interface Binary { + readonly offsets: Int32Array + readonly data: Uint8Array +} + +/** @internal */ +export type Dense = Uint8Array | Int32Array | BigInt64Array | Float32Array | Float64Array | Binary + +/** Bytes of BYTE_ARRAY data a column chunk may still produce. @internal */ +export interface Budget { + left: number +} + +const spend = (budget: Budget, n: number): void => { + if (n > budget.left) fail("Limit", "the column chunk's BYTE_ARRAY values exceed maxBinaryLength") + budget.left -= n +} + +const POW2: Array = [] +for (let i = 0; i <= 64; i++) POW2.push(2 ** i) + +const littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1 + +/** Reads `n` little-endian values of `width` bytes into a fresh typed array. */ +const fixedLE = ( + bytes: Uint8Array, + pos: number, + n: number, + width: number, + make: (buffer: ArrayBuffer, length: number) => A, + get: (view: DataView, at: number) => number | bigint +): A => { + const copy = bytes.slice(pos, pos + n * width) + const out = make(copy.buffer, n) + if (!littleEndian) { + const view = new DataView(copy.buffer) + const target = out as unknown as Array + for (let i = 0; i < n; i++) target[i] = get(view, i * width) + } + return out +} + +const fromLE = (kind: Kind, bytes: Uint8Array, pos: number, n: number): Dense => { + switch (kind.tag) { + case "i32": + return fixedLE(bytes, pos, n, 4, (b, l) => new Int32Array(b, 0, l), (v, at) => v.getInt32(at, true)) + case "f32": + return fixedLE(bytes, pos, n, 4, (b, l) => new Float32Array(b, 0, l), (v, at) => v.getFloat32(at, true)) + case "i64": + return fixedLE(bytes, pos, n, 8, (b, l) => new BigInt64Array(b, 0, l), (v, at) => v.getBigInt64(at, true)) + case "f64": + return fixedLE(bytes, pos, n, 8, (b, l) => new Float64Array(b, 0, l), (v, at) => v.getFloat64(at, true)) + default: + return bytes.slice(pos, pos + n * (kind as { readonly width: number }).width) + } +} + +/** + * Reads values of up to 32 bits packed from the least significant bit of each byte, as + * the RLE/bit-packed hybrid and DELTA_BINARY_PACKED store them. The caller checks that + * the bytes are there. + */ +class BitReader { + private buffer = 0 + private bits = 0 + private readonly bytes: Uint8Array + private pos: number + constructor(bytes: Uint8Array, pos: number) { + this.bytes = bytes + this.pos = pos + } + read(width: number): number { + while (this.bits < width) { + this.buffer += this.bytes[this.pos++] * POW2[this.bits] + this.bits += 8 + } + const value = this.buffer % POW2[width] + this.buffer = (this.buffer - value) / POW2[width] + this.bits -= width + return value + } +} + +/** An unsigned LEB128 varint of at most 32 bits, and where it ends. */ +const uleb32 = (bytes: Uint8Array, pos: number, end: number, what: string): readonly [number, number] => { + let result = 0 + for (let shift = 0; shift < 35; shift += 7) { + if (pos >= end) return fail("Malformed", `${what} is truncated`) + const b = bytes[pos++] + result += (b & 0x7f) * POW2[shift] + if (b < 0x80) { + if (result > 0xffffffff) break + return [result, pos] + } + } + return fail("Malformed", `${what} is longer than 32 bits`) +} + +const BIG_7 = BigInt(7) +const BIG_1 = BigInt(1) +const BIG_0x7f = BigInt(0x7f) + +/** A zigzag ULEB128 varint of at most 64 bits, and where it ends. */ +const zigzag64 = (bytes: Uint8Array, pos: number, end: number, what: string): readonly [bigint, number] => { + let result = BigInt(0) + for (let i = 0; i < 10; i++) { + if (pos >= end) return fail("Malformed", `${what} is truncated`) + const b = bytes[pos++] + if (i === 9 && b > 1) break + result |= (BigInt(b) & BIG_0x7f) << (BIG_7 * BigInt(i)) + if (b < 0x80) return [(result >> BIG_1) ^ -(result & BIG_1), pos] + } + return fail("Malformed", `${what} is longer than 64 bits`) +} + +/** The bit width of values from 0 to `max`. @internal */ +export const bitWidth = (max: number): number => max === 0 ? 0 : 32 - Math.clz32(max) + +/** + * Decodes `n` values of the RLE/bit-packed hybrid (no length prefix) into `out`, and + * returns where the runs ended. + * + * @internal + */ +export const hybrid = ( + bytes: Uint8Array, + pos: number, + end: number, + width: number, + n: number, + out: Uint8Array | Int32Array, + what: string +): number => { + const valueBytes = (width + 7) >> 3 + let i = 0 + while (i < n) { + if (pos >= end) fail("Malformed", `${what} end after ${i} of ${n} values`) + const [header, start] = uleb32(bytes, pos, end, `${what} run header`) + const runLength = Math.floor(header / 2) + if (runLength === 0) fail("Malformed", `${what} has a run of length 0`) + if (header % 2 === 1) { + // runLength groups of 8 values, each group `width` bytes + const take = Math.min(runLength * 8, n - i) + if (Math.ceil(take * width / 8) > end - start) fail("Malformed", `${what} has a truncated bit-packed run`) + const reader = new BitReader(bytes, start) + for (let k = 0; k < take; k++) out[i++] = reader.read(width) + pos = Math.min(start + runLength * width, end) + } else { + if (valueBytes > end - start) fail("Malformed", `${what} has a truncated RLE run`) + let value = 0 + for (let k = 0; k < valueBytes; k++) value += bytes[start + k] * POW2[8 * k] + if (value >= POW2[width]) fail("Malformed", `${what} has an RLE value wider than ${width} bits`) + const stop = Math.min(i + runLength, n) + out.fill(value, i, stop) + i = stop + pos = start + valueBytes + } + } + return pos +} + +/** + * Decodes `n` values of the deprecated BIT_PACKED level encoding (most significant bit + * first, no runs), and returns where they end. + * + * @internal + */ +export const bitPacked = ( + bytes: Uint8Array, + pos: number, + end: number, + width: number, + n: number, + out: Uint8Array, + what: string +): number => { + const size = Math.ceil(n * width / 8) + if (size > end - pos) fail("Malformed", `${what} need ${size} bytes`) + let bit = pos * 8 + for (let i = 0; i < n; i++) { + let value = 0 + for (let k = 0; k < width; k++, bit++) value = (value << 1) | ((bytes[bit >> 3] >> (7 - (bit & 7))) & 1) + out[i] = value + } + return pos + size +} + +/** Reads the 4-byte little-endian length that prefixes some hybrid runs, and returns where the runs end. @internal */ +export const lengthPrefix = (bytes: Uint8Array, pos: number, end: number, what: string): number => { + if (end - pos < 4) fail("Malformed", `${what} length is truncated`) + const length = (bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24)) >>> 0 + if (length > end - pos - 4) fail("Malformed", `${what} of ${length} bytes is longer than the page`) + return pos + 4 + length +} + +const readLength = (bytes: Uint8Array, pos: number): number => + (bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24)) >>> 0 + +/** + * Decodes `n` PLAIN values. + * + * @internal + */ +export const plain = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n: number, budget: Budget): Dense => { + switch (kind.tag) { + case "bool": { + if (Math.ceil(n / 8) > end - pos) fail("Malformed", `PLAIN BOOLEAN values need ${Math.ceil(n / 8)} bytes`) + const out = new Uint8Array(n) + for (let i = 0; i < n; i++) out[i] = (bytes[pos + (i >> 3)] >> (i & 7)) & 1 + return out + } + case "bytes": { + let total = 0 + let p = pos + for (let i = 0; i < n; i++) { + if (end - p < 4) fail("Malformed", `PLAIN BYTE_ARRAY value ${i} of ${n} is truncated`) + const length = readLength(bytes, p) + p += 4 + if (length > end - p) fail("Malformed", `PLAIN BYTE_ARRAY value ${i} of ${length} bytes is truncated`) + p += length + total += length + } + spend(budget, total) + const offsets = new Int32Array(n + 1) + const data = new Uint8Array(total) + p = pos + for (let i = 0, at = 0; i < n; i++) { + const length = readLength(bytes, p) + data.set(bytes.subarray(p + 4, p + 4 + length), at) + p += 4 + length + offsets[i + 1] = at += length + } + return { offsets, data } + } + default: { + const size = n * kind.width + if (size > end - pos) fail("Malformed", `PLAIN values need ${size} bytes`) + return fromLE(kind, bytes, pos, n) + } + } +} + +/** + * Decodes `n` BYTE_STREAM_SPLIT values, which fill the rest of the page. + * + * @internal + */ +export const byteStreamSplit = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n: number): Dense => { + const width = (kind as { readonly width: number }).width + if (end - pos !== n * width) { + fail("Malformed", `BYTE_STREAM_SPLIT data of ${end - pos} bytes is not ${n} values of ${width} bytes`) + } + const joined = new Uint8Array(n * width) + for (let k = 0; k < width; k++) { + const stream = pos + k * n + for (let i = 0; i < n; i++) joined[i * width + k] = bytes[stream + i] + } + return fromLE(kind, joined, 0, n) +} + +/** + * Decodes a DELTA_BINARY_PACKED stream that must hold exactly `n` values, and returns + * them with where the stream ends. + * + * @internal + */ +export const deltaBinaryPacked: { + (bytes: Uint8Array, pos: number, end: number, n: number, wide: false): readonly [Int32Array, number] + (bytes: Uint8Array, pos: number, end: number, n: number, wide: true): readonly [BigInt64Array, number] +} = ((bytes: Uint8Array, pos: number, end: number, n: number, wide: boolean) => { + const what = "DELTA_BINARY_PACKED header" + const [blockSize, p1] = uleb32(bytes, pos, end, what) + const [miniblocks, p2] = uleb32(bytes, p1, end, what) + const [total, p3] = uleb32(bytes, p2, end, what) + const [first, p4] = zigzag64(bytes, p3, end, what) + if (blockSize === 0 || blockSize % 128 !== 0) { + fail("Malformed", `DELTA_BINARY_PACKED block size ${blockSize} is not a multiple of 128`) + } + if (miniblocks === 0 || blockSize % miniblocks !== 0 || (blockSize / miniblocks) % 32 !== 0) { + fail( + "Malformed", + `DELTA_BINARY_PACKED blocks of ${blockSize} values can't hold ${miniblocks} miniblocks of a multiple of 32 values` + ) + } + if (total !== n) fail("Malformed", `DELTA_BINARY_PACKED holds ${total} values, expected ${n}`) + const perMiniblock = blockSize / miniblocks + const maxWidth = wide ? 64 : 32 + const out = wide ? new BigInt64Array(n) : new Int32Array(n) + pos = p4 + if (n === 0) return [out, pos] + let last32 = Number(BigInt.asIntN(32, first)) + let last64 = BigInt.asIntN(64, first) + if (wide) (out as BigInt64Array)[0] = last64 + else (out as Int32Array)[0] = last32 + let i = 1 + while (i < n) { + const [minDelta, widths] = zigzag64(bytes, pos, end, "DELTA_BINARY_PACKED block") + if (miniblocks > end - widths) fail("Malformed", "DELTA_BINARY_PACKED miniblock bit widths are truncated") + pos = widths + miniblocks + const minDelta32 = Number(BigInt.asIntN(32, minDelta)) + for (let m = 0; m < miniblocks && i < n; m++) { + const width = bytes[widths + m] + if (width > maxWidth) { + fail("Malformed", `DELTA_BINARY_PACKED miniblock bit width ${width} is wider than ${maxWidth}`) + } + const size = perMiniblock * width / 8 + if (size > end - pos) fail("Malformed", "DELTA_BINARY_PACKED miniblock is truncated") + const reader = new BitReader(bytes, pos) + const take = Math.min(perMiniblock, n - i) + if (wide) { + const target = out as BigInt64Array + for (let k = 0; k < take; k++) { + const delta = width <= 32 + ? BigInt(reader.read(width)) + : BigInt(reader.read(32)) | (BigInt(reader.read(width - 32)) << BigInt(32)) + target[i++] = last64 = BigInt.asIntN(64, last64 + minDelta + delta) + } + } else { + const target = out as Int32Array + for (let k = 0; k < take; k++) target[i++] = last32 = (last32 + minDelta32 + reader.read(width)) | 0 + } + pos += size + } + } + return [out, pos] +}) as never + +const lengths = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + what: string +): readonly [Int32Array, number] => { + const [values, next] = deltaBinaryPacked(bytes, pos, end, n, false) + for (let i = 0; i < n; i++) if (values[i] < 0) fail("Malformed", `${what} ${values[i]} is negative`) + return [values, next] +} + +/** + * Decodes `n` DELTA_LENGTH_BYTE_ARRAY values, and returns them with where they end. + * + * @internal + */ +export const deltaLengthByteArray = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + budget: Budget +): readonly [Binary, number] => { + const [sizes, start] = lengths(bytes, pos, end, n, "DELTA_LENGTH_BYTE_ARRAY length") + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + total += sizes[i] + if (total > end - start) fail("Malformed", "DELTA_LENGTH_BYTE_ARRAY data is truncated") + offsets[i + 1] = total + } + spend(budget, total) + return [{ offsets, data: bytes.slice(start, start + total) }, start + total] +} + +/** + * Decodes `n` DELTA_BYTE_ARRAY values. + * + * @internal + */ +export const deltaByteArray = (bytes: Uint8Array, pos: number, end: number, n: number, budget: Budget): Binary => { + const [prefixes, next] = lengths(bytes, pos, end, n, "DELTA_BYTE_ARRAY prefix length") + const [suffixes] = deltaLengthByteArray(bytes, next, end, n, budget) + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + const prefix = prefixes[i] + if (i === 0 ? prefix !== 0 : prefix > offsets[i] - offsets[i - 1]) { + fail("Malformed", `DELTA_BYTE_ARRAY value ${i} has a prefix of ${prefix} bytes, longer than the value before it`) + } + total += prefix + suffixes.offsets[i + 1] - suffixes.offsets[i] + offsets[i + 1] = total + } + spend(budget, total - suffixes.data.length) + const out = new Uint8Array(total) + for (let i = 0; i < n; i++) { + const at = offsets[i] + const prefix = prefixes[i] + if (prefix > 0) out.copyWithin(at, offsets[i - 1], offsets[i - 1] + prefix) + out.set(suffixes.data.subarray(suffixes.offsets[i], suffixes.offsets[i + 1]), at + prefix) + } + return { offsets, data: out } +} + +/** + * Converts BYTE_ARRAY values that must all be `width` bytes long to fixed-width values. + * + * @internal + */ +export const toFixed = (values: Binary, width: number, what: string): Uint8Array => { + const n = values.offsets.length - 1 + for (let i = 0; i < n; i++) { + const length = values.offsets[i + 1] - values.offsets[i] + if (length !== width) fail("Malformed", `${what} value ${i} has ${length} bytes, not ${width}`) + } + return values.data +} + +/** The number of values in `dense`. @internal */ +export const lengthOf = (kind: Kind, dense: Dense): number => + kind.tag === "bytes" + ? (dense as Binary).offsets.length - 1 + : kind.tag === "fixed" + ? (dense as Uint8Array).length / kind.width + : (dense as Uint8Array).length + +/** + * Looks up dictionary `indices` in `dictionary`. + * + * @internal + */ +export const gather = (kind: Kind, dictionary: Dense, indices: Int32Array, budget: Budget): Dense => { + const size = lengthOf(kind, dictionary) + const n = indices.length + for (let i = 0; i < n; i++) { + const index = indices[i] + if (index < 0 || index >= size) { + fail("Malformed", `dictionary index ${index >>> 0} is out of range for a dictionary of ${size} values`) + } + } + switch (kind.tag) { + case "bytes": { + const dict = dictionary as Binary + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) offsets[i + 1] = total += dict.offsets[indices[i] + 1] - dict.offsets[indices[i]] + spend(budget, total) + const data = new Uint8Array(total) + for (let i = 0; i < n; i++) { + data.set(dict.data.subarray(dict.offsets[indices[i]], dict.offsets[indices[i] + 1]), offsets[i]) + } + return { offsets, data } + } + case "fixed": { + const dict = dictionary as Uint8Array + const width = kind.width + const out = new Uint8Array(n * width) + for (let i = 0; i < n; i++) out.set(dict.subarray(indices[i] * width, (indices[i] + 1) * width), i * width) + return out + } + default: { + const dict = dictionary as Int32Array + const out = new (dict.constructor as Int32ArrayConstructor)(n) + for (let i = 0; i < n; i++) out[i] = dict[indices[i]] + return out + } + } +} + +/** + * Concatenates dense values decoded page by page. + * + * @internal + */ +export const concat = (kind: Kind, chunks: ReadonlyArray): Dense => { + if (chunks.length === 1) return chunks[0] + if (kind.tag === "bytes") { + const parts = chunks as ReadonlyArray + let count = 0 + let total = 0 + for (const part of parts) { + count += part.offsets.length - 1 + total += part.data.length + } + const offsets = new Int32Array(count + 1) + const data = new Uint8Array(total) + let i = 0 + let at = 0 + for (const part of parts) { + for (let k = 1; k < part.offsets.length; k++) offsets[++i] = at + part.offsets[k] + data.set(part.data, at) + at += part.data.length + } + return { offsets, data } + } + const parts = chunks as ReadonlyArray + let total = 0 + for (const part of parts) total += part.length + const out = new (empty(kind).constructor as Int32ArrayConstructor)(total) + let at = 0 + for (const part of parts) { + out.set(part, at) + at += part.length + } + return out +} + +/** No values of `kind`. @internal */ +export const empty = (kind: Kind): Dense => { + switch (kind.tag) { + case "i32": + return new Int32Array(0) + case "i64": + return new BigInt64Array(0) + case "f32": + return new Float32Array(0) + case "f64": + return new Float64Array(0) + case "bytes": + return { offsets: new Int32Array(1), data: new Uint8Array(0) } + default: + return new Uint8Array(0) + } +} diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts new file mode 100644 index 00000000000..2163fcc3c28 --- /dev/null +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -0,0 +1,1175 @@ +import { assert, describe, it } from "@effect/vitest" +import { Cause, Effect, Exit } from "effect" +import * as Hex from "effect/encoding/Hex" +import * as Parquet from "effect/encoding/Parquet" +import * as Thrift from "effect/encoding/Thrift" +import { readFileSync } from "node:fs" +import { gunzipSync, gzipSync } from "node:zlib" + +// Two kinds of evidence: +// - "pyarrow fixtures": files written by pyarrow, read back and compared with +// gold.json. Interoperability evidence; see fixtures/parquet/README.md. +// - "conformance": one test per assertion of apache-parquet-format 2.14.0, +// named after its id. fixtures/parquet/CONFORMANCE.md maps every assertion to +// its test, or says why it has none. +type Gold = null | boolean | number | string | ReadonlyArray +interface GoldFile { + readonly name: string + readonly numRows: number + readonly rowGroups: ReadonlyArray<{ + readonly numRows: number + readonly columns: ReadonlyArray<{ + readonly codec: Parquet.CompressionCodec + readonly encodings: ReadonlyArray + }> + }> + readonly columns: ReadonlyArray<{ + readonly path: ReadonlyArray + readonly physicalType: string + readonly typeLength: number | null + readonly maxDefinitionLevel: number + readonly maxRepetitionLevel: number + readonly convertedType: string | null + readonly logicalType: Parquet.LogicalType | null + readonly values: ReadonlyArray + }> +} + +const gold: { readonly writer: string; readonly files: ReadonlyArray } = JSON.parse( + readFileSync(new URL("../fixtures/parquet/gold.json", import.meta.url), "utf8") +) +const fixture = (name: string) => + Parquet.fromBytes(new Uint8Array(readFileSync(new URL("../fixtures/parquet/" + name, import.meta.url)))) + +const gzip: Parquet.Decompress = (data) => Effect.sync(() => new Uint8Array(gunzipSync(data))) + +/** Turns a column's layers and values back into one nested value per row, like gold.json. */ +const rows = (column: Parquet.Column): Array => { + const { layers, values } = column + const bit = (bits: Uint8Array, i: number) => (bits[i >> 3] >> (i & 7) & 1) === 1 + const leaf = (j: number): Gold => { + switch (values.type) { + case "BOOLEAN": + return bit(values.values, j) + case "INT64": + return String(values.values[j]) + case "INT32": + case "FLOAT": + case "DOUBLE": + return values.values[j] + case "BYTE_ARRAY": + return Hex.encode(values.values.subarray(values.offsets[j], values.offsets[j + 1])) + case "INT96": + return Hex.encode(values.values.subarray(j * 12, j * 12 + 12)) + case "FIXED_LEN_BYTE_ARRAY": + return Hex.encode(values.values.subarray(j * values.width, (j + 1) * values.width)) + } + } + // The value below node i, at slot j of node i's slots (i = -1 for the rows) + const below = (i: number, j: number): Gold => { + if (i === layers.length - 1) return leaf(j) + const next = layers[i + 1] + if (next.offsets === undefined) return at(i + 1, j) + const out: Array = [] + for (let k = next.offsets[j]; k < next.offsets[j + 1]; k++) out.push(below(i + 1, k)) + return out + } + const at = (i: number, j: number): Gold => { + const validity = layers[i].validity + return validity !== undefined && !bit(validity, j) ? null : below(i, j) + } + return Array.from({ length: column.numRows }, (_, row) => below(-1, row)) +} + +// --------------------------------------------------------------------------- +// Synthetic files, written with the Thrift writer +// --------------------------------------------------------------------------- + +type V = Thrift.Value +const I32 = (value: number): V => ({ _tag: "I32", value }) +const I64 = (value: number): V => ({ _tag: "I64", value: BigInt(value) }) +const Bool = (value: boolean): V => ({ _tag: "Bool", value }) +const Str = (value: string): V => ({ _tag: "Binary", value: new TextEncoder().encode(value) }) +const List = (elementType: Thrift.TypeName, elements: ReadonlyArray): V => ({ _tag: "List", elementType, elements }) +const Struct = (fields: { readonly [id: number]: V | undefined }): V => ({ + _tag: "Struct", + fields: Object.entries(fields).flatMap(([id, value]) => value === undefined ? [] : [{ id: Number(id), value }]) +}) +const thrift = (value: V): Uint8Array => { + const writer = Thrift.makeWriter() + writer.writeValue(value) + return writer.finish() +} +const bytes = (...parts: ReadonlyArray>): Uint8Array => { + const out = new Uint8Array(parts.reduce((n, part) => n + part.length, 0)) + let at = 0 + for (const part of parts) { + out.set(part, at) + at += part.length + } + return out +} +const u32 = (n: number) => [n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, n >>> 24] +const uleb = (n: number): Array => n < 0x80 ? [n] : [(n & 0x7f) | 0x80, ...uleb(Math.floor(n / 0x80))] +const text = (s: string) => new TextEncoder().encode(s) +const int32s = (...values: ReadonlyArray) => new Uint8Array(Int32Array.from(values).buffer) +const plainBinary = (...values: ReadonlyArray) => + bytes(...values.flatMap((v) => [u32(text(v).length), text(v)])) +/** RLE/bit-packed hybrid runs of repeated values: [value, count] pairs. */ +const runs = (width: number, ...pairs: ReadonlyArray) => + bytes(...pairs.map(([value, count]) => [...uleb(count * 2), ...u32(value).slice(0, (width + 7) >> 3)])) +const lengthPrefixed = (data: Uint8Array) => bytes(u32(data.length), data) + +const Type = { BOOLEAN: 0, INT32: 1, INT64: 2, INT96: 3, FLOAT: 4, DOUBLE: 5, BYTE_ARRAY: 6, FIXED_LEN_BYTE_ARRAY: 7 } +const REQUIRED = 0 +const OPTIONAL = 1 +const REPEATED = 2 +const PLAIN = 0 +const PLAIN_DICTIONARY = 2 +const RLE = 3 +const BIT_PACKED = 4 +const DELTA_BINARY_PACKED = 5 +const DELTA_LENGTH_BYTE_ARRAY = 6 +const DELTA_BYTE_ARRAY = 7 +const RLE_DICTIONARY = 8 +const BYTE_STREAM_SPLIT = 9 +const ALP = 10 +const GZIP = 2 + +const element = (name: string, options: { + readonly type?: number | undefined + readonly typeLength?: number | undefined + readonly repetition?: number | undefined + readonly children?: number | undefined +}): V => + Struct({ + 1: options.type === undefined ? undefined : I32(options.type), + 2: options.typeLength === undefined ? undefined : I32(options.typeLength), + 3: options.repetition === undefined ? undefined : I32(options.repetition), + 4: Str(name), + 5: options.children === undefined ? undefined : I32(options.children) + }) +const root = (children: number) => element("schema", { children }) + +const page = (type: number, header: { readonly [id: number]: V }, body: Uint8Array, uncompressedSize = body.length) => + bytes(thrift(Struct({ 1: I32(type), 2: I32(uncompressedSize), 3: I32(body.length), ...header })), body) +const dataPage = (numValues: number, body: Uint8Array, options?: { + readonly encoding?: number | undefined + readonly levels?: number | undefined + readonly uncompressedSize?: number | undefined +}) => + page( + 0, + { + 5: Struct({ + 1: I32(numValues), + 2: I32(options?.encoding ?? PLAIN), + 3: I32(options?.levels ?? RLE), + 4: I32(options?.levels ?? RLE) + }) + }, + body, + options?.uncompressedSize + ) +const dictionaryPage = (numValues: number, body: Uint8Array, encoding = PLAIN) => + page(2, { 7: Struct({ 1: I32(numValues), 2: I32(encoding) }) }, body) +const dataPageV2 = (options: { + readonly numValues: number + readonly numNulls: number + readonly numRows: number + readonly encoding?: number | undefined + readonly repetition?: Uint8Array + readonly definition?: Uint8Array + readonly values: Uint8Array + readonly compressed?: boolean | undefined + readonly uncompressedSize?: number | undefined +}) => { + const repetition = options.repetition ?? new Uint8Array(0) + const definition = options.definition ?? new Uint8Array(0) + const body = bytes(repetition, definition, options.values) + return page( + 3, + { + 8: Struct({ + 1: I32(options.numValues), + 2: I32(options.numNulls), + 3: I32(options.numRows), + 4: I32(options.encoding ?? PLAIN), + 5: I32(definition.length), + 6: I32(repetition.length), + 7: options.compressed === undefined ? undefined : Bool(options.compressed) + }) + }, + body, + options.uncompressedSize + ) +} + +interface Chunk { + readonly type: number + readonly path: ReadonlyArray + readonly pages: ReadonlyArray + readonly numValues: number + readonly codec?: number | undefined + /** Whether the first page is a dictionary page. */ + readonly dictionary?: boolean | undefined + readonly statistics?: V | undefined + readonly chunk?: { readonly [id: number]: V } +} + +/** Writes a Parquet file with one row group. */ +const file = (options: { + readonly schema: ReadonlyArray + readonly numRows: number + readonly chunks: ReadonlyArray + readonly metadata?: { readonly [id: number]: V } +}): Uint8Array => { + const body: Array = [] + let offset = 4 + const columns = options.chunks.map((chunk) => { + const data = bytes(...chunk.pages) + const meta = Struct({ + 1: I32(chunk.type), + 2: List("I32", [I32(PLAIN), I32(RLE)]), + 3: List("Binary", chunk.path.map(Str)), + 4: I32(chunk.codec ?? 0), + 5: I64(chunk.numValues), + 6: I64(data.length), + 7: I64(data.length), + 9: I64(chunk.dictionary === true ? offset + chunk.pages[0].length : offset), + 11: chunk.dictionary === true ? I64(offset) : undefined, + 12: chunk.statistics + }) + body.push(data) + const column = Struct({ 2: I64(offset), 3: meta, ...chunk.chunk }) + offset += data.length + return column + }) + const footer = thrift(Struct({ + 1: I32(1), + 2: List("Struct", options.schema), + 3: I64(options.numRows), + 4: List("Struct", [Struct({ 1: List("Struct", columns), 2: I64(offset - 4), 3: I64(options.numRows) })]), + ...options.metadata + })) + return bytes(text("PAR1"), ...body, footer, u32(footer.length), text("PAR1")) +} + +/** A file with one top-level column. */ +const flat = (options: { + readonly type: number + readonly repetition: number + readonly numRows: number + /** Defaults to `numRows`. */ + readonly numValues?: number | undefined + readonly pages: ReadonlyArray + readonly typeLength?: number | undefined + readonly codec?: number | undefined + readonly dictionary?: boolean | undefined +}) => + file({ + schema: [ + root(1), + element("x", { type: options.type, repetition: options.repetition, typeLength: options.typeLength }) + ], + numRows: options.numRows, + chunks: [{ ...options, path: ["x"], numValues: options.numValues ?? options.numRows }] + }) + +/** DELTA_BINARY_PACKED for at most 128 small values: one block of four miniblocks. */ +const delta = (values: ReadonlyArray): Uint8Array => { + const zigzag = (n: number) => n < 0 ? -2 * n - 1 : 2 * n + const header = [...uleb(128), 4, ...uleb(values.length), ...uleb(zigzag(values[0] ?? 0))] + if (values.length <= 1) return new Uint8Array(header) + const deltas = values.slice(1).map((v, i) => v - values[i]) + const min = Math.min(...deltas) + const relative = deltas.map((d) => d - min) + const width = 32 - Math.clz32(Math.max(...relative)) + const miniblocks = Math.ceil(relative.length / 32) + const packed = new Uint8Array(miniblocks * 32 * width / 8) + relative.forEach((value, i) => { + for (let b = 0; b < width; b++) if ((value >> b) & 1) packed[(i * width + b) >> 3] |= 1 << ((i * width + b) & 7) + }) + return bytes(header, uleb(zigzag(min)), [0, 1, 2, 3].map((m) => m < miniblocks ? width : 0), packed) +} + +const metadataOf = (bytes: Uint8Array) => Parquet.readMetadata(Parquet.fromBytes(bytes)) +type ReadOptions = Omit[2], "rowGroup" | "column"> +const read = Effect.fnUntraced(function*(bytes: Uint8Array, options?: ReadOptions) { + const source = Parquet.fromBytes(bytes) + const metadata = yield* Parquet.readMetadata(source) + return yield* Parquet.readColumn(source, metadata, { rowGroup: 0, column: 0, ...options }) +}) +const values = (bytes: Uint8Array, options?: ReadOptions) => Effect.map(read(bytes, options), rows) +const failsWith = Effect.fnUntraced(function*( + effect: Effect.Effect, + kind: Parquet.ParquetError["kind"], + message?: RegExp +) { + const error = yield* Effect.flip(effect) + assert.isTrue(Parquet.isParquetError(error)) + assert.strictEqual(error.kind, kind, error.message) + if (message !== undefined) assert.match(error.message, message) +}) + +describe("Parquet", () => { + describe("pyarrow " + gold.writer + " fixtures", () => { + for (const file of gold.files) { + it.effect(file.name, () => + Effect.gen(function*() { + const source = fixture(file.name) + const metadata = yield* Parquet.readMetadata(source) + assert.strictEqual(metadata.numRows, file.numRows) + assert.strictEqual(metadata.createdBy, "parquet-cpp-arrow version " + gold.writer) + assert.deepStrictEqual( + metadata.rowGroups.map((group) => ({ + numRows: group.numRows, + columns: group.columns.map((chunk) => ({ + codec: chunk.metaData!.codec, + encodings: [...chunk.metaData!.encodings].sort() + })) + })), + file.rowGroups.map((group) => ({ + numRows: group.numRows, + columns: group.columns.map((c) => ({ codec: c.codec, encodings: [...c.encodings].sort() })) + })) + ) + assert.strictEqual(metadata.columns.length, file.columns.length) + for (let c = 0; c < file.columns.length; c++) { + const expected = file.columns[c] + const descriptor = metadata.columns[c] + const leaf = descriptor.nodes[descriptor.nodes.length - 1] + assert.deepStrictEqual({ + path: descriptor.path, + physicalType: descriptor.physicalType, + typeLength: descriptor.typeLength ?? null, + maxDefinitionLevel: descriptor.maxDefinitionLevel, + maxRepetitionLevel: descriptor.maxRepetitionLevel, + convertedType: leaf.convertedType ?? null, + logicalType: leaf.logicalType ?? null + }, { + path: expected.path, + physicalType: expected.physicalType, + typeLength: expected.typeLength, + maxDefinitionLevel: expected.maxDefinitionLevel, + maxRepetitionLevel: expected.maxRepetitionLevel, + convertedType: expected.convertedType, + logicalType: expected.logicalType + }) + const values: Array = [] + for (let g = 0; g < metadata.rowGroups.length; g++) { + const column = yield* Parquet.readColumn(source, metadata, { + rowGroup: g, + column: c, + codecs: { GZIP: gzip } + }) + values.push(...rows(column)) + } + assert.deepStrictEqual(values, expected.values, expected.path.join(".")) + } + })) + } + }) + describe("conformance (apache-parquet-format-2.14.0)", () => { + const int32s3 = flat({ type: Type.INT32, repetition: REQUIRED, numRows: 3, pages: [dataPage(3, int32s(1, 2, 3))] }) + const optionalInt32 = (pages: ReadonlyArray, numRows: number, codec?: number) => + flat({ type: Type.INT32, repetition: OPTIONAL, numRows, pages, codec }) + const repeatedInt32 = (pages: ReadonlyArray, numRows: number, numValues: number) => + flat({ type: Type.INT32, repetition: REPEATED, numRows, numValues, pages }) + // def levels 1, 0, 1 as RLE runs, for a column with one optional node + const someNull = runs(1, [1, 1], [0, 1], [1, 1]) + + describe("README.md#file-format", () => { + it.effect("PQ-FILE-1 MUST start and end with the magic number PAR1", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(int32s3), [1, 2, 3]) + const badTrailer = int32s3.slice() + badTrailer[badTrailer.length - 1] = 0x30 + yield* failsWith(metadataOf(badTrailer), "Malformed", /end with the magic number/) + const badHeader = int32s3.slice() + badHeader[3] = 0x30 + yield* failsWith(metadataOf(badHeader), "Malformed", /start with the magic number/) + })) + + it.effect("PQ-FILE-2 MUST find the metadata from the 4-byte little-endian length before the final magic", () => + Effect.gen(function*() { + const n = int32s3.length + const metadata = yield* metadataOf(int32s3) + assert.strictEqual(metadata.rowGroups[0].columns[0].metaData?.dataPageOffset, 4) + const tooLong = int32s3.slice() + tooLong.set(u32(n - 11), n - 8) + yield* failsWith(metadataOf(tooLong), "Malformed", /footer length/) + const garbage = bytes(text("PAR1"), [0x19, 0xfc], u32(2), text("PAR1")) + yield* failsWith(metadataOf(garbage), "Malformed", /invalid Thrift metadata/) + })) + }) + + describe("parquet.thrift FileMetaData", () => { + it.effect("PQ-META-1 SHOULD accept versions 1 and 2 interchangeably (L1411-1413)", () => + Effect.gen(function*() { + const v2 = file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: 3, + chunks: [{ type: Type.INT32, path: ["x"], numValues: 3, pages: [dataPage(3, int32s(1, 2, 3))] }], + metadata: { 1: I32(2) } + }) + assert.strictEqual((yield* metadataOf(v2)).version, 2) + assert.deepStrictEqual(yield* values(v2), yield* values(int32s3)) + })) + + it.effect("PQ-META-2 MUST give a column order per leaf column, in schema order (L1440-1455)", () => + Effect.gen(function*() { + const make = (orders: ReadonlyArray) => + file({ + schema: [ + root(2), + element("a", { type: Type.INT32, repetition: REQUIRED }), + element("b", { type: Type.DOUBLE, repetition: REQUIRED }) + ], + numRows: 0, + chunks: [ + { type: Type.INT32, path: ["a"], numValues: 0, pages: [] }, + { type: Type.DOUBLE, path: ["b"], numValues: 0, pages: [] } + ], + metadata: { 7: List("Struct", orders) } + }) + const metadata = yield* metadataOf(make([Struct({ 1: Struct({}) }), Struct({ 2: Struct({}) })])) + assert.deepStrictEqual(metadata.columns.map((c) => c.columnOrder), ["TYPE_ORDER", "IEEE_754_TOTAL_ORDER"]) + yield* failsWith(metadataOf(make([Struct({ 1: Struct({}) })])), "Malformed", /column orders/) + })) + }) + + describe("parquet.thrift SchemaElement", () => { + it.effect("PQ-SCHEMA-1 MUST give every node but the root a repetition (L535-537)", () => + failsWith( + metadataOf(file({ + schema: [root(1), element("x", { type: Type.INT32 })], + numRows: 0, + chunks: [{ type: Type.INT32, path: ["x"], numValues: 0, pages: [] }] + })), + "Malformed", + /repetition/ + )) + + it.effect("PQ-SCHEMA-2 MUST rebuild the schema tree from its depth-first elements and num_children (L525-526)", () => + Effect.gen(function*() { + const schema = [ + root(2), + element("a", { repetition: OPTIONAL, children: 2 }), + element("b", { type: Type.INT32, repetition: REPEATED }), + element("c", { type: Type.BYTE_ARRAY, repetition: OPTIONAL }), + element("d", { type: Type.FIXED_LEN_BYTE_ARRAY, typeLength: 16, repetition: REQUIRED }) + ] + const chunks = [ + { type: Type.INT32, path: ["a", "b"], numValues: 0, pages: [] }, + { type: Type.BYTE_ARRAY, path: ["a", "c"], numValues: 0, pages: [] }, + { type: Type.FIXED_LEN_BYTE_ARRAY, path: ["d"], numValues: 0, pages: [] } + ] + const metadata = yield* metadataOf(file({ schema, numRows: 0, chunks })) + assert.deepStrictEqual(metadata.schema.children?.map((n) => n.name), ["a", "d"]) + assert.deepStrictEqual(metadata.schema.children?.[0].children?.map((n) => n.name), ["b", "c"]) + assert.deepStrictEqual(metadata.columns.map((c) => [c.path, c.physicalType, c.typeLength]), [ + [["a", "b"], "INT32", undefined], + [["a", "c"], "BYTE_ARRAY", undefined], + [["d"], "FIXED_LEN_BYTE_ARRAY", 16] + ]) + yield* failsWith( + metadataOf(file({ schema: schema.slice(0, 4), numRows: 0, chunks })), + "Malformed", + /ends inside a group/ + ) + })) + }) + + describe("README.md#nested-encoding", () => { + it.effect("PQ-NEST-1 MUST compute the maximum levels from the optional and repeated nodes of the path", () => + Effect.gen(function*() { + const metadata = yield* metadataOf(file({ + schema: [ + root(1), + element("a", { repetition: OPTIONAL, children: 1 }), + element("list", { repetition: REPEATED, children: 1 }), + element("b", { repetition: REQUIRED, children: 1 }), + element("c", { type: Type.INT32, repetition: REPEATED }) + ], + numRows: 0, + chunks: [{ type: Type.INT32, path: ["a", "list", "b", "c"], numValues: 0, pages: [] }] + })) + assert.strictEqual(metadata.columns[0].maxDefinitionLevel, 3) + assert.strictEqual(metadata.columns[0].maxRepetitionLevel, 2) + })) + + it.effect("PQ-NEST-2 MUST assemble lists from repetition and definition levels", () => + Effect.gen(function*() { + // rows [1, 2], [], [3]: levels (r, d) = (0, 1) (1, 1) (0, 0) (0, 1) + const page = dataPage( + 4, + bytes( + lengthPrefixed(runs(1, [0, 1], [1, 1], [0, 2])), + lengthPrefixed(runs(1, [1, 2], [0, 1], [1, 1])), + int32s(1, 2, 3) + ) + ) + const column = yield* read(repeatedInt32([page], 3, 4)) + assert.deepStrictEqual(rows(column), [[1, 2], [], [3]]) + assert.deepStrictEqual(Array.from(column.layers[0].offsets!), [0, 2, 2, 3]) + })) + }) + + describe("README.md#nulls", () => { + it.effect("PQ-NULL-1 MUST read nulls from the definition levels only, with no values in the data", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(optionalInt32([dataPage(3, lengthPrefixed(runs(1, [0, 3])))], 3)), [ + null, + null, + null + ]) + const column = yield* read(optionalInt32([dataPage(3, bytes(lengthPrefixed(someNull), int32s(5, 7)))], 3)) + assert.deepStrictEqual(rows(column), [5, null, 7]) + assert.strictEqual(column.layers[0].nullCount, 1) + })) + }) + + describe("README.md#data-pages", () => { + it.effect("PQ-PAGE-1 MUST read a required top-level column's page as values only", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(int32s3), [1, 2, 3]) + })) + + it.effect("PQ-PAGE-2 MUST read uncompressed_page_size as the size of levels and values together", () => + Effect.gen(function*() { + const body = bytes(lengthPrefixed(someNull), int32s(5, 7)) + const make = (uncompressedSize: number) => + optionalInt32([dataPage(3, gzipSync(body), { uncompressedSize })], 3, GZIP) + assert.deepStrictEqual(yield* values(make(body.length), { codecs: { GZIP: gzip } }), [5, null, 7]) + yield* failsWith(values(make(body.length + 1), { codecs: { GZIP: gzip } }), "Malformed", /decompressed to/) + })) + }) + + describe("README.md#column-chunks", () => { + const dictionary = dictionaryPage(2, int32s(10, 20)) + const indices = dataPage(3, bytes([1], runs(1, [0, 1], [1, 2])), { encoding: RLE_DICTIONARY }) + const make = (pages: ReadonlyArray, numRows = 3) => + flat({ type: Type.INT32, repetition: REQUIRED, numRows, pages, dictionary: true }) + + it.effect("PQ-CHUNK-1 MUST read the dictionary page first, and at most one (parquet.thrift L727-729)", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(make([dictionary, indices])), [10, 20, 20]) + yield* failsWith(values(make([dictionary, dictionary, indices])), "Malformed", /dictionary page/) + yield* failsWith(values(make([dictionary, indices, dictionary, indices], 6)), "Malformed", /dictionary page/) + })) + + it.effect("PQ-CHUNK-2 MAY skip pages it does not read: index pages and later page types", () => + Effect.gen(function*() { + const index = page(1, { 6: Struct({}) }, new Uint8Array([1, 2, 3])) + const future = page(42, {}, new Uint8Array([4, 5])) + assert.deepStrictEqual( + yield* values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [index, future, dataPage(3, int32s(1, 2, 3))] + }) + ), + [1, 2, 3] + ) + })) + }) + + describe("parquet.thrift DataPageHeaderV2", () => { + it.effect("PQ-V2-1 MUST begin every v2 page at a row boundary (L759-761)", () => + Effect.gen(function*() { + const part = (repetition: number, value: number, numRows: number) => + dataPageV2({ + numValues: 1, + numNulls: 0, + numRows, + repetition: runs(1, [repetition, 1]), + definition: runs(1, [1, 1]), + values: int32s(value) + }) + assert.deepStrictEqual(yield* values(repeatedInt32([part(0, 1, 1), part(0, 2, 1)], 2, 2)), [[1], [2]]) + yield* failsWith(values(repeatedInt32([part(0, 1, 1), part(1, 2, 0)], 1, 2)), "Malformed", /row boundary/) + })) + + it.effect("PQ-V2-2 MUST read v2 levels uncompressed and without a length prefix (L742-745, L763)", () => + Effect.gen(function*() { + const values_ = int32s(5, 7) + const decompress: Parquet.Decompress = (data, size) => { + assert.deepStrictEqual(gunzipSync(data), Buffer.from(values_)) + assert.strictEqual(size, 8) + return gzip(data, size) + } + const v2 = dataPageV2({ + numValues: 3, + numNulls: 1, + numRows: 3, + definition: someNull, + values: gzipSync(values_), + uncompressedSize: someNull.length + 8 + }) + assert.deepStrictEqual(yield* values(optionalInt32([v2], 3, GZIP), { codecs: { GZIP: decompress } }), [ + 5, + null, + 7 + ]) + })) + + it.effect("PQ-V2-3 MUST NOT decompress the values of a v2 page whose is_compressed is false (L745)", () => + Effect.gen(function*() { + const v2 = dataPageV2({ + numValues: 3, + numNulls: 1, + numRows: 3, + definition: someNull, + values: int32s(5, 7), + compressed: false + }) + assert.deepStrictEqual(yield* values(optionalInt32([v2], 3, GZIP)), [5, null, 7]) + })) + + it.effect("PQ-V2-4 MUST hold num_values - num_nulls values in the data section (L748-751)", () => + Effect.gen(function*() { + const v2 = dataPageV2({ numValues: 3, numNulls: 0, numRows: 3, definition: someNull, values: int32s(5, 7) }) + yield* failsWith(values(optionalInt32([v2], 3)), "Malformed", /nulls/) + })) + }) + + describe("parquet.thrift Statistics", () => { + const withStatistics = (statistics: V) => + file({ + schema: [root(1), element("x", { type: Type.DOUBLE, repetition: REQUIRED })], + numRows: 0, + chunks: [{ type: Type.DOUBLE, path: ["x"], numValues: 0, pages: [], statistics }] + }) + const statisticsOf = (bytes: Uint8Array) => + Effect.map(metadataOf(bytes), (m) => m.rowGroups[0].columns[0].metaData?.statistics) + + it.effect("PQ-STATS-1 MUST distinguish an absent null_count from a null_count of 0 (L286-289)", () => + Effect.gen(function*() { + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({ 3: I64(0) }))))?.nullCount, 0) + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({}))))?.nullCount, undefined) + })) + + it.effect("PQ-STATS-2 MUST NOT assume nan_count is 0 when it is absent (L315-316)", () => + Effect.gen(function*() { + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({ 9: I64(0) }))))?.nanCount, 0) + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({}))))?.nanCount, undefined) + })) + }) + + describe("Encodings.md#plain-plain--0", () => { + it.effect("PQ-PLAIN-1 MUST read PLAIN values of every physical type", () => + Effect.gen(function*() { + const cases: ReadonlyArray, number?]> = [ + // BOOLEAN: bit-packed, least significant bit first + [Type.BOOLEAN, 10, new Uint8Array([0b10001101, 0b01]), [ + true, + false, + true, + true, + false, + false, + false, + true, + true, + false + ]], + [Type.INT32, 2, int32s(-1, 0x7fffffff), [-1, 0x7fffffff]], + [Type.INT64, 2, new Uint8Array(BigInt64Array.from([-(2n ** 63n), 5n]).buffer), [String(-(2n ** 63n)), "5"]], + [Type.INT96, 1, new Uint8Array(12).fill(7), ["070707070707070707070707"]], + [Type.FLOAT, 2, new Uint8Array(Float32Array.from([1.5, -0]).buffer), [1.5, -0]], + [Type.DOUBLE, 1, new Uint8Array(Float64Array.from([Math.PI]).buffer), [Math.PI]], + [Type.BYTE_ARRAY, 3, plainBinary("", "ab", "é"), ["", "6162", "c3a9"]], + [Type.FIXED_LEN_BYTE_ARRAY, 2, new Uint8Array([1, 2, 3, 4]), ["0102", "0304"], 2] + ] + for (const [type, n, data, expected, typeLength] of cases) { + const bytes = flat({ type, typeLength, repetition: REQUIRED, numRows: n, pages: [dataPage(n, data)] }) + assert.deepStrictEqual(yield* values(bytes), expected) + } + })) + }) + + describe("Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8", () => { + const make = (dictionaryEncoding: number, encoding: number, indices: Uint8Array, numRows = 3) => + flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows, + dictionary: true, + pages: [ + dictionaryPage(2, plainBinary("a", "b"), dictionaryEncoding), + dataPage(numRows, indices, { encoding }) + ] + }) + + it.effect("PQ-DICT-1 MUST read indices as a bit-width byte followed by RLE/bit-packed runs (L84-85)", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(make(PLAIN, RLE_DICTIONARY, bytes([1], runs(1, [1, 2], [0, 1])))), [ + "62", + "62", + "61" + ]) + yield* failsWith(values(make(PLAIN, RLE_DICTIONARY, bytes([33], runs(33, [1, 3])))), "Malformed", /bit width/) + })) + + it.effect("PQ-DICT-2 SHOULD read the deprecated PLAIN_DICTIONARY in dictionary and data pages (L87-88)", () => + Effect.gen(function*() { + const indices = bytes([1], runs(1, [0, 3])) + assert.deepStrictEqual(yield* values(make(PLAIN_DICTIONARY, PLAIN_DICTIONARY, indices)), ["61", "61", "61"]) + })) + }) + + describe("Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3", () => { + it.effect("PQ-RLE-1 MUST unpack bit-packed runs from the least significant bit (note 1)", () => + Effect.gen(function*() { + // the spec's example: 0 to 7 with bit width 3 is 10001000 11000110 11111010 + const bytes_ = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 8, + dictionary: true, + pages: [ + dictionaryPage(8, int32s(10, 11, 12, 13, 14, 15, 16, 17)), + dataPage(8, new Uint8Array([3, 0b11, 0b10001000, 0b11000110, 0b11111010]), { encoding: RLE_DICTIONARY }) + ] + }) + assert.deepStrictEqual(yield* values(bytes_), [10, 11, 12, 13, 14, 15, 16, 17]) + })) + + it.effect("PQ-RLE-2 MUST reject runs shorter than 1 (note 3, L141)", () => + failsWith( + values(optionalInt32([dataPage(1, bytes(lengthPrefixed(new Uint8Array([0, 1])), int32s(1)))], 1)), + "Malformed", + /run of length 0/ + )) + + it.effect("PQ-RLE-3 MUST read RLE booleans with a 4-byte length in v1 and v2 pages (L155-170)", () => + Effect.gen(function*() { + const data = lengthPrefixed(runs(1, [1, 3], [0, 2])) + const expected = [true, true, true, false, false] + const v1 = dataPage(5, data, { encoding: RLE }) + const v2 = dataPageV2({ numValues: 5, numNulls: 0, numRows: 5, encoding: RLE, values: data }) + for (const page of [v1, v2]) { + assert.deepStrictEqual( + yield* values(flat({ type: Type.BOOLEAN, repetition: REQUIRED, numRows: 5, pages: [page] })), + expected + ) + } + })) + }) + + describe("Encodings.md#bit-packed-deprecated-bit_packed--4", () => { + // seven nested optional nodes: definition levels 0 to 7 need 3 bits + const schema = [ + root(1), + ...Array.from({ length: 6 }, (_, i) => element("g" + i, { repetition: OPTIONAL, children: 1 })), + element("x", { type: Type.INT32, repetition: OPTIONAL }) + ] + const make = (page: Uint8Array) => + file({ + schema, + numRows: 8, + chunks: [{ type: Type.INT32, path: ["g0", "g1", "g2", "g3", "g4", "g5", "x"], numValues: 8, pages: [page] }] + }) + + it.effect("PQ-BP-1 MUST unpack BIT_PACKED levels from the most significant bit (L188-197)", () => + Effect.gen(function*() { + // the spec's example: 0 to 7 with bit width 3 is 00000101 00111001 01110111 + const page = dataPage(8, bytes([0b00000101, 0b00111001, 0b01110111], int32s(42)), { levels: BIT_PACKED }) + const column = yield* read(make(page)) + assert.deepStrictEqual(rows(column), [null, null, null, null, null, null, null, 42]) + assert.deepStrictEqual(column.layers.map((l) => l.nullCount), [1, 2, 3, 4, 5, 6, 7]) + })) + + it.effect("PQ-BP-2 MUST NOT read BIT_PACKED as a values encoding (L198-199)", () => + failsWith( + values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + pages: [dataPage(1, int32s(1), { encoding: BIT_PACKED })] + }) + ), + "Malformed", + /not an encoding/ + )) + }) + + describe("Encodings.md#delta-encoding-delta_binary_packed--5", () => { + const int32 = (data: Uint8Array, n: number) => + values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: n, + pages: [dataPage(n, data, { encoding: DELTA_BINARY_PACKED })] + }) + ) + // Example 2 (7, 5, 3, 1, 2, 3, 4, 5), with a valid block size of 128 and four miniblocks of 32 + const example2 = (widths: ReadonlyArray, packed: ReadonlyArray) => + bytes([0x80, 0x01, 4, 8, 14], [3], widths, packed) + + it.effect("PQ-DELTA-1 MUST read the header and blocks of bit-packed miniblocks (examples 1 and 2)", () => + Effect.gen(function*() { + // Example 1: 1, 2, 3, 4, 5 is a min delta of 1 and bit width 0 + assert.deepStrictEqual(yield* int32(new Uint8Array([0x80, 0x01, 4, 5, 2, 2, 0, 0, 0, 0]), 5), [1, 2, 3, 4, 5]) + // relative deltas 0, 0, 0, 3, 3, 3, 3 on 2 bits, padded to 32 values + const packed = [0b11000000, 0b00111111, 0, 0, 0, 0, 0, 0] + assert.deepStrictEqual(yield* int32(example2([2, 0, 0, 0], packed), 8), [7, 5, 3, 1, 2, 3, 4, 5]) + })) + + it.effect("PQ-DELTA-2 MUST accept arbitrary padding bits in the last miniblock (L257-260)", () => + Effect.gen(function*() { + const packed = [0b11000000, 0b11111111, 0xff, 0xaa, 0x55, 0xff, 0xff, 0xff] + assert.deepStrictEqual(yield* int32(example2([2, 0, 0, 0], packed), 8), [7, 5, 3, 1, 2, 3, 4, 5]) + })) + + it.effect("PQ-DELTA-3 MUST accept arbitrary bit widths for unneeded miniblocks (L262-268)", () => + Effect.gen(function*() { + const packed = [0b11000000, 0b00111111, 0, 0, 0, 0, 0, 0] + assert.deepStrictEqual(yield* int32(example2([2, 0xff, 0x41, 7], packed), 8), [7, 5, 3, 1, 2, 3, 4, 5]) + })) + + it.effect("PQ-DELTA-4 MUST wrap deltas around in two's complement (L270-273)", () => + Effect.gen(function*() { + // INT32: 2^31 - 1, -2^31, 2^31 - 1 are deltas 1 and -1 after wrapping + const i32 = bytes([0x80, 0x01, 4, 3], uleb(0xfffffffe), [1], [2, 0, 0, 0], [0b0010, 0, 0, 0, 0, 0, 0, 0]) + assert.deepStrictEqual(yield* int32(i32, 3), [0x7fffffff, -0x80000000, 0x7fffffff]) + // INT64: 2^63 - 1, -2^63 is a delta of 1 after wrapping + const max64 = [...Array(9).fill(0xff).map((b, i) => i === 0 ? 0xfe : b), 0x01] + const i64 = bytes([0x80, 0x01, 4, 2], max64, [2], [0, 0, 0, 0]) + assert.deepStrictEqual( + yield* values( + flat({ + type: Type.INT64, + repetition: REQUIRED, + numRows: 2, + pages: [dataPage(2, i64, { encoding: DELTA_BINARY_PACKED })] + }) + ), + [String(2n ** 63n - 1n), String(-(2n ** 63n))] + ) + })) + + it.effect("PQ-DELTA-5 MUST reject miniblocks wider than the physical type (L274-276)", () => + failsWith( + int32(bytes([0x80, 0x01, 4, 2, 0], [0], [33, 0, 0, 0], new Uint8Array(132)), 2), + "Malformed", + /bit width 33/ + )) + }) + + describe("Encodings.md#delta-length-byte-array-delta_length_byte_array--6", () => { + it.effect("PQ-DLBA-1 MUST read delta-encoded lengths followed by the concatenated bytes (example)", () => + Effect.gen(function*() { + const data = bytes(delta([5, 5, 6, 6]), text("HelloWorldFoobarABCDEF")) + const page = dataPage(4, data, { encoding: DELTA_LENGTH_BYTE_ARRAY }) + assert.deepStrictEqual( + yield* values(flat({ type: Type.BYTE_ARRAY, repetition: REQUIRED, numRows: 4, pages: [page] })), + ["Hello", "World", "Foobar", "ABCDEF"].map((s) => Hex.encode(text(s))) + ) + })) + }) + + describe("Encodings.md#delta-strings-delta_byte_array--7", () => { + it.effect("PQ-DBA-1 MUST read prefix lengths then suffixes as DELTA_LENGTH_BYTE_ARRAY (example)", () => + Effect.gen(function*() { + const data = bytes(delta([0, 2, 0, 3]), delta([4, 2, 6, 5]), text("axislebabbleyhood")) + const page = dataPage(4, data, { encoding: DELTA_BYTE_ARRAY }) + assert.deepStrictEqual( + yield* values(flat({ type: Type.BYTE_ARRAY, repetition: REQUIRED, numRows: 4, pages: [page] })), + ["axis", "axle", "babble", "babyhood"].map((s) => Hex.encode(text(s))) + ) + })) + + it.effect("PQ-DBA-2 MUST read FIXED_LEN_BYTE_ARRAY values with all their lengths encoded", () => + Effect.gen(function*() { + const data = bytes(delta([0, 2, 1]), delta([4, 2, 3]), text("axislebcd")) + const page = dataPage(3, data, { encoding: DELTA_BYTE_ARRAY }) + const file_ = flat({ + type: Type.FIXED_LEN_BYTE_ARRAY, + typeLength: 4, + repetition: REQUIRED, + numRows: 3, + pages: [page] + }) + assert.deepStrictEqual(yield* values(file_), ["axis", "axle", "abcd"].map((s) => Hex.encode(text(s)))) + })) + }) + + describe("Encodings.md#byte-stream-split-byte_stream_split--9", () => { + const split = new Uint8Array([0xaa, 0x00, 0xa3, 0xbb, 0x11, 0xb4, 0xcc, 0x22, 0xc5, 0xdd, 0x33, 0xd6]) + const make = (data: Uint8Array) => + flat({ + type: Type.FLOAT, + repetition: REQUIRED, + numRows: 3, + pages: [dataPage(3, data, { encoding: BYTE_STREAM_SPLIT })] + }) + + it.effect("PQ-BSS-1 MUST gather byte k of every value from stream k (example)", () => + Effect.gen(function*() { + const raw = new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd, 0x00, 0x11, 0x22, 0x33, 0xa3, 0xb4, 0xc5, 0xd6]) + assert.deepStrictEqual(yield* values(make(split)), Array.from(new Float32Array(raw.buffer))) + })) + + it.effect("PQ-BSS-2 MUST hold exactly K * N bytes, with no padding", () => + failsWith(values(make(bytes(split, [0]))), "Malformed", /BYTE_STREAM_SPLIT/)) + }) + + describe("Compression.md", () => { + const body = int32s(1, 2, 3) + const make = (data: Uint8Array, codec: number) => + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + codec, + pages: [dataPage(3, data, { uncompressedSize: 12 })] + }) + + it.effect("PQ-COMP-1 MUST decompress each page with the column chunk's codec", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(make(gzipSync(body), GZIP), { codecs: { GZIP: gzip } }), [1, 2, 3]) + yield* failsWith(values(make(gzipSync(body), GZIP)), "Unsupported", /GZIP/) + const failing: Parquet.Decompress = () => Effect.fail("corrupt") + const error = yield* Effect.flip(values(make(gzipSync(body), GZIP), { codecs: { GZIP: failing } })) + assert.strictEqual(error.kind, "Codec") + assert.strictEqual(error.cause, "corrupt") + })) + + it.effect("PQ-COMP-2 SHOULD read GZIP pages holding several members (L61), given a Decompress that does", () => + Effect.gen(function*() { + const members = bytes(gzipSync(body.subarray(0, 4)), gzipSync(body.subarray(4))) + assert.deepStrictEqual(yield* values(make(members, GZIP), { codecs: { GZIP: gzip } }), [1, 2, 3]) + })) + }) + + describe("unsupported features", () => { + it.effect("PQ-UNSUP-1 fails with Unsupported on an encrypted footer (PARE)", () => + Effect.gen(function*() { + const encrypted = int32s3.slice() + encrypted.set(text("PARE"), encrypted.length - 4) + yield* failsWith(metadataOf(encrypted), "Unsupported", /encrypted footer/) + })) + + it.effect("PQ-UNSUP-2 fails with Unsupported on an encrypted column chunk", () => + failsWith( + read(file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: 3, + chunks: [{ + type: Type.INT32, + path: ["x"], + numValues: 3, + pages: [dataPage(3, int32s(1, 2, 3))], + chunk: { 8: Struct({ 1: Struct({}) }) } + }] + })), + "Unsupported", + /encrypted/ + )) + + it.effect("PQ-UNSUP-3 fails with Unsupported on the ALP encoding", () => + failsWith( + values( + flat({ + type: Type.DOUBLE, + repetition: REQUIRED, + numRows: 1, + pages: [dataPage(1, new Uint8Array(8), { encoding: ALP })] + }) + ), + "Unsupported", + /ALP/ + )) + + it.effect("PQ-UNSUP-4 fails with Unsupported on a column chunk in another file", () => + failsWith( + read(file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: 3, + chunks: [{ + type: Type.INT32, + path: ["x"], + numValues: 3, + pages: [dataPage(3, int32s(1, 2, 3))], + chunk: { 1: Str("other.parquet") } + }] + })), + "Unsupported", + /other\.parquet/ + )) + + it.effect("PQ-UNSUP-5 fails with Unsupported on a codec without a Decompress, or unknown", () => + Effect.gen(function*() { + for (const codec of [1, 3, 4, 5, 6, 7, 99]) { + const bytes_ = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + codec, + pages: [dataPage(3, int32s(1, 2, 3))] + }) + yield* failsWith(values(bytes_), "Unsupported") + } + })) + }) + }) + + describe("hostile input", () => { + const int32s3 = flat({ type: Type.INT32, repetition: REQUIRED, numRows: 3, pages: [dataPage(3, int32s(1, 2, 3))] }) + const strings = flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: 2, + pages: [dataPage(2, plainBinary("abc", "de"))] + }) + + it.effect("enforces maxFooterLength, maxDepth, maxPageSize, maxValues and maxBinaryLength", () => + Effect.gen(function*() { + yield* failsWith( + Parquet.readMetadata(Parquet.fromBytes(int32s3), { maxFooterLength: 10 }), + "Limit", + /maxFooterLength/ + ) + const deep = file({ + schema: [ + root(1), + element("a", { repetition: OPTIONAL, children: 1 }), + element("b", { type: Type.INT32, repetition: OPTIONAL }) + ], + numRows: 0, + chunks: [{ type: Type.INT32, path: ["a", "b"], numValues: 0, pages: [] }] + }) + assert.strictEqual((yield* Parquet.readMetadata(Parquet.fromBytes(deep), { maxDepth: 2 })).columns.length, 1) + yield* failsWith(Parquet.readMetadata(Parquet.fromBytes(deep), { maxDepth: 1 }), "Limit", /maxDepth/) + yield* failsWith(read(int32s3, { maxPageSize: 11 }), "Limit", /maxPageSize/) + assert.deepStrictEqual(rows(yield* read(int32s3, { maxPageSize: 12, maxValues: 3 })), [1, 2, 3]) + yield* failsWith(read(int32s3, { maxValues: 2 }), "Limit", /maxValues/) + assert.strictEqual(rows(yield* read(strings, { maxBinaryLength: 5 })).length, 2) + yield* failsWith(read(strings, { maxBinaryLength: 4 }), "Limit", /maxBinaryLength/) + })) + + it.effect("dies on invalid options and indexes", () => + Effect.gen(function*() { + const dies = (effect: Effect.Effect) => + Effect.map(Effect.exit(effect), (exit) => assert.isTrue(Exit.isFailure(exit) && Cause.hasDies(exit.cause))) + yield* dies(Parquet.readMetadata(Parquet.fromBytes(int32s3), { maxDepth: 0 })) + yield* dies(read(int32s3, { maxPageSize: 1.5 })) + const source = Parquet.fromBytes(int32s3) + const metadata = yield* Parquet.readMetadata(source) + yield* dies(Parquet.readColumn(source, metadata, { rowGroup: 1, column: 0 })) + yield* dies(Parquet.readColumn(source, metadata, { rowGroup: 0, column: -1 })) + })) + + it.effect("rejects sources, chunks and pages that don't add up", () => + Effect.gen(function*() { + const short: Parquet.Source = { + size: int32s3.length, + read: (offset, length) => Effect.succeed(int32s3.subarray(offset, offset + length - 1)) + } + yield* failsWith(Parquet.readMetadata(short), "Malformed", /source returned/) + yield* failsWith(metadataOf(new Uint8Array(11)), "Malformed", /too short/) + const make = (pages: ReadonlyArray, numRows = 3) => + flat({ type: Type.INT32, repetition: REQUIRED, numRows, pages }) + yield* failsWith(read(make([dataPage(4, int32s(1, 2, 3, 4))])), "Malformed", /more values/) + yield* failsWith(read(make([dataPage(2, int32s(1, 2))])), "Malformed", /ends after 2/) + yield* failsWith(read(make([dataPage(3, int32s(1, 2))])), "Malformed", /PLAIN values need/) + const truncated = make([dataPage(3, int32s(1, 2, 3))]) + yield* failsWith(read(truncated.subarray(0, 20)), "Malformed") + // a page header claiming more bytes than the chunk + yield* failsWith( + read( + make([ + page(0, { 5: Struct({ 1: I32(3), 2: I32(0), 3: I32(3), 4: I32(3) }) }, int32s(1, 2, 3), 12).subarray( + 0, + 20 + ) + ]) + ), + "Malformed" + ) + })) + + it.effect("rejects levels and dictionary indices out of range", () => + Effect.gen(function*() { + const optional = (data: Uint8Array) => + flat({ type: Type.INT32, repetition: OPTIONAL, numRows: 1, pages: [dataPage(1, data)] }) + yield* failsWith( + read(optional(bytes(lengthPrefixed(runs(1, [3, 1])), int32s(1)))), + "Malformed", + /RLE value wider/ + ) + yield* failsWith(read(optional(bytes(lengthPrefixed(runs(2, [2, 1])), int32s(1)))), "Malformed") + const dictionary = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + dictionary: true, + pages: [dictionaryPage(1, int32s(9)), dataPage(1, bytes([1], runs(1, [1, 1])), { encoding: RLE_DICTIONARY })] + }) + yield* failsWith(read(dictionary), "Malformed", /out of range/) + const noDictionary = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + pages: [dataPage(1, bytes([1], runs(1, [0, 1])), { encoding: RLE_DICTIONARY })] + }) + yield* failsWith(read(noDictionary), "Malformed", /no dictionary page/) + })) + + it.effect("rejects levels that repeat a missing list or disagree with the row count", () => + Effect.gen(function*() { + const repeated = ( + rep: ReadonlyArray, + def: ReadonlyArray, + n: number, + numRows: number + ) => + flat({ + type: Type.INT32, + repetition: REPEATED, + numRows, + numValues: n, + pages: [ + dataPage( + n, + bytes(lengthPrefixed(runs(1, ...rep)), lengthPrefixed(runs(1, ...def)), int32s(...Array(n).fill(1))) + ) + ] + }) + // the first level repeats a list that was never started + yield* failsWith(read(repeated([[1, 1]], [[1, 1]], 1, 0)), "Malformed", /repeats a list/) + // an empty list (d = 0) followed by a repetition into it + yield* failsWith(read(repeated([[0, 1], [1, 1]], [[0, 1], [1, 1]], 2, 1)), "Malformed", /repeats a list/) + yield* failsWith(read(repeated([[0, 2]], [[1, 2]], 2, 3)), "Malformed", /holds 2 rows/) + })) + + it.effect("rejects metadata that disagrees with the schema", () => + Effect.gen(function*() { + const make = (type: number, path: ReadonlyArray) => + metadataOf(file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: 0, + chunks: [{ type, path, numValues: 0, pages: [] }] + })) + yield* failsWith(make(Type.INT64, ["x"]), "Malformed", /is INT64/) + yield* failsWith(make(Type.INT32, ["y"]), "Malformed", /is for "y"/) + yield* failsWith( + metadataOf( + file({ + schema: [root(1), element("x", { type: Type.FIXED_LEN_BYTE_ARRAY, repetition: REQUIRED })], + numRows: 0, + chunks: [] + }) + ), + "Malformed", + /type_length/ + ) + })) + }) +}) diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md new file mode 100644 index 00000000000..28bcc3a40cf --- /dev/null +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -0,0 +1,95 @@ +# Parquet conformance + +The reader in `effect/encoding/Parquet` targets +[apache/parquet-format `apache-parquet-format-2.14.0`](https://github.com/apache/parquet-format/tree/apache-parquet-format-2.14.0). +References are section anchors of the Markdown specs and line numbers of +`src/main/thrift/parquet.thrift` at that tag, so they don't move. + +Every applicable assertion has an id and one test in +`packages/effect/test/encoding/Parquet.test.ts`, under +`conformance (apache-parquet-format-2.14.0)`, whose name starts with the id. +The pyarrow fixtures in this directory are interoperability evidence on top of +these tests, not a replacement for them. + +This is not a claim of full conformance: the sections below list what the +reader rejects, what it doesn't read yet, and what applies only to writers. + +## Tested + +| Id | Reference | Assertion | +| ----------- | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| PQ-FILE-1 | README.md#file-format | A file starts and ends with the magic number `PAR1` | +| PQ-FILE-2 | README.md#file-format | The metadata is found from the 4-byte little-endian length before the final magic | +| PQ-META-1 | parquet.thrift L1411-1413 | Readers should accept versions 1 and 2 interchangeably | +| PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | +| PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | +| PQ-SCHEMA-2 | parquet.thrift L525-526, L487-540 | The schema tree is the depth-first list of elements, groups giving `num_children` | +| PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | +| PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | +| PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | +| PQ-PAGE-1 | README.md#data-pages | A required, non-nested column's page holds only the values | +| PQ-PAGE-2 | README.md#data-pages | `uncompressed_page_size` covers levels and values together | +| PQ-CHUNK-1 | README.md#column-chunks, parquet.thrift L727-729 | The dictionary page comes first, and there is at most one | +| PQ-CHUNK-2 | README.md#column-chunks | Readers can skip pages they don't read (index pages, later page types) | +| PQ-V2-1 | parquet.thrift L759-761 | Every v2 data page begins at a row boundary | +| PQ-V2-2 | parquet.thrift L742-745, L763 | v2 levels are uncompressed RLE without a length prefix | +| PQ-V2-3 | parquet.thrift L745 | v2 values are compressed only when `is_compressed` is true | +| PQ-V2-4 | parquet.thrift L748-751 | A v2 page holds `num_values - num_nulls` values | +| PQ-STATS-1 | parquet.thrift L286-289 | Readers must distinguish an absent `null_count` from 0 | +| PQ-STATS-2 | parquet.thrift L315-316 | Readers must not assume `nan_count` is 0 when absent | +| PQ-PLAIN-1 | Encodings.md#plain-plain--0 | PLAIN encoding of every physical type | +| PQ-DICT-1 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | Indices are a bit-width byte (at most 32) and RLE/bit-packed runs | +| PQ-DICT-2 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | The deprecated `PLAIN_DICTIONARY` is still read | +| PQ-RLE-1 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 1 | Bit-packed runs are packed from the least significant bit | +| PQ-RLE-2 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3 | Run lengths are at least 1 | +| PQ-RLE-3 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, length table | RLE booleans carry a 4-byte length in v1 and v2 pages | +| PQ-BP-1 | Encodings.md#bit-packed-deprecated-bit_packed--4 | `BIT_PACKED` levels are packed from the most significant bit | +| PQ-BP-2 | Encodings.md#bit-packed-deprecated-bit_packed--4 | `BIT_PACKED` is only for levels | +| PQ-DELTA-1 | Encodings.md#delta-encoding-delta_binary_packed--5, examples 1 and 2 | Header, blocks and miniblocks | +| PQ-DELTA-2 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary padding bits | +| PQ-DELTA-3 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary bit widths of unneeded miniblocks | +| PQ-DELTA-4 | Encodings.md#delta-encoding-delta_binary_packed--5 | Deltas wrap around in two's complement | +| PQ-DELTA-5 | Encodings.md#delta-encoding-delta_binary_packed--5 | Miniblocks are no wider than the physical type (rejected when they are) | +| PQ-DLBA-1 | Encodings.md#delta-length-byte-array-delta_length_byte_array--6 | Delta-encoded lengths, then the concatenated bytes | +| PQ-DBA-1 | Encodings.md#delta-strings-delta_byte_array--7 | Prefix lengths, then suffixes as `DELTA_LENGTH_BYTE_ARRAY` | +| PQ-DBA-2 | Encodings.md#delta-strings-delta_byte_array--7 | `FIXED_LEN_BYTE_ARRAY` values encode all their lengths | +| PQ-BSS-1 | Encodings.md#byte-stream-split-byte_stream_split--9 | Byte k of every value is in stream k | +| PQ-BSS-2 | Encodings.md#byte-stream-split-byte_stream_split--9 | The streams are exactly K * N bytes, with no padding | +| PQ-COMP-1 | Compression.md#codecs | Pages are decompressed with the column chunk's codec | +| PQ-COMP-2 | Compression.md#gzip | Readers should read GZIP pages with several members (through the supplied `Decompress`) | + +## Unsupported: fails with `Unsupported` + +| Id | Reference | Feature | +| ---------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| PQ-UNSUP-1 | Encryption.md | Files with an encrypted footer (`PARE`) | +| PQ-UNSUP-2 | Encryption.md, parquet.thrift ColumnChunk fields 8-9 | Encrypted column chunks | +| PQ-UNSUP-3 | Encodings.md#adaptive-lossless-floating-point-alp--10 | The `ALP` encoding (in preview at 2.14.0) | +| PQ-UNSUP-4 | parquet.thrift ColumnChunk.file_path | Column chunks in another file | +| PQ-UNSUP-5 | Compression.md#codecs | Every codec but `UNCOMPRESSED` without a `Decompress`, and unknown codecs. The module has no codecs; the shared ones are planned in `effect/encoding/Compression` | + +## Deferred: not read yet + +- Page index: `ColumnIndex` and `OffsetIndex` (PageIndex.md, parquet.thrift L1240-1360). The reader reads every page in order. +- Bloom filters (BloomFilter.md). +- Page checksums (README.md#checksumming). `crc` is skipped, not verified. +- Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. +- Logical types (LogicalTypes.md): annotations are decoded into the schema, values stay in their physical type. This includes the LIST and MAP backward-compatibility rules, Variant and Geometry. +- Error recovery (README.md#error-recovery): each `readColumn` call is independent, so a corrupt column chunk doesn't affect the others, but nothing inside a chunk is recovered. +- Binary protocol extensions (BinaryProtocolExtensions.md). + +## Not applicable: writer obligations + +- Writing in general: this module only reads. +- parquet.thrift L286 and L1338: writers should always write `null_count` and `null_counts`. +- parquet.thrift L1411-1413: writers should write version 1. +- Encodings.md#delta-encoding-delta_binary_packed--5: padding and unused bit widths should be zero; the reader side is PQ-DELTA-2 and PQ-DELTA-3. +- Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3: writers must not emit runs longer than 2^31 - 1. The reader keeps the stricter 32-bit run header limit. +- README.md#configurations, and the dictionary fallback in Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8. +- parquet.thrift L36: new writers should not write INT96. The reader still reads it (pyarrow fixture `int96.parquet`). + +## Resource limits + +The limits (`maxFooterLength`, `maxDepth`, `maxPageSize`, `maxValues`, +`maxBinaryLength`) aren't spec requirements. Their tests are under +`hostile input`, with the other rejections of malformed input. diff --git a/packages/effect/test/fixtures/parquet/README.md b/packages/effect/test/fixtures/parquet/README.md new file mode 100644 index 00000000000..c3743a9061f --- /dev/null +++ b/packages/effect/test/fixtures/parquet/README.md @@ -0,0 +1,31 @@ +# Parquet fixtures + +`generate.py` writes these files with pyarrow 26.0.1 (parquet-cpp, `created_by` +`parquet-cpp-arrow version 26.0.1`) and `gold.json`, which records for every +file its row groups (row count, codec and encodings of each column chunk) and +its leaf columns (path, physical type, levels, converted and logical type, and +the stored value of every row). The gold values are computed from the source +data, not read back from the files; each file is also read back with pyarrow +and compared to its source table. Small pages (64 or 128 bytes) give most +column chunks several pages. + +| File | Covers | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `plain.parquet` | Data page v1, PLAIN, 2 row groups: bool, required int32, int64, float, double, string, binary, fixed_size_binary(4), int8, uint16, date32, timestamp[ms, UTC], time64[us], decimal128(9, 2) | +| `int96.parquet` | A nullable INT96 timestamp column. `use_deprecated_int96_timestamps` applies to every timestamp in a file, so it has its own file | +| `dictionary.parquet` | Data page v2, dictionary encoding for int32, int64, double, string and fixed_size_binary(4) | +| `encodings.parquet` | Data page v1: DELTA_BINARY_PACKED (int32, int64, with wrapping deltas between min and max and two blocks per page), DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY (string with shared prefixes, fixed_size_binary(3)), BYTE_STREAM_SPLIT (float, double, int32, int64, fixed_size_binary(3)), RLE booleans | +| `encodings_v2.parquet` | The same table and encodings with data page v2 | +| `nested.parquet` | Data page v1, 2 row groups: list with null, empty and null-element lists, list>, nullable struct>, map, required struct, list> | +| `nested_v2.parquet` | The same table with data page v2 | +| `gzip.parquet` | GZIP, data page v2, dictionary: int64, string, list | +| `empty.parquet` | An int32 and a string column, zero rows | + +All files are uncompressed except `gzip.parquet`. pyarrow accepted every +requested encoding combination. + +Regenerate from `packages/effect`; the output is byte for byte the same: + +```sh +uv run --no-project --with pyarrow==26.0.1 python test/fixtures/parquet/generate.py +``` diff --git a/packages/effect/test/fixtures/parquet/dictionary.parquet b/packages/effect/test/fixtures/parquet/dictionary.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cae0d3555d845aab0aaca4293b93a3a09ace26a4 GIT binary patch literal 3193 zcmb_f&2Jk;6rY{_h+~ZFn9g)vEX#*>WKdcX(^vte3hSh4fFw8(sA*)WvJ=NKN*v>) zgr`8Vq&(bUaPXss+HsEfX0at+sQa-DaruX6Thlm4wFB{xs-N(uf&}iJOFk8(W2&C+1tgz9 zdF_yPLcn}h8-XL>%waY>1%biLRlz;=(h~YmgQvK9y`;lfu9r$xpp3F;9^xE;Cjy@k z#7rQif`UmTqQ1XR_1Bag|E$kwDP1Bt(&|1{zYF?Wk|8!MVtdXCCDceVg{p<(;t~Kj z$z?Q?D`TWI+%z=;ViKbF@bC~4XyI!PHn&tOmLjG$79(K&&uR@r-GrN!(^BU?HD_1r zKb20Hh4U-{hfcU&_bnZ*Yw75RHx>O1R&&?7IT!^Gz;(3SbU|zL@aB*dd{W&!rfLqG ziFBsdB^^iyqkc#SqcUjfM}VYxlN8VCBx)l$-E4I^J?$0}uO&&<->CZ830E)-*o}C# z30A2Z#ctD*beL0^QJ~%#7Eo_bI{kP{(dT-Z7F!)6rm?ZOQd%ppAavKW)YL20&6m?q zg3dg1fv_OVdzM$%3!C^Af-hrZvtFr|&yuM)7!I@nM$NwE$_J&TYa5mA(zRlxTBzS3 zJQwSa zwcuthNZdVz?#ol4>f#$i{GtOFoTbLjl5irRYt#KRmkCT{A`;=`5qNNJR|-L#xtR2_IQyDGQyW*;GvLku)+>jXLuy+ z9fjXf#7VMiPsgA!}~#tt+qE?ZVtsU&3p!oV6@#79PCbw5%;FTec=Ft89&xEO(InG5>64=8h;x|hGB9kCl_r{ImK;TG`m)qUs+w4cxAgV zUt66|zfmi$P24EWT9@%0fCHn0H#zR&P1q!ox`}69Mak`@NyJ!$`7G+o{b~+WQG9k< z+AED{m=0RFcX?r>n4TDzc~^?q{AHUTcE_xl2LG7NFP+rFkN(Kkiq+Zl49u))SKi7> zL^upM7pSiypY$sz%Pmy2R!|9yhD-VnCsM_-3Ri{(rnlv(;kAYFm~=!@CTrVs@ggyk xwL<;9>!r9rtlj+R)f%Aj0Jhc7Rn@k&ot*AxpVG>)aw%PQ##e0LLL=#J_<;^Q+l9S|Lj zt7OR~2^k>1(dTa8C->=M9TE1qpR7}O;kh(_!ajCI1|;#Mh)gh%Q3iT~Zn{YP?M(WO z@m89O*l&GC)t^W<9D>nG!|d~GS#+W_?SxsjDO=ymEMIMsR{yel*6o>X+6k}arvWgh GRsR8q2W*7^ literal 0 HcmV?d00001 diff --git a/packages/effect/test/fixtures/parquet/encodings.parquet b/packages/effect/test/fixtures/parquet/encodings.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9e2fa62a59a386cae00358198668592794beee3a GIT binary patch literal 11604 zcmeG?dt6NE`)6*MlBAkxB6i#oq8c?qn3+OL*<6;&rDmqO-WpQ0$en6hs+>(Bvl9DW{WpWrWGYA=N*0Mv63 zxCh-7#$BMBKpz5K0{sr)A45L}{StInz*Pgh0o@tKV2AK=0DcDD7_D!x(K`hjBgh3((~-M;BTCM}TV!^E!Z1fJV@NgRv#RuF&tp82!Hs5dH21 z9qrb@ycfWJ&@aOnWl=BshW@?;+#rBe05L}N1#+W*3Ukzrexbh?p`V40bJRBwI?AC8 z^8X67z5xFKcpG{pbo3o##CXhL+y=S``gxeU0DJ~e2oP;`1Bf!1Ph5jYSd|8V=qm%M z5-yz&^tm12F}Gakm`k*Yaa%*jb$JMLq#+O5Mc>gM#H0VU0C7FvLdUg1U(~=z?NG1a zmUtCEG5T}I^VFVUPcmKVJa$&`|Img@6TH(%7C=?VdEjGOy17Rt#r9r7-(?4P65HguV8DUF% z!$7QYXlVl&I?K<_-!XJ3J0j2OW#wSWuwhCGeJ}%0F?iMvlp(gX-V7a5jt$dJ@M2~T zWl8329ISoT&P|41!a&R$G`$@IVTrNfaudk>h98cKvaO@Dst{up2G7PrPsuS$L+RTD2As;0RnW)u`1P z4cJ9*!8lC-9cUEENTLPd4lJ}D^b#);o?(XuGN2`MEhqp{wN{EL0AGm`Ai*b4)I}qJ ziPi?JFaR~QT4Y8w06?DRf;$iai|RB3r|Z-WuliZD0u_qTO61e05SpJlPywp**Fq zHN>Cnf`&G*FV=A=9=2{hRNXwiRtY;;bnURp(<@z=nGKj8s&vl`^R9rzK{ky9YL5;) z9vA@LZaqFv7vcvGqRYgR4=wWa$!mk3?t5~g^VxOZrylw_XUC!97hl~d|2<;t!5>Z{ zGIz@9%LI8aC->s;gBwqp{ks3{@yJsT3eH62?zhCfm722xD9frVbBf4w6(e)fKpu1< z+DM33oumtKf%$B0q$Vahgk(US6r+nt(77QxJSHYM7=>sVXW4rxjk>i6DBH;2u5eSST5OBCU1bjgZ z0*VudfXAPSz=%phz~#+BATUZrz~RnAz!#(=pg4;W@c2s*7*Wd*aCzAX1V*b7aJXv_ z@CEA;P@EhDJU)~M6wW1MxJcnxFz<~hTU$?fZ{*dfOl)h)Zg7*xR1*c8J>m#4I z<9mevw5HH2>hkmjB~IpE=5;v=uL56vo?e>QgUVKv6yzzW9*+|AdVBLisZ)7|LTGCq zq|g=Iq=E{mpjb*Ty_w;spel>>`bxdIz9`$Prd;oor!S;i>CQ7NpD&8&+TorJ!W8~WQofbct z_^$X&{0^yb=iTBfTQAH%!rOo5?lxYV`pzl!%6nx)j;`NoSsquaI_hC4ReE}Q7z_pv zPZa{gQG>x#=}`)k(o#=@XDP54R3084hB8l&QV&lLPvC*uQI!GYN`Z)Tav92$2GFQd zz|6x#RfSBe$D_|FrV_uRs5u4EPn5BxMBcuO2Jr z7CZ`iWHIU9*Fgp4;VHt@!t7eb?jPI}%yO;@C;8Tt2Ng}NbgI1MmyvDn7Fy$Wv&`>| z+qT0KsbIHzWx4C}^7Plm5?}6N>Gb}2HHX~Ygh4u8y=t3VdbVwL-m5KFlc`HqzOi1H z6%P7CiX>^JgOit+-&N|deXa3Nx2Bm2hMo7h=#h5P%P@HIFI6Yi{NNQ&zuA0Ox7kI% zHmASaxrrAYzqda zQxYRbMlXw4l(AGlKRwbXBWv-36jfS`Zh72-h11ey3zFt#>60R27pi9^j)+NL=&#QV z4x79zdVycs%$YM6rN%B^*mLHx`EwR6R!;9eFC!^_QKsKKeY9ptVp{l&g!s7&=Fgou zOP}d8KQl6G#FAMH{1#f5)Qm}!W-Sbl$%?g~KU1M7!3F>Z5C?xS$BT?;DvKgWngl~=O=CD|QbZ{ZBJ&o7ng{+^fA*C zOO6?Q|H?b^cO`sNSesF)c(p0Ke(cd*lEs;egXVf<(vB&>_;M4tT0DZ`{nV$wx?w)a zQH}DlDb_tZ231TrSvf-?Gjts4uWNT?>3zwgp`(&b3Ii4#|8vLDlzScM?o$G;OA${XLfq^Fc#y1D6g={3H02df1s*%|eHh96A1(!Jf`&P&Rt)Gz51 zF?vZ<+|z{{Baa_SICdbbsJeRQU$!Ui{@5@xcYBuN(vPbL=5tRE)D@hb8+6p#&+%^6 zv;&uNAIFwdm|=~F|W$G{q<^DWWc0RC(iXN8l8W;vmjX$uu(qFXXK5EUDUnP ziUf&sQ%@ya9pZLnYR1d6mW7Mb2UkTEnG9HIux~@<#Et6Id7hzq)veA0dsUnDsNOzh zxR?8piQ!YN#|tXEoi<8zDw=$5!t-ToM)#r`sm2DXp#eI@fjdm#Lnw#S0Qkm67{b>N zFr<(V8lVveV~Rumkc9%41`gE#c+f()fCyIu-jEfvazHU~(i#ETh(-Z|Ml_2El)(|e zMqmXj)u>NHTZjZY8PO%iAZZ+-ArtaJLjjr%DeRn9L9@|Rnh1AfXE>P&qn~J`$Apel z93VgHYlOnRY7Uo!D7)$Tj8f2=e<5W0$uETPMXN2mkm%C7r&5hpQ@zUNZ!SK&tQ*sB z`Xf*GLBTTdp3z;T8Ed1n?H*e8&d&QIb>pcQH`QOSusT zR{kEacvonC_2XT0c~z-)ze#)LK6cLga-ns{Rov@t<6gV1;$?$VQk796 zbz1$fbLm(5$$NM4>!mDT>Pc*DV#bXs$q3lr}MUF zu4M%ei7lD;LgJI0F@5h=k^9{K?xT+#Up>$!tjh)OwQG9vpSo6DyH|g1<+K?)wpyFo z+R5a;zVnWzes#cAFjZnbJMqoTz5jDFIQ{so?CnF>-6>zq+dhFa$L!$wVZy^D3p>fF z#zCcd&xJ0Q3%?2EJ#;-c_nTGis)iJL+Z;LmwrtPa_*2~KD+{eRgjno;7?HoZVcYrg z`_;1CufJWl>)T;9)~kp5R2RJbF6h`;@8uUBp1R)4!FlA_x=Q!ocb46A&^;Nt)~N3t zfuDnRgmpYO^zjkvUXNq}9am?rbTjvL@!8t#r#8<5f}y@2Z+Q0l>-#r)n{~f;-ZlT~ zfp^aLpVfEeKCf&TmmGM#=6fsSE%v*n58k}n;z7if6Pc$r>H+MJsk$>myv4vwt``We} zTz%olos`g510M%&d0;)~R@u8R17c4<+!pGhJ~Om*N$13U`R_VeNlRSgZf(AsI|OT; z-br|eM)nNZ1@u-x1fpR^lbR3&15=ujrqffDK`e@)6#d6y7#dN_5P^hxPy`077%htk zL=zqs&FdLCR3BB%FH=|2K2+vA}v5k5a$J>3#Q z!KZXU$sZXrYxYmkkbui=;A-|c zL2hEtYYZ!v@VM2rBu4zER|YZm`JjtAJC%It&Lg|pM|54fe)*QQg4YX%se}(6Z1;~0 zJ~`XY{3o4Nj{<4M*j~GC*@Qmva`Q939sgTI`s%kMC*S|B{NBa^$3=$)a)cSOClYZonE6#eatbrzMehc>3H^m%e^ zGvT`|hcE8c`Pcs`}^-~L#W?fzq8ig|0?^9K*^V`RMTRY}EY4NX=BslYFaq{tyMAJD}zg+k#OnP?a&j+q$e*SC0 z!_d1MMfC==HN%cPdVP70{~X)<>&gd*{(2*Pt4sUNt`}CNoPE4tzN`I|&0iceoATNf z${|=3Oo{=1?SsGf;cz4zN^D|JASnm@!V}A(e~S(;`jOB(t^WDl6~j~OpP$$?yrJ&J ziOjFG3Ad+Z$Pv<(r_(%J$A(!w|$n2d?hQQ{J=jjhcgJIYqX zwdrYOV$&?p0WuNN#m9}|vJ<-h?eD8>Y+-#&MEIWpD&v2sOzuJ}dubznX=7Jm>@NAp zk7hpkQ=0eM-Fq9^*hvalGp1n1)aIj0L7UreD&B8uSK8gZ_4lVlrzv>sE-o{+0ku{? zG*@e8BWbaMw7%u$;^pRcb&l@xk9_GwG@22OyW62g^Fmgm88`|WKeh0*@>-#|R%o|K z;Vy6aD!WN9Mvp&?qLVBHr53Pa&FY0#RAeG9GO?@kf+cHunc3>qCQ)mF?xI@TZLC_8 z57n}3S79lxu(T_j0BZ*xP#?UN>i>ziQj8fsZKnS!)(qAgc+XZYTWz%_;#w2ChR}ad zZts0>N(t6oGo`HOw?yJwBDABfn)^l7;F!J#oMz27wP@F5Yt60o& z|KO3;GU+D!S)G)P$jGK?W71lF;D~?V*j2CKxVJj%h#ojg=&aIu#Mvu3uwm_lIG}87 zC8o;pIA5DwX4)V;iGX&Qhwvlp~eOFr`Nr!KqYAr4NpP@pFVQ zXCchu5fcgQN-so@MY4w$HBM5F08$vtJtI*r4sTw>loJkt0Jr46j*f0H7%>XCVLl7* zWQYUw1CX;dqb7y=M-58U5S+@#G11X+g6}{_M`R!8MwEi{I3ahujl|)o(aBopK~ZYI za9zO2LCI>rSe>72bgU+7;K1v3@{>KPo0#qI?;D$9+jsM~bRhz6sN)@j8vy z_;9}gW1X~7x|so_JUo3czS$qhf4Kf5ke8L0CFwL#!A@$ZfnYCOe=X({{2`{t;p%~Z z5Vw|gjwYh7!19TDXrW&>GuYGLN$H);K0eZ(3iy2Hu+qw5neYEpdg zsFCmM1pI@22wTtZmvx-MDQCA9T>p7Wira@%+R`F-B^kN5OF=Q-!OKhJs2nQ3PH6l0wP zMuOk?@B|4e*a8X%69ANjt%%#)+?*leQWT{VQVy+F9XcS6#Vum(;{63o?Zwb|fO-M~ zx1gKCybE*_=tH23px*%eedwp5pNH-OxO#wBpgX}F><~T{z|Wu?L$`z889K_Gf{yww z0z~?s08w9in4^u$FmHr@7P<`9=pw^^4{&W^-2hMw&U+C{S=qI7$8ubl?j&dl2 z{J#LL55PYFUWZ--9eu|bF&;CRw}CEzeg@Xg03QM514LWh0HO@$6Zaq*cBKg*`f7kw z1)I(X`rHoim|He<%q7~yxUHe%zTAa1(vS!3qVMPr;?e)70C7KGL&v>AUsS+IZBZ>{ zS9p~@F#2=LlXSa?2ieXIo?9qoSurPx1%-A`1P>`R5nX8Rq|o?KL)bDz5D;lPp~&GQ zSW%1g@`Sqmw^sR$A*a~ zc%fkpg-O;-9E^R&&Q$}w!T@2DvFYp$5H=YbZa9IAU$%xd6bQjXp%cj1g@Pb!I1mIz z86X@MvWDWrmVp-pG#wmJvWCM(;I9UF$pHHs_@H>PWiW9vbS95*xC!0Cz|Yuc_5lt7 zp)>qU-k~@N9u6Ua-3;<@P_f|wB@iww0^zU`KDfjPWb(z>hf9dy;jj{ZX20M-5ImE2 zHv^s7AI2{m1D@G0W`CKwflC3Ku7?3KbSCd`z=<7Z|9A#PjDDK@wytR{Fm^T^0LFhN ze#VZ4!5$oFq8~0OY&y7j2xRt;(GM3X;e(5uKt}@~lQ$+#CN8)vi5|EJ34fgdPB7q^ zHIpy6Pzc|Z_aKvB#tyT8aKRIPCN8l-FXKN`mz@lFCLVf8E?Of0!k~xQUnbv-J-C3e z;R_exhfAKU89Pi~;9@6uMvk$|ygZjIuUWVl*UFRC>gTf8N7jh|a^ z^!KZd?XD`m6nHrGLFSrObpds?cdHIs6MywJus==B@4WcMOv(k!TO_TsHOyQx4FsjsGMl42+@Mu&T zK!79gz*4PJsnuW?y#?bm0d$~IBqNCygb`S1J?JGqBs~2VHDo|T)*4U%qAHCTQvkja zB|w6Ap{R>S028eZSYZNcXtl_UY5;&d%>^S60gLKLwumUM=p?9#3Sa{s%oXH^V1SKI z6HIWfA{l^S(LZp;kPmPj24L;U_z6vBT z6_7Z|rIA4G(SgH(69By3dVHSA$2T8Dmx(18TJ-Uwb%P)7dT^xk$^7rq%YQ1^QeO7# zt1H#NNBQpi{wN{~Cmp{)koyV>&kftR_Ndt}yI&uUK6a<%L{#B!OFUia1xtalxW1-f zFL`ans3dC0gf>hQ4Y8_HwPDV%p00^j$Hj(`=vAq4+PEaGE21Og;zC1Fh?a4ZzLip| zJ``ajq>@%LnrNxB!BUGMscUOfj9lmdX^Smj7$8IcTBL}Kv&7)wTPc)r#wgy2iunJb zk1kS1uEms#tW{X*=`zWOVvLZ?`JK>Y7{|L#4ulv;0vVCefe;3hL1M@iLiivqSLgTT zP|6juz?Rx%SQobfWlWv%7L?vVu|2%vdA4!!Df;F{|%=cPqQ+U-=W(-n~+o9 zN`g&v8Ic=<%`>rb@GFwf{)ZEytCfT~xTsRI*c=30o)H3yWsHEsHAP@VwMW3_SRmj* zfB;x*5dto+69S6W1p$ZK6@d}e0|A@U69LbtHv$&B9|A5+FC=UUPy%GVJw;BP(DnP*DLTNyqoG^xml#VUFhFA#P$Upa7%GTBsK1K5C zR3)@^Xtg4RWWj3F^zy(bM*KwKzpcr2@`fT^NrlAR%etrJ&QH0``T6s_F2Wg+T-|U(XQ3o zzB92c|7oZ7W|Qfc1q$|9)0L|Q9iQ}${_A4p^{VrgwRJZO71Y(?_trj(ysDq@BJ*U$ zi?YqK;N^kmI!3Q*-)X^v@o&mbByJJ&x85wfxbf_q1DxF_Zf@eVY3w|+QE{tk$idYc zEvpkMl?Ofal?qQ!54~RR;i*KRKd9GxDm*G-QCaD!_pAgKz0$+OLto|TQR(67;R!r2 z98~H-t`dm2CXc>Kp$CmhIjlT9l$D5-dw5iVXcf$rFxOXtEf1i85fJw9tOO1C2d%P7 zsfX&UHisujP|6nc#lb@0K&q2`;50LIiJboEDQoiL3SnP|k6=l)_I&j6&0pyUuD1sN zZ3X&2fBrKW@ZVdJniaCU-dD&jxfgQJV#2KzAtlw3Gx_PIxliTWzjsSAE4aj;;P<#X zWN%Q7q~^STR<46<_+!_rRsJVjH|-x!g}UCVD$Fk`(p?sc{Mh@&Q~DP@E_ZX~hiJ8p z%1y4Bxwg4QFE(6Cqt09T#d}?l_tlllQ^hrXC0<_sm#F&=PmO=PHrbTte#YmVN5)Yv z{osi|*B(`ILzh1Mdi_o9dS~6Lg8r_j$Decj&f)m1-620jo%{9Js4Lx1S)8w{ZP)Ah zUCGsmSFg^vcU^KUE3nI!O7w`>#ZmLK7U||>M*C#tESNh}nGvU5k}!APz|RFoIF20e!)Du{l$f-$*v*ym3l{`(C?K3AkI%oL8X>B{;r0v@}Fka$X~3BZ1g?2O|&3;LC7qR zY}zw9Sf6hKPs>|43qbw*Z*wZxEUH;nHObnpW5}s-M{B0arTUIT1GMc9EV?baH*{p0 zNonBR!+&l$IP=y_$NjZIwl!;hye>-VpEtOV%h)%3eRcCI=ggUv=dZ52URlTW?qD@{ zW^Ptv?_v99UhLj(f9HkOlNuNHjvBQvCgI_{wb6&mlMe04*;`+q{Fm*Kn?E!q7jDjx zpZ_6mU@`mnKyAtKSs@3l{T*-CPTq6A@P4A{=6@ zqa1rT%f7*l`ieYhbl`-MM^5+K zJF574XI`2*aII{N&xkAIyQq3)?B%7*N{)Ntqki+GVP0+r#zzKOkLA^LJ8qOB**o#{xF?HOj_OG@Q_W3O zQxkNG1tTory(o*-1o-A=n8J4vFr|yoq4gJtXCKRo{H*4-c<9;fG5FWhuXFs z@?+mZli8eh8SdQ`{Gf3$O?eTW&e)!)%LyG4Uorcc$R{mp%Fc}fw^{w&MjbkwH_#@c z%USPLEA6-sT~5{AYCOGc^3*LGtxaw1r7}Oi*$30V+T+3t5?N1Ad6m5LUsppj4`0jO zJT(7CbslH)IMxiaeP`VH`zz*kl2OfrDvO@*oh|2m9n87wa(dR+%iGlsDfPBFaQJoA zj@OCD*!36ZS+5DR*nT&vczx5RGu5~2rG+cL$=~*k`(x|8p+5B`&%X^hFR zd-io2ak8Pt?f0!!xB6-y3|(c^=LXNeuV%P)B0K#40qdUkq=6mtvX{A<`#Jk;?Dk`u zM}eVG;rG`(db#5Em0o7uZ=G=|zO?6!)9pu%UD;1+n#QCBUw-_ZmGK6LZBquX-)?ay z>f({?V{3MWk4@Szdgk66_hK(TZ9e#5Mts-qS2~W3oMfJp&`!27J#tO%qiLR2XG5Qv zbMAJi{fQGkZ+&W~+I?^CJ?p)p$Ja;9OFh5aGw#agM zaNq;484uTYH~N8ly!{n=b?7kKf%C?yNd7 zv~pqRlwHMdI$4P;ToSIWzgaj0tDQbeI6@<*hMWWXFdzcau%byV2!eqn%}CSfCCVTc z#ZZd=V=x99QEMOq3H6`|Oc*g*77>UhJPe=_#3qq6=rI5T9tHn_2Y4IN+W%7?zek9W6-|73st6ibK)U#dUEl9y`yG&E&ZFkbpB(u^-MrVh?B_NGjeGlh=M|1Qu=a57j{RAI3v0tlW>~yx=ke=+8$WvI*}Uj^ zrlx+8rKj8WV{1kI4_%bvtUC1)7}?zATx2`G)I()IPChTBuXhn+55dN zWNlS&#aj<-Yai8h(ds1|R`FiWbyxE5+}Ruu9eQ-Sz4?z?s~#odQ@%a7U9$;);N|LX zd_D2EsLZ_ABPQPdw))nh4lffY@eM&Q&8qne=Er_BHQ%BpzI^S>Wj+t;M!Yo6Ilor+=ypu^?YyN&oCfvd4ZZLD z#n~y^Z`{-!kDoj3-SgD*$6D#1yYg}m*N$`Tt>B*MI{Mf+d&|V)#}V~0;~&_2{WYx0 zKUCq=VD)fl-O7IdDlZrveCl%1n0ZQvjn__1?B}DpI@;U0sJ`Lo0(}P6r)yQWh;^$P z%D%{2@N{kTXP3;9Mvwb8r|%cH8Uk$RTwGJ_7Z&|lv9Za^vhGD*mPgBfst8R`2cMms zrn{Z?<<34iX;#I(M&>t+@|#?h?_Dy&z4E8SN9=;-OLyCKjBbvwJ`#5=)oEyTS@4zA zL#OoJosY`&?Qg%a4WoW{`PN!@`>WhnhprwQ_pEfj(X|GL{jzU{&uJh3&C}7}ipzeX zM4{Oa%hC>qrI^mR^yR!45#p1{Kkcc@{`{AcyWuz23L5ohE8P#=dwF3-zzo~l`PGBN zf4LI5(YbwRm$S=fp1ePIj*G*j^3+cjZ+chChuU-I2$m=$kWO_W{q^( zhJa}K&;kY0`~)qSjEmD!!V0#HtxdT-Wh-FY*cqAFv!W*qDtctE}P*;M~ed^Tk; z!==siU(T4pdIRs7%4MqUg^BQmiT$RCe^733eQ!w#)?F*5jOVum!dn9SH3=WDIBtkQaf)yr6LVD0%hp=@kLri!s+znD0fzS~46+Bjkxr0`HE6l0)c z8?8`GQz#=8BNU1Vg+~OzDU}L^56*z`bA+&FAgmG*6AkPNFGTwy*+YXGCn$ykDFW7> z(I}UIgBLL+!6^`6NbBS1=n9kJBY_*%({LnHoS+|ooUR@@Av_>vP>P!1ls=9rj*jE} z20A(-`xsZE6r9He8F3ha(~+anG){wJRQ{3Lz!8JeRQ~Z=f9a@rbpD9r%haMeT?iBB4) zQW=f$pgP4k96S>ZYo#I;*9r{|BwsjDD7 z7FZa0lK;Pxr~Ocj#C*g410+qU7%?6+k^S)_@sLcl+8-slri>1pm?{aF@Hcr@1YrGu ziU3qhw8Z!s{2lWpNgXMwR{17OBgbmhUSlKu2lz@fG1}xnQXZZ@7~k~wL>F}Qo+AH=Ppoui59E3kZ`9^TQfO%C-8kSM&<4EZwD z1L%RnM4y*w^f_XT65Wwx{OJA!j+~GfI&#F@IsyOS9K!xgqxBI(}N|>4) umefxZAKy=vkPtUboD!Ch7!?;Qmdg4|`a9W_!<&)6e<%um%HRjTwf;YAIzkx$ literal 0 HcmV?d00001 diff --git a/packages/effect/test/fixtures/parquet/generate.py b/packages/effect/test/fixtures/parquet/generate.py new file mode 100644 index 00000000000..bfae0379d62 --- /dev/null +++ b/packages/effect/test/fixtures/parquet/generate.py @@ -0,0 +1,319 @@ +""" +Generates the Parquet fixtures read by the effect Parquet reader tests, and +gold.json: for every file, its row groups (row count, codec and encodings of +each column chunk) and its leaf columns (schema facts and the stored value of +every row). README.md describes the files and how to regenerate them. + +A leaf value is the row's value projected onto the leaf path: a struct takes +one path component, a list or map two ("list", "element" / "key_value", +"key" | "value") and maps over its items, and a null anywhere is null. Leaf +scalars are encoded by Parquet physical type: + BOOLEAN true / false + INT32, FLOAT, DOUBLE number + INT64 decimal string + BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY hex of the stored bytes + INT96 hex of the 12 stored bytes +Gold values come from the source data below, not from reading the files back; +every file is also read back with pyarrow and compared to its source table. +""" + +import json +import random +import struct +from decimal import Decimal +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +HERE = Path(__file__).parent +I32_MIN, I32_MAX = -(2**31), 2**31 - 1 +I64_MIN, I64_MAX = -(2**63), 2**63 - 1 +NS_PER_DAY = 86_400 * 10**9 +JULIAN_UNIX_EPOCH = 2_440_588 + +rng = random.Random(0x9A4E) + + +def nulls(values, every, offset=0): + return [None if (i + offset) % every == 0 else v for i, v in enumerate(values)] + + +def f32(x): + return struct.unpack(" python values).""" + table = pa.table({f.name: pa.array(data[f.name], f.type) for f in schema}, schema=schema) + path = HERE / name + pq.write_table(table, path, write_batch_size=8, **options) + assert pq.read_table(path).equals(table), name + + meta = pq.ParquetFile(path).metadata + row_groups = [] + for i in range(meta.num_row_groups): + rg = meta.row_group(i) + row_groups.append({ + "numRows": rg.num_rows, + "columns": [ + {"codec": rg.column(j).compression, "encodings": sorted(set(rg.column(j).encodings))} + for j in range(rg.num_columns) + ], + }) + columns = [] + for i in range(meta.num_columns): + column = meta.schema.column(i) + top, *rest = column.path.split(".") + t = schema.field(top).type + columns.append({ + "path": [top, *rest], + "physicalType": column.physical_type, + "typeLength": column.length if column.physical_type == "FIXED_LEN_BYTE_ARRAY" else None, + "maxDefinitionLevel": column.max_definition_level, + "maxRepetitionLevel": column.max_repetition_level, + "convertedType": None if column.converted_type == "NONE" else column.converted_type, + "logicalType": logical_type(column), + "values": [project(v, t, rest, column) for v in data[top]], + }) + files.append({"name": name, "numRows": meta.num_rows, "rowGroups": row_groups, "columns": columns}) + + +# --- plain.parquet: one flat column per physical / logical type --------------- + +N = 40 +STRINGS = ["", "a", "hello", "héllo wörld", "日本語", "🌍", "parquet", "x" * 20] +plain = { + "bool": nulls([i % 3 == 0 for i in range(N)], 7), + "int32": [I32_MIN, I32_MAX, 0, -1] + [rng.randint(-10**6, 10**6) for _ in range(N - 4)], + "int64": nulls([I64_MIN, I64_MAX, 0, -1] + [rng.randint(-2**40, 2**40) for _ in range(N - 4)], 9, 5), + "float32": nulls([f32(rng.uniform(-1e3, 1e3)) for _ in range(N)], 8, 3), + "float64": nulls([rng.uniform(-1e9, 1e9) for _ in range(N - 2)] + [5e-324, 1.7976931348623157e308], 6, 2), + "string": nulls([STRINGS[i % len(STRINGS)] for i in range(N)], 11, 10), + "binary": nulls([bytes(rng.getrandbits(8) for _ in range(i % 6)) for i in range(N)], 5, 4), + "fixed_size_binary": nulls([struct.pack(">I", rng.getrandbits(32)) for _ in range(N)], 7, 1), + "int8": nulls([(-128, 127, 0)[i % 3] if i < 3 else rng.randint(-128, 127) for i in range(N)], 10, 6), + "uint16": nulls([65535 if i == 0 else rng.randint(0, 65535) for i in range(N)], 9), + "date32": nulls([rng.randint(-50_000, 50_000) for _ in range(N)], 8, 4), + "timestamp_ms_utc": nulls([rng.randint(-2**41, 2**41) for _ in range(N)], 7, 2), + "time64_us": nulls([rng.randint(0, 86_400 * 10**6 - 1) for _ in range(N)], 6), + "decimal": nulls([Decimal(rng.randint(-(10**9) + 1, 10**9 - 1)).scaleb(-2) for _ in range(N)], 5, 3), +} +PLAIN_SCHEMA = pa.schema([ + ("bool", pa.bool_()), + pa.field("int32", pa.int32(), nullable=False), + ("int64", pa.int64()), + ("float32", pa.float32()), + ("float64", pa.float64()), + ("string", pa.string()), + ("binary", pa.binary()), + ("fixed_size_binary", pa.binary(4)), + ("int8", pa.int8()), + ("uint16", pa.uint16()), + ("date32", pa.date32()), + ("timestamp_ms_utc", pa.timestamp("ms", tz="UTC")), + ("time64_us", pa.time64("us")), + ("decimal", pa.decimal128(9, 2)), +]) +FLAT = dict( + compression="NONE", data_page_version="1.0", use_dictionary=False, row_group_size=25, data_page_size=64 +) +write("plain.parquet", plain, PLAIN_SCHEMA, **FLAT) + +# use_deprecated_int96_timestamps applies to every timestamp column of a file. +int96 = {"timestamp_ns": nulls([I64_MIN + 1, I64_MAX, 0, -1, NS_PER_DAY - 1, -NS_PER_DAY] + + [rng.randint(-2**62, 2**62) for _ in range(N - 6)], 9, 7)} +write("int96.parquet", int96, pa.schema([("timestamp_ns", pa.timestamp("ns"))]), + use_deprecated_int96_timestamps=True, **FLAT) + +# --- dictionary.parquet ------------------------------------------------------- + +dictionary = { + "int32": nulls([(7, -3, I32_MAX)[i % 3] for i in range(N)], 5), + "int64": nulls([(I64_MIN, 42)[i % 4 // 3] for i in range(N)], 6, 1), + "float64": nulls([(0.5, -1.25, 3e10)[rng.randint(0, 2)] for _ in range(N)], 7, 2), + "string": nulls([("red", "green", "blue", "")[rng.randint(0, 3)] for _ in range(N)], 4, 3), + "fixed_size_binary": nulls([(b"\x00\x01\x02\x03", b"abcd")[i % 2] for i in range(N)], 8, 4), +} +write("dictionary.parquet", dictionary, pa.schema([ + ("int32", pa.int32()), + ("int64", pa.int64()), + ("float64", pa.float64()), + ("string", pa.string()), + ("fixed_size_binary", pa.binary(4)), +]), compression="NONE", data_page_version="2.0", use_dictionary=True, row_group_size=25, data_page_size=128) + +# --- encodings.parquet / encodings_v2.parquet --------------------------------- + +# Delta pages are cut only between 128-value blocks, so each delta column is +# one page of 160 values: a full block and a partial one. +M = 160 +WORDS = ["axis", "axle", "axolotl", "babble", "baby", "babyhood", "back", "backbone", "bad", "cab", "cabin"] + + +def walk(extremes): + values = [rng.randint(-1000, 1000)] + while len(values) < M: + values.append(values[-1] + rng.randint(-50, 60)) + values[100:100] = extremes # adjacent extremes give wrapping deltas + return values[:M] + + +encodings = { + "delta_int32": walk([I32_MIN, I32_MAX, I32_MIN, I32_MAX, 0]), + "delta_int64": nulls(walk([I64_MAX, I64_MIN, I64_MAX, 0, I64_MIN]), 13, 5), + "delta_length_string": nulls(["".join(rng.choice("abcé日 ") for _ in range(rng.randint(0, 5))) + for _ in range(M)], 17), + "delta_string": nulls([WORDS[i * 3 % len(WORDS)] + ("" if i % 4 else str(i)) for i in range(M)], 19, 3), + "delta_fixed_size_binary": [b"ab" + bytes([i % 7]) if i % 3 else bytes([i % 256, 0, 1]) for i in range(M)], + "bss_float32": nulls([rng.randint(-10**4, 10**4) / 16 for _ in range(M)], 11, 2), + "bss_float64": nulls([rng.randint(-10**6, 10**6) / 1024 for _ in range(M)], 7, 3), + "bss_int32": [I32_MIN, I32_MAX] + [rng.randint(-10**5, 10**5) for _ in range(M - 2)], + "bss_int64": nulls([I64_MIN, I64_MAX] + [rng.randint(-2**40, 2**40) for _ in range(M - 2)], 23, 9), + "bss_fixed_size_binary": nulls([bytes(rng.getrandbits(8) for _ in range(3)) for _ in range(M)], 29, 4), + "rle_bool": nulls([i % 10 < 7 for i in range(M)], 31, 6), +} +ENCODINGS_SCHEMA = pa.schema([ + pa.field("delta_int32", pa.int32(), nullable=False), + ("delta_int64", pa.int64()), + ("delta_length_string", pa.string()), + ("delta_string", pa.string()), + ("delta_fixed_size_binary", pa.binary(3)), + ("bss_float32", pa.float32()), + ("bss_float64", pa.float64()), + ("bss_int32", pa.int32()), + ("bss_int64", pa.int64()), + ("bss_fixed_size_binary", pa.binary(3)), + ("rle_bool", pa.bool_()), +]) +COLUMN_ENCODING = { + "delta_int32": "DELTA_BINARY_PACKED", + "delta_int64": "DELTA_BINARY_PACKED", + "delta_length_string": "DELTA_LENGTH_BYTE_ARRAY", + "delta_string": "DELTA_BYTE_ARRAY", + "delta_fixed_size_binary": "DELTA_BYTE_ARRAY", + "bss_float32": "BYTE_STREAM_SPLIT", + "bss_float64": "BYTE_STREAM_SPLIT", + "bss_int32": "BYTE_STREAM_SPLIT", + "bss_int64": "BYTE_STREAM_SPLIT", + "bss_fixed_size_binary": "BYTE_STREAM_SPLIT", + "rle_bool": "RLE", +} +for name, version in [("encodings.parquet", "1.0"), ("encodings_v2.parquet", "2.0")]: + write(name, encodings, ENCODINGS_SCHEMA, compression="NONE", data_page_version=version, use_dictionary=False, + column_encoding=COLUMN_ENCODING, data_page_size=1024) + +# --- nested.parquet / nested_v2.parquet --------------------------------------- + + +def maybe(i, every, value): + return None if i % every == 0 else value + + +nested = { + "ints": [[None, [], [None], [1, None, 3]][i] if i < 4 else + maybe(i, 9, [rng.randint(-100, 100) for _ in range(i % 4)]) for i in range(N)], + "nested_strings": [maybe(i, 7, [maybe(j + i, 5, [STRINGS[(i + j + k) % len(STRINGS)] for k in range(j)]) + for j in range(i % 3)]) for i in range(N)], + "struct": [maybe(i, 6, {"a": i * 1000 - 20_000, "b": maybe(i, 4, [maybe(k, 3, i * k - 2**40) for k in range(i % 5)])}) + for i in range(N)], + "map": [maybe(i, 8, [(f"k{j}", maybe(i + j, 4, i * j)) for j in range(i % 4)]) for i in range(N)], + "required_struct": [{"x": i * 0.5 - 7} for i in range(N)], + "structs": [maybe(i, 10, [maybe(i + j, 6, {"s": maybe(j, 4, STRINGS[(i * j) % len(STRINGS)]), "t": maybe(j, 3, i % 2 == 0)}) + for j in range(i % 4)]) for i in range(N)], +} +NESTED_SCHEMA = pa.schema([ + ("ints", pa.list_(pa.int32())), + ("nested_strings", pa.list_(pa.list_(pa.string()))), + ("struct", pa.struct([pa.field("a", pa.int32(), nullable=False), ("b", pa.list_(pa.int64()))])), + ("map", pa.map_(pa.string(), pa.int32())), + pa.field("required_struct", pa.struct([("x", pa.float64())]), nullable=False), + ("structs", pa.list_(pa.struct([("s", pa.string()), ("t", pa.bool_())]))), +]) +for name, version in [("nested.parquet", "1.0"), ("nested_v2.parquet", "2.0")]: + write(name, nested, NESTED_SCHEMA, compression="NONE", data_page_version=version, row_group_size=20, + data_page_size=128) + +# --- gzip.parquet / empty.parquet --------------------------------------------- + +gzip = { + "int64": nulls([rng.choice([5, -5, 2**50, I64_MIN]) for _ in range(N)], 7), + "string": nulls([STRINGS[i % 5] * (i % 3) for i in range(N)], 6, 2), + "ints": [maybe(i, 9, [i % 5, -(i % 3)] * (i % 3)) for i in range(N)], +} +write("gzip.parquet", gzip, pa.schema([("int64", pa.int64()), ("string", pa.string()), ("ints", pa.list_(pa.int32()))]), + compression="gzip", data_page_version="2.0", use_dictionary=True, row_group_size=25, data_page_size=128) + +write("empty.parquet", {"int32": [], "string": []}, pa.schema([("int32", pa.int32()), ("string", pa.string())]), + compression="NONE") + +(HERE / "gold.json").write_text(json.dumps({"writer": pa.__version__, "files": files}, separators=(",", ":")) + "\n") +print(", ".join(f"{f['name']} ({f['numRows']} rows)" for f in files)) diff --git a/packages/effect/test/fixtures/parquet/gold.json b/packages/effect/test/fixtures/parquet/gold.json new file mode 100644 index 00000000000..fdf2421640b --- /dev/null +++ b/packages/effect/test/fixtures/parquet/gold.json @@ -0,0 +1 @@ +{"writer":"26.0.1","files":[{"name":"plain.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]},{"numRows":15,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["bool"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,false,false,true,false,false,true,null,false,true,false,false,true,false,null,true,false,false,true,false,false,null,false,false,true,false,false,true,null,false,true,false,false,true,false,null,true,false,false,true]},{"path":["int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":0,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-2147483648,2147483647,0,-1,-2485,240748,289867,486454,-750849,-383303,507566,375400,941788,518714,-236855,259349,-852255,-744797,132897,-6338,-879326,775747,-937588,-650874,165936,462545,799320,294874,619374,742750,-907447,143913,-903073,-280663,813649,-693188,-809534,460108,-603483,545663]},{"path":["int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","9223372036854775807","0","-1",null,"-612517975526","1014778902415","311431409545","-202245119271","983832560846","-31291490818","837871651048","970588578440",null,"-235619389794","-721672261917","364739787284","229872738901","23654086610","-694745346433","-229461817390","702863645291",null,"-984134634385","816973333979","-1040843898097","351167913739","478024230911","941786591876","-636946374567","912800268601",null,"1056608115412","-202405809974","-827900779241","538401484164","822152175252","628224872673","297712530435","-866090127401"]},{"path":["float32"],"physicalType":"FLOAT","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-450.6831359863281,-299.3732604980469,600.4390869140625,-47.3121223449707,-141.6328125,null,538.801025390625,648.2783813476562,108.50069427490234,762.208984375,-365.00225830078125,-812.13525390625,113.71562194824219,null,-201.44508361816406,-50.45414733886719,-265.34210205078125,-3.591360092163086,874.5276489257812,-340.8632507324219,747.6163330078125,null,950.2943725585938,-831.8798217773438,-511.589599609375,-825.4498291015625,703.2380981445312,-100.73992919921875,-114.27555084228516,null,-923.2208862304688,481.6390380859375,-699.5462646484375,732.0736694335938,-855.2393188476562,-537.63330078125,-510.3900146484375,null,626.786865234375,-586.336669921875]},{"path":["float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-361948578.8402506,-39844526.547680736,462198887.2298949,-351572414.94397473,null,-540291037.2727698,-842337550.5208129,542572955.2318871,788407864.287821,167400813.1610577,null,-530703108.8940255,975408029.9216423,-977593887.954108,216775576.54046702,940647964.6998701,null,-758723214.8041923,309062622.1151371,683942435.2329497,767721777.7373397,38824400.26790285,null,588883746.7285874,-572492743.6244751,946169165.0968335,-625056550.9666383,-637809239.9030551,null,894357105.4446449,-399041737.85538054,785407526.5399792,-57431156.704161644,781951290.5817332,null,512244004.9062264,-637118366.8924098,274101705.2052095,5e-324,1.7976931348623157e+308]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["",null,"68656c6c6f","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574","7878787878787878787878787878787878787878","","61","68656c6c6f","68c3a96c6c6f2077c3b6726c64",null,"f09f8c8d","70617271756574","7878787878787878787878787878787878787878","","61","68656c6c6f","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574",null,"","61","68656c6c6f","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574","7878787878787878787878787878787878787878","","61",null,"68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574","7878787878787878787878787878787878787878"]},{"path":["binary"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["",null,"8531","328f2f","54d89635","132f7fc867",null,"8f","99b6","5806e2","4db42857",null,"","3c","1444","f2351c",null,"d2785dadef","","3a","bee2",null,"5ecc8687","6d325a5425","","c5",null,"534126","746cd2ba","56a07ecf0b","",null,"bcd4","29097d","3b4e7065","80db33b220",null,"cc","c2ef","b11652"]},{"path":["fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":4,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["be685304","2aeb3486","d46ddd41","a298ecdc","52e48c4d","024a48b2",null,"30df6448","1edd9397","332374af","e055f523","557e69aa","7c0bd114",null,"84cb4198","f3e69f34","92799511","81685de3","73b1ecc8","ca0f16c5",null,"3edeabef","c280635b","e2a57a95","7fcaf6e9","452e8b90","620d6b3a",null,"48de7b54","2976b061","3d8abd9e","29bdc553","92b8f589","2fe96d19",null,"0545516d","1895cb96","abe13fa9","1de21521","57a6237d"]},{"path":["int8"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"INT_8","logicalType":{"type":"INTEGER","bitWidth":8,"isSigned":true},"values":[-128,127,0,-65,null,-91,4,-17,49,-51,49,64,111,84,null,-65,0,-122,-57,-63,-102,-32,3,-86,null,-5,-42,116,-111,-32,-118,-111,77,67,null,-32,-52,7,96,13]},{"path":["uint16"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UINT_16","logicalType":{"type":"INTEGER","bitWidth":16,"isSigned":false},"values":[null,46435,43512,58879,34506,62158,57263,34426,19311,null,50138,13517,28563,52386,43984,62883,22592,1375,null,12247,34577,8629,45224,24098,23030,19699,3810,null,22697,6343,43936,35597,16595,55979,48814,25057,null,58807,44144,12998]},{"path":["date32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"DATE","logicalType":{"type":"DATE"},"values":[2011,-528,2237,-30841,null,16171,37455,29178,-44273,-23872,-9776,-11965,null,-14072,-47147,1770,10891,4855,-1796,-42665,null,-9862,-47911,-8056,36150,-34656,-20640,-13481,null,-3350,30965,-5844,-25409,35294,13387,21373,null,-26135,-10208,8331]},{"path":["timestamp_ms_utc"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"TIMESTAMP_MILLIS","logicalType":{"type":"TIMESTAMP","isAdjustedToUTC":true,"unit":"MILLIS"},"values":["-887934534497","-179353769780","-1745779504790","2101404986895","-1701787662337",null,"-1771210649084","-893324018340","2129597418792","-1532568076546","-1062417404954","-1531271410110",null,"-1030157107266","801765703572","-1772851035818","1806680405144","-369682922021","-96310095459",null,"-1565897530671","-2021567124130","-835828118937","-516351822760","678737883977","-1248421768682",null,"772843221303","1865999106498","-2088949775672","867424331385","23918359260","-564009072615",null,"-1263572046450","1463908715091","1346894376114","1611319502689","1680974754938","1857540507649"]},{"path":["time64_us"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":{"type":"TIME","isAdjustedToUTC":false,"unit":"MICROS"},"values":[null,"7658039589","55294262265","22172311358","50783133622","74027678352",null,"33811403730","22709832814","35332610090","80892681096","57056123678",null,"18690423441","35307860611","54300400780","27254171160","62724332812",null,"10448801251","28714564282","49131855093","78812007313","20133576206",null,"9680375702","885974325","35538329740","22092052588","26420901929",null,"24415876618","46612475354","34170021051","33122380013","21050186942",null,"72181531332","52804541502","70749633083"]},{"path":["decimal"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":4,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"DECIMAL","logicalType":{"type":"DECIMAL","scale":2,"precision":9},"values":["2f60c000","f0f3e297",null,"1fa02eb5","c88542a0","d0f13757","2ab91f51",null,"e6f4ae37","cbc68f2e","f41ce627","e1ab0a41",null,"03fe6e19","23fa4acb","0eb51767","06b45c7d",null,"2ccc5af0","0ea713c8","06ae9b0e","fd867600",null,"f3570659","f699f594","c4c92630","f482e0e4",null,"2196d40a","33cb2887","0a9be157","15f9c56a",null,"366507df","d6fd8482","00977fc7","e7770008",null,"07af039a","019c6c3c"]}]},{"name":"int96.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]},{"numRows":15,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["timestamp_ns"],"physicalType":"INT96","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["010000afb10000008c9c2300","ffff4ee2e24d00008bde2600",null,"ffff4e91944e00008b3d2500","ffff4e91944e00008c3d2500","00000000000000008b3d2500","27011d89872500008b602500","c29993cbcf3f0000d8dd2500","0f20813ac83f0000c94c2500","fba507097a4e00000ebb2500","385cd982a30c00005d822500",null,"5f3ba79f144100008f782500","33300b966f1b00000e892500","8111f96d972e0000cab92400","e868847c022d000059482500","d1b2337bbb2f000002702400","80dd67c2a8490000f4db2400","9c85791c3801000051782500","6cb80a75313800004e282500",null,"8cab2ff5711a000034fb2400","41654702fb18000009e12400","ae1b1276e107000006c72400","9e050cd9872d0000d7ec2500","01918e857f170000727c2500","e4ac16e4ce460000e6c62400","4231f8b1fd3b00005fbc2500","7da49dd10a460000f6082500",null,"27bdb965cf270000f2d82500","f3906b4280250000709a2400","fe37af707e33000029a02500","70fdb2322b1300003bf92400","eaa52d66c842000059dd2400","d39be29f2c23000093172500","dc2b142ac71d00002b5d2500","4787e840a91d00006dd02400",null,"c6b98d605d31000036aa2500"]}]},{"name":"dictionary.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]},{"numRows":15,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]}],"columns":[{"path":["int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,-3,2147483647,7,-3,null,7,-3,2147483647,7,null,2147483647,7,-3,2147483647,null,-3,2147483647,7,-3,null,7,-3,2147483647,7,null,2147483647,7,-3,2147483647,null,-3,2147483647,7,-3,null,7,-3,2147483647,7]},{"path":["int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","-9223372036854775808","-9223372036854775808","42","-9223372036854775808",null,"-9223372036854775808","42","-9223372036854775808","-9223372036854775808","-9223372036854775808",null,"-9223372036854775808","-9223372036854775808","-9223372036854775808","42","-9223372036854775808",null,"-9223372036854775808","42","-9223372036854775808","-9223372036854775808","-9223372036854775808",null,"-9223372036854775808","-9223372036854775808","-9223372036854775808","42","-9223372036854775808",null,"-9223372036854775808","42","-9223372036854775808","-9223372036854775808","-9223372036854775808",null,"-9223372036854775808","-9223372036854775808","-9223372036854775808","42"]},{"path":["float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[30000000000.0,-1.25,30000000000.0,0.5,0.5,null,-1.25,-1.25,0.5,30000000000.0,-1.25,-1.25,null,30000000000.0,30000000000.0,0.5,0.5,0.5,-1.25,null,30000000000.0,-1.25,0.5,30000000000.0,30000000000.0,30000000000.0,null,30000000000.0,0.5,0.5,-1.25,30000000000.0,0.5,null,30000000000.0,30000000000.0,0.5,0.5,30000000000.0,0.5]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["726564",null,"726564","726564","",null,"","726564","726564",null,"726564","726564","726564",null,"726564","626c7565","",null,"","677265656e","626c7565",null,"626c7565","626c7565","726564",null,"","726564","726564",null,"677265656e","626c7565","626c7565",null,"677265656e","","726564",null,"","726564"]},{"path":["fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":4,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364"]}]},{"name":"encodings.parquet","numRows":160,"rowGroups":[{"numRows":160,"columns":[{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_LENGTH_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]}],"columns":[{"path":["delta_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":0,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[346,352,338,311,324,355,366,426,451,420,399,351,308,362,343,307,357,362,387,389,421,431,469,499,451,402,413,373,405,395,374,390,450,413,410,365,420,421,432,392,369,381,374,407,414,385,405,439,413,419,391,422,450,480,465,505,551,510,543,544,554,559,548,606,564,589,605,623,605,625,685,645,601,551,611,651,707,765,790,808,774,818,801,769,722,733,723,678,693,701,723,677,730,714,723,705,740,693,674,660,-2147483648,2147483647,-2147483648,2147483647,0,705,716,705,719,757,760,792,760,791,815,869,845,848,886,894,844,855,831,871,858,827,822,774,758,750,768,780,751,702,677,646,690,748,712,747,777,754,799,845,814,868,905,895,882,836,818,850,885,836,791,847,859,868,819,863]},{"path":["delta_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["995","1000","1056","1014","1015","994","996","992",null,"985","982","1033","1012","1063","1058","1112","1113","1110","1063","1065","1123",null,"1069","1062","1035","1076","1086","1061","1083","1067","1025","1020","973","937",null,"873","904","942","995","1018","1020","1048","1061","1070","1092","1129","1174",null,"1279","1308","1293","1310","1288","1258","1227","1187","1233","1185","1160","1120",null,"1121","1083","1123","1168","1147","1128","1129","1126","1077","1036","1070","1085",null,"1144","1156","1110","1082","1120","1082","1032","1075","1048","1034","998","1058",null,"1016","1017","1042","1037","1047","1101","1158","1194","1204","1241","1221","1274",null,"9223372036854775807","-9223372036854775808","9223372036854775807","0","-9223372036854775808","1309","1259","1274","1310","1328","1322","1377",null,"1391","1361","1360","1352","1308","1355","1327","1352","1406","1410","1365","1326",null,"1412","1373","1331","1314","1277","1240","1204","1176","1235","1211","1226","1258",null,"1272","1257","1284","1254","1254","1243","1233","1276","1256","1228","1249","1229",null,"1248","1262","1254","1278","1250","1277","1295","1279"]},{"path":["delta_length_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,"e697a5e697a5","61626362","2062c3a9","e697a563e697a5","e697a56163","62c3a920","61c3a9","20","","6161e697a5","6162","61e697a563","e697a5c3a961","61","c3a9e697a563e697a5","63c3a92020",null,"61e697a5","63626120e697a5","6320622061","6261","c3a920","e697a562e697a5","c3a9e697a5","6262c3a9","6320612063","e697a5","e697a561616263","636263c3a9e697a5","63c3a920e697a563","e697a5c3a9","e697a520e697a56363","2061",null,"e697a5","e697a5c3a920","","e697a5c3a920","6162","20206120","20","62e697a5","e697a56361","636162","20c3a9","61c3a961","e697a562616162","62622061","2061c3a96162","61",null,"20c3a961","c3a9","632061","c3a9","6320e697a562","62206120e697a5","e697a5","","e697a5","c3a9","","6120","2062e697a56261","c3a9","61","c3a9",null,"6161","","","","61e697a5","c3a96262","","","63626361","","c3a96161","6320","c3a9c3a9636163","2063","62c3a96162","63",null,"20e697a5","6261c3a9","2020206262","20616263","20e697a520c3a9e697a5","c3a961c3a963e697a5","","616161","202062e697a5","20e697a5","","61","c3a961e697a520","6362","20","",null,"","c3a9e697a520c3a9","20206261c3a9","c3a961","6261e697a563","e697a5c3a9c3a961","62c3a9e697a563","6263e697a5","","6120c3a9","61c3a962c3a963","62636363e697a5","61","e697a5","63c3a9e697a5","e697a5e697a5636120",null,"62","e697a562","62","62c3a9636120","626262e697a5e697a5","6263","62c3a962e697a520","62","e697a5c3a962","62e697a5","62","e697a520616161","6361e697a561","61","e697a520","c3a962c3a9",null,"c3a9206363e697a5","e697a56162","6262e697a5","2062c3a963c3a9","616261e697a5e697a5","616362e697a5","c3a96162e697a5","61e697a5c3a9","6361","63c3a9626262","e697a5c3a9c3a9","e697a52062","20","","6120c3a92020","20e697a52061c3a9",null,"e697a561","636361","6362c3a9","e697a5c3a92061","","626262e697a562"]},{"path":["delta_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["6178697330","626162626c65","6261636b","636162","61786c6534","62616279","6261636b626f6e65","636162696e","61786f6c6f746c38","62616279686f6f64","626164","61786973","626162626c653132","6261636b","636162","61786c65",null,"6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643230","626164","61786973","626162626c65","6261636b3234","636162","61786c65","62616279","6261636b626f6e653238","636162696e","61786f6c6f746c","62616279686f6f64","6261643332","61786973","626162626c65",null,"6361623336","61786c65","62616279","6261636b626f6e65","636162696e3430","61786f6c6f746c","62616279686f6f64","626164","617869733434","626162626c65","6261636b","636162","61786c653438","62616279","6261636b626f6e65","636162696e","61786f6c6f746c3532","62616279686f6f64",null,"61786973","626162626c653536","6261636b","636162","61786c65","626162793630","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643634","626164","61786973","626162626c65","6261636b3638","636162","61786c65","62616279","6261636b626f6e653732",null,"61786f6c6f746c","62616279686f6f64","6261643736","61786973","626162626c65","6261636b","6361623830","61786c65","62616279","6261636b626f6e65","636162696e3834","61786f6c6f746c","62616279686f6f64","626164","617869733838","626162626c65","6261636b","636162",null,"62616279","6261636b626f6e65","636162696e","61786f6c6f746c3936","62616279686f6f64","626164","61786973","626162626c65313030","6261636b","636162","61786c65","62616279313034","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f64313038","626164","61786973",null,"6261636b313132","636162","61786c65","62616279","6261636b626f6e65313136","636162696e","61786f6c6f746c","62616279686f6f64","626164313230","61786973","626162626c65","6261636b","636162313234","61786c65","62616279","6261636b626f6e65","636162696e313238","61786f6c6f746c",null,"626164","61786973313332","626162626c65","6261636b","636162","61786c65313336","62616279","6261636b626f6e65","636162696e","61786f6c6f746c313430","62616279686f6f64","626164","61786973","626162626c65313434","6261636b","636162","61786c65","62616279313438",null,"636162696e","61786f6c6f746c","62616279686f6f64313532","626164","61786973","626162626c65","6261636b313536","636162","61786c65","62616279"]},{"path":["delta_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["000001","616201","616202","030001","616204","616205","060001","616200","616201","090001","616203","616204","0c0001","616206","616200","0f0001","616202","616203","120001","616205","616206","150001","616201","616202","180001","616204","616205","1b0001","616200","616201","1e0001","616203","616204","210001","616206","616200","240001","616202","616203","270001","616205","616206","2a0001","616201","616202","2d0001","616204","616205","300001","616200","616201","330001","616203","616204","360001","616206","616200","390001","616202","616203","3c0001","616205","616206","3f0001","616201","616202","420001","616204","616205","450001","616200","616201","480001","616203","616204","4b0001","616206","616200","4e0001","616202","616203","510001","616205","616206","540001","616201","616202","570001","616204","616205","5a0001","616200","616201","5d0001","616203","616204","600001","616206","616200","630001","616202","616203","660001","616205","616206","690001","616201","616202","6c0001","616204","616205","6f0001","616200","616201","720001","616203","616204","750001","616206","616200","780001","616202","616203","7b0001","616205","616206","7e0001","616201","616202","810001","616204","616205","840001","616200","616201","870001","616203","616204","8a0001","616206","616200","8d0001","616202","616203","900001","616205","616206","930001","616201","616202","960001","616204","616205","990001","616200","616201","9c0001","616203","616204","9f0001"]},{"path":["bss_float32"],"physicalType":"FLOAT","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-43.0,43.3125,-22.0625,316.5,-439.9375,-127.3125,18.8125,598.625,623.0,null,420.375,-139.0625,-436.5625,-220.0625,463.9375,586.4375,66.125,414.5625,-337.3125,-129.3125,null,-156.5,-192.625,-19.0625,-254.125,-182.625,512.5625,15.375,227.5625,-61.8125,46.6875,null,365.3125,-375.25,-34.8125,-98.0,596.5625,-375.8125,593.8125,-59.625,-31.1875,366.3125,null,-175.0625,-503.375,70.0625,360.375,307.5625,347.0625,-342.5,-552.125,624.4375,376.125,null,-90.6875,356.5,605.6875,331.3125,555.4375,-249.0,-436.75,-149.6875,193.0,521.5,null,-210.4375,-8.75,75.25,-306.25,599.4375,618.125,482.6875,-42.5625,235.0,-507.625,null,-431.4375,-389.0,-447.0,-393.625,425.0,-388.125,-427.0,435.4375,-455.125,-316.5625,null,-193.9375,-515.125,27.5625,-197.1875,116.3125,-19.6875,487.375,58.0,6.875,-391.5625,null,-214.0625,493.625,265.375,410.5,96.1875,-123.125,-372.8125,-331.0,182.9375,-223.0625,null,4.5625,218.625,422.3125,-617.8125,235.5625,-302.8125,-575.5,-621.125,-270.5625,-470.875,null,212.375,125.6875,5.8125,409.5625,243.1875,-339.25,-40.0,544.6875,340.625,-455.8125,null,372.5,-107.6875,-326.8125,-421.125,-260.0625,-382.6875,526.9375,365.375,-102.125,455.6875,null,329.125,526.625,-33.8125,-126.125,157.25,-252.8125,504.0625,-193.8125,457.0625,-393.875,null,-187.5,-386.9375,-306.875,-81.9375,-591.6875,100.375,-242.4375]},{"path":["bss_float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[367.146484375,216.3037109375,-45.404296875,-246.9521484375,null,-939.69140625,-200.38671875,-704.7138671875,775.865234375,922.1630859375,-521.6357421875,null,602.3759765625,-648.45703125,-219.9970703125,59.5732421875,-777.11328125,888.3837890625,null,-743.2919921875,-618.986328125,-459.5986328125,-37.6669921875,-508.603515625,251.0556640625,null,-224.1376953125,972.560546875,-296.939453125,-637.6865234375,-690.7109375,118.5244140625,null,-668.4931640625,18.4228515625,628.2099609375,387.2919921875,-560.029296875,-882.931640625,null,531.5068359375,-386.9130859375,-201.7763671875,268.939453125,-682.5419921875,-142.603515625,null,370.99609375,-362.736328125,42.3154296875,-254.3662109375,655.669921875,694.8515625,null,-101.5029296875,533.9765625,813.228515625,166.6044921875,191.0478515625,-95.1650390625,null,937.26953125,-246.8505859375,-612.4248046875,99.2333984375,-497.138671875,-349.3037109375,null,-349.6103515625,349.333984375,705.724609375,482.576171875,-271.2900390625,-764.9677734375,null,-678.041015625,-10.9296875,342.4619140625,959.15234375,607.236328125,-445.1103515625,null,711.8271484375,-740.19921875,35.1142578125,-123.166015625,9.7666015625,-555.3564453125,null,769.6357421875,394.380859375,-317.14453125,-705.2509765625,815.4677734375,-97.205078125,null,-555.1708984375,529.1474609375,244.80078125,-148.810546875,837.091796875,358.9521484375,null,-511.701171875,194.2822265625,277.2734375,327.8388671875,-315.7138671875,-464.7490234375,null,-614.7822265625,606.330078125,476.701171875,-336.1572265625,-378.59765625,515.888671875,null,-813.3203125,-59.9111328125,583.8701171875,-812.3291015625,217.8701171875,876.6845703125,null,-42.65625,851.103515625,-369.6923828125,-623.7890625,99.3505859375,720.28515625,null,-828.4794921875,90.01953125,-371.2607421875,10.7314453125,405.2333984375,237.5390625,null,284.154296875,90.720703125,-89.6220703125,-377.3857421875,676.6484375,199.1337890625,null,-255.5400390625,888.41796875,-268.052734375,12.4375,581.1142578125,341.697265625,null,755.982421875,813.00390625,-767.6181640625,-130.4541015625,476.11328125,-39.353515625,null,803.1201171875]},{"path":["bss_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-2147483648,2147483647,24630,-8252,-88741,279,-80859,-79846,-88224,11214,-72621,41673,94160,-72079,-75456,49714,-28221,-69862,35645,9817,15468,70413,76991,89996,-77851,-19432,36584,-36803,-18873,-27782,4874,16048,88154,-57213,-15930,97013,-64599,63937,-10116,31204,41596,-58577,8126,-65323,-19876,-37092,-43312,-89958,-19789,27103,22296,19318,69678,-10861,39994,-8405,-27595,37714,78587,27400,88771,22216,83967,51193,-12492,11652,47228,84674,42964,-8228,73119,-64033,-99646,90841,23044,-90804,79377,-44780,19843,18044,-74864,21640,-57604,-40662,-54977,-98373,-18308,67031,30243,97549,97982,51741,95370,-75576,80731,80124,55178,89130,-30615,-2998,52618,70509,-20366,-96531,33925,80541,-76437,28102,47292,68212,13253,-28233,50059,10424,67564,-65300,-25483,94973,83740,-58425,-57885,-32337,50174,-70539,-93538,-74075,73099,-51392,-55340,-70481,-61805,9271,-70489,-88830,23499,-49353,-46996,-50008,-16437,21629,26720,89281,71447,21080,1327,-12061,-56619,-13475,1719,95956,77982,-18198,87923,53004,-44195,-69179,36258,-27110,18966,-55393]},{"path":["bss_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","9223372036854775807","987863054100","-296709105316","797163188552","283198447560","210302952758","-242326649351","682448063443","-133756023825","1087324371926","992378598252","202450873419","-163663339987",null,"-851188577698","-175190867480","-568420662460","888396909884","-967582777543","-438169682332","464692519730","867134292246","278899198129","253340965962","-802603576546","-81558278880","356243668616","617686406039","219022649198","-990766883952","-32296480474","358083665127","-971286852589","-378225370839","797453355920","-807104191328",null,"-390682297362","-1092497215362","-978571312995","-91580169782","1066730189811","430481217755","307658968460","-302016529785","910846036120","-262418637231","-410538560962","695107625814","540223280584","337319050436","-997323342453","230232436340","-1062128698264","-298926060898","-410931378914","-736778008989","-65991568603","851673167648",null,"234018773466","235019331533","605069254338","-1014013373950","628920372329","-687878596681","-708899400047","-85900646935","312486659162","-567271953017","1019981739437","693375127142","-948621157977","-1073149451028","755298532586","-1081886191699","963462205821","-24009810685","847648551381","-527998211714","-762169547994","319743072226",null,"-755692925920","121169648679","-1040928921954","251238298346","1015370757681","-194980851552","822769963093","-133418024571","-564926299881","-660269138406","1056900410002","239753541378","-92796144166","-687408872136","-403448570799","987893209335","102662608696","528580901162","957056644830","-254655530643","-21421591238","366790029881",null,"-710728631032","483739032797","-225612438291","992590595094","-391556708723","441239395898","227561934972","420505691518","-219258864319","-836798736275","-263566729195","593315416161","127055598463","510154441724","-326140058306","-379289775153","1079181312388","118834101207","-1010855571923","-960810927306","-271852908247","-871532118498",null,"-609178482600","-145211802073","-407971053503","974973273544","-418132649076","-845833554356","-907607164810","-821233710697","-611538191768","-59725661798","-188190488612","1051278023789","-486014714752","-420053141405","670152470078","-878859760470","-522447514654","681864633933","562381935015","138329833268","875971386725","-967975647744",null,"-469328883599","701452300781","-1016314675471","-428111787833","1091174619886","-501130950046","514645087743"]},{"path":["bss_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["aacfc4","414c98","04274c","20cea8","aaf446","bff03e","ad6cc2","0f32b5","69f5c6","21b4c7","8c9325","99905f","8ec640","a2d58f","9753b6","053825","f72a1a","83416b","3e04ff","42cc67","082117","e64bc7","e6ca6f","a553de","592678",null,"6b1331","64733d","c9a7c8","79f0ba","26675f","10ecdb","b36d78","6c54c5","14ebce","2426c3","a84823","20fd34","ce3fc9","4ec436","2758c3","ad640f","de41e5","1931fb","66f81c","eb1465","48ed00","383de9","58d935","0996a6","df2d17","73bd80","e2bce5","699c6a",null,"844b90","935ff9","facec5","46dadc","57a0ca","e55153","a685f3","c842b0","e42886","8ca306","11dede","f7cef8","725728","cd2dbe","424d36","8f619b","aadc45","474b82","2a4c0f","7b3470","b1a09b","02ecf2","10f84b","94c9a6","438720","dac3ba","597fdd","590000",null,"2a2b18","2e2164","b2c489","9722ed","ff67ba","9c9f7a","7697c6","a1f865","abba76","c44f0c","9f887b","cb87d8","77f255","1b9d85","b16bf6","bc1f67","4ad820","aa0ecd","667475","094bb5","e053b7","15a6d9","298b89","632c49","974fdd","86e4a8","87f8e7","865966",null,"6e1ce3","9b829d","71d715","989afc","12c8c3","d0500e","706596","bc6b3e","9d1fbf","7ca7e8","9109f7","4d0ad6","ebf77b","d9ba59","46237b","f7a01c","0948e5","8b1398","d4c9c8","9dbc3a","36b566","efc0dd","e564db","6d7268","2362a3","ae5f34","05d20f","8f5d1d",null,"c77a35","3180d2","39e592","2d7a7c","0555cd","3db2ea","ea2b83","348b82","4f7a35","d21428","5ea75b","6c29a1","9b474f","3c98bb","62f80e","1a4a5b","cb50f7","d35d35"]},{"path":["rle_bool"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,null,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,null,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,null,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,null,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,null,true,true,true,true,true,true,true,false,false,false]}]},{"name":"encodings_v2.parquet","numRows":160,"rowGroups":[{"numRows":160,"columns":[{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_LENGTH_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]}],"columns":[{"path":["delta_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":0,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[346,352,338,311,324,355,366,426,451,420,399,351,308,362,343,307,357,362,387,389,421,431,469,499,451,402,413,373,405,395,374,390,450,413,410,365,420,421,432,392,369,381,374,407,414,385,405,439,413,419,391,422,450,480,465,505,551,510,543,544,554,559,548,606,564,589,605,623,605,625,685,645,601,551,611,651,707,765,790,808,774,818,801,769,722,733,723,678,693,701,723,677,730,714,723,705,740,693,674,660,-2147483648,2147483647,-2147483648,2147483647,0,705,716,705,719,757,760,792,760,791,815,869,845,848,886,894,844,855,831,871,858,827,822,774,758,750,768,780,751,702,677,646,690,748,712,747,777,754,799,845,814,868,905,895,882,836,818,850,885,836,791,847,859,868,819,863]},{"path":["delta_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["995","1000","1056","1014","1015","994","996","992",null,"985","982","1033","1012","1063","1058","1112","1113","1110","1063","1065","1123",null,"1069","1062","1035","1076","1086","1061","1083","1067","1025","1020","973","937",null,"873","904","942","995","1018","1020","1048","1061","1070","1092","1129","1174",null,"1279","1308","1293","1310","1288","1258","1227","1187","1233","1185","1160","1120",null,"1121","1083","1123","1168","1147","1128","1129","1126","1077","1036","1070","1085",null,"1144","1156","1110","1082","1120","1082","1032","1075","1048","1034","998","1058",null,"1016","1017","1042","1037","1047","1101","1158","1194","1204","1241","1221","1274",null,"9223372036854775807","-9223372036854775808","9223372036854775807","0","-9223372036854775808","1309","1259","1274","1310","1328","1322","1377",null,"1391","1361","1360","1352","1308","1355","1327","1352","1406","1410","1365","1326",null,"1412","1373","1331","1314","1277","1240","1204","1176","1235","1211","1226","1258",null,"1272","1257","1284","1254","1254","1243","1233","1276","1256","1228","1249","1229",null,"1248","1262","1254","1278","1250","1277","1295","1279"]},{"path":["delta_length_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,"e697a5e697a5","61626362","2062c3a9","e697a563e697a5","e697a56163","62c3a920","61c3a9","20","","6161e697a5","6162","61e697a563","e697a5c3a961","61","c3a9e697a563e697a5","63c3a92020",null,"61e697a5","63626120e697a5","6320622061","6261","c3a920","e697a562e697a5","c3a9e697a5","6262c3a9","6320612063","e697a5","e697a561616263","636263c3a9e697a5","63c3a920e697a563","e697a5c3a9","e697a520e697a56363","2061",null,"e697a5","e697a5c3a920","","e697a5c3a920","6162","20206120","20","62e697a5","e697a56361","636162","20c3a9","61c3a961","e697a562616162","62622061","2061c3a96162","61",null,"20c3a961","c3a9","632061","c3a9","6320e697a562","62206120e697a5","e697a5","","e697a5","c3a9","","6120","2062e697a56261","c3a9","61","c3a9",null,"6161","","","","61e697a5","c3a96262","","","63626361","","c3a96161","6320","c3a9c3a9636163","2063","62c3a96162","63",null,"20e697a5","6261c3a9","2020206262","20616263","20e697a520c3a9e697a5","c3a961c3a963e697a5","","616161","202062e697a5","20e697a5","","61","c3a961e697a520","6362","20","",null,"","c3a9e697a520c3a9","20206261c3a9","c3a961","6261e697a563","e697a5c3a9c3a961","62c3a9e697a563","6263e697a5","","6120c3a9","61c3a962c3a963","62636363e697a5","61","e697a5","63c3a9e697a5","e697a5e697a5636120",null,"62","e697a562","62","62c3a9636120","626262e697a5e697a5","6263","62c3a962e697a520","62","e697a5c3a962","62e697a5","62","e697a520616161","6361e697a561","61","e697a520","c3a962c3a9",null,"c3a9206363e697a5","e697a56162","6262e697a5","2062c3a963c3a9","616261e697a5e697a5","616362e697a5","c3a96162e697a5","61e697a5c3a9","6361","63c3a9626262","e697a5c3a9c3a9","e697a52062","20","","6120c3a92020","20e697a52061c3a9",null,"e697a561","636361","6362c3a9","e697a5c3a92061","","626262e697a562"]},{"path":["delta_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["6178697330","626162626c65","6261636b","636162","61786c6534","62616279","6261636b626f6e65","636162696e","61786f6c6f746c38","62616279686f6f64","626164","61786973","626162626c653132","6261636b","636162","61786c65",null,"6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643230","626164","61786973","626162626c65","6261636b3234","636162","61786c65","62616279","6261636b626f6e653238","636162696e","61786f6c6f746c","62616279686f6f64","6261643332","61786973","626162626c65",null,"6361623336","61786c65","62616279","6261636b626f6e65","636162696e3430","61786f6c6f746c","62616279686f6f64","626164","617869733434","626162626c65","6261636b","636162","61786c653438","62616279","6261636b626f6e65","636162696e","61786f6c6f746c3532","62616279686f6f64",null,"61786973","626162626c653536","6261636b","636162","61786c65","626162793630","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643634","626164","61786973","626162626c65","6261636b3638","636162","61786c65","62616279","6261636b626f6e653732",null,"61786f6c6f746c","62616279686f6f64","6261643736","61786973","626162626c65","6261636b","6361623830","61786c65","62616279","6261636b626f6e65","636162696e3834","61786f6c6f746c","62616279686f6f64","626164","617869733838","626162626c65","6261636b","636162",null,"62616279","6261636b626f6e65","636162696e","61786f6c6f746c3936","62616279686f6f64","626164","61786973","626162626c65313030","6261636b","636162","61786c65","62616279313034","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f64313038","626164","61786973",null,"6261636b313132","636162","61786c65","62616279","6261636b626f6e65313136","636162696e","61786f6c6f746c","62616279686f6f64","626164313230","61786973","626162626c65","6261636b","636162313234","61786c65","62616279","6261636b626f6e65","636162696e313238","61786f6c6f746c",null,"626164","61786973313332","626162626c65","6261636b","636162","61786c65313336","62616279","6261636b626f6e65","636162696e","61786f6c6f746c313430","62616279686f6f64","626164","61786973","626162626c65313434","6261636b","636162","61786c65","62616279313438",null,"636162696e","61786f6c6f746c","62616279686f6f64313532","626164","61786973","626162626c65","6261636b313536","636162","61786c65","62616279"]},{"path":["delta_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["000001","616201","616202","030001","616204","616205","060001","616200","616201","090001","616203","616204","0c0001","616206","616200","0f0001","616202","616203","120001","616205","616206","150001","616201","616202","180001","616204","616205","1b0001","616200","616201","1e0001","616203","616204","210001","616206","616200","240001","616202","616203","270001","616205","616206","2a0001","616201","616202","2d0001","616204","616205","300001","616200","616201","330001","616203","616204","360001","616206","616200","390001","616202","616203","3c0001","616205","616206","3f0001","616201","616202","420001","616204","616205","450001","616200","616201","480001","616203","616204","4b0001","616206","616200","4e0001","616202","616203","510001","616205","616206","540001","616201","616202","570001","616204","616205","5a0001","616200","616201","5d0001","616203","616204","600001","616206","616200","630001","616202","616203","660001","616205","616206","690001","616201","616202","6c0001","616204","616205","6f0001","616200","616201","720001","616203","616204","750001","616206","616200","780001","616202","616203","7b0001","616205","616206","7e0001","616201","616202","810001","616204","616205","840001","616200","616201","870001","616203","616204","8a0001","616206","616200","8d0001","616202","616203","900001","616205","616206","930001","616201","616202","960001","616204","616205","990001","616200","616201","9c0001","616203","616204","9f0001"]},{"path":["bss_float32"],"physicalType":"FLOAT","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-43.0,43.3125,-22.0625,316.5,-439.9375,-127.3125,18.8125,598.625,623.0,null,420.375,-139.0625,-436.5625,-220.0625,463.9375,586.4375,66.125,414.5625,-337.3125,-129.3125,null,-156.5,-192.625,-19.0625,-254.125,-182.625,512.5625,15.375,227.5625,-61.8125,46.6875,null,365.3125,-375.25,-34.8125,-98.0,596.5625,-375.8125,593.8125,-59.625,-31.1875,366.3125,null,-175.0625,-503.375,70.0625,360.375,307.5625,347.0625,-342.5,-552.125,624.4375,376.125,null,-90.6875,356.5,605.6875,331.3125,555.4375,-249.0,-436.75,-149.6875,193.0,521.5,null,-210.4375,-8.75,75.25,-306.25,599.4375,618.125,482.6875,-42.5625,235.0,-507.625,null,-431.4375,-389.0,-447.0,-393.625,425.0,-388.125,-427.0,435.4375,-455.125,-316.5625,null,-193.9375,-515.125,27.5625,-197.1875,116.3125,-19.6875,487.375,58.0,6.875,-391.5625,null,-214.0625,493.625,265.375,410.5,96.1875,-123.125,-372.8125,-331.0,182.9375,-223.0625,null,4.5625,218.625,422.3125,-617.8125,235.5625,-302.8125,-575.5,-621.125,-270.5625,-470.875,null,212.375,125.6875,5.8125,409.5625,243.1875,-339.25,-40.0,544.6875,340.625,-455.8125,null,372.5,-107.6875,-326.8125,-421.125,-260.0625,-382.6875,526.9375,365.375,-102.125,455.6875,null,329.125,526.625,-33.8125,-126.125,157.25,-252.8125,504.0625,-193.8125,457.0625,-393.875,null,-187.5,-386.9375,-306.875,-81.9375,-591.6875,100.375,-242.4375]},{"path":["bss_float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[367.146484375,216.3037109375,-45.404296875,-246.9521484375,null,-939.69140625,-200.38671875,-704.7138671875,775.865234375,922.1630859375,-521.6357421875,null,602.3759765625,-648.45703125,-219.9970703125,59.5732421875,-777.11328125,888.3837890625,null,-743.2919921875,-618.986328125,-459.5986328125,-37.6669921875,-508.603515625,251.0556640625,null,-224.1376953125,972.560546875,-296.939453125,-637.6865234375,-690.7109375,118.5244140625,null,-668.4931640625,18.4228515625,628.2099609375,387.2919921875,-560.029296875,-882.931640625,null,531.5068359375,-386.9130859375,-201.7763671875,268.939453125,-682.5419921875,-142.603515625,null,370.99609375,-362.736328125,42.3154296875,-254.3662109375,655.669921875,694.8515625,null,-101.5029296875,533.9765625,813.228515625,166.6044921875,191.0478515625,-95.1650390625,null,937.26953125,-246.8505859375,-612.4248046875,99.2333984375,-497.138671875,-349.3037109375,null,-349.6103515625,349.333984375,705.724609375,482.576171875,-271.2900390625,-764.9677734375,null,-678.041015625,-10.9296875,342.4619140625,959.15234375,607.236328125,-445.1103515625,null,711.8271484375,-740.19921875,35.1142578125,-123.166015625,9.7666015625,-555.3564453125,null,769.6357421875,394.380859375,-317.14453125,-705.2509765625,815.4677734375,-97.205078125,null,-555.1708984375,529.1474609375,244.80078125,-148.810546875,837.091796875,358.9521484375,null,-511.701171875,194.2822265625,277.2734375,327.8388671875,-315.7138671875,-464.7490234375,null,-614.7822265625,606.330078125,476.701171875,-336.1572265625,-378.59765625,515.888671875,null,-813.3203125,-59.9111328125,583.8701171875,-812.3291015625,217.8701171875,876.6845703125,null,-42.65625,851.103515625,-369.6923828125,-623.7890625,99.3505859375,720.28515625,null,-828.4794921875,90.01953125,-371.2607421875,10.7314453125,405.2333984375,237.5390625,null,284.154296875,90.720703125,-89.6220703125,-377.3857421875,676.6484375,199.1337890625,null,-255.5400390625,888.41796875,-268.052734375,12.4375,581.1142578125,341.697265625,null,755.982421875,813.00390625,-767.6181640625,-130.4541015625,476.11328125,-39.353515625,null,803.1201171875]},{"path":["bss_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-2147483648,2147483647,24630,-8252,-88741,279,-80859,-79846,-88224,11214,-72621,41673,94160,-72079,-75456,49714,-28221,-69862,35645,9817,15468,70413,76991,89996,-77851,-19432,36584,-36803,-18873,-27782,4874,16048,88154,-57213,-15930,97013,-64599,63937,-10116,31204,41596,-58577,8126,-65323,-19876,-37092,-43312,-89958,-19789,27103,22296,19318,69678,-10861,39994,-8405,-27595,37714,78587,27400,88771,22216,83967,51193,-12492,11652,47228,84674,42964,-8228,73119,-64033,-99646,90841,23044,-90804,79377,-44780,19843,18044,-74864,21640,-57604,-40662,-54977,-98373,-18308,67031,30243,97549,97982,51741,95370,-75576,80731,80124,55178,89130,-30615,-2998,52618,70509,-20366,-96531,33925,80541,-76437,28102,47292,68212,13253,-28233,50059,10424,67564,-65300,-25483,94973,83740,-58425,-57885,-32337,50174,-70539,-93538,-74075,73099,-51392,-55340,-70481,-61805,9271,-70489,-88830,23499,-49353,-46996,-50008,-16437,21629,26720,89281,71447,21080,1327,-12061,-56619,-13475,1719,95956,77982,-18198,87923,53004,-44195,-69179,36258,-27110,18966,-55393]},{"path":["bss_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","9223372036854775807","987863054100","-296709105316","797163188552","283198447560","210302952758","-242326649351","682448063443","-133756023825","1087324371926","992378598252","202450873419","-163663339987",null,"-851188577698","-175190867480","-568420662460","888396909884","-967582777543","-438169682332","464692519730","867134292246","278899198129","253340965962","-802603576546","-81558278880","356243668616","617686406039","219022649198","-990766883952","-32296480474","358083665127","-971286852589","-378225370839","797453355920","-807104191328",null,"-390682297362","-1092497215362","-978571312995","-91580169782","1066730189811","430481217755","307658968460","-302016529785","910846036120","-262418637231","-410538560962","695107625814","540223280584","337319050436","-997323342453","230232436340","-1062128698264","-298926060898","-410931378914","-736778008989","-65991568603","851673167648",null,"234018773466","235019331533","605069254338","-1014013373950","628920372329","-687878596681","-708899400047","-85900646935","312486659162","-567271953017","1019981739437","693375127142","-948621157977","-1073149451028","755298532586","-1081886191699","963462205821","-24009810685","847648551381","-527998211714","-762169547994","319743072226",null,"-755692925920","121169648679","-1040928921954","251238298346","1015370757681","-194980851552","822769963093","-133418024571","-564926299881","-660269138406","1056900410002","239753541378","-92796144166","-687408872136","-403448570799","987893209335","102662608696","528580901162","957056644830","-254655530643","-21421591238","366790029881",null,"-710728631032","483739032797","-225612438291","992590595094","-391556708723","441239395898","227561934972","420505691518","-219258864319","-836798736275","-263566729195","593315416161","127055598463","510154441724","-326140058306","-379289775153","1079181312388","118834101207","-1010855571923","-960810927306","-271852908247","-871532118498",null,"-609178482600","-145211802073","-407971053503","974973273544","-418132649076","-845833554356","-907607164810","-821233710697","-611538191768","-59725661798","-188190488612","1051278023789","-486014714752","-420053141405","670152470078","-878859760470","-522447514654","681864633933","562381935015","138329833268","875971386725","-967975647744",null,"-469328883599","701452300781","-1016314675471","-428111787833","1091174619886","-501130950046","514645087743"]},{"path":["bss_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["aacfc4","414c98","04274c","20cea8","aaf446","bff03e","ad6cc2","0f32b5","69f5c6","21b4c7","8c9325","99905f","8ec640","a2d58f","9753b6","053825","f72a1a","83416b","3e04ff","42cc67","082117","e64bc7","e6ca6f","a553de","592678",null,"6b1331","64733d","c9a7c8","79f0ba","26675f","10ecdb","b36d78","6c54c5","14ebce","2426c3","a84823","20fd34","ce3fc9","4ec436","2758c3","ad640f","de41e5","1931fb","66f81c","eb1465","48ed00","383de9","58d935","0996a6","df2d17","73bd80","e2bce5","699c6a",null,"844b90","935ff9","facec5","46dadc","57a0ca","e55153","a685f3","c842b0","e42886","8ca306","11dede","f7cef8","725728","cd2dbe","424d36","8f619b","aadc45","474b82","2a4c0f","7b3470","b1a09b","02ecf2","10f84b","94c9a6","438720","dac3ba","597fdd","590000",null,"2a2b18","2e2164","b2c489","9722ed","ff67ba","9c9f7a","7697c6","a1f865","abba76","c44f0c","9f887b","cb87d8","77f255","1b9d85","b16bf6","bc1f67","4ad820","aa0ecd","667475","094bb5","e053b7","15a6d9","298b89","632c49","974fdd","86e4a8","87f8e7","865966",null,"6e1ce3","9b829d","71d715","989afc","12c8c3","d0500e","706596","bc6b3e","9d1fbf","7ca7e8","9109f7","4d0ad6","ebf77b","d9ba59","46237b","f7a01c","0948e5","8b1398","d4c9c8","9dbc3a","36b566","efc0dd","e564db","6d7268","2362a3","ae5f34","05d20f","8f5d1d",null,"c77a35","3180d2","39e592","2d7a7c","0555cd","3db2ea","ea2b83","348b82","4f7a35","d21428","5ea75b","6c29a1","9b474f","3c98bb","62f80e","1a4a5b","cb50f7","d35d35"]},{"path":["rle_bool"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,null,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,null,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,null,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,null,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,null,true,true,true,true,true,true,true,false,false,false]}]},{"name":"nested.parquet","numRows":40,"rowGroups":[{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]},{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["ints","list","element"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[],[null],[1,null,3],[],[-63],[85,65],[66,-11,7],[],null,[-2,16],[-37,49,-92],[],[78],[-55,-28],[97,-7,-78],[],[37],null,[95,-78,2],[],[-36],[-63,10],[-24,-68,58],[],[62],[44,96],null,[],[12],[91,58],[64,-18,-59],[],[98],[44,100],[28,8,28],null,[-53],[-86,-49],[13,-52,50]]},{"path":["nested_strings","list","element","list","element"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":5,"maxRepetitionLevel":2,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[[]],[[],["68c3a96c6c6f2077c3b6726c64"]],[],[[]],[null,["70617271756574"]],[],null,[[],["61"]],[],[null],[[],["e697a5e69cace8aa9e"]],[],[[]],null,[],[[]],[[],["68656c6c6f"]],[],[[]],[null,["f09f8c8d"]],null,[[]],[[],[""]],[],[null],[[],["68c3a96c6c6f2077c3b6726c64"]],[],null,[[],null],[],[[]],[[],["61"]],[],[[]],null,[],[[]],[[],["7878787878787878787878787878787878787878"]],[]]},{"path":["struct","a"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,-19000,-18000,-17000,-16000,-15000,null,-13000,-12000,-11000,-10000,-9000,null,-7000,-6000,-5000,-4000,-3000,null,-1000,0,1000,2000,3000,null,5000,6000,7000,8000,9000,null,11000,12000,13000,14000,15000,null,17000,18000,19000]},{"path":["struct","b","list","element"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,"-1099511627774"],[null,"-1099511627773","-1099511627770"],null,[],null,[null,"-1099511627769"],null,[null,"-1099511627767","-1099511627758",null],[],[null],null,[null,"-1099511627763","-1099511627750"],[null,"-1099511627762","-1099511627748",null],[],null,[null,"-1099511627759"],null,[null,"-1099511627757","-1099511627738",null],null,[null],[null,"-1099511627754"],[null,"-1099511627753","-1099511627730"],null,[],[null],[null,"-1099511627749"],null,[null,"-1099511627747","-1099511627718",null],null,[null],null,[null,"-1099511627743","-1099511627710"],[null,"-1099511627742","-1099511627708",null],[],null,[null,"-1099511627739"],[null,"-1099511627738","-1099511627700"],[null,"-1099511627737","-1099511627698",null]]},{"path":["map","key_value","key"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":2,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"]]},{"path":["map","key_value","value"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[0],[0,2],[0,null,6],[],[0],[0,6],[0,null,14],null,[0],[0,10],[0,null,22],[],[0],[0,14],[0,null,30],null,[0],[0,18],[0,null,38],[],[0],[0,22],[0,null,46],null,[0],[0,26],[0,null,54],[],[0],[0,30],[0,null,62],null,[0],[0,34],[0,null,70],[],[0],[0,38],[0,null,78]]},{"path":["required_struct","x"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-7.0,-6.5,-6.0,-5.5,-5.0,-4.5,-4.0,-3.5,-3.0,-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0,12.5]},{"path":["structs","list","element","s"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],null,[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],null,[null],[null,"70617271756574"],[null,null,"70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],null,[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"]]},{"path":["structs","list","element","t"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,null,false],[],[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],null,[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,false,false],[],[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false]]}]},{"name":"nested_v2.parquet","numRows":40,"rowGroups":[{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]},{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]}],"columns":[{"path":["ints","list","element"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[],[null],[1,null,3],[],[-63],[85,65],[66,-11,7],[],null,[-2,16],[-37,49,-92],[],[78],[-55,-28],[97,-7,-78],[],[37],null,[95,-78,2],[],[-36],[-63,10],[-24,-68,58],[],[62],[44,96],null,[],[12],[91,58],[64,-18,-59],[],[98],[44,100],[28,8,28],null,[-53],[-86,-49],[13,-52,50]]},{"path":["nested_strings","list","element","list","element"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":5,"maxRepetitionLevel":2,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[[]],[[],["68c3a96c6c6f2077c3b6726c64"]],[],[[]],[null,["70617271756574"]],[],null,[[],["61"]],[],[null],[[],["e697a5e69cace8aa9e"]],[],[[]],null,[],[[]],[[],["68656c6c6f"]],[],[[]],[null,["f09f8c8d"]],null,[[]],[[],[""]],[],[null],[[],["68c3a96c6c6f2077c3b6726c64"]],[],null,[[],null],[],[[]],[[],["61"]],[],[[]],null,[],[[]],[[],["7878787878787878787878787878787878787878"]],[]]},{"path":["struct","a"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,-19000,-18000,-17000,-16000,-15000,null,-13000,-12000,-11000,-10000,-9000,null,-7000,-6000,-5000,-4000,-3000,null,-1000,0,1000,2000,3000,null,5000,6000,7000,8000,9000,null,11000,12000,13000,14000,15000,null,17000,18000,19000]},{"path":["struct","b","list","element"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,"-1099511627774"],[null,"-1099511627773","-1099511627770"],null,[],null,[null,"-1099511627769"],null,[null,"-1099511627767","-1099511627758",null],[],[null],null,[null,"-1099511627763","-1099511627750"],[null,"-1099511627762","-1099511627748",null],[],null,[null,"-1099511627759"],null,[null,"-1099511627757","-1099511627738",null],null,[null],[null,"-1099511627754"],[null,"-1099511627753","-1099511627730"],null,[],[null],[null,"-1099511627749"],null,[null,"-1099511627747","-1099511627718",null],null,[null],null,[null,"-1099511627743","-1099511627710"],[null,"-1099511627742","-1099511627708",null],[],null,[null,"-1099511627739"],[null,"-1099511627738","-1099511627700"],[null,"-1099511627737","-1099511627698",null]]},{"path":["map","key_value","key"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":2,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"]]},{"path":["map","key_value","value"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[0],[0,2],[0,null,6],[],[0],[0,6],[0,null,14],null,[0],[0,10],[0,null,22],[],[0],[0,14],[0,null,30],null,[0],[0,18],[0,null,38],[],[0],[0,22],[0,null,46],null,[0],[0,26],[0,null,54],[],[0],[0,30],[0,null,62],null,[0],[0,34],[0,null,70],[],[0],[0,38],[0,null,78]]},{"path":["required_struct","x"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-7.0,-6.5,-6.0,-5.5,-5.0,-4.5,-4.0,-3.5,-3.0,-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0,12.5]},{"path":["structs","list","element","s"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],null,[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],null,[null],[null,"70617271756574"],[null,null,"70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],null,[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"]]},{"path":["structs","list","element","t"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,null,false],[],[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],null,[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,false,false],[],[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false]]}]},{"name":"gzip.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]},{"numRows":15,"columns":[{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]}],"columns":[{"path":["int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,"-5","-5","-9223372036854775808","-9223372036854775808","-9223372036854775808","5",null,"1125899906842624","-9223372036854775808","1125899906842624","1125899906842624","1125899906842624","-9223372036854775808",null,"5","-9223372036854775808","-9223372036854775808","-9223372036854775808","-9223372036854775808","1125899906842624",null,"-5","-5","-9223372036854775808","-9223372036854775808","1125899906842624","-9223372036854775808",null,"5","-5","-9223372036854775808","-5","-5","1125899906842624",null,"-5","5","-9223372036854775808","-5"]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["","61","68656c6c6f68656c6c6f","",null,"","","68656c6c6f","68c3a96c6c6f2077c3b6726c6468c3a96c6c6f2077c3b6726c64","",null,"6161","","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9ee697a5e69cace8aa9e","",null,"68656c6c6f68656c6c6f","","e697a5e69cace8aa9e","","",null,"68c3a96c6c6f2077c3b6726c6468c3a96c6c6f2077c3b6726c64","","","6161","",null,"e697a5e69cace8aa9ee697a5e69cace8aa9e","","61","68656c6c6f68656c6c6f","",null,"","","68656c6c6f","68c3a96c6c6f2077c3b6726c6468c3a96c6c6f2077c3b6726c64",""]},{"path":["ints","list","element"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[1,-1],[2,-2,2,-2],[],[4,-1],[0,-2,0,-2],[],[2,-1],[3,-2,3,-2],null,[0,-1],[1,-2,1,-2],[],[3,-1],[4,-2,4,-2],[],[1,-1],[2,-2,2,-2],null,[4,-1],[0,-2,0,-2],[],[2,-1],[3,-2,3,-2],[],[0,-1],[1,-2,1,-2],null,[3,-1],[4,-2,4,-2],[],[1,-1],[2,-2,2,-2],[],[4,-1],[0,-2,0,-2],null,[2,-1],[3,-2,3,-2],[]]}]},{"name":"empty.parquet","numRows":0,"rowGroups":[{"numRows":0,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[]}]}]} diff --git a/packages/effect/test/fixtures/parquet/gzip.parquet b/packages/effect/test/fixtures/parquet/gzip.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7d148b5d440a7a1d6ea0ce4042fe81b8f89b8bc6 GIT binary patch literal 3056 zcmb_eZEO@p7@nQk+b+j(y&j#-ZgVLMk@YZqlu9p^l+eAFQsAtm5uv0c*ItjKdbG#& zu#KWXAc9B?9~u+WU^R+L_+cV}1P!8K{D@y^RER$yf)M>>gC-!f)OTj@dVAbal{oFq z?K?B?Gw<`v`_5jo*)pGsw3Y@#RHn#_(6kq1#2O~IMyFv_AKd=3puJat&_k!{R_QQ7 zCF-KCHtGRC5ATuJr^*iI1WmQv5!$AR{sN;VC9sG*i%IjOEE2XOaTp}pk+46FY9F)q7Nj?E@EtoL4h<*lm& zM}IwXX#64LcZ-nvpP3?X%y*o3K=e<_TYpL0n<^dRFop^g}Yz#8aW?AG@ zWMnHQzqny_87*sr)4-7F9tcrl&SKUX`)w9?b}SSeV;2b`By?^T0O(Y2qe?CS$pOe( z7C=*xV{BwGB924n!YV(sQd*kFz!n5B$XXc&qgx4aeM;!-G}GjGlfw`VcpyP z<(vYhxDCpK{drjL^#wo#WUC!m0aGG7p8z`^jIcn!uB8q6_e>;=c1ZH4pPY8KSB%^) z7SQ(as6z)Y2seGx*itrAf%cm~nc#l7j>*mkGe~ZT%grORRrS9`dK!}xE+BDCpa-C= z^D>u}HNJGVe!Biw{N_-s_s0*{R26>{7QfgeJiqt*fq^X*A3HBE+pg|fja!{pcAcPC z-W*Jzp*=$v)>>r$YkgiPMBZ5f+_+;7Hz@3~4QJkkn?Ub7b37UE%rP7e(XIjnVEDQku2d zT8)8xRMe`FFjam?Wh$aj5-|%Y7LSLg@J{#$E#Etvhp2JEi7HRvXdx#h;hOF|#hl$AYlnVfIvKn#;jc@L3jz literal 0 HcmV?d00001 diff --git a/packages/effect/test/fixtures/parquet/int96.parquet b/packages/effect/test/fixtures/parquet/int96.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e5fb4484b61c1208a434195d697a204c9651a4c3 GIT binary patch literal 1014 zcmaJ=ZAg<*6n-{!sfm60&Ml`1zVEEu)VVUrr0&EoEK?y&NOa1alG8cOzN}Od5$!{a zqOuIqzDbgPDApf;s1;<=kCZY{BqW5OZy9FiZW;I^5BI(2x#yhsJm;Q!Zzjt&2oRpg zXy}Bs1Qmip13*rnhf@ahJSNoI5r)OZjOppUK+6X$$88rg2q#GYa4X>@;r4J@aB~xx zS_&i#UTJ?hx(gVel<L`UN>O&o1Kizv$ad=DGqfQ(HNS|3DxuiOZ1b&6n?6uDCc_)Lj-6tuo(Y89Xu z7WwW(LDUKI<*k&%o?l)CWu{uE)@Y#-6lzHU z^>ViWWsGvhi-y)aO+$4;@W#%~3bj_Dqie-oFpcFyZ>5>Iy$PdYbn zUkJHKQx=sU%g)}Hv%_7ps!tZhWh>*7<(e5wXY2?wSy?h;R+eaWSxqL!_HvZSSQ^)~ zUedHm;->w4B1GEC|4-$4NV0gaTawaUk%=xU7j)ZsoGIFzV<;;!W|tHmFclY;bGls4 zm%_DMTqO`pNPsHTZgx~2wp8nj>~_7`;ixzgT4Qlkl~$C8#>7TOM;g=`{-|%$rOu=m F{|5ql|EB-| literal 0 HcmV?d00001 diff --git a/packages/effect/test/fixtures/parquet/nested.parquet b/packages/effect/test/fixtures/parquet/nested.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a93f81da53d48f92269a14f9b52425653f1f6595 GIT binary patch literal 9726 zcmb_iYj73S9Y6Q(xtk?iFa&np8*kIdW4+=v34z2YRI-;CqFz&oLL%){LlOuOA|&Az zKgb6gt&JVq>9o{wtm9Zmn6^XJIxW*y)M;zA5BfoeTF2po<2W6}3~ep7==S$Ndv?#> z2QS-Y_x|^sJ^%OP{CC3|w|j}n&BHOAc{6LWYQ|!WF%8#Cp69ItbaCZy{S)H~z)u0I z0RIZO1n?=qF2LUbzKg35_x}cb7S|%&--RoV%Vg$!bAHyW!av>A(>5nAHYYw-WhOG3 zGe@FmjkH92_no-5knpo+HfvUg_#~apNiK_LxEcw3!c5e}5(K>8YK&{E1|OIKBCTd? z)|?j-A#rU^N?7p>S0hEFLlZ>q^DbkF#MYm@S33nO34Q9!x-4`F^dW6+PQN6N8io4k z!m!#bmWa=VB#!As(aBIYC$m{B!__E6*L1R@t|hcP*5d|_8!<=nW~Ff3EX)kOa&UO~ zp1M7+yf8XEK)%}1H@b7z;CKwTeKYX$-gkfS-eW&}|Ip*4(a<0kLyShVF>BUIEn4i% zjBu45>l4grb)pRi(M%kF^x#W1KZ&N_MY^-yQe*`^BVROQ^ zIngCgs*!TVuQct!-^77wn|2hFXxp4PQ#h%S!u*hHyAK>m2B=0DJ|v3GNiafHBZWGX zYad)D9Dgik9)tchLVq#~L)cKr6VxRnZ_$`MPf&y{AcYEzYb2a4VOl~;KYBh_LPtV~ zT;Oio2sqhBAOkYF%&byjNW{VM)+U+K^=5q}rHw#D+X#fWl9@8If}lw8S7Rb5Uq0|n z3^lDkMo}q3I?Qn+P|P+0)ro59tfG2Y6I5$DzJXQbFjc=Q1BV%bo$T<=GBV-4npN5x zLi^VY#^cCZGh=45W*7iQU}xJ1ERX0bGpp#$7W5WWK*ZPKBqRnLlt)RTv>=ykYZSjr z#BUQ=7L?!?o{4X3vg6wt70Ds8s`kv*xK>5gvT%L8bp0~1{?J2my-+DA(Jh7Rg}-D; zKJ;{)#kKZaz+!fo?WHU_1dGLs5i(Re3aV}kD_!rG$+(CxZg!fTdDI)S2pwbz3R?>L zN?aDMT3qy)91sYkwkcX|lcAZVXSf;_+wAPUumP2X0-M>KI7xgI6E#x!z74%U#lEF9f@|Bv_xPy#Pv^kM! zBmopBHBtn0;icsW#r;@|P&^vPnMaTZre`dD6pwJ!92J1s5?T_@ldxLCDhZPk8WM5| znS^IvWa5~X@Pvf^EMq5D%I6ZEk&w0d^Slij7w!Ru|0H?TmUro+*p|!8h8C!b(Vp-z zC>8Dr66eIqfRYiR6rGJw;@W{DZ+{9>Vk=Z7A`mwqHN8>^#oN?uNXm#Mw8q!41Hd5@ z=0qOlhTIYE|L1EP=7HPZG6%#e*auoFGZnJah;ICpXpraZcG;kQpsTro04^21e-7;aA$FbK=@e;xDKKtsm-K zz}B8s(1a-kbe}*BRc{6nXfxZgW;<>~FC2^6F=j2!2q=)K&deNIHr1g~vpo|#CX|N~ z3zqI&$k^97C0Gn+9-AF`6#b1ytu)m%^f=xxaNU6W8*ycE-Hj`W>r1%WaGk<)^hw@E z+z;TQ-H;;nb<7>Y^(R~u9KQ!_0t|Fkb&;ssocK_aq((|-e=qsuzMxkUHANA)TM!_j zh=3X?0{4rWvhow!fj5u{Ma?@2ZgWzDS|&9T6hvt(Xx_ixyr*`G%t@pQH(yTr+MF~e zQfj1xF@!K@Swb7__DK}gNHUYn$zasLtC1oRHt=tJg(z@y+#C-Yc#fIr?@v7pz)G@G zbnxfN$HE@ICc}x6%>-xBy>lF*rCIz!$F{~n*?l#cO=_n>_g+31qHh5l?}viUw#i`- z*0cq}nEyjRAESQ$!5GdwcFth0cpt_-21Y-d$=HKvhjv`fSnm}G-On?Y0namwQPu0w z`@_tKTNpcl_#4VGwx)xz)?0*3h4q{hw%WneNO|W&ic_@IMfIB#ZJQHk3Jo<&r(uLzVFcResR@VP0zy{-p@o2-laTU` z)?FzfSc&m`3EL%HBq43xv_3E4RT6ed2ptM)v~2{OZ6lD0n*7pPrITOu|I%7rzR3?n zwv9j;YR5}w1wl%>glzKPI$x6?C~6ylDTr$6tfDHL{NBPQU(|)Lb20)u*+yVNYVu2G zi?x5bwVG>3P~1vNjS-mIHUi^QpI|IthB^Pjxb zK7Z?H?DKog-aLA^pny=UQjFH%LVXZDKJA5+7pM`U*C&(1Jw^RGVaY;)eqW80JLJB$ zT=x5g`ND~t>b2EK;mdyi9ZfshMgkNy{3NC#phkj#$Orin_2S>D;m;(~pdt36Fl;%g zsA#B>62foz?_Ej#y68xXb~7@Lq=3?;Mv8*!_@BU9YDh{uexnVKa3tFVU{w-UOE^zL zOF~=18VS=9)=F3>;gu53mvDiE3ng46p(Ei{5@sa4x(zlf+#zJ2UK6&_Mw@D`qqCs} zF3|BW3Mr){l%lf{O6TwRuMSB$5mM3FkkmOk{t6a;4#d9gfBY+HGtSdth;OgqS3*lQ zypc{>#OBa4VwE`Fy5tJ#)*qn&FUSzQE|l*xB9!7Fq4|xw{k@0d+T1WtgAk9tbOggJ0Th_~95wAlF10kMJ!KUKTvc~+|@sB!dUS|44-{zGPbcS1sckbFU>Z?QoK#DvKfq6HzEsnU!qC)lQ}+)uC64U4j@TeMY@So+@BjFff_bWS!PM(~Z9 zM#!8(E;^LcQ;tywYY=hnE7lnh-&m|gr}eCX(|NnfIXOFRM?PCpE>p*8)ki;O2D2rc zeY4i3)*HHBL%VmxuB{YoD!?ePpk#;j6vyejbGh^QDUi1IrkqS|szv9N zY!R~+k@jO$5I>!;PA8nF)6UCRz=96YfyZsm_)wiCxZ_L^v+*m!!WtIA=<}?9WN=lmsyW+ zoX*sB&e>+Nta>vqG$N&S9%-EvX{}!|OxifVG)bd{}DO~6iwHlSEB|}4> zFGJ&kX`Q!@8#tY(+nn#*h@hlCkdis_GKjP;oh{+)8+2i6y{_}xTFG9P%c9&XO6wo5 zc20F%WLk6mqSJa(wxqPOtk*fxlWxggT3S;?o(R*Lr)}x9i%#pez8s{r>XAy*+9z!Njnev~ zdO(&++ex>(`-?p{_-&FsRoOsCtK06n?pj=Q-0He}+}vij!*w^it2T?dT+VfGqq|kE zdow|d1BlU`G_?+B?iQN9jUIy@D2~z-LArfi!<+Z z4dm?>{P7S+gulQiAG;fSI$*3jAa6rs!*0RPZSNNA#gEUAT$>aA74%$kRc?6WNcX!^4vNKJ#p7fzY+h5p@>PhuW5a= zd-GQCyXK~VUwL``pz!6Tc+5=7%2Jt6?HWGirX(Citlj`k4-lc1< zL;PX5_GaZ71awj3)@PQu%lkR$Yah5&Yeo z1d-c=@!CDYsJ9RmA+%2wOmHm2mD{zUw+ngT$5~%b^GH9~ZzlYu6ZjGNbeoXUkIzWF z2>rykueW=6qZqCl15(4zkpH@ivd0oye|>u)zi9pv6~D;%{Cb)jNlqLXOH;wfkP}Y1 zlTu04A@NWwgWi7pMdCNWcU>hv;qRf+0TYR(w8#%~pUxMUT_^wAc5VNT9oP1aj^4AU jZuj8m*p_=n>Y7^W8|#;(9>a^>xlT4Y8{f(CKfnG9vCBs) literal 0 HcmV?d00001 diff --git a/packages/effect/test/fixtures/parquet/nested_v2.parquet b/packages/effect/test/fixtures/parquet/nested_v2.parquet new file mode 100644 index 0000000000000000000000000000000000000000..71c94692452220138fb09d72bd2548dd216c7880 GIT binary patch literal 9569 zcmb_iYiu0V6`q}YcQV;*jBz}}?6P%D9>Wrx*m-zsLP|UvXXDVtO$@O^Y84#YiIX@c zv7HCz2dHWxO+%GRm6{^dqN=S(M5`23L?c=Ps4az-rU;Nr5y}sQ@}oecwkoB8>38nE zvoq`6yimtGd*5>a4mxtJX?bNqid?ugf@` zR5_fK*@j>Y}oGmW#= zLl4b_gsheAv+6B9DuJwZIQd}fSssq0gaV7-?_Z_G<2!!wUc+GoN4QaI)%KD3I7@Dj zrw*rYvLqY{Iflk&YjJ%J7;MWXUJX{%LV^eZMabb4FI&sPJ^q6@tscT+ zC~veH`|#!9uhy!u5_OrJlbu;w>csk`_w6|1Q;!^bKK(PDy-RygZ`IQV*5I$!GHDNT zHQBh3>qsJP)@oVm$yTOQh;&QugTM4wOW)vuEgHkEgq7%{`(ZG0lqtHVGX`fLh~ZO& zrR45Qzx4eRzw-T~zxMqHkS@c&_5HR#`F`VHeBXS-_s>oF{?yyP|2T5+$XVY%@OR(8 z`<(B0{?qrJ^S%#tFq^FBIO!XfIGjW{oP^W9g(K<5Zx}oHI~}ZI%WH+5C05elaMDky z6OJV7hn(#@dNLXCn&34!-pXF z$Q+IYGX?Rve={U_V}iPdsK6R*JSC$toxBSLgd?fFz;0f}Sj%p5Ts>xFXI0!36UB|5 z6OJUe!EQ#fJ%BaPc}mBphzDj6Yfa0X!Yv%>Q-WxYqPC*-iqGk)m3=oagOuS9NxJ!UXNvl(&b+dj5__t z5T!bO6;Wqa1)|J|sOYQ`#o5u5Z+{9|B7;mk^uo@s4B+j7taGbEv!b&?mKlq)mY31j zu_u$(WD$aBM@kR#i(!Cz1mHlG19nw*_go@7Lt%%q2YIxMSLLd&qTzrO$dakGoXuIg zRcDJM6)7JDqAliID$Qv zn)Oig>@p}N?KarF7@FEl;$C5`=(9Skg{TR7_#}ZHVlgMC*$cCR3XWOB%MTaWEIjz& zFl?WwXK}8;^({D?aNdnGiSsKs+i`w? z`@lT@7F-YFq~{uC(<_)ej`PnrDIfj_x)d~UnF=D({z!$xNf`m2a3ozGW}n<2U>?y! zh&zJT?-tx-7IB9o$^CV<>JxgJZYJf`&^FoaaPop+PdF0n@egGoFYYtI~+-Vfw|R{Cj$7H0Cn?L>@@J&GqW`9ATZ4CYcy@swaCxU zYg!iCK3a%|y%=B}Aw7}Pw4=zc;k>4;FKAlZt->1Bs_ExX(5g9!2%k*hNN&f6l#pp; zLNyRh8XQjgDHFnxWc@E^4;_1DHh5JlE(76Yk;BP8;tfZVwM6r-s>I+v+ZY$^)YvGjN9>bmgp&MyUL)OQQ0?Psa=*K_V+>rAVOiP35Ru z)PD^+b_A6NF>=fxdW6a=X9K5SXshGwB-&QkPs)IK5M##-qE5iPa#qSy!2Eq|42tCc z0nCp94qrYnf8>gQ`H!v$nD6*3z8VDUk47NCImI?8 z%V_|eSXDOwKo3X3W`1(B0_dt7${uf|FNDb9NHPPUzr)z6c2Zppp_7#%cQ_K<0-^td z5PCM5fdKJfnTF8Gy-+|nlIjcWzEy;=1)``1(J3m59UUngNp=a*AHy1?T@{FKw&NC# zWV;|*t)g{`)+=f&>L{94G^1#PqK%4Pr|4Wo=P5d0(FKaSie9g1R?!>U5v>bD^mGXe zo9!VMj?N1AF9)J8D2d8ML`7$nsLO!pHHiI#Z%`~ zt(s$u4-60O>bH_wtM2gaqZ8wJCt(y^ow#W2AnfeuGxrakZ>*?>kNc%neIiW{aq6u_la8f*_wDTuZYq}*&n zDrayiwn*5fiIzDkgufoQUyr*duXW!{BZ4m2)N>H&5soh4M+4G>C+KG!=}O2bW~b+U z?10#-#|`^&!#y<5J>G~2f(J+V;wAc)C^}oAS?!&c1$!iTok2a7ZCz{}sypoSaeUmf z*SoK|ncTulHRZ~%ejGzwGpQbK8cwijTFNp~7tlH(v>uAthhpxlO)D$jSfny2G&F2c zV-dOb=|y(dH9@q?D($m|eb#WFX?CX;VeMa-V5~Tk`iIWedGG4o4Mk zsmw6us?4}#RTu4(CO+=eE$&Mzk&}e`sFou?=}^^`vlW_UK zRsF&m_iJk|v#Pmq*;RcuZl8_2=hwLh*CB!-p0!cc;phT>psKlXWmSE8gL|&4RMjC? zivr8&Y=!2}Qq>3J_QAOO^B(u;#!Rm1%Bq?&@=U3!dB#zjcDYsk*yfDUVfWb@Vsr_>TO~!pZC1m z=xVj+ts{zYKry}nYp?b$Kd2|??ZH8=0?xQUF!x~{df2u9!#Ofb>&I*3W})79FMmd(8_bmess z|73}OtzeS+CBIljmwCa^OY4{15$G#iWc?(uZl(20CvTMYd0hw^>=XV;`E)6q^48O> zfn7ZZ275>D-Q3%~bzoWdy@S17EnboAyK%gEtBj9#fL6+&%J_(}xEz9av@h5n5enQ{ z?RD)4?Clo%aT7;Gzu>1m_O|pE5Ue|3Z*xoYKB3R+=oah6m()jFn-}{l)357$58vTe~gL=iN@r^BXDk#8o$aNA^z)MNWGC zOM6h{W{koHnO<|hX`Z(VZdGHgIm@CW_|`6rS#l7C{;M5?YO*E@v0 z%hrDZ`G?_#b>hAZx?D4JzsBdbQZ3>iH<5x*9bj z^m}WBklTathW#R_w~`cLw3G@ZIF{qg@7=tq3w0p#tiQK)bO7qN5&yDDe2IFxO;{=O zGmjBR0uNc z#HQRyr6l^0xG9#wZ<&9Q{0;a$FSMWd_fzXYh{RG_B$Hgq`J(1Ds(%ezG_ZU3qW-b5 md-gZ(8yXwme$Qy*(%j;f#Y<9;;%D83KHAr7@X{Oqx$1xNcVeLc literal 0 HcmV?d00001 diff --git a/packages/effect/test/fixtures/parquet/plain.parquet b/packages/effect/test/fixtures/parquet/plain.parquet new file mode 100644 index 0000000000000000000000000000000000000000..136896d67b52b253bbd4bc6c2d3914a74f88b66b GIT binary patch literal 8857 zcmb_i30M?I7Ooi@7!gs_=`?;U5|vq0Rt^!5fZ8w|q8LGxL!)vDD3^d-Mq!N+5;3m# z5j8AE*DGG}T6YCGJU~2SqM`xujE0C8>U#CQS2I05FnH|uZGYWUU9Vof_t&deuU<8b z;e(u&M9C>R2c-+Vm?%xGIhl-;5msSOOG`^ah%r3v=9JdUzF*b}xbkFhJ1A|%4vZiO zxf}ybc`UKltI^I#Mg}Mel)}iTf{?J?h>T2Nh+Bdw5x#yR{8RNp*im9i;tm;;tCK9q zkb9=2D)N<}Y&IgG#ZT$ypzN$u=k-f_!ZC%r9}1{Ws!??fZypI7ik}D{o65=1JA%+- z`6B^YkbS=8Q^Bb~jw0{N;MN<10-?c`%P9~jTzM>G-v*T`F@kpErh3Y!86NGctkoXch3U-^t{)dZbALc;fDRsq5j(mGi3$T3nEv% zY=Clo((9g8K)=!NPg}G(XwL@Z8757sCMg<4g8h5o6L>BmzIgzL$8|2UHPttA;lm1JuWwruwd4dyJwsq)MnuWFlu#_Mhs0pT375b#9%d zWn?8U_8Pvm*WA%x%;m#9KU@4$VHtnw{+Sy`JC>D!cJ16_D8PW>CbwrEd{?-TBRT^c zy#_1Oun`xNkdWK~DsdHC;LSF(Vt;BvbUPs2Ub*r1+U>t>U60Oz{Y{}viAbF>GbX(& zyk@m=fyT`;xE+-5Qr-l;i42|ybn3Pd{BKH^$dpUoqm;(C_b7eeuwi&DT-lA-6FsZS zQ9K5(JuNk_Us^E$ZPLXt_e3m8TZAmD z_ajTF$#rB2*lQmG1+||$wsuuxE64}MamqN!@7x%BOY56T;>I$4es`UJA@Ntf zcva)opqqsO$mCQX?Pv4r(v|&trtiEquDR#9Iq_RRZ-1`qyp#MY6tyIKx#eHuKD&{& zyYWPINA~E@x@}G6c~KLtZ_HWVhRK=S>Zf-(`m1wjxeSTGTrPuKKknhYa}4fq5Gfd` zM!Vuzjhjk4$#gp)5K4h}BRn{)CMcbuJf$s%!+SD37@bRj1_hZ9<#Bk*;8_FDHF$7J zwn7Pr%$4R+@N+|3Yo4}EyP+B3P6l9-Y2Unl8MTj)=UZTdzC}p&d_qn?B4lShAvy4# zJd%*w3c%<}$Wj1TTtmp&ZG@nx;h}_}u#1i$FrScp9}}{97a=_+(YbBy&CJsHus!7P3nwqXa*m&?-5Dvq>ou4lVeTNZ(tS@E|(s8UH94Qur zvzGzp7cU^D&MR}k)*5VD{TAX7dh@U!a^; zm6RV3Jv%1ldfx?Eg6YDeX;5c^)_;oP$eP)+y;roD%S;@zNfe+{V*V~H}uBD!jLNAekK!JuXy+SV+-h6 zw>WF*T`138|0I4XNDw8(npH%506jEaaUAAM7}5W}>3YBeIO=3@ORMm;9=DyO z!nGr?z0ajrgE9XTb=pigV)3TKE<^*`wY#B| zhB;0-0*-p(OV1+K%$|{IR38nbvziVg`N8AsE4vr<+lQ=Y9}W!Z^Uv;qx1a17c=Fhy zeow4#|G8nC8E^C^$*SiI|C1K`K8iKjJ8`x{P51+ge{?xvvg2Ee*Yh*T;}DZD0MTl# zJYC(q4AwaO;#!ev6atM+G9*IB&dSaXN*j35J9jX8=MK6Wp3Pc@sehI-OOKZcoMsN= z9P#Hz^b8=?sFHIQrXunR|x+y0Zbt`Sdq;dJQfp;XfaedgiBBE@=e6Y?ZNgl3}-ZB#iIN-QJe6A`(ELU8olD+9GF`MNpIAvM`!WAz-}u zeik1ckuLGa~G$g-Oaf&sShL&y@DZ4JY=aE!5cV4^htFK2uATkyBY z4%3JD)y)d-ov|yzV%A}ieCVXrczO1fXqVV8IX{|cM~cej7|BiKje7A0ig zp7*p0=vQ4fE=Di0DoEM!IoR}@AKXt}hJIuH%P$6l&OHvjR}Y3VBG$GKI$@6MorruW zWhcz$?1B>PZh*J4Jp6PgkjW_q+&T?o)aR0(f==87|KhNFLqJqoW4^T8cF;dxar z#Mk@zX%+6VX8T9&#Cq)?fwg8({;SvSC+G)%#0}ezWy!={x6zkGZ9^x z!~}Sglu6Q3gJdzrw4P%st5i;yy-GC+-K=D6eKPmj&5u`HFzbKPKHu!yh7jd{j!r}F zUtZ6fOIGF{zmrK6raO(+$kryfD;cX-9S0E3BrPf~CNV;34SMb5921$GoIsRvJHi>q zC#CmyAxZ|w`0nOPlnQt^otlswL0impHEc77nn_xEYJ5_x9T{bfBcY_o_@s!`Y@)PK zDwJ~0a%y~5O!VZm_?(!@Vv8JlGN56Qoz1B2T@;v30RoUV6Nt`kb{tSUVJf7dMn|N_ z^mnl{AC-V4gx4uOJ~1XOJt8q>a$?%#ndwn>oktZ~S|bK6M?2(jbDcahE#dRd)()Bm z4I^gJBgj7Qn(v@-L%4U zEnp<>y-loC3OE%I>@L@I*O-&`FEC;Yj8s$IIlT{@nBahOYd?O%%xa~fr=S!@lH%q{ zrQ{j2dB&>yM($i2-xe{k1#x)keTUQ+OF^2TB~ZLVBVL?as-Vz_Ei_Wyhmq=ighLN( z@kvgHv!y^YeSuoUjSSn!s9v-O#NwAJy{}`6w|pIIsX%8|bBzM6$5Fs@P1w06s@t91 zVRw>tWo>^2ac4Kltzc$!gBC?-{E&)RW5lj8QvJrdyO~K}LL>z2Bots~ueF+m27{qb zQ(0H#>{YpHZZ`_2#fJzF#@rq?V0F!i%Mod!e?U=M)Y_MLbWQ{`?*@1hAr%z5H zp7TIM8`Sq|lGeS(01SJTQ5E$872?56?=Wcgmcw96?pGjtv{ihN_-Bi(yAP+-QYq!w%C;>zl2^jt%>Z3~tp~7r-W+ zaxm^lHOZd1)3p_MY}f~KxK*26!G_{V;{#4JEm6ubG`AghY}gAjxK%3#0#50uVQ}zh zDNr|T!?z`dLpJwjdLvHhWjpv0?u$gIl$UcQ;6vC4m@qbQWtB zb!-4F+^XxIVE=a{#4V-i#%2(8L`0Qpil{}#+fqhrL{iideIm~XFufzMeT%$zL>NH^IlacI`32gjytPkeBp=T_xdqts!P^^1lU5%ai zZ2a*L=3M^B9o4Vs3GT713|J|G2Z`=cnI@#6wB`EiaJj*d|C^wO9j z4BoX?!8acI^L)6kk5hPX0GQ3Iua+N>sQYT|u%xE>0$)||?yf>ZyB(WZ-kq6ELmt~2I0GKdB*6WGl_w~<+bfIem{RbdXRu972YX4;Wm#CDKzeJ>_CTH4a#H6OhCnwpuxb<`D V=WNvoX~PHJqyavljDdej`7if2;tc=* literal 0 HcmV?d00001 From a4c97152c05842fc0ede411e399e4b2593dde941 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 20:16:33 +0000 Subject: [PATCH 20/32] Cap fixed-width and per-page value allocations, fix spec references --- packages/effect/src/encoding/Parquet.ts | 36 ++++++++-- .../effect/src/internal/parquet/encodings.ts | 11 ++- packages/effect/test/encoding/Parquet.test.ts | 68 +++++++++++++++++-- .../test/fixtures/parquet/CONFORMANCE.md | 14 ++-- 4 files changed, 107 insertions(+), 22 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index 3b92ca35a98..2362937b189 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -463,6 +463,10 @@ export const fromBytes = (bytes: Uint8Array): Source => ({ * `GZIP`, pages may hold several gzip members. `LZ4` is the deprecated * Hadoop-framed codec; `LZ4_RAW` is the plain LZ4 block format. * + * **Gotchas** + * + * Report failures in the error channel: a function that throws is a defect. + * * @stability experimental * @category models * @since 4.1.5 @@ -1704,15 +1708,23 @@ const decompress = ( * {@link Decompress} for it in `codecs`. Dictionary-encoded values are looked * up in the dictionary, so the result never refers to it. * - * Limits for untrusted files: `maxPageSize` (default 256 MiB) caps each page's - * compressed and uncompressed size, `maxValues` (default 2^26) the values of - * the column chunk, nulls included, and `maxBinaryLength` (default 1 GiB) the - * total bytes of its `BYTE_ARRAY` values. Each is at most 2^31 - 1. + * Limits for untrusted files, each at most 2^31 - 1: + * + * - `maxPageSize` (default 256 MiB) caps each page's compressed and + * uncompressed size. + * - `maxValues` (default 2^24) caps the values of the column chunk and of + * each page, nulls included. + * - `maxBinaryLength` (default 1 GiB) caps the total bytes of the chunk's + * `BYTE_ARRAY`, `FIXED_LEN_BYTE_ARRAY` and `INT96` values. Fixed-width + * values are counted for every value, nulls included. * * **Gotchas** * - * `rowGroup` and `column` must be indexes into `metadata`: anything else is - * a defect, not a failure. + * - Memory and time grow with the values times the nodes on the column's + * path: every optional node has a validity bitmap. Lower `maxValues` for + * deeply nested columns from untrusted files. + * - `rowGroup` and `column` must be indexes into `metadata`: anything else + * is a defect, not a failure. * * @stability experimental * @category decoding @@ -1732,7 +1744,7 @@ export const readColumn = ( ): Effect.Effect => Effect.gen(function*() { const maxPageSize = limit("maxPageSize", options.maxPageSize, 256 * 1024 * 1024, MAX_INT32) - const maxValues = limit("maxValues", options.maxValues, 2 ** 26, MAX_INT32) + const maxValues = limit("maxValues", options.maxValues, 2 ** 24, MAX_INT32) const maxBinaryLength = limit("maxBinaryLength", options.maxBinaryLength, 1024 * 1024 * 1024, MAX_INT32) const group = Number.isInteger(options.rowGroup) ? metadata.rowGroups[options.rowGroup] : undefined const column = Number.isInteger(options.column) ? metadata.columns[options.column] : undefined @@ -1750,6 +1762,13 @@ export const readColumn = ( if (meta.numValues > maxValues) { return yield* error("Limit", `${where} has ${meta.numValues} values, more than maxValues (${maxValues})`) } + const width = column.physicalType === "INT96" ? 12 : column.typeLength ?? 0 + if (meta.numValues * width > maxBinaryLength) { + return yield* error( + "Limit", + `${where} has ${meta.numValues} values of ${width} bytes, more than maxBinaryLength (${maxBinaryLength})` + ) + } const decoder = new ChunkDecoder(column, meta.numValues, { left: maxBinaryLength }) // A chunk without values needs no pages; parquet-cpp writes data_page_offset 0 for one. if (meta.numValues === 0) return yield* Effect.mapError(attempt(() => decoder.finish(group.numRows)), locate(where)) @@ -1782,6 +1801,9 @@ export const readColumn = ( if (header.compressedSize > maxPageSize || header.uncompressedSize > maxPageSize) { return yield* error("Limit", `a page of ${where} is larger than maxPageSize (${maxPageSize})`) } + if (header.numValues > maxValues) { + return yield* error("Limit", `a page of ${where} has more values than maxValues (${maxValues})`) + } pos = body + header.compressedSize const page = bytes.subarray(body, pos) const decompressed = (data: Uint8Array, size: number, codec = meta.codec) => diff --git a/packages/effect/src/internal/parquet/encodings.ts b/packages/effect/src/internal/parquet/encodings.ts index 9d49a026238..d10fd9a5533 100644 --- a/packages/effect/src/internal/parquet/encodings.ts +++ b/packages/effect/src/internal/parquet/encodings.ts @@ -216,10 +216,17 @@ export const bitPacked = ( ): number => { const size = Math.ceil(n * width / 8) if (size > end - pos) fail("Malformed", `${what} need ${size} bytes`) - let bit = pos * 8 + let at = pos + let bit = 0 for (let i = 0; i < n; i++) { let value = 0 - for (let k = 0; k < width; k++, bit++) value = (value << 1) | ((bytes[bit >> 3] >> (7 - (bit & 7))) & 1) + for (let k = 0; k < width; k++) { + value = (value << 1) | ((bytes[at] >> (7 - bit)) & 1) + if (++bit === 8) { + bit = 0 + at++ + } + } out[i] = value } return pos + size diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index 2163fcc3c28..f6caa744f7d 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -451,7 +451,7 @@ describe("Parquet", () => { /repetition/ )) - it.effect("PQ-SCHEMA-2 MUST rebuild the schema tree from its depth-first elements and num_children (L525-526)", () => + it.effect("PQ-SCHEMA-2 MUST rebuild the schema tree from its depth-first elements and num_children (L518-523, L542-547)", () => Effect.gen(function*() { const schema = [ root(2), @@ -593,7 +593,7 @@ describe("Parquet", () => { yield* failsWith(values(repeatedInt32([part(0, 1, 1), part(1, 2, 0)], 1, 2)), "Malformed", /row boundary/) })) - it.effect("PQ-V2-2 MUST read v2 levels uncompressed and without a length prefix (L742-745, L763)", () => + it.effect("PQ-V2-2 MUST read v2 levels uncompressed and without a length prefix (L744-745, L767)", () => Effect.gen(function*() { const values_ = int32s(5, 7) const decompress: Parquet.Decompress = (data, size) => { @@ -616,7 +616,7 @@ describe("Parquet", () => { ]) })) - it.effect("PQ-V2-3 MUST NOT decompress the values of a v2 page whose is_compressed is false (L745)", () => + it.effect("PQ-V2-3 MUST NOT decompress the values of a v2 page whose is_compressed is false (L745, L779)", () => Effect.gen(function*() { const v2 = dataPageV2({ numValues: 3, @@ -629,7 +629,7 @@ describe("Parquet", () => { assert.deepStrictEqual(yield* values(optionalInt32([v2], 3, GZIP)), [5, null, 7]) })) - it.effect("PQ-V2-4 MUST hold num_values - num_nulls values in the data section (L748-751)", () => + it.effect("PQ-V2-4 MUST hold num_values - num_nulls values in the data section (L755-756)", () => Effect.gen(function*() { const v2 = dataPageV2({ numValues: 3, numNulls: 0, numRows: 3, definition: someNull, values: int32s(5, 7) }) yield* failsWith(values(optionalInt32([v2], 3)), "Malformed", /nulls/) @@ -774,7 +774,7 @@ describe("Parquet", () => { chunks: [{ type: Type.INT32, path: ["g0", "g1", "g2", "g3", "g4", "g5", "x"], numValues: 8, pages: [page] }] }) - it.effect("PQ-BP-1 MUST unpack BIT_PACKED levels from the most significant bit (L188-197)", () => + it.effect("PQ-BP-1 MUST unpack BIT_PACKED levels from the most significant bit (L182-195)", () => Effect.gen(function*() { // the spec's example: 0 to 7 with bit width 3 is 00000101 00111001 01110111 const page = dataPage(8, bytes([0b00000101, 0b00111001, 0b01110111], int32s(42)), { levels: BIT_PACKED }) @@ -783,7 +783,7 @@ describe("Parquet", () => { assert.deepStrictEqual(column.layers.map((l) => l.nullCount), [1, 2, 3, 4, 5, 6, 7]) })) - it.effect("PQ-BP-2 MUST NOT read BIT_PACKED as a values encoding (L198-199)", () => + it.effect("PQ-BP-2 MUST NOT read BIT_PACKED as a values encoding (L197-198)", () => failsWith( values( flat({ @@ -939,6 +939,7 @@ describe("Parquet", () => { const error = yield* Effect.flip(values(make(gzipSync(body), GZIP), { codecs: { GZIP: failing } })) assert.strictEqual(error.kind, "Codec") assert.strictEqual(error.cause, "corrupt") + assert.match(error.message, /^column "x" of row group 0, page at byte 4: GZIP decompression failed/) })) it.effect("PQ-COMP-2 SHOULD read GZIP pages holding several members (L61), given a Decompress that does", () => @@ -1054,6 +1055,61 @@ describe("Parquet", () => { yield* failsWith(read(strings, { maxBinaryLength: 4 }), "Limit", /maxBinaryLength/) })) + it.effect("caps fixed-width values and per-page value counts", () => + Effect.gen(function*() { + const wide = flat({ + type: Type.FIXED_LEN_BYTE_ARRAY, + typeLength: 0x7fffffff, + repetition: OPTIONAL, + numRows: 1000, + pages: [dataPage(1000, lengthPrefixed(runs(1, [0, 1000])))] + }) + yield* failsWith(read(wide), "Limit", /maxBinaryLength/) + const dictionary = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + dictionary: true, + pages: [ + dictionaryPage(8, int32s(0, 1, 2, 3, 4, 5, 6, 7)), + dataPage(3, bytes([3], runs(3, [7, 3])), { encoding: RLE_DICTIONARY }) + ] + }) + assert.deepStrictEqual(rows(yield* read(dictionary)), [7, 7, 7]) + yield* failsWith(read(dictionary, { maxValues: 5 }), "Limit", /maxValues/) + })) + + it.effect("checks page sizes before calling a codec", () => + Effect.gen(function*() { + const calls: Array = [] + const codecs = { + GZIP: (data: Uint8Array, size: number) => { + calls.push(size) + return gzip(data, size) + } + } + const v2 = (definitionLength: number, uncompressedSize: number) => + page( + 3, + { + 8: Struct({ 1: I32(1), 2: I32(0), 3: I32(1), 4: I32(PLAIN), 5: I32(definitionLength), 6: I32(0) }) + }, + new Uint8Array(4), + uncompressedSize + ) + const make = (page_: Uint8Array) => + flat({ type: Type.INT32, repetition: OPTIONAL, numRows: 1, codec: GZIP, pages: [page_] }) + yield* failsWith( + read(make(dataPage(1, new Uint8Array(4), { uncompressedSize: -1 })), { codecs }), + "Malformed", + /negative size/ + ) + yield* failsWith(read(make(v2(5, 8)), { codecs }), "Malformed", /levels of a v2 data page/) + yield* failsWith(read(make(v2(3, 2)), { codecs }), "Malformed", /levels of a v2 data page/) + yield* failsWith(read(make(v2(-1, 8)), { codecs }), "Malformed", /negative/) + assert.deepStrictEqual(calls, []) + })) + it.effect("dies on invalid options and indexes", () => Effect.gen(function*() { const dies = (effect: Effect.Effect) => diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index 28bcc3a40cf..712cb0c9fef 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -23,7 +23,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-META-1 | parquet.thrift L1411-1413 | Readers should accept versions 1 and 2 interchangeably | | PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | | PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | -| PQ-SCHEMA-2 | parquet.thrift L525-526, L487-540 | The schema tree is the depth-first list of elements, groups giving `num_children` | +| PQ-SCHEMA-2 | parquet.thrift L518-523, L542-547 | The schema tree is the depth-first list of elements, groups giving `num_children` | | PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | | PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | | PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | @@ -32,9 +32,9 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-CHUNK-1 | README.md#column-chunks, parquet.thrift L727-729 | The dictionary page comes first, and there is at most one | | PQ-CHUNK-2 | README.md#column-chunks | Readers can skip pages they don't read (index pages, later page types) | | PQ-V2-1 | parquet.thrift L759-761 | Every v2 data page begins at a row boundary | -| PQ-V2-2 | parquet.thrift L742-745, L763 | v2 levels are uncompressed RLE without a length prefix | -| PQ-V2-3 | parquet.thrift L745 | v2 values are compressed only when `is_compressed` is true | -| PQ-V2-4 | parquet.thrift L748-751 | A v2 page holds `num_values - num_nulls` values | +| PQ-V2-2 | parquet.thrift L744-745, L767 | v2 levels are uncompressed RLE without a length prefix | +| PQ-V2-3 | parquet.thrift L745, L779 | v2 values are compressed only when `is_compressed` is true | +| PQ-V2-4 | parquet.thrift L755-756 | A v2 page holds `num_values - num_nulls` values | | PQ-STATS-1 | parquet.thrift L286-289 | Readers must distinguish an absent `null_count` from 0 | | PQ-STATS-2 | parquet.thrift L315-316 | Readers must not assume `nan_count` is 0 when absent | | PQ-PLAIN-1 | Encodings.md#plain-plain--0 | PLAIN encoding of every physical type | @@ -43,8 +43,8 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-RLE-1 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 1 | Bit-packed runs are packed from the least significant bit | | PQ-RLE-2 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3 | Run lengths are at least 1 | | PQ-RLE-3 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, length table | RLE booleans carry a 4-byte length in v1 and v2 pages | -| PQ-BP-1 | Encodings.md#bit-packed-deprecated-bit_packed--4 | `BIT_PACKED` levels are packed from the most significant bit | -| PQ-BP-2 | Encodings.md#bit-packed-deprecated-bit_packed--4 | `BIT_PACKED` is only for levels | +| PQ-BP-1 | Encodings.md#bit-packed-deprecated-bit_packed--4, L182-195 | `BIT_PACKED` levels are packed from the most significant bit | +| PQ-BP-2 | Encodings.md#bit-packed-deprecated-bit_packed--4, L197-198 | `BIT_PACKED` is only for levels | | PQ-DELTA-1 | Encodings.md#delta-encoding-delta_binary_packed--5, examples 1 and 2 | Header, blocks and miniblocks | | PQ-DELTA-2 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary padding bits | | PQ-DELTA-3 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary bit widths of unneeded miniblocks | @@ -70,7 +70,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. ## Deferred: not read yet -- Page index: `ColumnIndex` and `OffsetIndex` (PageIndex.md, parquet.thrift L1240-1360). The reader reads every page in order. +- Page index: `ColumnIndex` and `OffsetIndex` (PageIndex.md, parquet.thrift L1235-1373). The reader reads every page in order. - Bloom filters (BloomFilter.md). - Page checksums (README.md#checksumming). `crc` is skipped, not verified. - Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. From 3ebf8d110727ae599eb28ec77d4e17ef692a4d84 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 20:36:19 +0000 Subject: [PATCH 21/32] Simplify Parquet reader and tests --- packages/effect/src/encoding/Parquet.ts | 900 ++++++++---------- .../effect/src/internal/parquet/encodings.ts | 247 ++--- packages/effect/test/encoding/Parquet.test.ts | 194 ++-- 3 files changed, 590 insertions(+), 751 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index 2362937b189..a5862525a24 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -548,10 +548,13 @@ export interface Column { // Errors and options // --------------------------------------------------------------------------- -const { Failure, fail } = Encodings +const { Failure, fail, u32 } = Encodings + +const error = (kind: ParquetError["kind"], message: string, cause?: unknown): ParquetError => + new ParquetError(cause === undefined ? { kind, message } : { kind, message, cause }) const usage = (message: string): never => { - throw new ParquetError({ kind: "Usage", message }) + throw error("Usage", message) } const limit = (name: string, value: number | undefined, fallback: number, max: number): number => { @@ -570,28 +573,18 @@ const attempt = (f: () => A): Effect.Effect => try { return Effect.succeed(f()) } catch (cause) { - if (cause instanceof Failure) return Effect.fail(new ParquetError({ kind: cause.kind, message: cause.message })) + if (cause instanceof Failure) return Effect.fail(error(cause.kind, cause.message)) if (Thrift.isThriftError(cause) && cause.kind !== "Usage") { return Effect.fail( - new ParquetError({ - kind: cause.kind === "Limit" ? "Limit" : "Malformed", - message: `invalid Thrift metadata: ${cause.message}` - }) + error(cause.kind === "Limit" ? "Limit" : "Malformed", `invalid Thrift metadata: ${cause.message}`) ) } return Effect.die(cause) } }) -const error = (kind: ParquetError["kind"], message: string) => new ParquetError({ kind, message }) - /** Prefixes an error's message with where in the file it happened. */ -const locate = (where: string) => (e: ParquetError): ParquetError => - new ParquetError({ - kind: e.kind, - message: `${where}: ${e.message}`, - ...(e.cause === undefined ? {} : { cause: e.cause }) - }) +const locate = (where: string) => (e: ParquetError): ParquetError => error(e.kind, `${where}: ${e.message}`, e.cause) const readRange = ( source: Source, @@ -683,6 +676,32 @@ const struct = (r: Reader, field: (id: number, type: Thrift.Type) => void): void r.readStructEnd() } +/** A struct field: its id and how to read it. */ +type Field = readonly [id: number, read: (r: Reader, type: Thrift.Type) => A] + +/** + * Reads a struct into an object with a key per entry of `fields`: the value of + * the field with that id, or `undefined` when it is absent. Other fields are + * skipped. + */ +const readStruct = }>( + r: Reader, + fields: F +): { [K in keyof F]: ReturnType | undefined } => { + const out: Record = {} + const keys: Array = [] + for (const key in fields) { + out[key] = undefined + keys[fields[key][0]] = key + } + struct(r, (id, type) => { + const key = keys[id] + if (key === undefined) r.skip(type) + else out[key] = fields[key][1](r, type) + }) + return out as { [K in keyof F]: ReturnType | undefined } +} + // Generated Thrift code skips a field of an unexpected type, as these do. const skip = (r: Reader, type: Thrift.Type): undefined => { r.skip(type) @@ -694,77 +713,44 @@ const i64 = (r: Reader, type: Thrift.Type) => type === T.I64 ? r.readI64Number() const bool = (r: Reader, type: Thrift.Type) => type === T.Bool ? r.readBool() : skip(r, type) const string = (r: Reader, type: Thrift.Type) => type === T.Binary ? r.readString() : skip(r, type) const binary = (r: Reader, type: Thrift.Type) => type === T.Binary ? r.readBinary() : skip(r, type) -const list = (r: Reader, type: Thrift.Type, element: Thrift.Type, read: () => A): Array | undefined => { - if (type !== T.List) return skip(r, type) - const out: Array = [] - for (let n = r.readListBegin(), elementType = r.elementType; n > 0; n--) { - if (elementType !== element) return fail("Malformed", "a metadata list has elements of the wrong type") - out.push(read()) +const structOf = (read: (r: Reader) => A) => (r: Reader, type: Thrift.Type): A | undefined => + type === T.Struct ? read(r) : skip(r, type) +const list = + (element: Thrift.Type, read: (r: Reader) => A) => (r: Reader, type: Thrift.Type): Array | undefined => { + if (type !== T.List) return skip(r, type) + const out: Array = [] + for (let n = r.readListBegin(), elementType = r.elementType; n > 0; n--) { + if (elementType !== element) return fail("Malformed", "a metadata list has elements of the wrong type") + out.push(read(r)) + } + return out } - return out +/** For a field whose presence is all that matters. */ +const present = (r: Reader, type: Thrift.Type): true => { + r.skip(type) + return true } const required = (value: A | undefined, what: string): A => value === undefined ? fail("Malformed", `${what} is missing`) : value const keyValue = (r: Reader): KeyValue => { - let key: string | undefined - let value: string | undefined - struct(r, (id, type) => { - if (id === 1) key = string(r, type) - else if (id === 2) value = string(r, type) - else r.skip(type) - }) - return { key: required(key, "KeyValue.key"), value } + const f = readStruct(r, { key: [1, string], value: [2, string] }) + return { key: required(f.key, "KeyValue.key"), value: f.value } } -const statistics = (r: Reader): Statistics => { - const s = { - nullCount: undefined as number | undefined, - distinctCount: undefined as number | undefined, - nanCount: undefined as number | undefined, - minValue: undefined as Uint8Array | undefined, - maxValue: undefined as Uint8Array | undefined, - isMinValueExact: undefined as boolean | undefined, - isMaxValueExact: undefined as boolean | undefined, - min: undefined as Uint8Array | undefined, - max: undefined as Uint8Array | undefined - } - struct(r, (id, type) => { - switch (id) { - case 1: - s.max = binary(r, type) - break - case 2: - s.min = binary(r, type) - break - case 3: - s.nullCount = i64(r, type) - break - case 4: - s.distinctCount = i64(r, type) - break - case 5: - s.maxValue = binary(r, type) - break - case 6: - s.minValue = binary(r, type) - break - case 7: - s.isMaxValueExact = bool(r, type) - break - case 8: - s.isMinValueExact = bool(r, type) - break - case 9: - s.nanCount = i64(r, type) - break - default: - r.skip(type) - } +const statistics = (r: Reader): Statistics => + readStruct(r, { + nullCount: [3, i64], + distinctCount: [4, i64], + nanCount: [9, i64], + minValue: [6, binary], + maxValue: [5, binary], + isMinValueExact: [8, bool], + isMaxValueExact: [7, bool], + min: [2, binary], + max: [1, binary] }) - return s -} const simpleLogicalTypes = { 1: "STRING", @@ -780,22 +766,14 @@ const simpleLogicalTypes = { 19: "FILE" } as const -/** Reads a struct's fields into an array indexed by field id; unread field types are skipped. */ -const fields = (r: Reader, read: (id: number, type: Thrift.Type) => unknown): Array => { - const out: Array = [] - struct(r, (id, type) => { - out[id] = read(id, type) - }) - return out -} - -const timeUnit = (r: Reader): "MILLIS" | "MICROS" | "NANOS" | undefined => { - let unit: "MILLIS" | "MICROS" | "NANOS" | undefined +/** Reads a union of empty structs, such as TimeUnit or ColumnOrder, as the name of its field. */ +const unionName = (r: Reader, names: ReadonlyArray): A | undefined => { + let out: A | undefined struct(r, (id, type) => { r.skip(type) - if (type === T.Struct) unit = timeUnits[id as 1 | 2 | 3] + if (type === T.Struct) out = names[id] }) - return unit + return out } const logicalType = (r: Reader): LogicalType | undefined => { @@ -810,48 +788,44 @@ const logicalType = (r: Reader): LogicalType | undefined => { } switch (id) { case 5: { - const f = fields(r, (_, t) => i32(r, t)) + const f = readStruct(r, { scale: [1, i32], precision: [2, i32] }) out = { type: "DECIMAL", - scale: required(f[1] as number | undefined, "DecimalType.scale"), - precision: required(f[2] as number | undefined, "DecimalType.precision") + scale: required(f.scale, "DecimalType.scale"), + precision: required(f.precision, "DecimalType.precision") } return } case 7: case 8: { - const f = fields(r, (fid, t) => fid === 1 ? bool(r, t) : fid === 2 && t === T.Struct ? timeUnit(r) : skip(r, t)) - const unit = f[2] as "MILLIS" | "MICROS" | "NANOS" | undefined - const isAdjustedToUTC = required(f[1] as boolean | undefined, "TimeType.isAdjustedToUTC") - out = unit === undefined ? undefined : { type: id === 7 ? "TIME" : "TIMESTAMP", isAdjustedToUTC, unit } + const f = readStruct(r, { isAdjustedToUTC: [1, bool], unit: [2, structOf((r) => unionName(r, timeUnits))] }) + const isAdjustedToUTC = required(f.isAdjustedToUTC, "TimeType.isAdjustedToUTC") + out = f.unit === undefined + ? undefined + : { type: id === 7 ? "TIME" : "TIMESTAMP", isAdjustedToUTC, unit: f.unit } return } case 10: { - const f = fields(r, (fid, t) => fid === 1 ? i8(r, t) : fid === 2 ? bool(r, t) : skip(r, t)) + const f = readStruct(r, { bitWidth: [1, i8], isSigned: [2, bool] }) out = { type: "INTEGER", - bitWidth: required(f[1] as number | undefined, "IntType.bitWidth"), - isSigned: required(f[2] as boolean | undefined, "IntType.isSigned") + bitWidth: required(f.bitWidth, "IntType.bitWidth"), + isSigned: required(f.isSigned, "IntType.isSigned") } return } - case 16: { - const f = fields(r, (fid, t) => fid === 1 ? i8(r, t) : skip(r, t)) - out = { type: "VARIANT", specificationVersion: f[1] as number | undefined } + case 16: + out = { type: "VARIANT", specificationVersion: readStruct(r, { version: [1, i8] }).version } return - } - case 17: { - const f = fields(r, (fid, t) => fid === 1 ? string(r, t) : skip(r, t)) - out = { type: "GEOMETRY", crs: f[1] as string | undefined } + case 17: + out = { type: "GEOMETRY", crs: readStruct(r, { crs: [1, string] }).crs } return - } case 18: { - const f = fields(r, (fid, t) => fid === 1 ? string(r, t) : fid === 2 ? i32(r, t) : skip(r, t)) - const algorithm = f[2] as number | undefined + const f = readStruct(r, { crs: [1, string], algorithm: [2, i32] }) out = { type: "GEOGRAPHY", - crs: f[1] as string | undefined, - algorithm: algorithm === undefined ? undefined : name(edgeAlgorithms, algorithm) + crs: f.crs, + algorithm: f.algorithm === undefined ? undefined : name(edgeAlgorithms, f.algorithm) } return } @@ -863,144 +837,77 @@ const logicalType = (r: Reader): LogicalType | undefined => { return out } -interface Element { - readonly type: number | undefined - readonly typeLength: number | undefined - readonly repetition: number | undefined - readonly name: string - readonly numChildren: number | undefined - readonly convertedType: number | undefined - readonly scale: number | undefined - readonly precision: number | undefined - readonly fieldId: number | undefined - readonly logicalType: LogicalType | undefined -} - -const schemaElement = (r: Reader): Element => { - const f = fields(r, (id, type) => { - switch (id) { - case 4: - return string(r, type) - case 10: - return type === T.Struct ? logicalType(r) : skip(r, type) - case 1: - case 2: - case 3: - case 5: - case 6: - case 7: - case 8: - case 9: - return i32(r, type) - default: - return skip(r, type) - } +const schemaElement = (r: Reader) => { + const f = readStruct(r, { + type: [1, i32], + typeLength: [2, i32], + repetition: [3, i32], + name: [4, string], + numChildren: [5, i32], + convertedType: [6, i32], + scale: [7, i32], + precision: [8, i32], + fieldId: [9, i32], + logicalType: [10, structOf(logicalType)] }) - return { - type: f[1] as number | undefined, - typeLength: f[2] as number | undefined, - repetition: f[3] as number | undefined, - name: required(f[4] as string | undefined, "SchemaElement.name"), - numChildren: f[5] as number | undefined, - convertedType: f[6] as number | undefined, - scale: f[7] as number | undefined, - precision: f[8] as number | undefined, - fieldId: f[9] as number | undefined, - logicalType: f[10] as LogicalType | undefined - } + return { ...f, name: required(f.name, "SchemaElement.name") } } +type Element = ReturnType + const columnMetaData = (r: Reader): ColumnMetaData => { - const f = fields(r, (id, type) => { - switch (id) { - case 1: - case 4: - return i32(r, type) - case 2: - return list(r, type, T.I32, () => name(encodings, r.readI32())) - case 3: - return list(r, type, T.Binary, () => r.readString()) - case 5: - case 6: - case 7: - case 9: - case 10: - case 11: - return i64(r, type) - case 8: - return list(r, type, T.Struct, () => keyValue(r)) - case 12: - return type === T.Struct ? statistics(r) : skip(r, type) - default: - return skip(r, type) - } + const f = readStruct(r, { + type: [1, i32], + encodings: [2, list(T.I32, (r) => name(encodings, r.readI32()))], + pathInSchema: [3, list(T.Binary, (r) => r.readString())], + codec: [4, i32], + numValues: [5, i64], + totalUncompressedSize: [6, i64], + totalCompressedSize: [7, i64], + keyValueMetadata: [8, list(T.Struct, keyValue)], + dataPageOffset: [9, i64], + indexPageOffset: [10, i64], + dictionaryPageOffset: [11, i64], + statistics: [12, structOf(statistics)] }) - const type = required(f[1] as number | undefined, "ColumnMetaData.type") + const type = required(f.type, "ColumnMetaData.type") return { physicalType: physicalTypes[type] ?? fail("Unsupported", `physical type ${type} is not supported`), - encodings: required(f[2] as Array | undefined, "ColumnMetaData.encodings"), - pathInSchema: required(f[3] as Array | undefined, "ColumnMetaData.path_in_schema"), - codec: name(codecs, required(f[4] as number | undefined, "ColumnMetaData.codec")), - numValues: required(f[5] as number | undefined, "ColumnMetaData.num_values"), - totalUncompressedSize: required(f[6] as number | undefined, "ColumnMetaData.total_uncompressed_size"), - totalCompressedSize: required(f[7] as number | undefined, "ColumnMetaData.total_compressed_size"), - keyValueMetadata: (f[8] as Array | undefined) ?? [], - dataPageOffset: required(f[9] as number | undefined, "ColumnMetaData.data_page_offset"), - indexPageOffset: f[10] as number | undefined, - dictionaryPageOffset: f[11] as number | undefined, - statistics: f[12] as Statistics | undefined + encodings: required(f.encodings, "ColumnMetaData.encodings"), + pathInSchema: required(f.pathInSchema, "ColumnMetaData.path_in_schema"), + codec: name(codecs, required(f.codec, "ColumnMetaData.codec")), + numValues: required(f.numValues, "ColumnMetaData.num_values"), + totalUncompressedSize: required(f.totalUncompressedSize, "ColumnMetaData.total_uncompressed_size"), + totalCompressedSize: required(f.totalCompressedSize, "ColumnMetaData.total_compressed_size"), + keyValueMetadata: f.keyValueMetadata ?? [], + dataPageOffset: required(f.dataPageOffset, "ColumnMetaData.data_page_offset"), + indexPageOffset: f.indexPageOffset, + dictionaryPageOffset: f.dictionaryPageOffset, + statistics: f.statistics } } const columnChunk = (r: Reader): ColumnChunk => { - const f = fields(r, (id, type) => { - switch (id) { - case 1: - return string(r, type) - case 3: - return type === T.Struct ? columnMetaData(r) : skip(r, type) - case 8: - case 9: - r.skip(type) - return true - default: - return skip(r, type) - } + const f = readStruct(r, { + filePath: [1, string], + metaData: [3, structOf(columnMetaData)], + cryptoMetadata: [8, present], + encryptedColumnMetadata: [9, present] }) - const encrypted = f[8] === true || f[9] === true - const metaData = f[3] as ColumnMetaData | undefined - if (!encrypted && metaData === undefined) fail("Malformed", "ColumnChunk.meta_data is missing") - return { filePath: f[1] as string | undefined, encrypted, metaData } + const encrypted = f.cryptoMetadata === true || f.encryptedColumnMetadata === true + if (!encrypted && f.metaData === undefined) fail("Malformed", "ColumnChunk.meta_data is missing") + return { filePath: f.filePath, encrypted, metaData: f.metaData } } const rowGroup = (r: Reader): RowGroup => { - const f = fields(r, (id, type) => { - switch (id) { - case 1: - return list(r, type, T.Struct, () => columnChunk(r)) - case 2: - case 3: - return i64(r, type) - default: - return skip(r, type) - } - }) + const f = readStruct(r, { columns: [1, list(T.Struct, columnChunk)], totalByteSize: [2, i64], numRows: [3, i64] }) return { - columns: required(f[1] as Array | undefined, "RowGroup.columns"), - totalByteSize: required(f[2] as number | undefined, "RowGroup.total_byte_size"), - numRows: required(f[3] as number | undefined, "RowGroup.num_rows") + columns: required(f.columns, "RowGroup.columns"), + totalByteSize: required(f.totalByteSize, "RowGroup.total_byte_size"), + numRows: required(f.numRows, "RowGroup.num_rows") } } -const columnOrder = (r: Reader): ColumnOrder | undefined => { - let order: ColumnOrder | undefined - struct(r, (id, type) => { - r.skip(type) - if (type === T.Struct) order = columnOrders[id as 1 | 2 | 3] - }) - return order -} - interface Schema { readonly root: SchemaNode readonly columns: Array @@ -1076,30 +983,19 @@ const schemaTree = ( const fileMetaData = (footer: Uint8Array, maxDepth: number): FileMetaData => { const r = Thrift.makeReader(footer) - const f = fields(r, (id, type) => { - switch (id) { - case 1: - return i32(r, type) - case 2: - return list(r, type, T.Struct, () => schemaElement(r)) - case 3: - return i64(r, type) - case 4: - return list(r, type, T.Struct, () => rowGroup(r)) - case 5: - return list(r, type, T.Struct, () => keyValue(r)) - case 6: - return string(r, type) - case 7: - return list(r, type, T.Struct, () => columnOrder(r)) - default: - return skip(r, type) - } + const f = readStruct(r, { + version: [1, i32], + schema: [2, list(T.Struct, schemaElement)], + numRows: [3, i64], + rowGroups: [4, list(T.Struct, rowGroup)], + keyValueMetadata: [5, list(T.Struct, keyValue)], + createdBy: [6, string], + orders: [7, list(T.Struct, (r) => unionName(r, columnOrders))] }) - const elements = required(f[2] as Array | undefined, "FileMetaData.schema") + const elements = required(f.schema, "FileMetaData.schema") if (elements.length === 0) fail("Malformed", "FileMetaData.schema is empty") - const { columns, root } = schemaTree(elements, f[7] as Array | undefined, maxDepth) - const rowGroups = required(f[4] as Array | undefined, "FileMetaData.row_groups") + const { columns, root } = schemaTree(elements, f.orders, maxDepth) + const rowGroups = required(f.rowGroups, "FileMetaData.row_groups") for (const group of rowGroups) { if (group.columns.length !== columns.length) { fail("Malformed", `a row group has ${group.columns.length} column chunks for ${columns.length} columns`) @@ -1120,10 +1016,10 @@ const fileMetaData = (footer: Uint8Array, maxDepth: number): FileMetaData => { }) } return { - version: required(f[1] as number | undefined, "FileMetaData.version"), - numRows: required(f[3] as number | undefined, "FileMetaData.num_rows"), - createdBy: f[6] as string | undefined, - keyValueMetadata: (f[5] as Array | undefined) ?? [], + version: required(f.version, "FileMetaData.version"), + numRows: required(f.numRows, "FileMetaData.num_rows"), + createdBy: f.createdBy, + keyValueMetadata: f.keyValueMetadata ?? [], schema: root, columns, rowGroups @@ -1133,9 +1029,6 @@ const fileMetaData = (footer: Uint8Array, maxDepth: number): FileMetaData => { const MAGIC = 0x31524150 // "PAR1" little-endian const MAGIC_ENCRYPTED = 0x45524150 // "PARE" -const u32 = (bytes: Uint8Array, at: number): number => - (bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16) | (bytes[at + 3] << 24)) >>> 0 - /** * Reads the metadata in the footer of a Parquet file. * @@ -1184,16 +1077,6 @@ export const readMetadata = ( // Pages // --------------------------------------------------------------------------- -const PLAIN = 0 -const PLAIN_DICTIONARY = 2 -const RLE = 3 -const BIT_PACKED = 4 -const DELTA_BINARY_PACKED = 5 -const DELTA_LENGTH_BYTE_ARRAY = 6 -const DELTA_BYTE_ARRAY = 7 -const RLE_DICTIONARY = 8 -const BYTE_STREAM_SPLIT = 9 - const DATA_PAGE = 0 const DICTIONARY_PAGE = 2 const DATA_PAGE_V2 = 3 @@ -1203,9 +1086,9 @@ interface PageHeader { readonly uncompressedSize: number readonly compressedSize: number readonly numValues: number - readonly encoding: number - readonly definitionEncoding: number - readonly repetitionEncoding: number + readonly encoding: Encoding | number + readonly definitionEncoding: Encoding | number + readonly repetitionEncoding: Encoding | number readonly numNulls: number readonly numRows: number readonly definitionLength: number @@ -1214,81 +1097,100 @@ interface PageHeader { } const pageHeader = (r: Reader): PageHeader => { - const ints = (id: number, type: Thrift.Type) => id === 7 ? bool(r, type) : i32(r, type) - const f = fields( - r, - (id, type) => id <= 3 ? i32(r, type) : id >= 5 && id <= 8 && type === T.Struct ? fields(r, ints) : skip(r, type) - ) - const type = required(f[1] as number | undefined, "PageHeader.type") - const uncompressedSize = required(f[2] as number | undefined, "PageHeader.uncompressed_page_size") - const compressedSize = required(f[3] as number | undefined, "PageHeader.compressed_page_size") + const f = readStruct(r, { + type: [1, i32], + uncompressedSize: [2, i32], + compressedSize: [3, i32], + dataPage: [ + 5, + structOf((r) => + readStruct(r, { + numValues: [1, i32], + encoding: [2, i32], + definitionEncoding: [3, i32], + repetitionEncoding: [4, i32] + }) + ) + ], + dictionaryPage: [7, structOf((r) => readStruct(r, { numValues: [1, i32], encoding: [2, i32] }))], + dataPageV2: [ + 8, + structOf((r) => + readStruct(r, { + numValues: [1, i32], + numNulls: [2, i32], + numRows: [3, i32], + encoding: [4, i32], + definitionLength: [5, i32], + repetitionLength: [6, i32], + compressed: [7, bool] + }) + ) + ] + }) + const type = required(f.type, "PageHeader.type") + const uncompressedSize = required(f.uncompressedSize, "PageHeader.uncompressed_page_size") + const compressedSize = required(f.compressedSize, "PageHeader.compressed_page_size") if (uncompressedSize < 0 || compressedSize < 0) fail("Malformed", "a page header has a negative size") - const page = (id: number, what: string) => required(f[id] as Array | undefined, what) - const header = { type, uncompressedSize, compressedSize } - const count = (n: number | boolean | undefined, what: string): number => { - const value = required(n as number | undefined, what) + const count = (n: number | undefined, what: string): number => { + const value = required(n, what) if (value < 0) fail("Malformed", `${what} is negative`) return value } + const encoding = (code: number | undefined, what: string) => name(encodings, required(code, what)) + const header: PageHeader = { + type, + uncompressedSize, + compressedSize, + numValues: 0, + encoding: "PLAIN", + definitionEncoding: "RLE", + repetitionEncoding: "RLE", + numNulls: 0, + numRows: 0, + definitionLength: 0, + repetitionLength: 0, + compressed: true + } switch (type) { case DATA_PAGE: { - const p = page(5, "PageHeader.data_page_header") + const p = required(f.dataPage, "PageHeader.data_page_header") return { ...header, - numValues: count(p[1], "DataPageHeader.num_values"), - encoding: required(p[2] as number | undefined, "DataPageHeader.encoding"), - definitionEncoding: required(p[3] as number | undefined, "DataPageHeader.definition_level_encoding"), - repetitionEncoding: required(p[4] as number | undefined, "DataPageHeader.repetition_level_encoding"), - numNulls: 0, - numRows: 0, - definitionLength: 0, - repetitionLength: 0, - compressed: true + numValues: count(p.numValues, "DataPageHeader.num_values"), + encoding: encoding(p.encoding, "DataPageHeader.encoding"), + definitionEncoding: encoding(p.definitionEncoding, "DataPageHeader.definition_level_encoding"), + repetitionEncoding: encoding(p.repetitionEncoding, "DataPageHeader.repetition_level_encoding") } } case DATA_PAGE_V2: { - const p = page(8, "PageHeader.data_page_header_v2") - return { + const p = required(f.dataPageV2, "PageHeader.data_page_header_v2") + const v2 = { ...header, - numValues: count(p[1], "DataPageHeaderV2.num_values"), - numNulls: count(p[2], "DataPageHeaderV2.num_nulls"), - numRows: count(p[3], "DataPageHeaderV2.num_rows"), - encoding: required(p[4] as number | undefined, "DataPageHeaderV2.encoding"), - definitionLength: count(p[5], "DataPageHeaderV2.definition_levels_byte_length"), - repetitionLength: count(p[6], "DataPageHeaderV2.repetition_levels_byte_length"), - compressed: p[7] !== false, - definitionEncoding: RLE, - repetitionEncoding: RLE + numValues: count(p.numValues, "DataPageHeaderV2.num_values"), + numNulls: count(p.numNulls, "DataPageHeaderV2.num_nulls"), + numRows: count(p.numRows, "DataPageHeaderV2.num_rows"), + encoding: encoding(p.encoding, "DataPageHeaderV2.encoding"), + definitionLength: count(p.definitionLength, "DataPageHeaderV2.definition_levels_byte_length"), + repetitionLength: count(p.repetitionLength, "DataPageHeaderV2.repetition_levels_byte_length"), + compressed: p.compressed !== false + } + const levels = v2.repetitionLength + v2.definitionLength + if (levels > compressedSize || levels > uncompressedSize) { + fail("Malformed", "the levels of a v2 data page are larger than the page") } + return v2 } case DICTIONARY_PAGE: { - const p = page(7, "PageHeader.dictionary_page_header") + const p = required(f.dictionaryPage, "PageHeader.dictionary_page_header") return { ...header, - numValues: count(p[1], "DictionaryPageHeader.num_values"), - encoding: required(p[2] as number | undefined, "DictionaryPageHeader.encoding"), - definitionEncoding: RLE, - repetitionEncoding: RLE, - numNulls: 0, - numRows: 0, - definitionLength: 0, - repetitionLength: 0, - compressed: true + numValues: count(p.numValues, "DictionaryPageHeader.num_values"), + encoding: encoding(p.encoding, "DictionaryPageHeader.encoding") } } default: - return { - ...header, - numValues: 0, - encoding: PLAIN, - definitionEncoding: RLE, - repetitionEncoding: RLE, - numNulls: 0, - numRows: 0, - definitionLength: 0, - repetitionLength: 0, - compressed: true - } + return header } } @@ -1305,7 +1207,7 @@ const kindOf = (column: ColumnDescriptor): Encodings.Kind => { case "DOUBLE": return { tag: "f64", width: 8 } case "INT96": - return { tag: "fixed", width: 12 } + return { tag: "int96", width: 12 } case "FIXED_LEN_BYTE_ARRAY": return { tag: "fixed", width: column.typeLength! } case "BYTE_ARRAY": @@ -1313,185 +1215,176 @@ const kindOf = (column: ColumnDescriptor): Encodings.Kind => { } } -const encodingName = (code: number): string => String(name(encodings, code)) +/** Decodes RLE/bit-packed hybrid levels from 0 to `max` into `out`, unless the column has none. */ +const decodeLevels = ( + bytes: Uint8Array, + start: number, + end: number, + max: number, + out: Uint8Array | undefined, + what: string +): void => { + if (out !== undefined) Encodings.hybrid(bytes, start, end, Encodings.bitWidth(max), max, out, what) +} /** Decodes the pages of one column chunk and assembles its {@link Column}. */ class ChunkDecoder { + readonly column: ColumnDescriptor readonly kind: Encodings.Kind - readonly maxDefinition: number - readonly maxRepetition: number + readonly numValues: number + private readonly budget: Encodings.Budget + /** The levels of every value, nulls included; `undefined` when the column has none. */ + private readonly repetitions: Uint8Array | undefined + private readonly definitions: Uint8Array | undefined + private readonly dense: Array = [] private dictionary: Encodings.Dense | undefined private seenData = false - private readonly definitions: Array = [] - private readonly repetitions: Array = [] - private readonly dense: Array = [] /** Levels decoded so far: values, nulls included. */ count = 0 - readonly column: ColumnDescriptor - readonly numValues: number - private readonly budget: Encodings.Budget - - constructor(column: ColumnDescriptor, numValues: number, budget: Encodings.Budget) { + constructor(column: ColumnDescriptor, kind: Encodings.Kind, numValues: number, maxBinaryLength: number) { this.column = column + this.kind = kind this.numValues = numValues - this.budget = budget - this.kind = kindOf(column) - this.maxDefinition = column.maxDefinitionLevel - this.maxRepetition = column.maxRepetitionLevel + this.budget = { left: maxBinaryLength } + // Fixed-width values take their bytes in every slot, nulls included. + if (kind.tag === "fixed" || kind.tag === "int96") Encodings.spend(this.budget, numValues * kind.width) + this.repetitions = column.maxRepetitionLevel === 0 ? undefined : new Uint8Array(numValues) + this.definitions = column.maxDefinitionLevel === 0 ? undefined : new Uint8Array(numValues) } dictionaryPage(header: PageHeader, data: Uint8Array): void { if (this.dictionary !== undefined || this.seenData) { fail("Malformed", "a dictionary page must be the first page of a column chunk, and the only one") } - if (header.encoding !== PLAIN && header.encoding !== PLAIN_DICTIONARY) { - fail("Malformed", `a dictionary page is encoded with ${encodingName(header.encoding)}, not PLAIN`) + if (header.encoding !== "PLAIN" && header.encoding !== "PLAIN_DICTIONARY") { + fail("Malformed", `a dictionary page is encoded with ${header.encoding}, not PLAIN`) } this.dictionary = Encodings.plain(this.kind, data, 0, data.length, header.numValues, this.budget) } - private start(n: number): void { - if (n > this.numValues - this.count) fail("Malformed", "the pages hold more values than their column chunk") - this.seenData = true - } - /** A v1 data page: repetition levels, definition levels and values, all after decompression. */ dataPage(header: PageHeader, page: Uint8Array): void { - const n = header.numValues - this.start(n) + const [repetition, definition] = this.start(header.numValues) let pos = 0 - const levels = (encoding: number, max: number, what: string): Uint8Array | undefined => { - if (max === 0) return undefined - const out = new Uint8Array(n) - const width = Encodings.bitWidth(max) - if (encoding === RLE) { + const levels = (out: Uint8Array | undefined, encoding: Encoding | number, max: number, what: string) => { + if (out === undefined) return + if (encoding === "RLE") { const end = Encodings.lengthPrefix(page, pos, page.length, what) - Encodings.hybrid(page, pos + 4, end, width, n, out, what) + decodeLevels(page, pos + 4, end, max, out, what) pos = end - } else if (encoding === BIT_PACKED) { - pos = Encodings.bitPacked(page, pos, page.length, width, n, out, what) + } else if (encoding === "BIT_PACKED") { + pos = Encodings.bitPacked(page, pos, page.length, Encodings.bitWidth(max), max, out, what) } else { - fail("Unsupported", `${what} encoded with ${encodingName(encoding)} are not supported`) + fail("Unsupported", `${what} encoded with ${encoding} are not supported`) } - return out } - const repetition = levels(header.repetitionEncoding, this.maxRepetition, "repetition levels") - const definition = levels(header.definitionEncoding, this.maxDefinition, "definition levels") - this.levels(n, repetition, definition) - this.values(header.encoding, page, pos, this.nonNull(n, definition)) + levels(repetition, header.repetitionEncoding, this.column.maxRepetitionLevel, "repetition levels") + levels(definition, header.definitionEncoding, this.column.maxDefinitionLevel, "definition levels") + this.values(header.encoding, page, pos, this.tally(header.numValues, repetition, definition).nonNull) } /** A v2 data page: the levels as written and the values after decompression. */ dataPageV2(header: PageHeader, levels: Uint8Array, values: Uint8Array): void { const n = header.numValues - this.start(n) - const decode = (start: number, end: number, max: number, what: string) => { - if (max === 0) return undefined - const out = new Uint8Array(n) - Encodings.hybrid(levels, start, end, Encodings.bitWidth(max), n, out, what) - return out - } - const repetitionEnd = header.repetitionLength - const repetition = decode(0, repetitionEnd, this.maxRepetition, "repetition levels") - const definition = decode(repetitionEnd, levels.length, this.maxDefinition, "definition levels") - this.levels(n, repetition, definition) - const nonNull = this.nonNull(n, definition) + const [repetition, definition] = this.start(n) + const split = header.repetitionLength + decodeLevels(levels, 0, split, this.column.maxRepetitionLevel, repetition, "repetition levels") + decodeLevels(levels, split, levels.length, this.column.maxDefinitionLevel, definition, "definition levels") + const { nonNull, rows } = this.tally(n, repetition, definition) if (n - nonNull !== header.numNulls) { fail("Malformed", `a v2 data page has ${n - nonNull} nulls, but its header says ${header.numNulls}`) } if (repetition !== undefined && n > 0 && repetition[0] !== 0) { fail("Malformed", "a v2 data page does not start at a row boundary") } - let rows = n - if (repetition !== undefined) { - rows = 0 - for (let i = 0; i < n; i++) if (repetition[i] === 0) rows++ - } if (rows !== header.numRows) { fail("Malformed", `a v2 data page has ${rows} rows, but its header says ${header.numRows}`) } this.values(header.encoding, values, 0, nonNull) } - private levels(n: number, repetition: Uint8Array | undefined, definition: Uint8Array | undefined): void { - const check = (levels: Uint8Array | undefined, max: number, what: string) => { - if (levels === undefined) return - for (let i = 0; i < n; i++) { - if (levels[i] > max) fail("Malformed", `${what} ${levels[i]} is above the column's maximum of ${max}`) - } - } - check(repetition, this.maxRepetition, "repetition level") - check(definition, this.maxDefinition, "definition level") - if (repetition !== undefined) this.repetitions.push(repetition) - if (definition !== undefined) this.definitions.push(definition) - this.count += n + /** Starts a data page of `n` values, and returns its parts of the chunk's repetition and definition levels. */ + private start(n: number): readonly [Uint8Array | undefined, Uint8Array | undefined] { + if (n > this.numValues - this.count) fail("Malformed", "the pages hold more values than their column chunk") + this.seenData = true + const end = this.count + n + return [this.repetitions?.subarray(this.count, end), this.definitions?.subarray(this.count, end)] } - private nonNull(n: number, definition: Uint8Array | undefined): number { - if (definition === undefined) return n - let count = 0 - for (let i = 0; i < n; i++) if (definition[i] === this.maxDefinition) count++ - return count + /** Counts a page's non-null values and rows, and adds its levels to the chunk's. */ + private tally( + n: number, + repetition: Uint8Array | undefined, + definition: Uint8Array | undefined + ): { readonly nonNull: number; readonly rows: number } { + const max = this.column.maxDefinitionLevel + let nonNull = 0 + let rows = 0 + for (let i = 0; i < n; i++) { + if (definition === undefined || definition[i] === max) nonNull++ + if (repetition === undefined || repetition[i] === 0) rows++ + } + this.count += n + return { nonNull, rows } } - private values(encoding: number, bytes: Uint8Array, pos: number, n: number): void { + private values(encoding: Encoding | number, bytes: Uint8Array, pos: number, n: number): void { const kind = this.kind const end = bytes.length - const type = this.column.physicalType - const invalid = (): never => fail("Malformed", `${encodingName(encoding)} is not an encoding for ${type} values`) + const invalid = (): never => + fail("Malformed", `${encoding} is not an encoding for ${this.column.physicalType} values`) let dense: Encodings.Dense switch (encoding) { - case PLAIN: + case "PLAIN": dense = Encodings.plain(kind, bytes, pos, end, n, this.budget) break - case PLAIN_DICTIONARY: - case RLE_DICTIONARY: { - if (this.dictionary === undefined) fail("Malformed", "a dictionary-encoded page has no dictionary page") + case "PLAIN_DICTIONARY": + case "RLE_DICTIONARY": { + const dictionary = this.dictionary + if (dictionary === undefined) return fail("Malformed", "a dictionary-encoded page has no dictionary page") if (n === 0) return if (pos >= end) fail("Malformed", "dictionary indices are truncated") const width = bytes[pos] if (width > 32) fail("Malformed", `dictionary indices have a bit width of ${width}, more than 32`) const indices = new Int32Array(n) - Encodings.hybrid(bytes, pos + 1, end, width, n, indices, "dictionary indices") - dense = Encodings.gather(kind, this.dictionary!, indices, this.budget) + const max = Encodings.lengthOf(kind, dictionary) - 1 + if (max < 0) fail("Malformed", "a dictionary-encoded page has values but the dictionary is empty") + Encodings.hybrid(bytes, pos + 1, end, width, max, indices, "dictionary indices") + dense = Encodings.gather(kind, dictionary, indices, this.budget) break } - case RLE: { + case "RLE": { if (kind.tag !== "bool") return invalid() const out = new Uint8Array(n) const stop = Encodings.lengthPrefix(bytes, pos, end, "RLE BOOLEAN values") - Encodings.hybrid(bytes, pos + 4, stop, 1, n, out, "RLE BOOLEAN values") + Encodings.hybrid(bytes, pos + 4, stop, 1, 1, out, "RLE BOOLEAN values") dense = out break } - case DELTA_BINARY_PACKED: + case "DELTA_BINARY_PACKED": if (kind.tag === "i32") dense = Encodings.deltaBinaryPacked(bytes, pos, end, n, false)[0] else if (kind.tag === "i64") dense = Encodings.deltaBinaryPacked(bytes, pos, end, n, true)[0] else return invalid() break - case DELTA_LENGTH_BYTE_ARRAY: + case "DELTA_LENGTH_BYTE_ARRAY": if (kind.tag !== "bytes") return invalid() - dense = Encodings.deltaLengthByteArray(bytes, pos, end, n, this.budget)[0] + dense = Encodings.deltaLengthByteArray(bytes, pos, end, n, this.budget) break - case DELTA_BYTE_ARRAY: + case "DELTA_BYTE_ARRAY": if (kind.tag === "bytes") dense = Encodings.deltaByteArray(bytes, pos, end, n, this.budget) - else if (type === "FIXED_LEN_BYTE_ARRAY") { - dense = Encodings.toFixed( - Encodings.deltaByteArray(bytes, pos, end, n, this.budget), - kind.tag === "fixed" ? kind.width : 0, - "DELTA_BYTE_ARRAY" - ) + else if (kind.tag === "fixed") { + dense = Encodings.deltaByteArray(bytes, pos, end, n, this.budget, kind.width).data } else return invalid() break - case BYTE_STREAM_SPLIT: - if (kind.tag === "bool" || kind.tag === "bytes" || type === "INT96") return invalid() + case "BYTE_STREAM_SPLIT": + if (kind.tag === "bool" || kind.tag === "bytes" || kind.tag === "int96") return invalid() dense = Encodings.byteStreamSplit(kind, bytes, pos, end, n) break - case BIT_PACKED: + case "BIT_PACKED": return invalid() default: - return fail("Unsupported", `the ${encodingName(encoding)} encoding is not supported`) + return fail("Unsupported", `the ${encoding} encoding is not supported`) } this.dense.push(dense) } @@ -1500,21 +1393,9 @@ class ChunkDecoder { if (this.count !== this.numValues) { fail("Malformed", `the column chunk ends after ${this.count} of its ${this.numValues} values`) } - const join = (parts: Array) => - parts.length === 1 ? parts[0] : parts.length === 0 ? undefined : concatBytes(parts, this.count) - const dense = this.dense.length === 0 ? Encodings.empty(this.kind) : Encodings.concat(this.kind, this.dense) - return assemble(this.column, this.kind, numRows, this.count, join(this.repetitions), join(this.definitions), dense) - } -} - -const concatBytes = (parts: ReadonlyArray, total: number): Uint8Array => { - const out = new Uint8Array(total) - let at = 0 - for (const part of parts) { - out.set(part, at) - at += part.length + const dense = Encodings.concat(this.kind, this.dense) + return assemble(this.column, this.kind, numRows, this.count, this.repetitions, this.definitions, dense) } - return out } /** @@ -1648,25 +1529,25 @@ const spread = ( } return { type: "BYTE_ARRAY", offsets, values: values.data } } + case "int96": case "fixed": { const width = kind.width let values = dense as Uint8Array if (!full) { + const from = values values = new Uint8Array(slots * width) - const from = dense as Uint8Array for (let j = 0, k = 0; j < slots; j++) { - if (present[j] === 1) values.set(from.subarray(k * width, ++k * width), j * width) + if (present[j] !== 1) continue + for (let at = j * width, stop = at + width; at < stop; at++) values[at] = from[k++] } } - return column.physicalType === "INT96" - ? { type: "INT96", values } - : { type: "FIXED_LEN_BYTE_ARRAY", width, values } + return kind.tag === "int96" ? { type: "INT96", values } : { type: "FIXED_LEN_BYTE_ARRAY", width, values } } default: { let values = dense as Int32Array if (!full) { const from = values - values = new (from.constructor as Int32ArrayConstructor)(slots) + values = Encodings.alloc(kind, slots) as Int32Array for (let j = 0, k = 0; j < slots; j++) if (present[j] === 1) values[j] = from[k++] } return { type: column.physicalType, values } as Values @@ -1690,10 +1571,7 @@ const decompress = ( return Effect.fail(error("Unsupported", `${codec} pages need a Decompress for ${codec} in the codecs option`)) } return Effect.flatMap( - Effect.mapError( - f(data, size), - (cause) => new ParquetError({ kind: "Codec", message: `${codec} decompression failed`, cause }) - ), + Effect.mapError(f(data, size), (cause) => error("Codec", `${codec} decompression failed`, cause)), checked ) } @@ -1762,78 +1640,64 @@ export const readColumn = ( if (meta.numValues > maxValues) { return yield* error("Limit", `${where} has ${meta.numValues} values, more than maxValues (${maxValues})`) } - const width = column.physicalType === "INT96" ? 12 : column.typeLength ?? 0 - if (meta.numValues * width > maxBinaryLength) { - return yield* error( - "Limit", - `${where} has ${meta.numValues} values of ${width} bytes, more than maxBinaryLength (${maxBinaryLength})` - ) - } - const decoder = new ChunkDecoder(column, meta.numValues, { left: maxBinaryLength }) + const decoder = yield* Effect.mapError( + attempt(() => new ChunkDecoder(column, kindOf(column), meta.numValues, maxBinaryLength)), + locate(where) + ) // A chunk without values needs no pages; parquet-cpp writes data_page_offset 0 for one. - if (meta.numValues === 0) return yield* Effect.mapError(attempt(() => decoder.finish(group.numRows)), locate(where)) - let start = meta.dataPageOffset - const dictionary = meta.dictionaryPageOffset - if (dictionary !== undefined && dictionary > 0 && dictionary < start) start = dictionary - const length = meta.totalCompressedSize - if (start < 4 || length < 0 || length > source.size - start) { - return yield* error("Malformed", `${where} has pages outside the file`) - } - const bytes = yield* readRange(source, start, length) - const thrift = Thrift.makeReader(bytes) - let pos = 0 - while (decoder.count < meta.numValues) { - if (pos >= bytes.length) { - return yield* error("Malformed", `${where} ends after ${decoder.count} of its ${meta.numValues} values`) - } - const headerAt = pos - const header = yield* Effect.mapError( - attempt(() => { - thrift.position = pos - return pageHeader(thrift) - }), - locate(`${where}, page header at byte ${start + headerAt}`) - ) - const body = thrift.position - if (header.compressedSize > bytes.length - body) { - return yield* error("Malformed", `a page of ${where} is larger than its column chunk`) - } - if (header.compressedSize > maxPageSize || header.uncompressedSize > maxPageSize) { - return yield* error("Limit", `a page of ${where} is larger than maxPageSize (${maxPageSize})`) - } - if (header.numValues > maxValues) { - return yield* error("Limit", `a page of ${where} has more values than maxValues (${maxValues})`) + if (meta.numValues > 0) { + let start = meta.dataPageOffset + const dictionary = meta.dictionaryPageOffset + if (dictionary !== undefined && dictionary > 0 && dictionary < start) start = dictionary + const length = meta.totalCompressedSize + if (start < 4 || length < 0 || length > source.size - start) { + return yield* error("Malformed", `${where} has pages outside the file`) } - pos = body + header.compressedSize - const page = bytes.subarray(body, pos) - const decompressed = (data: Uint8Array, size: number, codec = meta.codec) => - decompress(codec, options.codecs, data, size) - const decode = Effect.gen(function*() { - switch (header.type) { - case DICTIONARY_PAGE: { - const data = yield* decompressed(page, header.uncompressedSize) - return yield* attempt(() => decoder.dictionaryPage(header, data)) - } - case DATA_PAGE: { - const data = yield* decompressed(page, header.uncompressedSize) - return yield* attempt(() => decoder.dataPage(header, data)) - } - case DATA_PAGE_V2: { - const levels = header.repetitionLength + header.definitionLength - if (levels > page.length || levels > header.uncompressedSize) { - return yield* error("Malformed", "the levels of a v2 data page are larger than the page") - } - const data = yield* decompressed( - page.subarray(levels), - header.uncompressedSize - levels, - header.compressed ? meta.codec : "UNCOMPRESSED" - ) - return yield* attempt(() => decoder.dataPageV2(header, page.subarray(0, levels), data)) - } - // Index pages and page types added later are skipped. + const bytes = yield* readRange(source, start, length) + const thrift = Thrift.makeReader(bytes) + let pos = 0 + while (decoder.count < meta.numValues) { + if (pos >= bytes.length) { + return yield* error("Malformed", `${where} ends after ${decoder.count} of its ${meta.numValues} values`) + } + const headerAt = start + pos + const header = yield* Effect.mapError( + attempt(() => { + thrift.position = pos + return pageHeader(thrift) + }), + locate(`${where}, page header at byte ${headerAt}`) + ) + const body = thrift.position + if (header.compressedSize > bytes.length - body) { + return yield* error("Malformed", `a page of ${where} is larger than its column chunk`) + } + if (header.compressedSize > maxPageSize || header.uncompressedSize > maxPageSize) { + return yield* error("Limit", `a page of ${where} is larger than maxPageSize (${maxPageSize})`) } - }) - yield* Effect.mapError(decode, locate(`${where}, page at byte ${start + headerAt}`)) + // Caps dictionary pages too, and rejects a page before decompressing it. + if (header.numValues > maxValues) { + return yield* error("Limit", `a page of ${where} has more values than maxValues (${maxValues})`) + } + pos = body + header.compressedSize + // Index pages and page types added later are skipped. + if (header.type !== DATA_PAGE && header.type !== DATA_PAGE_V2 && header.type !== DICTIONARY_PAGE) continue + const page = bytes.subarray(body, pos) + const levels = header.type === DATA_PAGE_V2 ? header.repetitionLength + header.definitionLength : 0 + const codec = header.compressed ? meta.codec : "UNCOMPRESSED" + yield* decompress(codec, options.codecs, page.subarray(levels), header.uncompressedSize - levels).pipe( + Effect.flatMap((data) => + attempt(() => + header.type === DICTIONARY_PAGE + ? decoder.dictionaryPage(header, data) + : header.type === DATA_PAGE + ? decoder.dataPage(header, data) + : decoder.dataPageV2(header, page.subarray(0, levels), data) + ) + ), + Effect.mapError(locate(`${where}, page at byte ${headerAt}`)) + ) + } } return yield* Effect.mapError(attempt(() => decoder.finish(group.numRows)), locate(where)) }) diff --git a/packages/effect/src/internal/parquet/encodings.ts b/packages/effect/src/internal/parquet/encodings.ts index d10fd9a5533..26268373511 100644 --- a/packages/effect/src/internal/parquet/encodings.ts +++ b/packages/effect/src/internal/parquet/encodings.ts @@ -7,6 +7,7 @@ * * @internal */ +import * as Bytes from "../../Bytes.ts" /** @internal */ export class Failure { @@ -25,8 +26,8 @@ export const fail = (kind: Failure["kind"], message: string): never => { /** * How a physical type's values are held while decoding: `bool` one byte (0 or 1) per - * value, `fixed` `width` bytes per value (INT96 and FIXED_LEN_BYTE_ARRAY), `bytes` - * offsets into concatenated data (BYTE_ARRAY). + * value, `int96` 12 bytes and `fixed` `width` bytes per value (FIXED_LEN_BYTE_ARRAY), + * `bytes` offsets into concatenated data (BYTE_ARRAY). * * @internal */ @@ -34,9 +35,12 @@ export type Kind = | { readonly tag: "bool" } | { readonly tag: "i32" | "f32"; readonly width: 4 } | { readonly tag: "i64" | "f64"; readonly width: 8 } + | { readonly tag: "int96"; readonly width: 12 } | { readonly tag: "fixed"; readonly width: number } | { readonly tag: "bytes" } +type Fixed = Extract + /** @internal */ export interface Binary { readonly offsets: Int32Array @@ -46,13 +50,14 @@ export interface Binary { /** @internal */ export type Dense = Uint8Array | Int32Array | BigInt64Array | Float32Array | Float64Array | Binary -/** Bytes of BYTE_ARRAY data a column chunk may still produce. @internal */ +/** Bytes of binary values (BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY and INT96) a column chunk may still produce. @internal */ export interface Budget { left: number } -const spend = (budget: Budget, n: number): void => { - if (n > budget.left) fail("Limit", "the column chunk's BYTE_ARRAY values exceed maxBinaryLength") +/** Takes `n` bytes from `budget`. @internal */ +export const spend = (budget: Budget, n: number): void => { + if (n > budget.left) fail("Limit", "the column chunk's binary values exceed maxBinaryLength") budget.left -= n } @@ -61,37 +66,40 @@ for (let i = 0; i <= 64; i++) POW2.push(2 ** i) const littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1 -/** Reads `n` little-endian values of `width` bytes into a fresh typed array. */ +/** The 4-byte little-endian unsigned integer at `at`. @internal */ +export const u32 = (bytes: Uint8Array, at: number): number => + (bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16) | (bytes[at + 3] << 24)) >>> 0 + +/** Views `n` little-endian values of `width` bytes in `owned`, a fresh copy, as a typed array. */ const fixedLE = ( - bytes: Uint8Array, - pos: number, + owned: Uint8Array, n: number, width: number, make: (buffer: ArrayBuffer, length: number) => A, get: (view: DataView, at: number) => number | bigint ): A => { - const copy = bytes.slice(pos, pos + n * width) - const out = make(copy.buffer, n) + const out = make(owned.buffer, n) if (!littleEndian) { - const view = new DataView(copy.buffer) + const view = new DataView(owned.buffer) const target = out as unknown as Array for (let i = 0; i < n; i++) target[i] = get(view, i * width) } return out } -const fromLE = (kind: Kind, bytes: Uint8Array, pos: number, n: number): Dense => { +/** The `n` little-endian values in `owned`, a fresh copy that the result may take over. */ +const fromLE = (kind: Fixed, owned: Uint8Array, n: number): Dense => { switch (kind.tag) { case "i32": - return fixedLE(bytes, pos, n, 4, (b, l) => new Int32Array(b, 0, l), (v, at) => v.getInt32(at, true)) + return fixedLE(owned, n, 4, (b, l) => new Int32Array(b, 0, l), (v, at) => v.getInt32(at, true)) case "f32": - return fixedLE(bytes, pos, n, 4, (b, l) => new Float32Array(b, 0, l), (v, at) => v.getFloat32(at, true)) + return fixedLE(owned, n, 4, (b, l) => new Float32Array(b, 0, l), (v, at) => v.getFloat32(at, true)) case "i64": - return fixedLE(bytes, pos, n, 8, (b, l) => new BigInt64Array(b, 0, l), (v, at) => v.getBigInt64(at, true)) + return fixedLE(owned, n, 8, (b, l) => new BigInt64Array(b, 0, l), (v, at) => v.getBigInt64(at, true)) case "f64": - return fixedLE(bytes, pos, n, 8, (b, l) => new Float64Array(b, 0, l), (v, at) => v.getFloat64(at, true)) + return fixedLE(owned, n, 8, (b, l) => new Float64Array(b, 0, l), (v, at) => v.getFloat64(at, true)) default: - return bytes.slice(pos, pos + n * (kind as { readonly width: number }).width) + return owned } } @@ -138,6 +146,7 @@ const uleb32 = (bytes: Uint8Array, pos: number, end: number, what: string): read const BIG_7 = BigInt(7) const BIG_1 = BigInt(1) +const BIG_32 = BigInt(32) const BIG_0x7f = BigInt(0x7f) /** A zigzag ULEB128 varint of at most 64 bits, and where it ends. */ @@ -156,9 +165,12 @@ const zigzag64 = (bytes: Uint8Array, pos: number, end: number, what: string): re /** The bit width of values from 0 to `max`. @internal */ export const bitWidth = (max: number): number => max === 0 ? 0 : 32 - Math.clz32(max) +const outOfRange = (what: string, value: number, max: number): never => + fail("Malformed", `${what} include ${value}, out of range for a maximum of ${max}`) + /** - * Decodes `n` values of the RLE/bit-packed hybrid (no length prefix) into `out`, and - * returns where the runs ended. + * Decodes `out.length` values from 0 to `max` of the RLE/bit-packed hybrid (no length + * prefix) into `out`. * * @internal */ @@ -167,10 +179,11 @@ export const hybrid = ( pos: number, end: number, width: number, - n: number, + max: number, out: Uint8Array | Int32Array, what: string -): number => { +): void => { + const n = out.length const valueBytes = (width + 7) >> 3 let i = 0 while (i < n) { @@ -183,25 +196,29 @@ export const hybrid = ( const take = Math.min(runLength * 8, n - i) if (Math.ceil(take * width / 8) > end - start) fail("Malformed", `${what} has a truncated bit-packed run`) const reader = new BitReader(bytes, start) - for (let k = 0; k < take; k++) out[i++] = reader.read(width) + for (let k = 0; k < take; k++) { + const value = reader.read(width) + if (value > max) outOfRange(what, value, max) + out[i++] = value + } pos = Math.min(start + runLength * width, end) } else { if (valueBytes > end - start) fail("Malformed", `${what} has a truncated RLE run`) let value = 0 for (let k = 0; k < valueBytes; k++) value += bytes[start + k] * POW2[8 * k] if (value >= POW2[width]) fail("Malformed", `${what} has an RLE value wider than ${width} bits`) + if (value > max) outOfRange(what, value, max) const stop = Math.min(i + runLength, n) out.fill(value, i, stop) i = stop pos = start + valueBytes } } - return pos } /** - * Decodes `n` values of the deprecated BIT_PACKED level encoding (most significant bit - * first, no runs), and returns where they end. + * Decodes `out.length` values from 0 to `max` of the deprecated BIT_PACKED level + * encoding (most significant bit first, no runs) into `out`, and returns where they end. * * @internal */ @@ -210,10 +227,11 @@ export const bitPacked = ( pos: number, end: number, width: number, - n: number, + max: number, out: Uint8Array, what: string ): number => { + const n = out.length const size = Math.ceil(n * width / 8) if (size > end - pos) fail("Malformed", `${what} need ${size} bytes`) let at = pos @@ -227,6 +245,7 @@ export const bitPacked = ( at++ } } + if (value > max) outOfRange(what, value, max) out[i] = value } return pos + size @@ -235,14 +254,11 @@ export const bitPacked = ( /** Reads the 4-byte little-endian length that prefixes some hybrid runs, and returns where the runs end. @internal */ export const lengthPrefix = (bytes: Uint8Array, pos: number, end: number, what: string): number => { if (end - pos < 4) fail("Malformed", `${what} length is truncated`) - const length = (bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24)) >>> 0 + const length = u32(bytes, pos) if (length > end - pos - 4) fail("Malformed", `${what} of ${length} bytes is longer than the page`) return pos + 4 + length } -const readLength = (bytes: Uint8Array, pos: number): number => - (bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24)) >>> 0 - /** * Decodes `n` PLAIN values. * @@ -261,7 +277,7 @@ export const plain = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n let p = pos for (let i = 0; i < n; i++) { if (end - p < 4) fail("Malformed", `PLAIN BYTE_ARRAY value ${i} of ${n} is truncated`) - const length = readLength(bytes, p) + const length = u32(bytes, p) p += 4 if (length > end - p) fail("Malformed", `PLAIN BYTE_ARRAY value ${i} of ${length} bytes is truncated`) p += length @@ -272,7 +288,7 @@ export const plain = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n const data = new Uint8Array(total) p = pos for (let i = 0, at = 0; i < n; i++) { - const length = readLength(bytes, p) + const length = u32(bytes, p) data.set(bytes.subarray(p + 4, p + 4 + length), at) p += 4 + length offsets[i + 1] = at += length @@ -282,7 +298,7 @@ export const plain = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n default: { const size = n * kind.width if (size > end - pos) fail("Malformed", `PLAIN values need ${size} bytes`) - return fromLE(kind, bytes, pos, n) + return fromLE(kind, bytes.slice(pos, pos + size), n) } } } @@ -292,8 +308,8 @@ export const plain = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n * * @internal */ -export const byteStreamSplit = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n: number): Dense => { - const width = (kind as { readonly width: number }).width +export const byteStreamSplit = (kind: Fixed, bytes: Uint8Array, pos: number, end: number, n: number): Dense => { + const width = kind.width if (end - pos !== n * width) { fail("Malformed", `BYTE_STREAM_SPLIT data of ${end - pos} bytes is not ${n} values of ${width} bytes`) } @@ -302,7 +318,7 @@ export const byteStreamSplit = (kind: Kind, bytes: Uint8Array, pos: number, end: const stream = pos + k * n for (let i = 0; i < n; i++) joined[i * width + k] = bytes[stream + i] } - return fromLE(kind, joined, 0, n) + return fromLE(kind, joined, n) } /** @@ -352,14 +368,25 @@ export const deltaBinaryPacked: { } const size = perMiniblock * width / 8 if (size > end - pos) fail("Malformed", "DELTA_BINARY_PACKED miniblock is truncated") - const reader = new BitReader(bytes, pos) const take = Math.min(perMiniblock, n - i) + if (width === 0) { + // every delta is minDelta + if (wide) { + const target = out as BigInt64Array + for (let k = 0; k < take; k++) target[i++] = last64 = BigInt.asIntN(64, last64 + minDelta) + } else { + const target = out as Int32Array + for (let k = 0; k < take; k++) target[i++] = last32 = (last32 + minDelta32) | 0 + } + continue + } + const reader = new BitReader(bytes, pos) if (wide) { const target = out as BigInt64Array for (let k = 0; k < take; k++) { const delta = width <= 32 ? BigInt(reader.read(width)) - : BigInt(reader.read(32)) | (BigInt(reader.read(width - 32)) << BigInt(32)) + : BigInt(reader.read(32)) | (BigInt(reader.read(width - 32)) << BIG_32) target[i++] = last64 = BigInt.asIntN(64, last64 + minDelta + delta) } } else { @@ -385,17 +412,10 @@ const lengths = ( } /** - * Decodes `n` DELTA_LENGTH_BYTE_ARRAY values, and returns them with where they end. - * - * @internal + * Reads the lengths of `n` DELTA_LENGTH_BYTE_ARRAY values, checks that their bytes are + * there, and returns their offsets with where the bytes start. */ -export const deltaLengthByteArray = ( - bytes: Uint8Array, - pos: number, - end: number, - n: number, - budget: Budget -): readonly [Binary, number] => { +const lengthOffsets = (bytes: Uint8Array, pos: number, end: number, n: number): readonly [Int32Array, number] => { const [sizes, start] = lengths(bytes, pos, end, n, "DELTA_LENGTH_BYTE_ARRAY length") const offsets = new Int32Array(n + 1) let total = 0 @@ -404,18 +424,44 @@ export const deltaLengthByteArray = ( if (total > end - start) fail("Malformed", "DELTA_LENGTH_BYTE_ARRAY data is truncated") offsets[i + 1] = total } - spend(budget, total) - return [{ offsets, data: bytes.slice(start, start + total) }, start + total] + return [offsets, start] +} + +/** + * Decodes `n` DELTA_LENGTH_BYTE_ARRAY values. + * + * @internal + */ +export const deltaLengthByteArray = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + budget: Budget +): Binary => { + const [offsets, start] = lengthOffsets(bytes, pos, end, n) + spend(budget, offsets[n]) + return { offsets, data: bytes.slice(start, start + offsets[n]) } } /** * Decodes `n` DELTA_BYTE_ARRAY values. * + * With a `width`, every value must be `width` bytes long (FIXED_LEN_BYTE_ARRAY), and the + * values are not taken from `budget`: the caller has already counted fixed-width values. + * * @internal */ -export const deltaByteArray = (bytes: Uint8Array, pos: number, end: number, n: number, budget: Budget): Binary => { +export const deltaByteArray = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + budget: Budget, + width?: number +): Binary => { const [prefixes, next] = lengths(bytes, pos, end, n, "DELTA_BYTE_ARRAY prefix length") - const [suffixes] = deltaLengthByteArray(bytes, next, end, n, budget) + const [suffixes, start] = lengthOffsets(bytes, next, end, n) const offsets = new Int32Array(n + 1) let total = 0 for (let i = 0; i < n; i++) { @@ -423,56 +469,60 @@ export const deltaByteArray = (bytes: Uint8Array, pos: number, end: number, n: n if (i === 0 ? prefix !== 0 : prefix > offsets[i] - offsets[i - 1]) { fail("Malformed", `DELTA_BYTE_ARRAY value ${i} has a prefix of ${prefix} bytes, longer than the value before it`) } - total += prefix + suffixes.offsets[i + 1] - suffixes.offsets[i] - offsets[i + 1] = total + const length = prefix + suffixes[i + 1] - suffixes[i] + if (width !== undefined && length !== width) { + fail("Malformed", `DELTA_BYTE_ARRAY value ${i} has ${length} bytes, not ${width}`) + } + offsets[i + 1] = total += length } - spend(budget, total - suffixes.data.length) + if (width === undefined) spend(budget, total) const out = new Uint8Array(total) for (let i = 0; i < n; i++) { const at = offsets[i] const prefix = prefixes[i] if (prefix > 0) out.copyWithin(at, offsets[i - 1], offsets[i - 1] + prefix) - out.set(suffixes.data.subarray(suffixes.offsets[i], suffixes.offsets[i + 1]), at + prefix) + out.set(bytes.subarray(start + suffixes[i], start + suffixes[i + 1]), at + prefix) } return { offsets, data: out } } -/** - * Converts BYTE_ARRAY values that must all be `width` bytes long to fixed-width values. - * - * @internal - */ -export const toFixed = (values: Binary, width: number, what: string): Uint8Array => { - const n = values.offsets.length - 1 - for (let i = 0; i < n; i++) { - const length = values.offsets[i + 1] - values.offsets[i] - if (length !== width) fail("Malformed", `${what} value ${i} has ${length} bytes, not ${width}`) - } - return values.data -} - /** The number of values in `dense`. @internal */ export const lengthOf = (kind: Kind, dense: Dense): number => kind.tag === "bytes" ? (dense as Binary).offsets.length - 1 - : kind.tag === "fixed" + : kind.tag === "fixed" || kind.tag === "int96" ? (dense as Uint8Array).length / kind.width : (dense as Uint8Array).length +/** `n` values of `kind`: zeros, or empty byte arrays. @internal */ +export const alloc = (kind: Kind, n: number): Dense => { + switch (kind.tag) { + case "bool": + return new Uint8Array(n) + case "i32": + return new Int32Array(n) + case "i64": + return new BigInt64Array(n) + case "f32": + return new Float32Array(n) + case "f64": + return new Float64Array(n) + case "int96": + case "fixed": + return new Uint8Array(n * kind.width) + case "bytes": + return { offsets: new Int32Array(n + 1), data: new Uint8Array(0) } + } +} + /** - * Looks up dictionary `indices` in `dictionary`. + * Looks up dictionary `indices` in `dictionary`. The caller checks that they are in + * range. * * @internal */ export const gather = (kind: Kind, dictionary: Dense, indices: Int32Array, budget: Budget): Dense => { - const size = lengthOf(kind, dictionary) const n = indices.length - for (let i = 0; i < n; i++) { - const index = indices[i] - if (index < 0 || index >= size) { - fail("Malformed", `dictionary index ${index >>> 0} is out of range for a dictionary of ${size} values`) - } - } switch (kind.tag) { case "bytes": { const dict = dictionary as Binary @@ -486,16 +536,19 @@ export const gather = (kind: Kind, dictionary: Dense, indices: Int32Array, budge } return { offsets, data } } + case "int96": case "fixed": { const dict = dictionary as Uint8Array const width = kind.width const out = new Uint8Array(n * width) - for (let i = 0; i < n; i++) out.set(dict.subarray(indices[i] * width, (indices[i] + 1) * width), i * width) + for (let i = 0, at = 0; i < n; i++) { + for (let from = indices[i] * width, stop = from + width; from < stop; from++) out[at++] = dict[from] + } return out } default: { const dict = dictionary as Int32Array - const out = new (dict.constructor as Int32ArrayConstructor)(n) + const out = alloc(kind, n) as Int32Array for (let i = 0; i < n; i++) out[i] = dict[indices[i]] return out } @@ -512,26 +565,20 @@ export const concat = (kind: Kind, chunks: ReadonlyArray): Dense => { if (kind.tag === "bytes") { const parts = chunks as ReadonlyArray let count = 0 - let total = 0 - for (const part of parts) { - count += part.offsets.length - 1 - total += part.data.length - } + for (const part of parts) count += part.offsets.length - 1 const offsets = new Int32Array(count + 1) - const data = new Uint8Array(total) let i = 0 let at = 0 for (const part of parts) { for (let k = 1; k < part.offsets.length; k++) offsets[++i] = at + part.offsets[k] - data.set(part.data, at) at += part.data.length } - return { offsets, data } + return { offsets, data: Bytes.concat(parts.map((part) => part.data)) } } const parts = chunks as ReadonlyArray let total = 0 - for (const part of parts) total += part.length - const out = new (empty(kind).constructor as Int32ArrayConstructor)(total) + for (const part of parts) total += lengthOf(kind, part) + const out = alloc(kind, total) as Int32Array let at = 0 for (const part of parts) { out.set(part, at) @@ -539,21 +586,3 @@ export const concat = (kind: Kind, chunks: ReadonlyArray): Dense => { } return out } - -/** No values of `kind`. @internal */ -export const empty = (kind: Kind): Dense => { - switch (kind.tag) { - case "i32": - return new Int32Array(0) - case "i64": - return new BigInt64Array(0) - case "f32": - return new Float32Array(0) - case "f64": - return new Float64Array(0) - case "bytes": - return { offsets: new Int32Array(1), data: new Uint8Array(0) } - default: - return new Uint8Array(0) - } -} diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index f6caa744f7d..c9a74b676c1 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { Cause, Effect, Exit } from "effect" +import { Bytes, Cause, Effect, Exit } from "effect" import * as Hex from "effect/encoding/Hex" import * as Parquet from "effect/encoding/Parquet" import * as Thrift from "effect/encoding/Thrift" @@ -85,11 +85,13 @@ const rows = (column: Parquet.Column): Array => { // Synthetic files, written with the Thrift writer // --------------------------------------------------------------------------- +const text = (s: string) => new TextEncoder().encode(s) + type V = Thrift.Value const I32 = (value: number): V => ({ _tag: "I32", value }) const I64 = (value: number): V => ({ _tag: "I64", value: BigInt(value) }) const Bool = (value: boolean): V => ({ _tag: "Bool", value }) -const Str = (value: string): V => ({ _tag: "Binary", value: new TextEncoder().encode(value) }) +const Str = (value: string): V => ({ _tag: "Binary", value: text(value) }) const List = (elementType: Thrift.TypeName, elements: ReadonlyArray): V => ({ _tag: "List", elementType, elements }) const Struct = (fields: { readonly [id: number]: V | undefined }): V => ({ _tag: "Struct", @@ -100,18 +102,10 @@ const thrift = (value: V): Uint8Array => { writer.writeValue(value) return writer.finish() } -const bytes = (...parts: ReadonlyArray>): Uint8Array => { - const out = new Uint8Array(parts.reduce((n, part) => n + part.length, 0)) - let at = 0 - for (const part of parts) { - out.set(part, at) - at += part.length - } - return out -} +const bytes = (...parts: ReadonlyArray>): Uint8Array => + Bytes.concat(parts.map((part) => Uint8Array.from(part))) const u32 = (n: number) => [n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, n >>> 24] const uleb = (n: number): Array => n < 0x80 ? [n] : [(n & 0x7f) | 0x80, ...uleb(Math.floor(n / 0x80))] -const text = (s: string) => new TextEncoder().encode(s) const int32s = (...values: ReadonlyArray) => new Uint8Array(Int32Array.from(values).buffer) const plainBinary = (...values: ReadonlyArray) => bytes(...values.flatMap((v) => [u32(text(v).length), text(v)])) @@ -183,6 +177,8 @@ const dataPageV2 = (options: { readonly values: Uint8Array readonly compressed?: boolean | undefined readonly uncompressedSize?: number | undefined + /** Defaults to the length of `definition`. */ + readonly definitionLength?: number | undefined }) => { const repetition = options.repetition ?? new Uint8Array(0) const definition = options.definition ?? new Uint8Array(0) @@ -195,7 +191,7 @@ const dataPageV2 = (options: { 2: I32(options.numNulls), 3: I32(options.numRows), 4: I32(options.encoding ?? PLAIN), - 5: I32(definition.length), + 5: I32(options.definitionLength ?? definition.length), 6: I32(repetition.length), 7: options.compressed === undefined ? undefined : Bool(options.compressed) }) @@ -214,7 +210,7 @@ interface Chunk { /** Whether the first page is a dictionary page. */ readonly dictionary?: boolean | undefined readonly statistics?: V | undefined - readonly chunk?: { readonly [id: number]: V } + readonly chunk?: { readonly [id: number]: V } | undefined } /** Writes a Parquet file with one row group. */ @@ -222,7 +218,7 @@ const file = (options: { readonly schema: ReadonlyArray readonly numRows: number readonly chunks: ReadonlyArray - readonly metadata?: { readonly [id: number]: V } + readonly metadata?: { readonly [id: number]: V } | undefined }): Uint8Array => { const body: Array = [] let offset = 4 @@ -266,6 +262,9 @@ const flat = (options: { readonly typeLength?: number | undefined readonly codec?: number | undefined readonly dictionary?: boolean | undefined + readonly statistics?: V | undefined + readonly chunk?: { readonly [id: number]: V } | undefined + readonly metadata?: { readonly [id: number]: V } | undefined }) => file({ schema: [ @@ -273,9 +272,28 @@ const flat = (options: { element("x", { type: options.type, repetition: options.repetition, typeLength: options.typeLength }) ], numRows: options.numRows, - chunks: [{ ...options, path: ["x"], numValues: options.numValues ?? options.numRows }] + chunks: [{ ...options, path: ["x"], numValues: options.numValues ?? options.numRows }], + metadata: options.metadata + }) + +/** A file with one required top-level column of `n` values in one data page. */ +const requiredColumn = (type: number, n: number, data: Uint8Array, options?: { + readonly encoding?: number | undefined + readonly typeLength?: number | undefined + readonly codec?: number | undefined + readonly uncompressedSize?: number | undefined +}) => + flat({ + type, + repetition: REQUIRED, + numRows: n, + typeLength: options?.typeLength, + codec: options?.codec, + pages: [dataPage(n, data, options)] }) +const int32s3 = requiredColumn(Type.INT32, 3, int32s(1, 2, 3)) + /** DELTA_BINARY_PACKED for at most 128 small values: one block of four miniblocks. */ const delta = (values: ReadonlyArray): Uint8Array => { const zigzag = (n: number) => n < 0 ? -2 * n - 1 : 2 * n @@ -371,7 +389,6 @@ describe("Parquet", () => { } }) describe("conformance (apache-parquet-format-2.14.0)", () => { - const int32s3 = flat({ type: Type.INT32, repetition: REQUIRED, numRows: 3, pages: [dataPage(3, int32s(1, 2, 3))] }) const optionalInt32 = (pages: ReadonlyArray, numRows: number, codec?: number) => flat({ type: Type.INT32, repetition: OPTIONAL, numRows, pages, codec }) const repeatedInt32 = (pages: ReadonlyArray, numRows: number, numValues: number) => @@ -407,10 +424,11 @@ describe("Parquet", () => { describe("parquet.thrift FileMetaData", () => { it.effect("PQ-META-1 SHOULD accept versions 1 and 2 interchangeably (L1411-1413)", () => Effect.gen(function*() { - const v2 = file({ - schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + const v2 = flat({ + type: Type.INT32, + repetition: REQUIRED, numRows: 3, - chunks: [{ type: Type.INT32, path: ["x"], numValues: 3, pages: [dataPage(3, int32s(1, 2, 3))] }], + pages: [dataPage(3, int32s(1, 2, 3))], metadata: { 1: I32(2) } }) assert.strictEqual((yield* metadataOf(v2)).version, 2) @@ -638,11 +656,7 @@ describe("Parquet", () => { describe("parquet.thrift Statistics", () => { const withStatistics = (statistics: V) => - file({ - schema: [root(1), element("x", { type: Type.DOUBLE, repetition: REQUIRED })], - numRows: 0, - chunks: [{ type: Type.DOUBLE, path: ["x"], numValues: 0, pages: [], statistics }] - }) + flat({ type: Type.DOUBLE, repetition: REQUIRED, numRows: 0, pages: [], statistics }) const statisticsOf = (bytes: Uint8Array) => Effect.map(metadataOf(bytes), (m) => m.rowGroups[0].columns[0].metaData?.statistics) @@ -685,8 +699,7 @@ describe("Parquet", () => { [Type.FIXED_LEN_BYTE_ARRAY, 2, new Uint8Array([1, 2, 3, 4]), ["0102", "0304"], 2] ] for (const [type, n, data, expected, typeLength] of cases) { - const bytes = flat({ type, typeLength, repetition: REQUIRED, numRows: n, pages: [dataPage(n, data)] }) - assert.deepStrictEqual(yield* values(bytes), expected) + assert.deepStrictEqual(yield* values(requiredColumn(type, n, data, { typeLength })), expected) } })) }) @@ -785,14 +798,7 @@ describe("Parquet", () => { it.effect("PQ-BP-2 MUST NOT read BIT_PACKED as a values encoding (L197-198)", () => failsWith( - values( - flat({ - type: Type.INT32, - repetition: REQUIRED, - numRows: 1, - pages: [dataPage(1, int32s(1), { encoding: BIT_PACKED })] - }) - ), + values(requiredColumn(Type.INT32, 1, int32s(1), { encoding: BIT_PACKED })), "Malformed", /not an encoding/ )) @@ -800,14 +806,7 @@ describe("Parquet", () => { describe("Encodings.md#delta-encoding-delta_binary_packed--5", () => { const int32 = (data: Uint8Array, n: number) => - values( - flat({ - type: Type.INT32, - repetition: REQUIRED, - numRows: n, - pages: [dataPage(n, data, { encoding: DELTA_BINARY_PACKED })] - }) - ) + values(requiredColumn(Type.INT32, n, data, { encoding: DELTA_BINARY_PACKED })) // Example 2 (7, 5, 3, 1, 2, 3, 4, 5), with a valid block size of 128 and four miniblocks of 32 const example2 = (widths: ReadonlyArray, packed: ReadonlyArray) => bytes([0x80, 0x01, 4, 8, 14], [3], widths, packed) @@ -842,14 +841,7 @@ describe("Parquet", () => { const max64 = [...Array(9).fill(0xff).map((b, i) => i === 0 ? 0xfe : b), 0x01] const i64 = bytes([0x80, 0x01, 4, 2], max64, [2], [0, 0, 0, 0]) assert.deepStrictEqual( - yield* values( - flat({ - type: Type.INT64, - repetition: REQUIRED, - numRows: 2, - pages: [dataPage(2, i64, { encoding: DELTA_BINARY_PACKED })] - }) - ), + yield* values(requiredColumn(Type.INT64, 2, i64, { encoding: DELTA_BINARY_PACKED })), [String(2n ** 63n - 1n), String(-(2n ** 63n))] ) })) @@ -866,9 +858,8 @@ describe("Parquet", () => { it.effect("PQ-DLBA-1 MUST read delta-encoded lengths followed by the concatenated bytes (example)", () => Effect.gen(function*() { const data = bytes(delta([5, 5, 6, 6]), text("HelloWorldFoobarABCDEF")) - const page = dataPage(4, data, { encoding: DELTA_LENGTH_BYTE_ARRAY }) assert.deepStrictEqual( - yield* values(flat({ type: Type.BYTE_ARRAY, repetition: REQUIRED, numRows: 4, pages: [page] })), + yield* values(requiredColumn(Type.BYTE_ARRAY, 4, data, { encoding: DELTA_LENGTH_BYTE_ARRAY })), ["Hello", "World", "Foobar", "ABCDEF"].map((s) => Hex.encode(text(s))) ) })) @@ -878,9 +869,8 @@ describe("Parquet", () => { it.effect("PQ-DBA-1 MUST read prefix lengths then suffixes as DELTA_LENGTH_BYTE_ARRAY (example)", () => Effect.gen(function*() { const data = bytes(delta([0, 2, 0, 3]), delta([4, 2, 6, 5]), text("axislebabbleyhood")) - const page = dataPage(4, data, { encoding: DELTA_BYTE_ARRAY }) assert.deepStrictEqual( - yield* values(flat({ type: Type.BYTE_ARRAY, repetition: REQUIRED, numRows: 4, pages: [page] })), + yield* values(requiredColumn(Type.BYTE_ARRAY, 4, data, { encoding: DELTA_BYTE_ARRAY })), ["axis", "axle", "babble", "babyhood"].map((s) => Hex.encode(text(s))) ) })) @@ -888,13 +878,9 @@ describe("Parquet", () => { it.effect("PQ-DBA-2 MUST read FIXED_LEN_BYTE_ARRAY values with all their lengths encoded", () => Effect.gen(function*() { const data = bytes(delta([0, 2, 1]), delta([4, 2, 3]), text("axislebcd")) - const page = dataPage(3, data, { encoding: DELTA_BYTE_ARRAY }) - const file_ = flat({ - type: Type.FIXED_LEN_BYTE_ARRAY, - typeLength: 4, - repetition: REQUIRED, - numRows: 3, - pages: [page] + const file_ = requiredColumn(Type.FIXED_LEN_BYTE_ARRAY, 3, data, { + encoding: DELTA_BYTE_ARRAY, + typeLength: 4 }) assert.deepStrictEqual(yield* values(file_), ["axis", "axle", "abcd"].map((s) => Hex.encode(text(s)))) })) @@ -902,13 +888,7 @@ describe("Parquet", () => { describe("Encodings.md#byte-stream-split-byte_stream_split--9", () => { const split = new Uint8Array([0xaa, 0x00, 0xa3, 0xbb, 0x11, 0xb4, 0xcc, 0x22, 0xc5, 0xdd, 0x33, 0xd6]) - const make = (data: Uint8Array) => - flat({ - type: Type.FLOAT, - repetition: REQUIRED, - numRows: 3, - pages: [dataPage(3, data, { encoding: BYTE_STREAM_SPLIT })] - }) + const make = (data: Uint8Array) => requiredColumn(Type.FLOAT, 3, data, { encoding: BYTE_STREAM_SPLIT }) it.effect("PQ-BSS-1 MUST gather byte k of every value from stream k (example)", () => Effect.gen(function*() { @@ -923,13 +903,7 @@ describe("Parquet", () => { describe("Compression.md", () => { const body = int32s(1, 2, 3) const make = (data: Uint8Array, codec: number) => - flat({ - type: Type.INT32, - repetition: REQUIRED, - numRows: 3, - codec, - pages: [dataPage(3, data, { uncompressedSize: 12 })] - }) + requiredColumn(Type.INT32, 3, data, { codec, uncompressedSize: 12 }) it.effect("PQ-COMP-1 MUST decompress each page with the column chunk's codec", () => Effect.gen(function*() { @@ -959,16 +933,12 @@ describe("Parquet", () => { it.effect("PQ-UNSUP-2 fails with Unsupported on an encrypted column chunk", () => failsWith( - read(file({ - schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + read(flat({ + type: Type.INT32, + repetition: REQUIRED, numRows: 3, - chunks: [{ - type: Type.INT32, - path: ["x"], - numValues: 3, - pages: [dataPage(3, int32s(1, 2, 3))], - chunk: { 8: Struct({ 1: Struct({}) }) } - }] + pages: [dataPage(3, int32s(1, 2, 3))], + chunk: { 8: Struct({ 1: Struct({}) }) } })), "Unsupported", /encrypted/ @@ -976,30 +946,19 @@ describe("Parquet", () => { it.effect("PQ-UNSUP-3 fails with Unsupported on the ALP encoding", () => failsWith( - values( - flat({ - type: Type.DOUBLE, - repetition: REQUIRED, - numRows: 1, - pages: [dataPage(1, new Uint8Array(8), { encoding: ALP })] - }) - ), + values(requiredColumn(Type.DOUBLE, 1, new Uint8Array(8), { encoding: ALP })), "Unsupported", /ALP/ )) it.effect("PQ-UNSUP-4 fails with Unsupported on a column chunk in another file", () => failsWith( - read(file({ - schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + read(flat({ + type: Type.INT32, + repetition: REQUIRED, numRows: 3, - chunks: [{ - type: Type.INT32, - path: ["x"], - numValues: 3, - pages: [dataPage(3, int32s(1, 2, 3))], - chunk: { 1: Str("other.parquet") } - }] + pages: [dataPage(3, int32s(1, 2, 3))], + chunk: { 1: Str("other.parquet") } })), "Unsupported", /other\.parquet/ @@ -1008,27 +967,14 @@ describe("Parquet", () => { it.effect("PQ-UNSUP-5 fails with Unsupported on a codec without a Decompress, or unknown", () => Effect.gen(function*() { for (const codec of [1, 3, 4, 5, 6, 7, 99]) { - const bytes_ = flat({ - type: Type.INT32, - repetition: REQUIRED, - numRows: 3, - codec, - pages: [dataPage(3, int32s(1, 2, 3))] - }) - yield* failsWith(values(bytes_), "Unsupported") + yield* failsWith(values(requiredColumn(Type.INT32, 3, int32s(1, 2, 3), { codec })), "Unsupported") } })) }) }) describe("hostile input", () => { - const int32s3 = flat({ type: Type.INT32, repetition: REQUIRED, numRows: 3, pages: [dataPage(3, int32s(1, 2, 3))] }) - const strings = flat({ - type: Type.BYTE_ARRAY, - repetition: REQUIRED, - numRows: 2, - pages: [dataPage(2, plainBinary("abc", "de"))] - }) + const strings = requiredColumn(Type.BYTE_ARRAY, 2, plainBinary("abc", "de")) it.effect("enforces maxFooterLength, maxDepth, maxPageSize, maxValues and maxBinaryLength", () => Effect.gen(function*() { @@ -1089,14 +1035,14 @@ describe("Parquet", () => { } } const v2 = (definitionLength: number, uncompressedSize: number) => - page( - 3, - { - 8: Struct({ 1: I32(1), 2: I32(0), 3: I32(1), 4: I32(PLAIN), 5: I32(definitionLength), 6: I32(0) }) - }, - new Uint8Array(4), + dataPageV2({ + numValues: 1, + numNulls: 0, + numRows: 1, + values: new Uint8Array(4), + definitionLength, uncompressedSize - ) + }) const make = (page_: Uint8Array) => flat({ type: Type.INT32, repetition: OPTIONAL, numRows: 1, codec: GZIP, pages: [page_] }) yield* failsWith( From c789f79ce7faa55deef3a10fc4d5524d34c5faf7 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:04:39 +0000 Subject: [PATCH 22/32] Reject a negative num_values in column chunk metadata --- packages/effect/src/encoding/Parquet.ts | 1 + packages/effect/test/encoding/Parquet.test.ts | 72 ++++++++++++++++++- .../test/fixtures/parquet/CONFORMANCE.md | 2 + 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index a5862525a24..ac62e1c5238 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -1637,6 +1637,7 @@ export const readColumn = ( return yield* error("Unsupported", `${where} is encrypted`) } const meta = chunk.metaData + if (meta.numValues < 0) return yield* error("Malformed", `${where} has ${meta.numValues} values`) if (meta.numValues > maxValues) { return yield* error("Limit", `${where} has ${meta.numValues} values, more than maxValues (${maxValues})`) } diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index c9a74b676c1..5f5eca6a92e 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -179,6 +179,8 @@ const dataPageV2 = (options: { readonly uncompressedSize?: number | undefined /** Defaults to the length of `definition`. */ readonly definitionLength?: number | undefined + /** Defaults to the length of `repetition`. */ + readonly repetitionLength?: number | undefined }) => { const repetition = options.repetition ?? new Uint8Array(0) const definition = options.definition ?? new Uint8Array(0) @@ -192,7 +194,7 @@ const dataPageV2 = (options: { 3: I32(options.numRows), 4: I32(options.encoding ?? PLAIN), 5: I32(options.definitionLength ?? definition.length), - 6: I32(repetition.length), + 6: I32(options.repetitionLength ?? repetition.length), 7: options.compressed === undefined ? undefined : Bool(options.compressed) }) }, @@ -562,6 +564,50 @@ describe("Parquet", () => { assert.deepStrictEqual(yield* values(make(body.length), { codecs: { GZIP: gzip } }), [5, null, 7]) yield* failsWith(values(make(body.length + 1), { codecs: { GZIP: gzip } }), "Malformed", /decompressed to/) })) + + it.effect("PQ-PAGE-3 MUST reject a negative count or size in a page header (parquet.thrift PageHeader)", () => + Effect.gen(function*() { + const present = runs(1, [1, 1]) + const column = (repetition: number, ...pages: ReadonlyArray) => + flat({ type: Type.INT32, repetition, numRows: 1, pages }) + const v2 = (options: Partial[0]>) => + dataPageV2({ numValues: 1, numNulls: 0, numRows: 1, values: int32s(7), ...options }) + const rejects = (bytes_: Uint8Array, message: RegExp) => failsWith(values(bytes_), "Malformed", message) + // with and without levels + yield* rejects( + column(OPTIONAL, dataPage(-1, bytes(lengthPrefixed(present), int32s(7)))), + /DataPageHeader\.num_values is negative/ + ) + yield* rejects(column(REQUIRED, dataPage(-1, int32s(7))), /DataPageHeader\.num_values is negative/) + yield* rejects( + column(OPTIONAL, v2({ numValues: -1, definition: present })), + /DataPageHeaderV2\.num_values is negative/ + ) + yield* rejects(column(REQUIRED, v2({ numValues: -1 })), /DataPageHeaderV2\.num_values is negative/) + yield* rejects(column(REQUIRED, v2({ numNulls: -1 })), /num_nulls is negative/) + yield* rejects(column(REQUIRED, v2({ numRows: -1 })), /num_rows is negative/) + yield* rejects(column(REQUIRED, v2({ definitionLength: -1 })), /definition_levels_byte_length is negative/) + yield* rejects(column(REQUIRED, v2({ repetitionLength: -1 })), /repetition_levels_byte_length is negative/) + yield* rejects( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + dictionary: true, + pages: [ + dictionaryPage(-1, int32s(7)), + dataPage(1, bytes([1], runs(1, [0, 1])), { encoding: RLE_DICTIONARY }) + ] + }), + /DictionaryPageHeader\.num_values is negative/ + ) + yield* rejects(column(REQUIRED, dataPage(1, int32s(7), { uncompressedSize: -1 })), /negative size/) + const header = Struct({ 1: I32(1), 2: I32(PLAIN), 3: I32(RLE), 4: I32(RLE) }) + yield* rejects( + column(REQUIRED, bytes(thrift(Struct({ 1: I32(0), 2: I32(4), 3: I32(-1), 5: header })), int32s(7))), + /negative size/ + ) + })) }) describe("README.md#column-chunks", () => { @@ -593,6 +639,30 @@ describe("Parquet", () => { [1, 2, 3] ) })) + + it.effect("PQ-CHUNK-3 MUST reject a column chunk whose num_values is negative (parquet.thrift L922-923)", () => + Effect.gen(function*() { + const levels = lengthPrefixed(runs(1, [0, 1])) + // with repetition and definition levels, and without any + yield* failsWith( + values(repeatedInt32([dataPage(1, bytes(levels, levels))], 1, -1)), + "Malformed", + /has -1 values/ + ) + yield* failsWith( + values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + numValues: -1, + pages: [dataPage(1, int32s(7))] + }) + ), + "Malformed", + /has -1 values/ + ) + })) }) describe("parquet.thrift DataPageHeaderV2", () => { diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index 712cb0c9fef..4c1416199b0 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -29,8 +29,10 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | | PQ-PAGE-1 | README.md#data-pages | A required, non-nested column's page holds only the values | | PQ-PAGE-2 | README.md#data-pages | `uncompressed_page_size` covers levels and values together | +| PQ-PAGE-3 | parquet.thrift L707, L733, L754-772, L835-838 | A page header's counts and sizes are not negative (rejected when they are) | | PQ-CHUNK-1 | README.md#column-chunks, parquet.thrift L727-729 | The dictionary page comes first, and there is at most one | | PQ-CHUNK-2 | README.md#column-chunks | Readers can skip pages they don't read (index pages, later page types) | +| PQ-CHUNK-3 | parquet.thrift L922-923 | A column chunk's `num_values` is not negative (rejected when it is) | | PQ-V2-1 | parquet.thrift L759-761 | Every v2 data page begins at a row boundary | | PQ-V2-2 | parquet.thrift L744-745, L767 | v2 levels are uncompressed RLE without a length prefix | | PQ-V2-3 | parquet.thrift L745, L779 | v2 values are compressed only when `is_compressed` is true | From cf57dc8ea29f5fb4b84481560f8a920f5ec50b93 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:54:42 +0000 Subject: [PATCH 23/32] Reject unknown and missing time units and ignore the order of unknown logical types --- packages/effect/src/encoding/Parquet.ts | 59 +++++++----- packages/effect/test/encoding/Parquet.test.ts | 89 ++++++++++++++++++- .../test/fixtures/parquet/CONFORMANCE.md | 4 + 3 files changed, 131 insertions(+), 21 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index ac62e1c5238..d34ce36203b 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -40,7 +40,9 @@ * * - Physical types: all eight. Values stay in their physical type; logical * types and converted types are decoded into the schema but not applied - * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). + * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). A + * logical type added after 2.14.0 is left out of the schema, and so is the + * order of its column's statistics. * - Encodings: `PLAIN`, `PLAIN_DICTIONARY` and `RLE_DICTIONARY` (values * are looked up in the dictionary), `RLE` (booleans), `DELTA_BINARY_PACKED`, * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; @@ -52,9 +54,9 @@ * * Not supported, failing with a {@link ParquetError} of kind `Unsupported`: * encrypted files and columns, the `ALP` encoding, column chunks in other - * files (`file_path`), and compressed pages without a matching - * {@link Decompress}. Not read: page indexes, bloom filters, page checksums, - * size and geospatial statistics. + * files (`file_path`), `TIME` and `TIMESTAMP` units added after 2.14.0, and + * compressed pages without a matching {@link Decompress}. Not read: page + * indexes, bloom filters, page checksums, size and geospatial statistics. * * Input is untrusted: every offset and length is checked against the bytes * that are there, and the sizes a file claims are capped by options @@ -300,7 +302,10 @@ export interface ColumnDescriptor { readonly maxDefinitionLevel: number /** The number of repeated nodes on the path. */ readonly maxRepetitionLevel: number - /** `undefined` when the file has no column orders, or one this module doesn't know. */ + /** + * `undefined` when the file has no column orders, or one this module doesn't + * know, or when the column's logical type is one this module doesn't know. + */ readonly columnOrder: ColumnOrder | undefined } @@ -766,20 +771,30 @@ const simpleLogicalTypes = { 19: "FILE" } as const -/** Reads a union of empty structs, such as TimeUnit or ColumnOrder, as the name of its field. */ -const unionName = (r: Reader, names: ReadonlyArray): A | undefined => { - let out: A | undefined +/** + * Reads a union of empty structs, such as TimeUnit or ColumnOrder, as the name + * of its field, or as the id of a field this module doesn't know. + */ +const unionName = (r: Reader, names: ReadonlyArray): A | number | undefined => { + let out: A | number | undefined struct(r, (id, type) => { r.skip(type) - if (type === T.Struct) out = names[id] + const known = names[id] + if (known === undefined) out = id + else if (type === T.Struct) out = known }) return out } -const logicalType = (r: Reader): LogicalType | undefined => { - let out: LogicalType | undefined +/** Reads a LogicalType union, or the id of a member this module doesn't know. */ +const logicalType = (r: Reader): LogicalType | number | undefined => { + let out: LogicalType | number | undefined struct(r, (id, type) => { - if (type !== T.Struct) return r.skip(type) + if (type !== T.Struct) { + r.skip(type) + out = id + return + } const simple = simpleLogicalTypes[id as keyof typeof simpleLogicalTypes] if (simple !== undefined) { r.skip(type) @@ -798,11 +813,13 @@ const logicalType = (r: Reader): LogicalType | undefined => { } case 7: case 8: { + const kind = id === 7 ? "TIME" : "TIMESTAMP" + const what = id === 7 ? "TimeType" : "TimestampType" const f = readStruct(r, { isAdjustedToUTC: [1, bool], unit: [2, structOf((r) => unionName(r, timeUnits))] }) - const isAdjustedToUTC = required(f.isAdjustedToUTC, "TimeType.isAdjustedToUTC") - out = f.unit === undefined - ? undefined - : { type: id === 7 ? "TIME" : "TIMESTAMP", isAdjustedToUTC, unit: f.unit } + const isAdjustedToUTC = required(f.isAdjustedToUTC, `${what}.isAdjustedToUTC`) + const unit = required(f.unit, `${what}.unit`) + if (typeof unit === "number") return fail("Unsupported", `${kind} unit ${unit} is not supported`) + out = { type: kind, isAdjustedToUTC, unit } return } case 10: { @@ -831,7 +848,7 @@ const logicalType = (r: Reader): LogicalType | undefined => { } default: r.skip(type) - out = undefined + out = id } }) return out @@ -916,7 +933,7 @@ interface Schema { /** Rebuilds the schema tree from its depth-first list of elements. */ const schemaTree = ( elements: ReadonlyArray, - orders: ReadonlyArray | undefined, + orders: ReadonlyArray | undefined, maxDepth: number ): Schema => { const columns: Array = [] @@ -938,7 +955,7 @@ const schemaTree = ( repetition, typeLength: e.typeLength, convertedType: e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType), - logicalType: e.logicalType, + logicalType: typeof e.logicalType === "number" ? undefined : e.logicalType, scale: e.scale, precision: e.precision, fieldId: e.fieldId @@ -962,6 +979,7 @@ const schemaTree = ( } const self: SchemaNode = { ...base, physicalType, children: undefined } const nodes = [...path, self] + const order = orders?.[columns.length] columns.push({ path: nodes.map((n) => n.name), nodes, @@ -969,7 +987,8 @@ const schemaTree = ( typeLength: physicalType === "FIXED_LEN_BYTE_ARRAY" ? e.typeLength : undefined, maxDefinitionLevel: definition, maxRepetitionLevel: repetitionLevel, - columnOrder: orders?.[columns.length] + // The order of a logical type this module doesn't know is unknown too. + columnOrder: typeof order === "number" || typeof e.logicalType === "number" ? undefined : order }) return self } diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index 5f5eca6a92e..ceb430d4534 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -88,6 +88,7 @@ const rows = (column: Parquet.Column): Array => { const text = (s: string) => new TextEncoder().encode(s) type V = Thrift.Value +const I8 = (value: number): V => ({ _tag: "I8", value }) const I32 = (value: number): V => ({ _tag: "I32", value }) const I64 = (value: number): V => ({ _tag: "I64", value: BigInt(value) }) const Bool = (value: boolean): V => ({ _tag: "Bool", value }) @@ -135,13 +136,22 @@ const element = (name: string, options: { readonly typeLength?: number | undefined readonly repetition?: number | undefined readonly children?: number | undefined + readonly convertedType?: number | undefined + readonly scale?: number | undefined + readonly precision?: number | undefined + /** A LogicalType union: a struct with one field. */ + readonly logicalType?: V | undefined }): V => Struct({ 1: options.type === undefined ? undefined : I32(options.type), 2: options.typeLength === undefined ? undefined : I32(options.typeLength), 3: options.repetition === undefined ? undefined : I32(options.repetition), 4: Str(name), - 5: options.children === undefined ? undefined : I32(options.children) + 5: options.children === undefined ? undefined : I32(options.children), + 6: options.convertedType === undefined ? undefined : I32(options.convertedType), + 7: options.scale === undefined ? undefined : I32(options.scale), + 8: options.precision === undefined ? undefined : I32(options.precision), + 10: options.logicalType }) const root = (children: number) => element("schema", { children }) @@ -397,6 +407,27 @@ describe("Parquet", () => { flat({ type: Type.INT32, repetition: REPEATED, numRows, numValues, pages }) // def levels 1, 0, 1 as RLE runs, for a column with one optional node const someNull = runs(1, [1, 1], [0, 1], [1, 1]) + type Annotation = Omit[1], "type" | "repetition" | "children"> + /** A file with one required column `x` that has statistics in TYPE_ORDER, and no rows. */ + const annotated = (type: number, annotation: Annotation) => + file({ + schema: [root(1), element("x", { type, repetition: REQUIRED, ...annotation })], + numRows: 0, + chunks: [{ type, path: ["x"], numValues: 0, pages: [] }], + metadata: { 7: List("Struct", [Struct({ 1: Struct({}) })]) } + }) + /** What the reader says about column `x`. */ + const annotationOf = (bytes: Uint8Array) => + Effect.map(metadataOf(bytes), ({ columns: [column] }) => ({ + logicalType: column.nodes[0].logicalType, + convertedType: column.nodes[0].convertedType, + columnOrder: column.columnOrder + })) + const empty = Struct({}) + const TIME = 7 + const TIMESTAMP = 8 + /** A TimeType or TimestampType with a TimeUnit union, or without its unit. */ + const temporal = (member: number, unit: V | undefined) => Struct({ [member]: Struct({ 1: Bool(true), 2: unit }) }) describe("README.md#file-format", () => { it.effect("PQ-FILE-1 MUST start and end with the magic number PAR1", () => @@ -501,6 +532,50 @@ describe("Parquet", () => { })) }) + describe("LogicalTypes.md", () => { + it.effect("PQ-LT-1 MUST handle an unknown TIMESTAMP unit as an unsupported feature (#timestamp)", () => + Effect.gen(function*() { + const TIMESTAMP_MICROS = 10 + const timestamp = (unit: number) => + annotated(Type.INT64, { + logicalType: temporal(TIMESTAMP, Struct({ [unit]: empty })), + convertedType: TIMESTAMP_MICROS + }) + assert.deepStrictEqual((yield* annotationOf(timestamp(3))).logicalType, { + type: "TIMESTAMP", + isAdjustedToUTC: true, + unit: "NANOS" + }) + // a fourth unit, which must not read as the legacy TIMESTAMP_MICROS + yield* failsWith(metadataOf(timestamp(4)), "Unsupported", /TIMESTAMP unit 4/) + })) + + it.effect("PQ-LT-2 MUST reject a TIME or TIMESTAMP without its unit (parquet.thrift L370-373, L380-383)", () => + Effect.gen(function*() { + const make = (member: number, unit: V | undefined) => + metadataOf(annotated(Type.INT64, { logicalType: temporal(member, unit) })) + yield* failsWith(make(TIMESTAMP, undefined), "Malformed", /TimestampType\.unit is missing/) + yield* failsWith(make(TIME, undefined), "Malformed", /TimeType\.unit is missing/) + // a TimeUnit union with no member + yield* failsWith(make(TIMESTAMP, empty), "Malformed", /TimestampType\.unit is missing/) + })) + + it.effect("PQ-LT-3 SHOULD ignore an unrecognized logical type and the column's order (#unsupported-logical-types)", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT64, {})), { + logicalType: undefined, + convertedType: undefined, + columnOrder: "TYPE_ORDER" + }) + // member 99 of the LogicalType union, which 2.14.0 doesn't have + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT64, { logicalType: Struct({ 99: empty }) })), { + logicalType: undefined, + convertedType: undefined, + columnOrder: undefined + }) + })) + }) + describe("README.md#nested-encoding", () => { it.effect("PQ-NEST-1 MUST compute the maximum levels from the optional and repeated nodes of the path", () => Effect.gen(function*() { @@ -1040,6 +1115,18 @@ describe("Parquet", () => { yield* failsWith(values(requiredColumn(Type.INT32, 3, int32s(1, 2, 3), { codec })), "Unsupported") } })) + + it.effect("PQ-UNSUP-6 fails with Unsupported on a TIME unit added after 2.14.0", () => + Effect.gen(function*() { + const time = (unit: number) => + annotated(Type.INT64, { logicalType: temporal(TIME, Struct({ [unit]: empty })) }) + assert.deepStrictEqual((yield* annotationOf(time(2))).logicalType, { + type: "TIME", + isAdjustedToUTC: true, + unit: "MICROS" + }) + yield* failsWith(metadataOf(time(4)), "Unsupported", /TIME unit 4/) + })) }) }) diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index 4c1416199b0..b6004203997 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -24,6 +24,9 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | | PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | | PQ-SCHEMA-2 | parquet.thrift L518-523, L542-547 | The schema tree is the depth-first list of elements, groups giving `num_children` | +| PQ-LT-1 | LogicalTypes.md#timestamp | Unknown `TIMESTAMP` units are unsupported features, not errors in the file | +| PQ-LT-2 | parquet.thrift L370-373, L380-383 | `TimestampType.unit` and `TimeType.unit` are required (rejected when missing) | +| PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, and so is its column's order | | PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | | PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | | PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | @@ -69,6 +72,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-UNSUP-3 | Encodings.md#adaptive-lossless-floating-point-alp--10 | The `ALP` encoding (in preview at 2.14.0) | | PQ-UNSUP-4 | parquet.thrift ColumnChunk.file_path | Column chunks in another file | | PQ-UNSUP-5 | Compression.md#codecs | Every codec but `UNCOMPRESSED` without a `Decompress`, and unknown codecs. The module has no codecs; the shared ones are planned in `effect/encoding/Compression` | +| PQ-UNSUP-6 | LogicalTypes.md#time | A `TIME` unit added after 2.14.0. Local policy: the spec states the rule for `TIMESTAMP` only (PQ-LT-1) | ## Deferred: not read yet From 323e9e11856ce0274762462a2ec97add0ca9ef66 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:01:51 +0000 Subject: [PATCH 24/32] Check annotations against the physical type and parameters the spec defines them for --- packages/effect/src/encoding/Parquet.ts | 161 +++++++++++++- packages/effect/test/encoding/Parquet.test.ts | 210 +++++++++++++++++- .../test/fixtures/parquet/CONFORMANCE.md | 6 +- 3 files changed, 360 insertions(+), 17 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index d34ce36203b..fe2c9a3d73e 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -40,9 +40,12 @@ * * - Physical types: all eight. Values stay in their physical type; logical * types and converted types are decoded into the schema but not applied - * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). A - * logical type added after 2.14.0 is left out of the schema, and so is the - * order of its column's statistics. + * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). + * - Annotations: an annotation added after 2.14.0, or one the spec doesn't + * define for the physical type it is on (a `STRING` on an `INT32`), is left + * out of the schema, and so is the order of the column's statistics. A + * `DECIMAL` precision or scale and an `INTEGER` bit width that the spec + * forbids fail with `Malformed`. * - Encodings: `PLAIN`, `PLAIN_DICTIONARY` and `RLE_DICTIONARY` (values * are looked up in the dictionary), `RLE` (booleans), `DELTA_BINARY_PACKED`, * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; @@ -261,6 +264,10 @@ export type ColumnOrder = "TYPE_ORDER" | "IEEE_754_TOTAL_ORDER" | "INT96_TIMESTA * Enum-valued fields hold the spec's name, or the numeric code of a value this * module doesn't know. * + * `convertedType` and `logicalType` are each `undefined` when the spec doesn't + * define the annotation for this node: for its physical type and + * `typeLength`, or for a group. They are not compared with each other. + * * @stability experimental * @category models * @since 4.1.5 @@ -304,7 +311,8 @@ export interface ColumnDescriptor { readonly maxRepetitionLevel: number /** * `undefined` when the file has no column orders, or one this module doesn't - * know, or when the column's logical type is one this module doesn't know. + * know, or when the leaf has an annotation this module doesn't know or the + * spec doesn't define for its physical type. */ readonly columnOrder: ColumnOrder | undefined } @@ -925,6 +933,140 @@ const rowGroup = (r: Reader): RowGroup => { } } +/** A logical type, or a converted type as the logical type that replaced it. */ +type Annotation = LogicalType | { readonly type: Exclude } + +/** + * The physical type LogicalTypes.md defines each annotation for, `undefined` + * for a group. The annotations that are not here depend on more than that. + */ +const annotated: { + readonly [ + K in Exclude + ]: PhysicalType | undefined +} = { + MAP: undefined, + MAP_KEY_VALUE: undefined, + LIST: undefined, + VARIANT: undefined, + FILE: undefined, + STRING: "BYTE_ARRAY", + UTF8: "BYTE_ARRAY", + ENUM: "BYTE_ARRAY", + JSON: "BYTE_ARRAY", + BSON: "BYTE_ARRAY", + GEOMETRY: "BYTE_ARRAY", + GEOGRAPHY: "BYTE_ARRAY", + DATE: "INT32", + TIME_MILLIS: "INT32", + UINT_8: "INT32", + UINT_16: "INT32", + UINT_32: "INT32", + INT_8: "INT32", + INT_16: "INT32", + INT_32: "INT32", + TIME_MICROS: "INT64", + TIMESTAMP: "INT64", + TIMESTAMP_MILLIS: "INT64", + TIMESTAMP_MICROS: "INT64", + UINT_64: "INT64", + INT_64: "INT64" +} +const annotatedLengths = { UUID: 16, FLOAT16: 2, INTERVAL: 12 } as const + +/** + * Whether LogicalTypes.md defines `annotation` for the node of schema element + * `e`: a leaf of `physicalType`, or a group. Fails on parameters it forbids + * for every node. + */ +const annotates = (annotation: Annotation, e: Element, physicalType: PhysicalType | undefined): boolean => { + switch (annotation.type) { + case "UNKNOWN": + // "a column that is always null", of any type + return true + case "UUID": + case "FLOAT16": + case "INTERVAL": + return physicalType === "FIXED_LEN_BYTE_ARRAY" && e.typeLength === annotatedLengths[annotation.type] + case "TIME": + return physicalType === (annotation.unit === "MILLIS" ? "INT32" : "INT64") + case "INTEGER": { + const { bitWidth } = annotation + if (bitWidth !== 8 && bitWidth !== 16 && bitWidth !== 32 && bitWidth !== 64) { + fail("Malformed", `schema element "${e.name}" is an INTEGER with a bit width of ${bitWidth}`) + } + return physicalType === (bitWidth === 64 ? "INT64" : "INT32") + } + case "DECIMAL": { + const { precision, scale } = annotation + const invalid = (reason: string): never => + fail("Malformed", `schema element "${e.name}" is a DECIMAL with ${reason}`) + if (precision < 1) invalid(`a precision of ${precision}`) + if (scale < 0 || scale > precision) invalid(`a scale of ${scale} for a precision of ${precision}`) + let digits: number + switch (physicalType) { + case "INT32": + digits = 9 + break + case "INT64": + digits = 18 + break + case "FIXED_LEN_BYTE_ARRAY": + // floor(log10(2^(8n - 1) - 1)), which is this as no power of 2 is a power of 10 + digits = Math.floor((8 * e.typeLength! - 1) * Math.log10(2)) + break + case "BYTE_ARRAY": + return true + default: + return false + } + if (precision > digits) { + invalid(`a precision of ${precision}, more than the ${digits} digits of its ${physicalType}`) + } + return true + } + default: + return physicalType === annotated[annotation.type] + } +} + +/** + * The annotations of a schema element's node, without those the spec doesn't + * define for it. `known` is whether the node kept all it had: when it didn't, + * the order of its values is unknown too. + */ +const annotations = (e: Element, physicalType: PhysicalType | undefined): { + readonly logicalType: LogicalType | undefined + readonly convertedType: ConvertedType | number | undefined + readonly known: boolean +} => { + let logicalType: LogicalType | undefined + let convertedType = e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType) + let known = typeof e.logicalType !== "number" && typeof convertedType !== "number" + if (typeof e.logicalType === "object") { + if (annotates(e.logicalType, e, physicalType)) logicalType = e.logicalType + else known = false + } + if (typeof convertedType === "string") { + let annotation: Annotation + if (convertedType !== "DECIMAL") annotation = { type: convertedType } + // A DecimalType supersedes the scale and precision of the schema element. + else if (typeof e.logicalType === "object" && e.logicalType.type === "DECIMAL") annotation = e.logicalType + else { + annotation = { + type: "DECIMAL", + scale: e.scale ?? 0, + precision: required(e.precision, `the precision of DECIMAL schema element "${e.name}"`) + } + } + if (!annotates(annotation, e, physicalType)) { + convertedType = undefined + known = false + } + } + return { logicalType, convertedType, known } +} + interface Schema { readonly root: SchemaNode readonly columns: Array @@ -954,8 +1096,6 @@ const schemaTree = ( name: e.name, repetition, typeLength: e.typeLength, - convertedType: e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType), - logicalType: typeof e.logicalType === "number" ? undefined : e.logicalType, scale: e.scale, precision: e.precision, fieldId: e.fieldId @@ -964,7 +1104,8 @@ const schemaTree = ( const count = required(e.numChildren, `the type or children of schema element "${e.name}"`) if (count < 0) fail("Malformed", `schema element "${e.name}" has ${count} children`) const children: Array = [] - const self: SchemaNode = { ...base, physicalType: undefined, children } + const { convertedType, logicalType } = annotations(e, undefined) + const self: SchemaNode = { ...base, convertedType, logicalType, physicalType: undefined, children } const nodes = isRoot ? path : [...path, self] for (let i = 0; i < count; i++) children.push(node(nodes, definition, repetitionLevel)) return self @@ -977,7 +1118,8 @@ const schemaTree = ( if (physicalType === "FIXED_LEN_BYTE_ARRAY" && !(e.typeLength !== undefined && e.typeLength > 0)) { fail("Malformed", `FIXED_LEN_BYTE_ARRAY column "${e.name}" has no positive type_length`) } - const self: SchemaNode = { ...base, physicalType, children: undefined } + const { convertedType, known, logicalType } = annotations(e, physicalType) + const self: SchemaNode = { ...base, convertedType, logicalType, physicalType, children: undefined } const nodes = [...path, self] const order = orders?.[columns.length] columns.push({ @@ -987,8 +1129,7 @@ const schemaTree = ( typeLength: physicalType === "FIXED_LEN_BYTE_ARRAY" ? e.typeLength : undefined, maxDefinitionLevel: definition, maxRepetitionLevel: repetitionLevel, - // The order of a logical type this module doesn't know is unknown too. - columnOrder: typeof order === "number" || typeof e.logicalType === "number" ? undefined : order + columnOrder: typeof order === "number" || !known ? undefined : order }) return self } diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index ceb430d4534..b2717e8fde7 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -408,15 +408,26 @@ describe("Parquet", () => { // def levels 1, 0, 1 as RLE runs, for a column with one optional node const someNull = runs(1, [1, 1], [0, 1], [1, 1]) type Annotation = Omit[1], "type" | "repetition" | "children"> - /** A file with one required column `x` that has statistics in TYPE_ORDER, and no rows. */ - const annotated = (type: number, annotation: Annotation) => - file({ - schema: [root(1), element("x", { type, repetition: REQUIRED, ...annotation })], + /** + * A file with one required column `x` that has statistics in TYPE_ORDER, and no rows. The annotation is on `x`, + * or without a `type` on a group around it. + */ + const annotated = (type: number | undefined, annotation: Annotation) => { + const leaf = type ?? Type.BYTE_ARRAY + return file({ + schema: type === undefined + ? [ + root(1), + element("g", { repetition: REQUIRED, children: 1, ...annotation }), + element("x", { type: leaf, repetition: REQUIRED }) + ] + : [root(1), element("x", { type, repetition: REQUIRED, ...annotation })], numRows: 0, - chunks: [{ type, path: ["x"], numValues: 0, pages: [] }], + chunks: [{ type: leaf, path: type === undefined ? ["g", "x"] : ["x"], numValues: 0, pages: [] }], metadata: { 7: List("Struct", [Struct({ 1: Struct({}) })]) } }) - /** What the reader says about column `x`. */ + } + /** What the reader says about the annotated node and the order of column `x`. */ const annotationOf = (bytes: Uint8Array) => Effect.map(metadataOf(bytes), ({ columns: [column] }) => ({ logicalType: column.nodes[0].logicalType, @@ -574,6 +585,193 @@ describe("Parquet", () => { columnOrder: undefined }) })) + + const GROUP = undefined + const FIXED = Type.FIXED_LEN_BYTE_ARRAY + const logical = (member: number, fields: { readonly [id: number]: V } = {}): Annotation => ({ + logicalType: Struct({ [member]: Struct(fields) }) + }) + const integer = (bitWidth: number, isSigned: boolean) => logical(10, { 1: I8(bitWidth), 2: Bool(isSigned) }) + const time = (unit: number) => ({ logicalType: temporal(TIME, Struct({ [unit]: empty })) }) + const utc = { isAdjustedToUTC: true } + + it.effect("PQ-LT-4 SHOULD ignore an annotation on a physical type it is not defined for, and the column's order (#unsupported-logical-types)", () => + Effect.gen(function*() { + type Node = readonly [type: number | undefined, typeLength?: number] + // the annotation as written and as read, a node the spec defines it for, and one it doesn't + const cases: ReadonlyArray< + readonly [Annotation, Parquet.LogicalType | Parquet.ConvertedType, defined: Node, other: Node] + > = [ + [logical(1), { type: "STRING" }, [Type.BYTE_ARRAY], [Type.INT32]], + [logical(2), { type: "MAP" }, [GROUP], [Type.BYTE_ARRAY]], + [logical(3), { type: "LIST" }, [GROUP], [Type.INT32]], + [logical(4), { type: "ENUM" }, [Type.BYTE_ARRAY], [FIXED, 4]], + [logical(5, { 1: I32(2), 2: I32(9) }), { type: "DECIMAL", scale: 2, precision: 9 }, [Type.INT32], [ + Type.DOUBLE + ]], + [logical(6), { type: "DATE" }, [Type.INT32], [Type.INT64]], + [time(1), { type: "TIME", ...utc, unit: "MILLIS" }, [Type.INT32], [Type.INT64]], + [time(2), { type: "TIME", ...utc, unit: "MICROS" }, [Type.INT64], [Type.INT32]], + [time(3), { type: "TIME", ...utc, unit: "NANOS" }, [Type.INT64], [Type.INT32]], + [ + { logicalType: temporal(TIMESTAMP, Struct({ 1: empty })) }, + { type: "TIMESTAMP", ...utc, unit: "MILLIS" }, + [ + Type.INT64 + ], + [Type.INT96] + ], + [integer(8, true), { type: "INTEGER", bitWidth: 8, isSigned: true }, [Type.INT32], [Type.INT64]], + [integer(16, false), { type: "INTEGER", bitWidth: 16, isSigned: false }, [Type.INT32], [Type.INT64]], + [integer(32, true), { type: "INTEGER", bitWidth: 32, isSigned: true }, [Type.INT32], [Type.INT64]], + [integer(64, false), { type: "INTEGER", bitWidth: 64, isSigned: false }, [Type.INT64], [Type.INT32]], + [logical(12), { type: "JSON" }, [Type.BYTE_ARRAY], [Type.INT64]], + [logical(13), { type: "BSON" }, [Type.BYTE_ARRAY], [Type.INT32]], + [logical(14), { type: "UUID" }, [FIXED, 16], [FIXED, 4]], + [logical(15), { type: "FLOAT16" }, [FIXED, 2], [Type.FLOAT]], + [logical(16), { type: "VARIANT", specificationVersion: undefined }, [GROUP], [Type.BYTE_ARRAY]], + [logical(17), { type: "GEOMETRY", crs: undefined }, [Type.BYTE_ARRAY], [GROUP]], + [logical(18), { type: "GEOGRAPHY", crs: undefined, algorithm: undefined }, [Type.BYTE_ARRAY], [FIXED, 21]], + [logical(19), { type: "FILE" }, [GROUP], [Type.BYTE_ARRAY]], + // converted types, by their code in parquet.thrift + [{ convertedType: 0 }, "UTF8", [Type.BYTE_ARRAY], [Type.INT32]], + [{ convertedType: 1 }, "MAP", [GROUP], [Type.BYTE_ARRAY]], + [{ convertedType: 2 }, "MAP_KEY_VALUE", [GROUP], [Type.BYTE_ARRAY]], + [{ convertedType: 3 }, "LIST", [GROUP], [Type.INT32]], + [{ convertedType: 4 }, "ENUM", [Type.BYTE_ARRAY], [Type.INT32]], + [{ convertedType: 5, scale: 2, precision: 9 }, "DECIMAL", [Type.INT32], [Type.FLOAT]], + [{ convertedType: 6 }, "DATE", [Type.INT32], [Type.INT64]], + [{ convertedType: 7 }, "TIME_MILLIS", [Type.INT32], [Type.INT64]], + [{ convertedType: 8 }, "TIME_MICROS", [Type.INT64], [Type.INT32]], + [{ convertedType: 9 }, "TIMESTAMP_MILLIS", [Type.INT64], [Type.INT32]], + [{ convertedType: 10 }, "TIMESTAMP_MICROS", [Type.INT64], [Type.INT96]], + [{ convertedType: 11 }, "UINT_8", [Type.INT32], [Type.INT64]], + [{ convertedType: 12 }, "UINT_16", [Type.INT32], [Type.INT64]], + [{ convertedType: 13 }, "UINT_32", [Type.INT32], [Type.INT64]], + [{ convertedType: 14 }, "UINT_64", [Type.INT64], [Type.INT32]], + [{ convertedType: 15 }, "INT_8", [Type.INT32], [Type.INT64]], + [{ convertedType: 16 }, "INT_16", [Type.INT32], [Type.INT64]], + [{ convertedType: 17 }, "INT_32", [Type.INT32], [Type.INT64]], + [{ convertedType: 18 }, "INT_64", [Type.INT64], [Type.INT32]], + [{ convertedType: 19 }, "JSON", [Type.BYTE_ARRAY], [Type.INT32]], + [{ convertedType: 20 }, "BSON", [Type.BYTE_ARRAY], [GROUP]], + [{ convertedType: 21 }, "INTERVAL", [FIXED, 12], [FIXED, 4]] + ] + for (const [annotation, expected, [type, typeLength], [otherType, otherLength]] of cases) { + const label = JSON.stringify(expected) + const kept = yield* annotationOf(annotated(type, { ...annotation, typeLength })) + assert.deepStrictEqual( + kept, + typeof expected === "string" + ? { logicalType: undefined, convertedType: expected, columnOrder: "TYPE_ORDER" } + : { logicalType: expected, convertedType: undefined, columnOrder: "TYPE_ORDER" }, + label + ) + const ignored = yield* annotationOf(annotated(otherType, { ...annotation, typeLength: otherLength })) + assert.deepStrictEqual(ignored, { + logicalType: undefined, + convertedType: undefined, + // the order belongs to the leaf, so it stays when the annotation was on the group around it + columnOrder: otherType === GROUP ? "TYPE_ORDER" : undefined + }, label) + } + // UNKNOWN is for "a column that is always null", whatever its type + for (const type of [Type.BOOLEAN, Type.INT32, Type.BYTE_ARRAY]) { + assert.deepStrictEqual(yield* annotationOf(annotated(type, logical(11))), { + logicalType: { type: "UNKNOWN" }, + convertedType: undefined, + columnOrder: "TYPE_ORDER" + }) + } + })) + + it.effect("PQ-LT-5 reads a node's logical type and converted type independently of each other", () => + Effect.gen(function*() { + // both defined for INT32, though DATE corresponds to the converted type DATE + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { ...logical(6), convertedType: 7 })), { + logicalType: { type: "DATE" }, + convertedType: "TIME_MILLIS", + columnOrder: "TYPE_ORDER" + }) + // STRING is not defined for INT32, INT_32 is + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { ...logical(1), convertedType: 17 })), { + logicalType: undefined, + convertedType: "INT_32", + columnOrder: undefined + }) + // the DecimalType supersedes the scale and precision of the schema element (parquet.thrift L557-564) + assert.deepStrictEqual( + yield* annotationOf(annotated(Type.INT32, { ...logical(5, { 1: I32(2), 2: I32(9) }), convertedType: 5 })), + { + logicalType: { type: "DECIMAL", scale: 2, precision: 9 }, + convertedType: "DECIMAL", + columnOrder: "TYPE_ORDER" + } + ) + // a converted type 2.14.0 doesn't have + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { convertedType: 99 })), { + logicalType: undefined, + convertedType: 99, + columnOrder: undefined + }) + })) + + it.effect("PQ-LT-6 MUST reject a DECIMAL whose precision or scale breaks the rules (#decimal)", () => + Effect.gen(function*() { + type Case = readonly [ + type: number, + scale: number | undefined, + precision: number | undefined, + typeLength?: number + ] + const forms = ( + [type, scale, precision, typeLength]: Case + ) => [ + // the converted type takes its parameters from the schema element, where the scale defaults to 0 + annotated(type, { convertedType: 5, scale, precision, typeLength }), + ...(scale === undefined || precision === undefined + ? [] + : [annotated(type, { ...logical(5, { 1: I32(scale), 2: I32(precision) }), typeLength })]) + ] + const valid: ReadonlyArray = [ + [Type.INT32, 0, 1], + [Type.INT32, 9, 9], + [Type.INT32, undefined, 9], + [Type.INT64, 2, 18], + // floor(log10(2^(8n - 1) - 1)) digits: 2^7 - 1 = 127, 2^31 - 1 = 2147483647, 2^127 - 1 is about 1.7e38 + [FIXED, 0, 2, 1], + [FIXED, 0, 9, 4], + [FIXED, 10, 38, 16], + [Type.BYTE_ARRAY, 0, 1000] + ] + for (const case_ of valid) { + for (const bytes_ of forms(case_)) { + assert.strictEqual((yield* annotationOf(bytes_)).columnOrder, "TYPE_ORDER", JSON.stringify(case_)) + } + } + const invalid: ReadonlyArray = [ + [[Type.INT32, 5, 2], /scale/], + [[Type.INT32, -1, 2], /scale/], + [[Type.INT32, 0, 0], /precision/], + [[Type.BYTE_ARRAY, 0, -1], /precision/], + [[Type.BYTE_ARRAY, 0, undefined], /precision/], + [[Type.INT32, 0, 10], /precision/], + [[Type.INT64, 0, 19], /precision/], + [[FIXED, 0, 3, 1], /precision/], + [[FIXED, 0, 10, 4], /precision/], + [[FIXED, 0, 39, 16], /precision/] + ] + for (const [case_, message] of invalid) { + for (const bytes_ of forms(case_)) yield* failsWith(metadataOf(bytes_), "Malformed", message) + } + })) + + it.effect("PQ-LT-7 MUST reject an INTEGER whose bit width is not 8, 16, 32 or 64 (#signed-integers)", () => + Effect.gen(function*() { + for (const bitWidth of [0, 24, 65, -8]) { + yield* failsWith(metadataOf(annotated(Type.INT32, integer(bitWidth, true))), "Malformed", /bit width/) + } + })) }) describe("README.md#nested-encoding", () => { diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index b6004203997..3e800fda251 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -27,6 +27,10 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-LT-1 | LogicalTypes.md#timestamp | Unknown `TIMESTAMP` units are unsupported features, not errors in the file | | PQ-LT-2 | parquet.thrift L370-373, L380-383 | `TimestampType.unit` and `TimeType.unit` are required (rejected when missing) | | PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, and so is its column's order | +| PQ-LT-4 | LogicalTypes.md#unsupported-logical-types | An annotation on a physical type it isn't defined for is ignored, with the column order | +| PQ-LT-5 | LogicalTypes.md#convertedtype | Local policy: a logical type and a converted type are checked independently | +| PQ-LT-6 | LogicalTypes.md#decimal | `DECIMAL` precision and scale fit the rules and the physical type (rejected otherwise) | +| PQ-LT-7 | LogicalTypes.md#signed-integers, parquet.thrift L388 | `INTEGER` bit widths are 8, 16, 32 or 64 (rejected otherwise) | | PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | | PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | | PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | @@ -80,7 +84,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. - Bloom filters (BloomFilter.md). - Page checksums (README.md#checksumming). `crc` is skipped, not verified. - Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. -- Logical types (LogicalTypes.md): annotations are decoded into the schema, values stay in their physical type. This includes the LIST and MAP backward-compatibility rules, Variant and Geometry. +- Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type and are not checked against the annotation: the range of a narrow `INTEGER`, which the spec leaves to the implementation, the digits of a `DECIMAL`, the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. - Error recovery (README.md#error-recovery): each `readColumn` call is independent, so a corrupt column chunk doesn't affect the others, but nothing inside a chunk is recovered. - Binary protocol extensions (BinaryProtocolExtensions.md). From af329cd93b14a95aa9d1ef00e9c5351095132aea Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:07:27 +0000 Subject: [PATCH 25/32] Reject STRING, ENUM and JSON values that are not UTF-8 --- packages/effect/src/encoding/Parquet.ts | 16 ++ .../effect/src/internal/parquet/encodings.ts | 41 ++++ packages/effect/test/encoding/Parquet.test.ts | 186 +++++++++++++++++- .../test/fixtures/parquet/CONFORMANCE.md | 107 +++++----- 4 files changed, 297 insertions(+), 53 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index fe2c9a3d73e..44bb4488908 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -41,6 +41,10 @@ * - Physical types: all eight. Values stay in their physical type; logical * types and converted types are decoded into the schema but not applied * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). + * - Text: every `STRING`, `ENUM` and `JSON` value, and every value of their + * converted types (`UTF8`, `ENUM`, `JSON`), is checked to be UTF-8, and so + * is every value of such a column's dictionary. Anything else fails with + * `Malformed`. `JSON` values are not parsed. * - Annotations: an annotation added after 2.14.0, or one the spec doesn't * define for the physical type it is on (a `STRING` on an `INT32`), is left * out of the schema, and so is the order of the column's statistics. A @@ -1393,6 +1397,8 @@ class ChunkDecoder { readonly kind: Encodings.Kind readonly numValues: number private readonly budget: Encodings.Budget + /** Whether the values are annotated as text, which has to be UTF-8. */ + private readonly utf8: boolean /** The levels of every value, nulls included; `undefined` when the column has none. */ private readonly repetitions: Uint8Array | undefined private readonly definitions: Uint8Array | undefined @@ -1407,6 +1413,11 @@ class ChunkDecoder { this.kind = kind this.numValues = numValues this.budget = { left: maxBinaryLength } + // A logical type supersedes the converted type. + const leaf = column.nodes[column.nodes.length - 1] + const annotation = leaf.logicalType?.type ?? leaf.convertedType + this.utf8 = kind.tag === "bytes" && + (annotation === "STRING" || annotation === "UTF8" || annotation === "ENUM" || annotation === "JSON") // Fixed-width values take their bytes in every slot, nulls included. if (kind.tag === "fixed" || kind.tag === "int96") Encodings.spend(this.budget, numValues * kind.width) this.repetitions = column.maxRepetitionLevel === 0 ? undefined : new Uint8Array(numValues) @@ -1421,6 +1432,7 @@ class ChunkDecoder { fail("Malformed", `a dictionary page is encoded with ${header.encoding}, not PLAIN`) } this.dictionary = Encodings.plain(this.kind, data, 0, data.length, header.numValues, this.budget) + if (this.utf8) Encodings.utf8(this.dictionary as Encodings.Binary, "dictionary value") } /** A v1 data page: repetition levels, definition levels and values, all after decompression. */ @@ -1546,6 +1558,10 @@ class ChunkDecoder { default: return fail("Unsupported", `the ${encoding} encoding is not supported`) } + // The values of a dictionary were checked with its page. + if (this.utf8 && encoding !== "PLAIN_DICTIONARY" && encoding !== "RLE_DICTIONARY") { + Encodings.utf8(dense as Encodings.Binary, "value") + } this.dense.push(dense) } diff --git a/packages/effect/src/internal/parquet/encodings.ts b/packages/effect/src/internal/parquet/encodings.ts index 26268373511..5e3bc0b8d07 100644 --- a/packages/effect/src/internal/parquet/encodings.ts +++ b/packages/effect/src/internal/parquet/encodings.ts @@ -486,6 +486,47 @@ export const deltaByteArray = ( return { offsets, data: out } } +/** + * Whether `data[pos, end)` is well-formed UTF-8 (the Unicode Standard, table + * 3-7): no overlong forms, no surrogates, nothing after U+10FFFF. + */ +const wellFormed = (data: Uint8Array, pos: number, end: number): boolean => { + while (pos < end) { + const lead = data[pos++] + if (lead < 0x80) continue + if (lead < 0xc2 || lead > 0xf4) return false + // The second byte's range is what excludes the forms above. + let more = 1 + let low = 0x80 + let high = 0xbf + if (lead >= 0xf0) { + more = 3 + if (lead === 0xf0) low = 0x90 + else if (lead === 0xf4) high = 0x8f + } else if (lead >= 0xe0) { + more = 2 + if (lead === 0xe0) low = 0xa0 + else if (lead === 0xed) high = 0x9f + } + if (more > end - pos) return false + const second = data[pos++] + if (second < low || second > high) return false + while (--more > 0) if ((data[pos++] & 0xc0) !== 0x80) return false + } + return true +} + +/** + * Fails unless every one of `values` is UTF-8 on its own. No strings are made. + * + * @internal + */ +export const utf8 = ({ data, offsets }: Binary, what: string): void => { + for (let i = 0, n = offsets.length - 1; i < n; i++) { + if (!wellFormed(data, offsets[i], offsets[i + 1])) fail("Malformed", `${what} ${i} is not UTF-8`) + } +} + /** The number of values in `dense`. @internal */ export const lengthOf = (kind: Kind, dense: Dense): number => kind.tag === "bytes" diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index b2717e8fde7..b94b9da768f 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -263,6 +263,8 @@ const file = (options: { return bytes(text("PAR1"), ...body, footer, u32(footer.length), text("PAR1")) } +type Annotation = Omit[1], "type" | "repetition" | "children"> + /** A file with one top-level column. */ const flat = (options: { readonly type: number @@ -277,11 +279,17 @@ const flat = (options: { readonly statistics?: V | undefined readonly chunk?: { readonly [id: number]: V } | undefined readonly metadata?: { readonly [id: number]: V } | undefined + readonly annotation?: Annotation | undefined }) => file({ schema: [ root(1), - element("x", { type: options.type, repetition: options.repetition, typeLength: options.typeLength }) + element("x", { + type: options.type, + repetition: options.repetition, + typeLength: options.typeLength, + ...options.annotation + }) ], numRows: options.numRows, chunks: [{ ...options, path: ["x"], numValues: options.numValues ?? options.numRows }], @@ -294,6 +302,7 @@ const requiredColumn = (type: number, n: number, data: Uint8Array, options?: { readonly typeLength?: number | undefined readonly codec?: number | undefined readonly uncompressedSize?: number | undefined + readonly annotation?: Annotation | undefined }) => flat({ type, @@ -301,6 +310,7 @@ const requiredColumn = (type: number, n: number, data: Uint8Array, options?: { numRows: n, typeLength: options?.typeLength, codec: options?.codec, + annotation: options?.annotation, pages: [dataPage(n, data, options)] }) @@ -407,7 +417,6 @@ describe("Parquet", () => { flat({ type: Type.INT32, repetition: REPEATED, numRows, numValues, pages }) // def levels 1, 0, 1 as RLE runs, for a column with one optional node const someNull = runs(1, [1, 1], [0, 1], [1, 1]) - type Annotation = Omit[1], "type" | "repetition" | "children"> /** * A file with one required column `x` that has statistics in TYPE_ORDER, and no rows. The annotation is on `x`, * or without a `type` on a group around it. @@ -772,6 +781,179 @@ describe("Parquet", () => { yield* failsWith(metadataOf(annotated(Type.INT32, integer(bitWidth, true))), "Malformed", /bit width/) } })) + type Binaries = ReadonlyArray> + const forms = ["PLAIN", "DELTA_LENGTH_BYTE_ARRAY", "DELTA_BYTE_ARRAY", "dictionary"] as const + /** A required BYTE_ARRAY column of `values`, each of which is its own dictionary entry in the last form. */ + const binaries = (form: typeof forms[number], values: Binaries, annotation?: Annotation) => { + const n = values.length + const lengths = values.map((value) => value.length) + const plain = bytes(...values.map((value) => lengthPrefixed(Uint8Array.from(value)))) + switch (form) { + case "PLAIN": + return requiredColumn(Type.BYTE_ARRAY, n, plain, { annotation }) + case "DELTA_LENGTH_BYTE_ARRAY": + return requiredColumn(Type.BYTE_ARRAY, n, bytes(delta(lengths), ...values), { + encoding: DELTA_LENGTH_BYTE_ARRAY, + annotation + }) + case "DELTA_BYTE_ARRAY": + return requiredColumn(Type.BYTE_ARRAY, n, bytes(delta(lengths.map(() => 0)), delta(lengths), ...values), { + encoding: DELTA_BYTE_ARRAY, + annotation + }) + case "dictionary": + return flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: n, + dictionary: true, + annotation, + pages: [ + dictionaryPage(n, plain), + dataPage(n, bytes([8], runs(8, ...values.map((_, i) => [i, 1] as const))), { encoding: RLE_DICTIONARY }) + ] + }) + } + } + const hex = (values: Binaries) => values.map((value) => Hex.encode(Uint8Array.from(value))) + const STRING = logical(1) + const utf8 = [ + ["STRING", STRING], + ["ENUM", logical(4)], + ["JSON", logical(12)], + ["UTF8", { convertedType: 0 }], + ["ENUM (converted)", { convertedType: 4 }], + ["JSON (converted)", { convertedType: 19 }] + ] as const + // "é" is C3 A9 + const split: Binaries = [[0x61, 0xc3], [0xa9]] + + it.effect("PQ-LT-8 MUST reject a STRING, ENUM or JSON value that is not UTF-8 (#string, #enum, #json)", () => + Effect.gen(function*() { + const valid: Binaries = [[], [0x61], [0xc3, 0xa9], [], [0xe2, 0x82, 0xac], [0xf0, 0x9f, 0x98, 0x80]] + for (const [label, annotation] of utf8) { + for (const form of forms) { + const where = `${label} ${form}` + assert.deepStrictEqual(yield* values(binaries(form, valid, annotation)), hex(valid), where) + yield* failsWith(values(binaries(form, [[0x61], [0xff]], annotation)), "Malformed", /value 1 .*UTF-8/) + // every value is UTF-8 on its own: here the two are only when joined + yield* failsWith(values(binaries(form, split, annotation)), "Malformed", /value 0 .*UTF-8/) + } + } + // v2 data pages + const v2 = (value: ReadonlyArray) => + flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: 1, + annotation: STRING, + pages: [dataPageV2({ + numValues: 1, + numNulls: 0, + numRows: 1, + values: lengthPrefixed(Uint8Array.from(value)) + })] + }) + assert.deepStrictEqual(yield* values(v2([0xc3, 0xa9])), ["c3a9"]) + yield* failsWith(values(v2([0xc3])), "Malformed", /value 0 .*UTF-8/) + })) + + it.effect("PQ-LT-9 MUST reject every sequence of bytes that is not well-formed UTF-8 (#string)", () => + Effect.gen(function*() { + const string = (value: ReadonlyArray) => values(binaries("PLAIN", [value], STRING)) + // the first and last code points of every length, and those around the surrogates + const valid: Binaries = [ + [0x00], + [0x7f], + [0xc2, 0x80], + [0xdf, 0xbf], + [0xe0, 0xa0, 0x80], + [0xed, 0x9f, 0xbf], + [0xee, 0x80, 0x80], + [0xef, 0xbf, 0xbf], + [0xf0, 0x90, 0x80, 0x80], + [0xf4, 0x8f, 0xbf, 0xbf], + // a byte order mark is a character like any other + [0xef, 0xbb, 0xbf, 0x61] + ] + for (const value of valid) assert.deepStrictEqual(yield* string(value), hex([value])) + const invalid: Binaries = [ + // a continuation byte first, and bytes that are never in UTF-8 + [0x80], + [0xbf], + [0xc0, 0x80], + [0xc1, 0xbf], + [0xf5, 0x80, 0x80, 0x80], + [0xff], + // sequences that end early + [0xc2], + [0xe2, 0x82], + [0xf0, 0x9f, 0x98], + [0xc2, 0x41], + [0xe2, 0x82, 0x41], + [0xf0, 0x9f, 0x98, 0x41], + // overlong forms + [0xe0, 0x9f, 0xbf], + [0xf0, 0x8f, 0xbf, 0xbf], + // surrogates, and code points after U+10FFFF + [0xed, 0xa0, 0x80], + [0xed, 0xbf, 0xbf], + [0xf4, 0x90, 0x80, 0x80] + ] + const strict = new TextDecoder("utf-8", { fatal: true }) + for (const value of invalid) { + yield* failsWith(string(value), "Malformed", /value 0 is not UTF-8/) + assert.throws(() => strict.decode(Uint8Array.from(value)), TypeError, undefined, hex([value])[0]) + } + })) + + it.effect("PQ-LT-10 checks DELTA_BYTE_ARRAY values after their prefixes are added (Encodings.md#delta-strings-delta_byte_array--7)", () => + Effect.gen(function*() { + const make = (prefixes: ReadonlyArray, suffixes: Binaries) => + values(requiredColumn( + Type.BYTE_ARRAY, + suffixes.length, + bytes(delta(prefixes), delta(suffixes.map((suffix) => suffix.length)), ...suffixes), + { encoding: DELTA_BYTE_ARRAY, annotation: STRING } + )) + // the suffix A9 is not UTF-8, the value C3 A9 is + assert.deepStrictEqual(yield* make([0, 1], [[0xc3, 0xa9], [0xa9]]), ["c3a9", "c3a9"]) + // the suffix "(" is UTF-8, the value C3 28 is not + yield* failsWith(make([0, 1], [[0xc3, 0xa9], [0x28]]), "Malformed", /value 1 .*UTF-8/) + })) + + it.effect("PQ-LT-11 checks a dictionary once, with the values no page refers to", () => + Effect.gen(function*() { + const make = (dictionary: Binaries) => + values(flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: 2, + dictionary: true, + annotation: STRING, + pages: [ + dictionaryPage( + dictionary.length, + bytes(...dictionary.map((value) => lengthPrefixed(Uint8Array.from(value)))) + ), + dataPage(2, bytes([1], runs(1, [0, 2])), { encoding: RLE_DICTIONARY }) + ] + })) + assert.deepStrictEqual(yield* make([[0x61], [0x62]]), ["61", "61"]) + yield* failsWith(make([[0x61], [0xff]]), "Malformed", /dictionary value 1 .*UTF-8/) + })) + + it.effect("PQ-LT-12 doesn't check the bytes of other BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns", () => + Effect.gen(function*() { + const binary: Binaries = [[0xff], ...split] + for (const annotation of [undefined, logical(13), { convertedType: 20 }, logical(17)]) { + for (const form of forms) { + assert.deepStrictEqual(yield* values(binaries(form, binary, annotation)), hex(binary)) + } + } + const fixed = requiredColumn(FIXED, 2, new Uint8Array([0xff, 0xc3, 0xa9, 0xff]), { typeLength: 2 }) + assert.deepStrictEqual(yield* values(fixed), ["ffc3", "a9ff"]) + })) }) describe("README.md#nested-encoding", () => { diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index 3e800fda251..25e2ed5d6f0 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -16,56 +16,61 @@ reader rejects, what it doesn't read yet, and what applies only to writers. ## Tested -| Id | Reference | Assertion | -| ----------- | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| PQ-FILE-1 | README.md#file-format | A file starts and ends with the magic number `PAR1` | -| PQ-FILE-2 | README.md#file-format | The metadata is found from the 4-byte little-endian length before the final magic | -| PQ-META-1 | parquet.thrift L1411-1413 | Readers should accept versions 1 and 2 interchangeably | -| PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | -| PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | -| PQ-SCHEMA-2 | parquet.thrift L518-523, L542-547 | The schema tree is the depth-first list of elements, groups giving `num_children` | -| PQ-LT-1 | LogicalTypes.md#timestamp | Unknown `TIMESTAMP` units are unsupported features, not errors in the file | -| PQ-LT-2 | parquet.thrift L370-373, L380-383 | `TimestampType.unit` and `TimeType.unit` are required (rejected when missing) | -| PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, and so is its column's order | -| PQ-LT-4 | LogicalTypes.md#unsupported-logical-types | An annotation on a physical type it isn't defined for is ignored, with the column order | -| PQ-LT-5 | LogicalTypes.md#convertedtype | Local policy: a logical type and a converted type are checked independently | -| PQ-LT-6 | LogicalTypes.md#decimal | `DECIMAL` precision and scale fit the rules and the physical type (rejected otherwise) | -| PQ-LT-7 | LogicalTypes.md#signed-integers, parquet.thrift L388 | `INTEGER` bit widths are 8, 16, 32 or 64 (rejected otherwise) | -| PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | -| PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | -| PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | -| PQ-PAGE-1 | README.md#data-pages | A required, non-nested column's page holds only the values | -| PQ-PAGE-2 | README.md#data-pages | `uncompressed_page_size` covers levels and values together | -| PQ-PAGE-3 | parquet.thrift L707, L733, L754-772, L835-838 | A page header's counts and sizes are not negative (rejected when they are) | -| PQ-CHUNK-1 | README.md#column-chunks, parquet.thrift L727-729 | The dictionary page comes first, and there is at most one | -| PQ-CHUNK-2 | README.md#column-chunks | Readers can skip pages they don't read (index pages, later page types) | -| PQ-CHUNK-3 | parquet.thrift L922-923 | A column chunk's `num_values` is not negative (rejected when it is) | -| PQ-V2-1 | parquet.thrift L759-761 | Every v2 data page begins at a row boundary | -| PQ-V2-2 | parquet.thrift L744-745, L767 | v2 levels are uncompressed RLE without a length prefix | -| PQ-V2-3 | parquet.thrift L745, L779 | v2 values are compressed only when `is_compressed` is true | -| PQ-V2-4 | parquet.thrift L755-756 | A v2 page holds `num_values - num_nulls` values | -| PQ-STATS-1 | parquet.thrift L286-289 | Readers must distinguish an absent `null_count` from 0 | -| PQ-STATS-2 | parquet.thrift L315-316 | Readers must not assume `nan_count` is 0 when absent | -| PQ-PLAIN-1 | Encodings.md#plain-plain--0 | PLAIN encoding of every physical type | -| PQ-DICT-1 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | Indices are a bit-width byte (at most 32) and RLE/bit-packed runs | -| PQ-DICT-2 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | The deprecated `PLAIN_DICTIONARY` is still read | -| PQ-RLE-1 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 1 | Bit-packed runs are packed from the least significant bit | -| PQ-RLE-2 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3 | Run lengths are at least 1 | -| PQ-RLE-3 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, length table | RLE booleans carry a 4-byte length in v1 and v2 pages | -| PQ-BP-1 | Encodings.md#bit-packed-deprecated-bit_packed--4, L182-195 | `BIT_PACKED` levels are packed from the most significant bit | -| PQ-BP-2 | Encodings.md#bit-packed-deprecated-bit_packed--4, L197-198 | `BIT_PACKED` is only for levels | -| PQ-DELTA-1 | Encodings.md#delta-encoding-delta_binary_packed--5, examples 1 and 2 | Header, blocks and miniblocks | -| PQ-DELTA-2 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary padding bits | -| PQ-DELTA-3 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary bit widths of unneeded miniblocks | -| PQ-DELTA-4 | Encodings.md#delta-encoding-delta_binary_packed--5 | Deltas wrap around in two's complement | -| PQ-DELTA-5 | Encodings.md#delta-encoding-delta_binary_packed--5 | Miniblocks are no wider than the physical type (rejected when they are) | -| PQ-DLBA-1 | Encodings.md#delta-length-byte-array-delta_length_byte_array--6 | Delta-encoded lengths, then the concatenated bytes | -| PQ-DBA-1 | Encodings.md#delta-strings-delta_byte_array--7 | Prefix lengths, then suffixes as `DELTA_LENGTH_BYTE_ARRAY` | -| PQ-DBA-2 | Encodings.md#delta-strings-delta_byte_array--7 | `FIXED_LEN_BYTE_ARRAY` values encode all their lengths | -| PQ-BSS-1 | Encodings.md#byte-stream-split-byte_stream_split--9 | Byte k of every value is in stream k | -| PQ-BSS-2 | Encodings.md#byte-stream-split-byte_stream_split--9 | The streams are exactly K * N bytes, with no padding | -| PQ-COMP-1 | Compression.md#codecs | Pages are decompressed with the column chunk's codec | -| PQ-COMP-2 | Compression.md#gzip | Readers should read GZIP pages with several members (through the supplied `Decompress`) | +| Id | Reference | Assertion | +| ----------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| PQ-FILE-1 | README.md#file-format | A file starts and ends with the magic number `PAR1` | +| PQ-FILE-2 | README.md#file-format | The metadata is found from the 4-byte little-endian length before the final magic | +| PQ-META-1 | parquet.thrift L1411-1413 | Readers should accept versions 1 and 2 interchangeably | +| PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | +| PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | +| PQ-SCHEMA-2 | parquet.thrift L518-523, L542-547 | The schema tree is the depth-first list of elements, groups giving `num_children` | +| PQ-LT-1 | LogicalTypes.md#timestamp | Unknown `TIMESTAMP` units are unsupported features, not errors in the file | +| PQ-LT-2 | parquet.thrift L370-373, L380-383 | `TimestampType.unit` and `TimeType.unit` are required (rejected when missing) | +| PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, and so is its column's order | +| PQ-LT-4 | LogicalTypes.md#unsupported-logical-types | An annotation on a physical type it isn't defined for is ignored, with the column order | +| PQ-LT-5 | LogicalTypes.md#convertedtype | Local policy: a logical type and a converted type are checked independently | +| PQ-LT-6 | LogicalTypes.md#decimal | `DECIMAL` precision and scale fit the rules and the physical type (rejected otherwise) | +| PQ-LT-7 | LogicalTypes.md#signed-integers, parquet.thrift L388 | `INTEGER` bit widths are 8, 16, 32 or 64 (rejected otherwise) | +| PQ-LT-8 | LogicalTypes.md#string, #enum, #json, parquet.thrift L50, L322, L326 | `STRING`, `ENUM` and `JSON` values, and those of their converted types, are UTF-8 (rejected otherwise) | +| PQ-LT-9 | LogicalTypes.md#string | Local policy: UTF-8 is the well-formed sequences of the Unicode Standard, which the spec doesn't define | +| PQ-LT-10 | Encodings.md#delta-strings-delta_byte_array--7 | `DELTA_BYTE_ARRAY` values are checked whole, after their prefix is added | +| PQ-LT-11 | LogicalTypes.md#string | Local policy: a dictionary is checked once with its page, values no page refers to included | +| PQ-LT-12 | LogicalTypes.md#bson, README.md#types | Unannotated, `BSON` and geospatial `BYTE_ARRAY` values and `FIXED_LEN_BYTE_ARRAY` values are any bytes | +| PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | +| PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | +| PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | +| PQ-PAGE-1 | README.md#data-pages | A required, non-nested column's page holds only the values | +| PQ-PAGE-2 | README.md#data-pages | `uncompressed_page_size` covers levels and values together | +| PQ-PAGE-3 | parquet.thrift L707, L733, L754-772, L835-838 | A page header's counts and sizes are not negative (rejected when they are) | +| PQ-CHUNK-1 | README.md#column-chunks, parquet.thrift L727-729 | The dictionary page comes first, and there is at most one | +| PQ-CHUNK-2 | README.md#column-chunks | Readers can skip pages they don't read (index pages, later page types) | +| PQ-CHUNK-3 | parquet.thrift L922-923 | A column chunk's `num_values` is not negative (rejected when it is) | +| PQ-V2-1 | parquet.thrift L759-761 | Every v2 data page begins at a row boundary | +| PQ-V2-2 | parquet.thrift L744-745, L767 | v2 levels are uncompressed RLE without a length prefix | +| PQ-V2-3 | parquet.thrift L745, L779 | v2 values are compressed only when `is_compressed` is true | +| PQ-V2-4 | parquet.thrift L755-756 | A v2 page holds `num_values - num_nulls` values | +| PQ-STATS-1 | parquet.thrift L286-289 | Readers must distinguish an absent `null_count` from 0 | +| PQ-STATS-2 | parquet.thrift L315-316 | Readers must not assume `nan_count` is 0 when absent | +| PQ-PLAIN-1 | Encodings.md#plain-plain--0 | PLAIN encoding of every physical type | +| PQ-DICT-1 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | Indices are a bit-width byte (at most 32) and RLE/bit-packed runs | +| PQ-DICT-2 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | The deprecated `PLAIN_DICTIONARY` is still read | +| PQ-RLE-1 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 1 | Bit-packed runs are packed from the least significant bit | +| PQ-RLE-2 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3 | Run lengths are at least 1 | +| PQ-RLE-3 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, length table | RLE booleans carry a 4-byte length in v1 and v2 pages | +| PQ-BP-1 | Encodings.md#bit-packed-deprecated-bit_packed--4, L182-195 | `BIT_PACKED` levels are packed from the most significant bit | +| PQ-BP-2 | Encodings.md#bit-packed-deprecated-bit_packed--4, L197-198 | `BIT_PACKED` is only for levels | +| PQ-DELTA-1 | Encodings.md#delta-encoding-delta_binary_packed--5, examples 1 and 2 | Header, blocks and miniblocks | +| PQ-DELTA-2 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary padding bits | +| PQ-DELTA-3 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary bit widths of unneeded miniblocks | +| PQ-DELTA-4 | Encodings.md#delta-encoding-delta_binary_packed--5 | Deltas wrap around in two's complement | +| PQ-DELTA-5 | Encodings.md#delta-encoding-delta_binary_packed--5 | Miniblocks are no wider than the physical type (rejected when they are) | +| PQ-DLBA-1 | Encodings.md#delta-length-byte-array-delta_length_byte_array--6 | Delta-encoded lengths, then the concatenated bytes | +| PQ-DBA-1 | Encodings.md#delta-strings-delta_byte_array--7 | Prefix lengths, then suffixes as `DELTA_LENGTH_BYTE_ARRAY` | +| PQ-DBA-2 | Encodings.md#delta-strings-delta_byte_array--7 | `FIXED_LEN_BYTE_ARRAY` values encode all their lengths | +| PQ-BSS-1 | Encodings.md#byte-stream-split-byte_stream_split--9 | Byte k of every value is in stream k | +| PQ-BSS-2 | Encodings.md#byte-stream-split-byte_stream_split--9 | The streams are exactly K * N bytes, with no padding | +| PQ-COMP-1 | Compression.md#codecs | Pages are decompressed with the column chunk's codec | +| PQ-COMP-2 | Compression.md#gzip | Readers should read GZIP pages with several members (through the supplied `Decompress`) | ## Unsupported: fails with `Unsupported` @@ -84,7 +89,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. - Bloom filters (BloomFilter.md). - Page checksums (README.md#checksumming). `crc` is skipped, not verified. - Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. -- Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type and are not checked against the annotation: the range of a narrow `INTEGER`, which the spec leaves to the implementation, the digits of a `DECIMAL`, the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. +- Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type and, but for the UTF-8 of text (PQ-LT-8), are not checked against the annotation: that `JSON` is valid JSON, the range of a narrow `INTEGER`, which the spec leaves to the implementation, the digits of a `DECIMAL`, the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. - Error recovery (README.md#error-recovery): each `readColumn` call is independent, so a corrupt column chunk doesn't affect the others, but nothing inside a chunk is recovered. - Binary protocol extensions (BinaryProtocolExtensions.md). From 300b34c051afc9627d6d7dde23089baf52c61d5e Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:11:17 +0000 Subject: [PATCH 26/32] Check pages against the CRC in their header --- packages/effect/src/encoding/Parquet.ts | 21 ++- packages/effect/src/encoding/Zip.ts | 57 +------ packages/effect/src/internal/crc32.ts | 63 +++++++ packages/effect/test/encoding/Parquet.test.ts | 160 +++++++++++++++++- .../test/fixtures/parquet/CONFORMANCE.md | 6 +- 5 files changed, 238 insertions(+), 69 deletions(-) create mode 100644 packages/effect/src/internal/crc32.ts diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index 44bb4488908..cd8a6ec9560 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -55,7 +55,9 @@ * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; * levels in `RLE` or the deprecated `BIT_PACKED`. * - Pages: data pages v1 and v2 and dictionary pages; index pages and page - * types added after 2.14.0 are skipped. + * types added after 2.14.0 are skipped. A page with a CRC-32 in its header + * is checked against it, the skipped ones too, and fails with `Malformed` + * when its bytes don't match. A page without one is read as it is. * - Nesting: any depth of optional, required and repeated fields, assembled from * repetition and definition levels into {@link Layer}s. * @@ -63,7 +65,7 @@ * encrypted files and columns, the `ALP` encoding, column chunks in other * files (`file_path`), `TIME` and `TIMESTAMP` units added after 2.14.0, and * compressed pages without a matching {@link Decompress}. Not read: page - * indexes, bloom filters, page checksums, size and geospatial statistics. + * indexes, bloom filters, size and geospatial statistics. * * Input is untrusted: every offset and length is checked against the bytes * that are there, and the sizes a file claims are capped by options @@ -77,6 +79,7 @@ */ import * as Data from "../Data.ts" import * as Effect from "../Effect.ts" +import { crc32 } from "../internal/crc32.ts" import * as Encodings from "../internal/parquet/encodings.ts" import * as Predicate from "../Predicate.ts" import * as Thrift from "./Thrift.ts" @@ -91,8 +94,8 @@ const ParquetErrorTypeId = "~effect/encoding/Parquet/ParquetError" * The `kind` field identifies the failure: * * - `Malformed`: the file breaks the Parquet format, for example a bad magic - * number, invalid Thrift metadata, an offset outside the file, or pages that - * don't add up to their column chunk. + * number, invalid Thrift metadata, an offset outside the file, pages that + * don't add up to their column chunk, or a page that doesn't match its CRC. * - `Unsupported`: the file uses a feature this module doesn't read, or a * compression codec with no {@link Decompress} in the options. * - `Limit`: the file needs more than an option such as `maxPageSize` allows. @@ -1249,6 +1252,8 @@ interface PageHeader { readonly type: number readonly uncompressedSize: number readonly compressedSize: number + /** The CRC-32 as the header holds it, a signed i32. */ + readonly crc: number | undefined readonly numValues: number readonly encoding: Encoding | number readonly definitionEncoding: Encoding | number @@ -1265,6 +1270,7 @@ const pageHeader = (r: Reader): PageHeader => { type: [1, i32], uncompressedSize: [2, i32], compressedSize: [3, i32], + crc: [4, i32], dataPage: [ 5, structOf((r) => @@ -1306,6 +1312,7 @@ const pageHeader = (r: Reader): PageHeader => { type, uncompressedSize, compressedSize, + crc: f.crc, numValues: 0, encoding: "PLAIN", definitionEncoding: "RLE", @@ -1857,9 +1864,13 @@ export const readColumn = ( return yield* error("Limit", `a page of ${where} has more values than maxValues (${maxValues})`) } pos = body + header.compressedSize + const page = bytes.subarray(body, pos) + // The CRC is of the page as it is stored: with its levels, and compressed. + if (header.crc !== undefined && crc32(0, page) !== header.crc >>> 0) { + return yield* error("Malformed", `${where}, page at byte ${headerAt}: the page does not match its CRC`) + } // Index pages and page types added later are skipped. if (header.type !== DATA_PAGE && header.type !== DATA_PAGE_V2 && header.type !== DICTIONARY_PAGE) continue - const page = bytes.subarray(body, pos) const levels = header.type === DATA_PAGE_V2 ? header.repetitionLength + header.definitionLength : 0 const codec = header.compressed ? meta.codec : "UNCOMPRESSED" yield* decompress(codec, options.codecs, page.subarray(levels), header.uncompressedSize - levels).pipe( diff --git a/packages/effect/src/encoding/Zip.ts b/packages/effect/src/encoding/Zip.ts index cb9eadd459e..3be685aa28f 100644 --- a/packages/effect/src/encoding/Zip.ts +++ b/packages/effect/src/encoding/Zip.ts @@ -27,6 +27,7 @@ import * as Channel from "../Channel.ts" import * as Data from "../Data.ts" import * as Effect from "../Effect.ts" import * as Exit from "../Exit.ts" +import { crc32 } from "../internal/crc32.ts" import { ByteQueue, InflateError, InflateMemory, Inflater, NeedInput } from "../internal/inflate.ts" import * as Predicate from "../Predicate.ts" import * as Pull from "../Pull.ts" @@ -837,62 +838,6 @@ const decodeName = (bytes: Uint8Array, utf8: boolean): string => { } } -// CRC-32 (IEEE) lookup tables for sixteen bytes at a time -const CRC_TABLE = (() => { - const table = new Int32Array(16 * 256) - for (let n = 0; n < 256; n++) { - let c = n - for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 - table[n] = c - } - for (let n = 0; n < 256; n++) { - let c = table[n] - for (let k = 1; k < 16; k++) { - c = table[c & 0xff] ^ (c >>> 8) - table[k * 256 + n] = c - } - } - return table -})() - -const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1 - -const crc32 = (crc: number, bytes: Uint8Array): number => { - const t = CRC_TABLE - let c = ~crc - let i = 0 - const length = bytes.length - // Reading whole words is faster than combining bytes, but needs aligned, - // little-endian words - if (littleEndian && length >= 64) { - for (; ((bytes.byteOffset + i) & 3) !== 0; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) - const count = ((length - i) >>> 4) << 2 - const words = new Int32Array(bytes.buffer, bytes.byteOffset + i, count) - for (let j = 0; j < count; j += 4) { - const a = c ^ words[j] - const b = words[j + 1] - const d = words[j + 2] - const e = words[j + 3] - c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ - t[3072 + (a >>> 24)] ^ t[2816 + (b & 0xff)] ^ t[2560 + ((b >>> 8) & 0xff)] ^ - t[2304 + ((b >>> 16) & 0xff)] ^ t[2048 + (b >>> 24)] ^ t[1792 + (d & 0xff)] ^ - t[1536 + ((d >>> 8) & 0xff)] ^ t[1280 + ((d >>> 16) & 0xff)] ^ t[1024 + (d >>> 24)] ^ - t[768 + (e & 0xff)] ^ t[512 + ((e >>> 8) & 0xff)] ^ t[256 + ((e >>> 16) & 0xff)] ^ t[e >>> 24] - } - i += count * 4 - } - for (; i + 16 <= length; i += 16) { - const a = c ^ (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24)) - c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ - t[3072 + (a >>> 24)] ^ t[2816 + bytes[i + 4]] ^ t[2560 + bytes[i + 5]] ^ t[2304 + bytes[i + 6]] ^ - t[2048 + bytes[i + 7]] ^ t[1792 + bytes[i + 8]] ^ t[1536 + bytes[i + 9]] ^ t[1280 + bytes[i + 10]] ^ - t[1024 + bytes[i + 11]] ^ t[768 + bytes[i + 12]] ^ t[512 + bytes[i + 13]] ^ t[256 + bytes[i + 14]] ^ - t[bytes[i + 15]] - } - for (; i < length; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) - return ~c >>> 0 -} - const validate = (header: Header): void => { const { method, mode, mtime, path, size, type } = header if (path === "") throw invalidHeader("path is empty") diff --git a/packages/effect/src/internal/crc32.ts b/packages/effect/src/internal/crc32.ts new file mode 100644 index 00000000000..1957790cc46 --- /dev/null +++ b/packages/effect/src/internal/crc32.ts @@ -0,0 +1,63 @@ +// CRC-32 (IEEE, the one of GZIP and ZIP) for the binary codecs. + +// Lookup tables for sixteen bytes at a time +const CRC_TABLE = (() => { + const table = new Int32Array(16 * 256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + table[n] = c + } + for (let n = 0; n < 256; n++) { + let c = table[n] + for (let k = 1; k < 16; k++) { + c = table[c & 0xff] ^ (c >>> 8) + table[k * 256 + n] = c + } + } + return table +})() + +const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1 + +/** + * The CRC-32 of `bytes` as an unsigned number, continuing from `crc`: 0, or + * the CRC-32 of the bytes before them. + * + * @internal + */ +export const crc32 = (crc: number, bytes: Uint8Array): number => { + const t = CRC_TABLE + let c = ~crc + let i = 0 + const length = bytes.length + // Reading whole words is faster than combining bytes, but needs aligned, + // little-endian words + if (littleEndian && length >= 64) { + for (; ((bytes.byteOffset + i) & 3) !== 0; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) + const count = ((length - i) >>> 4) << 2 + const words = new Int32Array(bytes.buffer, bytes.byteOffset + i, count) + for (let j = 0; j < count; j += 4) { + const a = c ^ words[j] + const b = words[j + 1] + const d = words[j + 2] + const e = words[j + 3] + c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ + t[3072 + (a >>> 24)] ^ t[2816 + (b & 0xff)] ^ t[2560 + ((b >>> 8) & 0xff)] ^ + t[2304 + ((b >>> 16) & 0xff)] ^ t[2048 + (b >>> 24)] ^ t[1792 + (d & 0xff)] ^ + t[1536 + ((d >>> 8) & 0xff)] ^ t[1280 + ((d >>> 16) & 0xff)] ^ t[1024 + (d >>> 24)] ^ + t[768 + (e & 0xff)] ^ t[512 + ((e >>> 8) & 0xff)] ^ t[256 + ((e >>> 16) & 0xff)] ^ t[e >>> 24] + } + i += count * 4 + } + for (; i + 16 <= length; i += 16) { + const a = c ^ (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24)) + c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ + t[3072 + (a >>> 24)] ^ t[2816 + bytes[i + 4]] ^ t[2560 + bytes[i + 5]] ^ t[2304 + bytes[i + 6]] ^ + t[2048 + bytes[i + 7]] ^ t[1792 + bytes[i + 8]] ^ t[1536 + bytes[i + 9]] ^ t[1280 + bytes[i + 10]] ^ + t[1024 + bytes[i + 11]] ^ t[768 + bytes[i + 12]] ^ t[512 + bytes[i + 13]] ^ t[256 + bytes[i + 14]] ^ + t[bytes[i + 15]] + } + for (; i < length; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) + return ~c >>> 0 +} diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index b94b9da768f..4e4fc8fbc75 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -4,7 +4,7 @@ import * as Hex from "effect/encoding/Hex" import * as Parquet from "effect/encoding/Parquet" import * as Thrift from "effect/encoding/Thrift" import { readFileSync } from "node:fs" -import { gunzipSync, gzipSync } from "node:zlib" +import { crc32, gunzipSync, gzipSync } from "node:zlib" // Two kinds of evidence: // - "pyarrow fixtures": files written by pyarrow, read back and compared with @@ -155,12 +155,29 @@ const element = (name: string, options: { }) const root = (children: number) => element("schema", { children }) -const page = (type: number, header: { readonly [id: number]: V }, body: Uint8Array, uncompressedSize = body.length) => - bytes(thrift(Struct({ 1: I32(type), 2: I32(uncompressedSize), 3: I32(body.length), ...header })), body) +/** `crc` is the unsigned CRC-32, which the header holds as a signed i32. */ +const page = ( + type: number, + header: { readonly [id: number]: V }, + body: Uint8Array, + uncompressedSize = body.length, + crc?: number +) => + bytes( + thrift(Struct({ + 1: I32(type), + 2: I32(uncompressedSize), + 3: I32(body.length), + 4: crc === undefined ? undefined : I32(crc | 0), + ...header + })), + body + ) const dataPage = (numValues: number, body: Uint8Array, options?: { readonly encoding?: number | undefined readonly levels?: number | undefined readonly uncompressedSize?: number | undefined + readonly crc?: number | undefined }) => page( 0, @@ -173,10 +190,15 @@ const dataPage = (numValues: number, body: Uint8Array, options?: { }) }, body, - options?.uncompressedSize + options?.uncompressedSize, + options?.crc ) -const dictionaryPage = (numValues: number, body: Uint8Array, encoding = PLAIN) => - page(2, { 7: Struct({ 1: I32(numValues), 2: I32(encoding) }) }, body) +const dictionaryPage = ( + numValues: number, + body: Uint8Array, + encoding = PLAIN, + options?: { readonly uncompressedSize?: number | undefined; readonly crc?: number | undefined } +) => page(2, { 7: Struct({ 1: I32(numValues), 2: I32(encoding) }) }, body, options?.uncompressedSize, options?.crc) const dataPageV2 = (options: { readonly numValues: number readonly numNulls: number @@ -191,6 +213,8 @@ const dataPageV2 = (options: { readonly definitionLength?: number | undefined /** Defaults to the length of `repetition`. */ readonly repetitionLength?: number | undefined + /** Computes the CRC from the page's bytes: levels, then values. */ + readonly crc?: ((body: Uint8Array) => number) | undefined }) => { const repetition = options.repetition ?? new Uint8Array(0) const definition = options.definition ?? new Uint8Array(0) @@ -209,7 +233,8 @@ const dataPageV2 = (options: { }) }, body, - options.uncompressedSize + options.uncompressedSize, + options.crc?.(body) ) } @@ -303,6 +328,7 @@ const requiredColumn = (type: number, n: number, data: Uint8Array, options?: { readonly codec?: number | undefined readonly uncompressedSize?: number | undefined readonly annotation?: Annotation | undefined + readonly crc?: number | undefined }) => flat({ type, @@ -1448,6 +1474,126 @@ describe("Parquet", () => { })) }) + describe("README.md#checksumming", () => { + const options = { codecs: { GZIP: gzip } } + const wrong = (body: Uint8Array) => (crc32(body) ^ 1) >>> 0 + + it.effect("PQ-CRC-1 MUST check a v1 data page against its CRC, of the bytes after the header (parquet.thrift L840-852)", () => + Effect.gen(function*() { + const seven = int32s(7) + const eight = int32s(8) + const make = (body: Uint8Array, crc?: number) => values(requiredColumn(Type.INT32, 1, body, { crc })) + assert.deepStrictEqual(yield* make(seven, crc32(seven)), [7]) + // the value changed and the CRC didn't + yield* failsWith(make(eight, crc32(seven)), "Malformed", /CRC/) + // a page without a CRC is not checked + assert.deepStrictEqual(yield* make(eight), [8]) + // of the compressed bytes, with the levels + const body = bytes(lengthPrefixed(someNull), int32s(5, 7)) + const compressed = gzipSync(body) + const gzipped = (crc: number) => + values( + optionalInt32([dataPage(3, compressed, { uncompressedSize: body.length, crc })], 3, GZIP), + options + ) + assert.deepStrictEqual(yield* gzipped(crc32(compressed)), [5, null, 7]) + yield* failsWith(gzipped(crc32(body)), "Malformed", /CRC/) + })) + + it.effect("PQ-CRC-2 MUST check a v2 data page against its CRC, of the levels and the stored values (parquet.thrift L840-852)", () => + Effect.gen(function*() { + const values_ = int32s(5, 7) + const make = (compressed: boolean, crc: (body: Uint8Array) => number) => + values( + optionalInt32( + [dataPageV2({ + numValues: 3, + numNulls: 1, + numRows: 3, + definition: someNull, + values: compressed ? gzipSync(values_) : values_, + compressed, + uncompressedSize: someNull.length + values_.length, + crc + })], + 3, + GZIP + ), + options + ) + for (const compressed of [false, true]) { + assert.deepStrictEqual(yield* make(compressed, crc32), [5, null, 7]) + yield* failsWith(make(compressed, wrong), "Malformed", /CRC/) + // not of the values alone + yield* failsWith( + make(compressed, (body) => crc32(body.subarray(someNull.length))), + "Malformed", + /CRC/ + ) + } + })) + + it.effect("PQ-CRC-3 MUST check a dictionary page against its CRC (parquet.thrift L840-852)", () => + Effect.gen(function*() { + const dictionary = int32s(10, 11) + const make = (compressed: boolean, crc: (body: Uint8Array) => number) => { + const stored = (body: Uint8Array) => compressed ? gzipSync(body) : body + const indices = bytes([1], runs(1, [1, 2], [0, 1])) + return values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + dictionary: true, + codec: compressed ? GZIP : undefined, + pages: [ + dictionaryPage(2, stored(dictionary), PLAIN, { + uncompressedSize: dictionary.length, + crc: crc(stored(dictionary)) + }), + dataPage(3, stored(indices), { encoding: RLE_DICTIONARY, uncompressedSize: indices.length }) + ] + }), + options + ) + } + for (const compressed of [false, true]) { + assert.deepStrictEqual(yield* make(compressed, crc32), [11, 11, 10]) + yield* failsWith(make(compressed, wrong), "Malformed", /CRC/) + } + })) + + it.effect("PQ-CRC-4 MUST compute the standard CRC-32 of GZIP, as an unsigned number (#checksumming)", () => + Effect.gen(function*() { + // the check value of CRC-32, which is negative as the i32 of the page header + const check = 0xcbf43926 + const body = text("123456789") + const make = (crc: number) => + values(requiredColumn(Type.FIXED_LEN_BYTE_ARRAY, 1, body, { typeLength: 9, crc })) + assert.strictEqual(check | 0, -873187034) + assert.deepStrictEqual(yield* make(check), [Hex.encode(body)]) + yield* failsWith(make(check ^ 0x80000000), "Malformed", /CRC/) + // CRC-32C, of the same bytes + yield* failsWith(make(0xe3069283), "Malformed", /CRC/) + })) + + it.effect("PQ-CRC-5 checks the CRC of the pages it skips", () => + Effect.gen(function*() { + const index = text("index") + const make = (crc: number) => + values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [page(1, { 6: Struct({}) }, index, index.length, crc), dataPage(3, int32s(1, 2, 3))] + }) + ) + assert.deepStrictEqual(yield* make(crc32(index)), [1, 2, 3]) + yield* failsWith(make(wrong(index)), "Malformed", /CRC/) + })) + }) + describe("unsupported features", () => { it.effect("PQ-UNSUP-1 fails with Unsupported on an encrypted footer (PARE)", () => Effect.gen(function*() { diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index 25e2ed5d6f0..efd27f1a7c2 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -71,6 +71,11 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-BSS-2 | Encodings.md#byte-stream-split-byte_stream_split--9 | The streams are exactly K * N bytes, with no padding | | PQ-COMP-1 | Compression.md#codecs | Pages are decompressed with the column chunk's codec | | PQ-COMP-2 | Compression.md#gzip | Readers should read GZIP pages with several members (through the supplied `Decompress`) | +| PQ-CRC-1 | README.md#checksumming, parquet.thrift L840-852 | A v1 data page's `crc` is of its stored bytes after the header (rejected on a mismatch) | +| PQ-CRC-2 | parquet.thrift L840-852 | A v2 data page's `crc` is of its levels and its values as stored | +| PQ-CRC-3 | parquet.thrift L840-852 | A dictionary page's `crc` is checked the same way | +| PQ-CRC-4 | README.md#checksumming, parquet.thrift L842-843 | The CRC is the standard CRC-32 of GZIP; the i32 of the header is read as unsigned | +| PQ-CRC-5 | parquet.thrift L844-845 | Local policy: the `crc` of a skipped page is checked too | ## Unsupported: fails with `Unsupported` @@ -87,7 +92,6 @@ reader rejects, what it doesn't read yet, and what applies only to writers. - Page index: `ColumnIndex` and `OffsetIndex` (PageIndex.md, parquet.thrift L1235-1373). The reader reads every page in order. - Bloom filters (BloomFilter.md). -- Page checksums (README.md#checksumming). `crc` is skipped, not verified. - Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. - Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type and, but for the UTF-8 of text (PQ-LT-8), are not checked against the annotation: that `JSON` is valid JSON, the range of a narrow `INTEGER`, which the spec leaves to the implementation, the digits of a `DECIMAL`, the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. - Error recovery (README.md#error-recovery): each `readColumn` call is independent, so a corrupt column chunk doesn't affect the others, but nothing inside a chunk is recovered. From 492b33b6132467df771a061cba17ecb6e2fb5baf Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:12:14 +0000 Subject: [PATCH 27/32] Simplify the DECIMAL precision check --- packages/effect/src/encoding/Parquet.ts | 27 +++++++++---------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index cd8a6ec9560..46f550eda4b 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -1010,23 +1010,16 @@ const annotates = (annotation: Annotation, e: Element, physicalType: PhysicalTyp fail("Malformed", `schema element "${e.name}" is a DECIMAL with ${reason}`) if (precision < 1) invalid(`a precision of ${precision}`) if (scale < 0 || scale > precision) invalid(`a scale of ${scale} for a precision of ${precision}`) - let digits: number - switch (physicalType) { - case "INT32": - digits = 9 - break - case "INT64": - digits = 18 - break - case "FIXED_LEN_BYTE_ARRAY": - // floor(log10(2^(8n - 1) - 1)), which is this as no power of 2 is a power of 10 - digits = Math.floor((8 * e.typeLength! - 1) * Math.log10(2)) - break - case "BYTE_ARRAY": - return true - default: - return false - } + if (physicalType === "BYTE_ARRAY") return true + const digits = physicalType === "INT32" + ? 9 + : physicalType === "INT64" + ? 18 + : physicalType === "FIXED_LEN_BYTE_ARRAY" + // floor(log10(2^(8n - 1) - 1)), which is this as no power of 2 is a power of 10 + ? Math.floor((8 * e.typeLength! - 1) * Math.log10(2)) + : undefined + if (digits === undefined) return false if (precision > digits) { invalid(`a precision of ${precision}, more than the ${digits} digits of its ${physicalType}`) } From 13df7481c793f95709919a6457ede3b6287a7ef5 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:22:01 +0000 Subject: [PATCH 28/32] Share the CRC-32 file with the compression branch verbatim --- packages/effect/src/internal/crc32.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/effect/src/internal/crc32.ts b/packages/effect/src/internal/crc32.ts index 1957790cc46..d6ededffad9 100644 --- a/packages/effect/src/internal/crc32.ts +++ b/packages/effect/src/internal/crc32.ts @@ -1,6 +1,7 @@ -// CRC-32 (IEEE, the one of GZIP and ZIP) for the binary codecs. +// CRC-32 shared by `effect/encoding/Zip` and the gzip decoder of +// `effect/encoding/Compression`. -// Lookup tables for sixteen bytes at a time +// CRC-32 (IEEE) lookup tables for sixteen bytes at a time const CRC_TABLE = (() => { const table = new Int32Array(16 * 256) for (let n = 0; n < 256; n++) { @@ -21,8 +22,8 @@ const CRC_TABLE = (() => { const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1 /** - * The CRC-32 of `bytes` as an unsigned number, continuing from `crc`: 0, or - * the CRC-32 of the bytes before them. + * Continues the CRC-32 (IEEE, as in zip and gzip) `crc` over `bytes`; start + * from 0. * * @internal */ From e1795054c0aacdea045376d7b8e00e893831423f Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:26:02 +0000 Subject: [PATCH 29/32] Say that statistics have no known order when the column order is undefined --- packages/effect/src/encoding/Parquet.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index 46f550eda4b..77db0a9e200 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -346,6 +346,12 @@ export interface KeyValue { * {@link ColumnDescriptor.columnOrder}; `min` and `max` are the deprecated * fields, ordered by signed comparison. * + * **Gotchas** + * + * - When the column order is `undefined`, `minValue` and `maxValue` have no + * known order. They are returned as written: don't compare values with + * them. + * * @stability experimental * @category models * @since 4.1.5 From 126a31f09e1e9f6cbf5f9260dd185c12cbc503c0 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 11:19:58 +0000 Subject: [PATCH 30/32] Ignore the converted type of a node whose logical type is ignored, and reject negative row counts LogicalTypes.md uses a ConvertedType only when no LogicalType is present, and an ignored LogicalType leaves only the physical type, so UTF8 next to an unknown or misplaced logical type no longer validates text. num_rows of the file and of a row group must not be negative. The module docs now say which pages have their CRC checked. --- packages/effect/src/encoding/Parquet.ts | 40 ++++++++++++----- packages/effect/test/encoding/Parquet.test.ts | 44 +++++++++++++++++-- .../test/fixtures/parquet/CONFORMANCE.md | 9 ++-- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index 77db0a9e200..d07925185c5 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -42,22 +42,27 @@ * types and converted types are decoded into the schema but not applied * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). * - Text: every `STRING`, `ENUM` and `JSON` value, and every value of their - * converted types (`UTF8`, `ENUM`, `JSON`), is checked to be UTF-8, and so - * is every value of such a column's dictionary. Anything else fails with - * `Malformed`. `JSON` values are not parsed. + * converted types (`UTF8`, `ENUM`, `JSON`) on a node without a logical + * type, is checked to be UTF-8, and so is every value of such a column's + * dictionary. Anything else fails with `Malformed`. `JSON` values are not + * parsed. * - Annotations: an annotation added after 2.14.0, or one the spec doesn't * define for the physical type it is on (a `STRING` on an `INT32`), is left - * out of the schema, and so is the order of the column's statistics. A - * `DECIMAL` precision or scale and an `INTEGER` bit width that the spec - * forbids fail with `Malformed`. + * out of the schema, and so is the order of the column's statistics. When + * that annotation is a logical type, the node's converted type is left out + * too, so only the physical type applies. A `DECIMAL` precision or scale and + * an `INTEGER` bit width that the spec forbids fail with `Malformed`. * - Encodings: `PLAIN`, `PLAIN_DICTIONARY` and `RLE_DICTIONARY` (values * are looked up in the dictionary), `RLE` (booleans), `DELTA_BINARY_PACKED`, * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; * levels in `RLE` or the deprecated `BIT_PACKED`. * - Pages: data pages v1 and v2 and dictionary pages; index pages and page - * types added after 2.14.0 are skipped. A page with a CRC-32 in its header - * is checked against it, the skipped ones too, and fails with `Malformed` - * when its bytes don't match. A page without one is read as it is. + * types added after 2.14.0 are skipped. Every page read on the way to the + * column chunk's last value, skipped ones included, is checked against the + * CRC-32 in its header when it has one, and fails with `Malformed` when its + * bytes don't match. Pages after the last value, and the pages of a chunk + * with no values, are not read and not checked. A page without a CRC-32 is + * read as it is. * - Nesting: any depth of optional, required and repeated fields, assembled from * repetition and definition levels into {@link Layer}s. * @@ -273,7 +278,9 @@ export type ColumnOrder = "TYPE_ORDER" | "IEEE_754_TOTAL_ORDER" | "INT96_TIMESTA * * `convertedType` and `logicalType` are each `undefined` when the spec doesn't * define the annotation for this node: for its physical type and - * `typeLength`, or for a group. They are not compared with each other. + * `typeLength`, or for a group. `convertedType` is also `undefined` when the + * node has a logical type that is ignored, as the spec then says to use only + * the physical type. Otherwise they are not compared with each other. * * @stability experimental * @category models @@ -942,10 +949,16 @@ const rowGroup = (r: Reader): RowGroup => { return { columns: required(f.columns, "RowGroup.columns"), totalByteSize: required(f.totalByteSize, "RowGroup.total_byte_size"), - numRows: required(f.numRows, "RowGroup.num_rows") + numRows: rowCount(f.numRows, "RowGroup.num_rows") } } +const rowCount = (n: number | undefined, what: string): number => { + const value = required(n, what) + if (value < 0) fail("Malformed", `${what} is ${value}`) + return value +} + /** A logical type, or a converted type as the logical type that replaced it. */ type Annotation = LogicalType | { readonly type: Exclude } @@ -1053,6 +1066,9 @@ const annotations = (e: Element, physicalType: PhysicalType | undefined): { if (annotates(e.logicalType, e, physicalType)) logicalType = e.logicalType else known = false } + // The converted type stands in for a logical type only when there is none. + // An ignored one leaves only the physical type (#unsupported-logical-types). + if (e.logicalType !== undefined && logicalType === undefined) convertedType = undefined if (typeof convertedType === "string") { let annotation: Annotation if (convertedType !== "DECIMAL") annotation = { type: convertedType } @@ -1183,7 +1199,7 @@ const fileMetaData = (footer: Uint8Array, maxDepth: number): FileMetaData => { } return { version: required(f.version, "FileMetaData.version"), - numRows: required(f.numRows, "FileMetaData.num_rows"), + numRows: rowCount(f.numRows, "FileMetaData.num_rows"), createdBy: f.createdBy, keyValueMetadata: f.keyValueMetadata ?? [], schema: root, diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts index 4e4fc8fbc75..32f49de94e1 100644 --- a/packages/effect/test/encoding/Parquet.test.ts +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -534,6 +534,21 @@ describe("Parquet", () => { assert.deepStrictEqual(metadata.columns.map((c) => c.columnOrder), ["TYPE_ORDER", "IEEE_754_TOTAL_ORDER"]) yield* failsWith(metadataOf(make([Struct({ 1: Struct({}) })])), "Malformed", /column orders/) })) + + it.effect("PQ-META-3 MUST reject a negative num_rows in the file or a row group (L1058-1059, L1426-1427)", () => + Effect.gen(function*() { + const make = (file_: number, rowGroup: number) => + metadataOf(file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: rowGroup, + chunks: [{ type: Type.INT32, path: ["x"], numValues: 0, pages: [] }], + metadata: { 3: I64(file_) } + })) + const metadata = yield* make(0, 0) + assert.deepStrictEqual([metadata.numRows, metadata.rowGroups[0].numRows], [0, 0]) + yield* failsWith(make(0, -1), "Malformed", /RowGroup\.num_rows is -1/) + yield* failsWith(make(-1, 0), "Malformed", /FileMetaData\.num_rows is -1/) + })) }) describe("parquet.thrift SchemaElement", () => { @@ -619,6 +634,15 @@ describe("Parquet", () => { convertedType: undefined, columnOrder: undefined }) + // the converted type doesn't stand in for a logical type that is there but ignored (#convertedtype) + for (const type of [Type.BYTE_ARRAY, undefined]) { + const annotation = { logicalType: Struct({ 99: empty }), convertedType: type === undefined ? 3 : 0 } + assert.deepStrictEqual(yield* annotationOf(annotated(type, annotation)), { + logicalType: undefined, + convertedType: undefined, + columnOrder: type === undefined ? "TYPE_ORDER" : undefined + }) + } })) const GROUP = undefined @@ -720,7 +744,7 @@ describe("Parquet", () => { } })) - it.effect("PQ-LT-5 reads a node's logical type and converted type independently of each other", () => + it.effect("PQ-LT-5 reads a node's logical type and converted type independently of each other, unless the logical type is ignored", () => Effect.gen(function*() { // both defined for INT32, though DATE corresponds to the converted type DATE assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { ...logical(6), convertedType: 7 })), { @@ -728,10 +752,11 @@ describe("Parquet", () => { convertedType: "TIME_MILLIS", columnOrder: "TYPE_ORDER" }) - // STRING is not defined for INT32, INT_32 is + // STRING is not defined for INT32, so it is ignored, and so is INT_32, which only stands in for a + // logical type when there is none (#convertedtype, #unsupported-logical-types) assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { ...logical(1), convertedType: 17 })), { logicalType: undefined, - convertedType: "INT_32", + convertedType: undefined, columnOrder: undefined }) // the DecimalType supersedes the scale and precision of the schema element (parquet.thrift L557-564) @@ -972,7 +997,18 @@ describe("Parquet", () => { it.effect("PQ-LT-12 doesn't check the bytes of other BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns", () => Effect.gen(function*() { const binary: Binaries = [[0xff], ...split] - for (const annotation of [undefined, logical(13), { convertedType: 20 }, logical(17)]) { + const annotations: ReadonlyArray = [ + undefined, + logical(13), + { convertedType: 20 }, + logical(17), + // a logical type supersedes UTF8, also when it is ignored for being unknown or not for BYTE_ARRAY + // (#convertedtype, #unsupported-logical-types); without one, UTF8 is checked (PQ-LT-8) + { ...logical(13), convertedType: 0 }, + { logicalType: Struct({ 99: empty }), convertedType: 0 }, + { ...logical(6), convertedType: 0 } + ] + for (const annotation of annotations) { for (const form of forms) { assert.deepStrictEqual(yield* values(binaries(form, binary, annotation)), hex(binary)) } diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index efd27f1a7c2..cadc1f3ac77 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -22,20 +22,21 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-FILE-2 | README.md#file-format | The metadata is found from the 4-byte little-endian length before the final magic | | PQ-META-1 | parquet.thrift L1411-1413 | Readers should accept versions 1 and 2 interchangeably | | PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | +| PQ-META-3 | parquet.thrift L1058-1059, L1426-1427 | `num_rows` of the file and of a row group is not negative (rejected when it is) | | PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | | PQ-SCHEMA-2 | parquet.thrift L518-523, L542-547 | The schema tree is the depth-first list of elements, groups giving `num_children` | | PQ-LT-1 | LogicalTypes.md#timestamp | Unknown `TIMESTAMP` units are unsupported features, not errors in the file | | PQ-LT-2 | parquet.thrift L370-373, L380-383 | `TimestampType.unit` and `TimeType.unit` are required (rejected when missing) | -| PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, and so is its column's order | +| PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, with its column's order and its node's converted type | | PQ-LT-4 | LogicalTypes.md#unsupported-logical-types | An annotation on a physical type it isn't defined for is ignored, with the column order | -| PQ-LT-5 | LogicalTypes.md#convertedtype | Local policy: a logical type and a converted type are checked independently | +| PQ-LT-5 | LogicalTypes.md#convertedtype | Local policy: both are checked independently; an ignored logical type drops the converted type | | PQ-LT-6 | LogicalTypes.md#decimal | `DECIMAL` precision and scale fit the rules and the physical type (rejected otherwise) | | PQ-LT-7 | LogicalTypes.md#signed-integers, parquet.thrift L388 | `INTEGER` bit widths are 8, 16, 32 or 64 (rejected otherwise) | | PQ-LT-8 | LogicalTypes.md#string, #enum, #json, parquet.thrift L50, L322, L326 | `STRING`, `ENUM` and `JSON` values, and those of their converted types, are UTF-8 (rejected otherwise) | | PQ-LT-9 | LogicalTypes.md#string | Local policy: UTF-8 is the well-formed sequences of the Unicode Standard, which the spec doesn't define | | PQ-LT-10 | Encodings.md#delta-strings-delta_byte_array--7 | `DELTA_BYTE_ARRAY` values are checked whole, after their prefix is added | | PQ-LT-11 | LogicalTypes.md#string | Local policy: a dictionary is checked once with its page, values no page refers to included | -| PQ-LT-12 | LogicalTypes.md#bson, README.md#types | Unannotated, `BSON` and geospatial `BYTE_ARRAY` values and `FIXED_LEN_BYTE_ARRAY` values are any bytes | +| PQ-LT-12 | LogicalTypes.md#bson, README.md#types | `BYTE_ARRAY` values with no text annotation in effect and `FIXED_LEN_BYTE_ARRAY` values are any bytes | | PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | | PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | | PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | @@ -75,7 +76,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. | PQ-CRC-2 | parquet.thrift L840-852 | A v2 data page's `crc` is of its levels and its values as stored | | PQ-CRC-3 | parquet.thrift L840-852 | A dictionary page's `crc` is checked the same way | | PQ-CRC-4 | README.md#checksumming, parquet.thrift L842-843 | The CRC is the standard CRC-32 of GZIP; the i32 of the header is read as unsigned | -| PQ-CRC-5 | parquet.thrift L844-845 | Local policy: the `crc` of a skipped page is checked too | +| PQ-CRC-5 | parquet.thrift L844-845 | Local policy: the `crc` of a skipped page before the chunk's last value is checked too | ## Unsupported: fails with `Unsupported` From 862fc626bd5e8ea6f88552fea5d17d7d4ca3df3a Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 11:57:21 +0000 Subject: [PATCH 31/32] Keep the converted type to logical type table in one place and return early in annotations --- packages/effect/src/encoding/Parquet.ts | 64 +++++++------------ .../src/internal/parquet/annotations.ts | 52 +++++++++++++++ 2 files changed, 74 insertions(+), 42 deletions(-) create mode 100644 packages/effect/src/internal/parquet/annotations.ts diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index d07925185c5..b45a9f5e4ec 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -85,6 +85,7 @@ import * as Data from "../Data.ts" import * as Effect from "../Effect.ts" import { crc32 } from "../internal/crc32.ts" +import { type Annotation, logicalOf } from "../internal/parquet/annotations.ts" import * as Encodings from "../internal/parquet/encodings.ts" import * as Predicate from "../Predicate.ts" import * as Thrift from "./Thrift.ts" @@ -959,9 +960,6 @@ const rowCount = (n: number | undefined, what: string): number => { return value } -/** A logical type, or a converted type as the logical type that replaced it. */ -type Annotation = LogicalType | { readonly type: Exclude } - /** * The physical type LogicalTypes.md defines each annotation for, `undefined` * for a group. The annotations that are not here depend on more than that. @@ -977,26 +975,13 @@ const annotated: { VARIANT: undefined, FILE: undefined, STRING: "BYTE_ARRAY", - UTF8: "BYTE_ARRAY", ENUM: "BYTE_ARRAY", JSON: "BYTE_ARRAY", BSON: "BYTE_ARRAY", GEOMETRY: "BYTE_ARRAY", GEOGRAPHY: "BYTE_ARRAY", DATE: "INT32", - TIME_MILLIS: "INT32", - UINT_8: "INT32", - UINT_16: "INT32", - UINT_32: "INT32", - INT_8: "INT32", - INT_16: "INT32", - INT_32: "INT32", - TIME_MICROS: "INT64", - TIMESTAMP: "INT64", - TIMESTAMP_MILLIS: "INT64", - TIMESTAMP_MICROS: "INT64", - UINT_64: "INT64", - INT_64: "INT64" + TIMESTAMP: "INT64" } const annotatedLengths = { UUID: 16, FLOAT16: 2, INTERVAL: 12 } as const @@ -1059,34 +1044,29 @@ const annotations = (e: Element, physicalType: PhysicalType | undefined): { readonly convertedType: ConvertedType | number | undefined readonly known: boolean } => { - let logicalType: LogicalType | undefined - let convertedType = e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType) - let known = typeof e.logicalType !== "number" && typeof convertedType !== "number" - if (typeof e.logicalType === "object") { - if (annotates(e.logicalType, e, physicalType)) logicalType = e.logicalType - else known = false - } + const converted = e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType) + const logicalType = typeof e.logicalType === "object" && annotates(e.logicalType, e, physicalType) + ? e.logicalType + : undefined // The converted type stands in for a logical type only when there is none. - // An ignored one leaves only the physical type (#unsupported-logical-types). - if (e.logicalType !== undefined && logicalType === undefined) convertedType = undefined - if (typeof convertedType === "string") { - let annotation: Annotation - if (convertedType !== "DECIMAL") annotation = { type: convertedType } + // An ignored logical type leaves only the physical type (#unsupported-logical-types). + if (e.logicalType !== undefined && logicalType === undefined) { + return { logicalType, convertedType: undefined, known: false } + } + if (typeof converted !== "string") return { logicalType, convertedType: converted, known: converted === undefined } + const annotation: Annotation = converted !== "DECIMAL" + ? logicalOf[converted] // A DecimalType supersedes the scale and precision of the schema element. - else if (typeof e.logicalType === "object" && e.logicalType.type === "DECIMAL") annotation = e.logicalType - else { - annotation = { - type: "DECIMAL", - scale: e.scale ?? 0, - precision: required(e.precision, `the precision of DECIMAL schema element "${e.name}"`) - } - } - if (!annotates(annotation, e, physicalType)) { - convertedType = undefined - known = false + : logicalType?.type === "DECIMAL" + ? logicalType + : { + type: "DECIMAL", + scale: e.scale ?? 0, + precision: required(e.precision, `the precision of DECIMAL schema element "${e.name}"`) } - } - return { logicalType, convertedType, known } + return annotates(annotation, e, physicalType) + ? { logicalType, convertedType: converted, known: true } + : { logicalType, convertedType: undefined, known: false } } interface Schema { diff --git a/packages/effect/src/internal/parquet/annotations.ts b/packages/effect/src/internal/parquet/annotations.ts new file mode 100644 index 00000000000..ba9fdcba963 --- /dev/null +++ b/packages/effect/src/internal/parquet/annotations.ts @@ -0,0 +1,52 @@ +/** + * The logical type that LogicalTypes.md gives for each converted type + * (https://github.com/apache/parquet-format/blob/apache-parquet-format-2.14.0/LogicalTypes.md). + * + * @internal + */ +import type { ConvertedType, LogicalType } from "../../encoding/Parquet.ts" + +/** + * A logical type, or a converted type as the logical type that replaced it. + * `INTERVAL` and `MAP_KEY_VALUE` have no logical type, so they keep their names. + * + * @internal + */ +export type Annotation = LogicalType | { readonly type: "INTERVAL" | "MAP_KEY_VALUE" } + +const utc = (type: "TIME" | "TIMESTAMP", unit: "MILLIS" | "MICROS"): Annotation => ({ + type, + isAdjustedToUTC: true, + unit +}) + +const integer = (bitWidth: number, isSigned: boolean): Annotation => ({ type: "INTEGER", bitWidth, isSigned }) + +/** + * `DECIMAL` is not here: its scale and precision come from the schema element. + * + * @internal + */ +export const logicalOf: { readonly [K in Exclude]: Annotation } = { + UTF8: { type: "STRING" }, + MAP: { type: "MAP" }, + MAP_KEY_VALUE: { type: "MAP_KEY_VALUE" }, + LIST: { type: "LIST" }, + ENUM: { type: "ENUM" }, + DATE: { type: "DATE" }, + TIME_MILLIS: utc("TIME", "MILLIS"), + TIME_MICROS: utc("TIME", "MICROS"), + TIMESTAMP_MILLIS: utc("TIMESTAMP", "MILLIS"), + TIMESTAMP_MICROS: utc("TIMESTAMP", "MICROS"), + UINT_8: integer(8, false), + UINT_16: integer(16, false), + UINT_32: integer(32, false), + UINT_64: integer(64, false), + INT_8: integer(8, true), + INT_16: integer(16, true), + INT_32: integer(32, true), + INT_64: integer(64, true), + JSON: { type: "JSON" }, + BSON: { type: "BSON" }, + INTERVAL: { type: "INTERVAL" } +} From 0d758b517d27168397123b27e6ccbb6fe69f2ee5 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 12:00:33 +0000 Subject: [PATCH 32/32] Correct the module doc on unknown converted types and tighten the page, text and logical type wording --- packages/effect/src/encoding/Parquet.ts | 31 ++++++++++--------- .../test/fixtures/parquet/CONFORMANCE.md | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts index b45a9f5e4ec..e35174f1bd5 100644 --- a/packages/effect/src/encoding/Parquet.ts +++ b/packages/effect/src/encoding/Parquet.ts @@ -44,25 +44,26 @@ * - Text: every `STRING`, `ENUM` and `JSON` value, and every value of their * converted types (`UTF8`, `ENUM`, `JSON`) on a node without a logical * type, is checked to be UTF-8, and so is every value of such a column's - * dictionary. Anything else fails with `Malformed`. `JSON` values are not - * parsed. - * - Annotations: an annotation added after 2.14.0, or one the spec doesn't - * define for the physical type it is on (a `STRING` on an `INT32`), is left - * out of the schema, and so is the order of the column's statistics. When - * that annotation is a logical type, the node's converted type is left out - * too, so only the physical type applies. A `DECIMAL` precision or scale and - * an `INTEGER` bit width that the spec forbids fail with `Malformed`. + * dictionary. A value that is not UTF-8 fails with `Malformed`. `JSON` + * values are not parsed. + * - Annotations: a logical type added after 2.14.0, or an annotation the + * spec doesn't define for the physical type it is on (a `STRING` on an + * `INT32`), is left out of the schema, and so is the order of the column's + * statistics. A converted type added after 2.14.0 stays as its numeric + * code, also without the order. When the annotation left out is a logical + * type, the node's converted type is left out too, so only the physical + * type applies. A `DECIMAL` precision or scale and an `INTEGER` bit width + * that the spec forbids fail with `Malformed`. * - Encodings: `PLAIN`, `PLAIN_DICTIONARY` and `RLE_DICTIONARY` (values * are looked up in the dictionary), `RLE` (booleans), `DELTA_BINARY_PACKED`, * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; * levels in `RLE` or the deprecated `BIT_PACKED`. * - Pages: data pages v1 and v2 and dictionary pages; index pages and page - * types added after 2.14.0 are skipped. Every page read on the way to the - * column chunk's last value, skipped ones included, is checked against the - * CRC-32 in its header when it has one, and fails with `Malformed` when its - * bytes don't match. Pages after the last value, and the pages of a chunk - * with no values, are not read and not checked. A page without a CRC-32 is - * read as it is. + * types added after 2.14.0 are skipped. Every page read to decode the + * column chunk's values, skipped ones included, is checked when its header + * has a CRC-32, and fails with `Malformed` on a mismatch. A page without + * one is not checked. Pages after the last value, and the pages of a chunk + * with no values, are not read and so not checked. * - Nesting: any depth of optional, required and repeated fields, assembled from * repetition and definition levels into {@link Layer}s. * @@ -1020,7 +1021,7 @@ const annotates = (annotation: Annotation, e: Element, physicalType: PhysicalTyp : physicalType === "INT64" ? 18 : physicalType === "FIXED_LEN_BYTE_ARRAY" - // floor(log10(2^(8n - 1) - 1)), which is this as no power of 2 is a power of 10 + // floor(log10(2^(8n - 1) - 1)), computed without the last - 1, as no power of 2 is a power of 10 ? Math.floor((8 * e.typeLength! - 1) * Math.log10(2)) : undefined if (digits === undefined) return false diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md index cadc1f3ac77..2cf93f888bb 100644 --- a/packages/effect/test/fixtures/parquet/CONFORMANCE.md +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -94,7 +94,7 @@ reader rejects, what it doesn't read yet, and what applies only to writers. - Page index: `ColumnIndex` and `OffsetIndex` (PageIndex.md, parquet.thrift L1235-1373). The reader reads every page in order. - Bloom filters (BloomFilter.md). - Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. -- Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type and, but for the UTF-8 of text (PQ-LT-8), are not checked against the annotation: that `JSON` is valid JSON, the range of a narrow `INTEGER`, which the spec leaves to the implementation, the digits of a `DECIMAL`, the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. +- Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type. Apart from the UTF-8 of text (PQ-LT-8) they are not checked against the annotation: not that `JSON` parses, nor the range of a narrow `INTEGER` (which the spec leaves to the implementation), the digits of a `DECIMAL` or the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. - Error recovery (README.md#error-recovery): each `readColumn` call is independent, so a corrupt column chunk doesn't affect the others, but nothing inside a chunk is recovered. - Binary protocol extensions (BinaryProtocolExtensions.md).