diff --git a/.changeset/encoding-parquet.md b/.changeset/encoding-parquet.md new file mode 100644 index 00000000000..3f50b37ccae --- /dev/null +++ b/.changeset/encoding-parquet.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add the experimental `effect/encoding/Parquet` module, which reads Apache Parquet metadata and column chunks into Arrow-style buffers. diff --git a/.changeset/encoding-thrift.md b/.changeset/encoding-thrift.md new file mode 100644 index 00000000000..f5e20a9605a --- /dev/null +++ b/.changeset/encoding-thrift.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add the experimental `effect/encoding/Thrift` module, which reads and writes the Apache Thrift compact protocol without an IDL. diff --git a/.changeset/fix-layer-ref-preload-default.md b/.changeset/fix-layer-ref-preload-default.md new file mode 100644 index 00000000000..c9abaaae7ba --- /dev/null +++ b/.changeset/fix-layer-ref-preload-default.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Fix `LayerRef.make` and `LayerRef.Service` exposing layer build errors during creation when `preload` is omitted. diff --git a/.changeset/openai-websocket-error-details.md b/.changeset/openai-websocket-error-details.md new file mode 100644 index 00000000000..879f105bc0a --- /dev/null +++ b/.changeset/openai-websocket-error-details.md @@ -0,0 +1,5 @@ +--- +"@effect/ai-openai": patch +--- + +Fix OpenAI WebSocket streams hanging on errors with missing messages or nullable type/code fields, and preserve provider error details. diff --git a/dprint.json b/dprint.json index 0622d2e2642..4a1e34baa16 100644 --- a/dprint.json +++ b/dprint.json @@ -22,6 +22,8 @@ "**/CHANGELOG.md", "packages/effect/src/StandardSchema.ts", "packages/tools/graphql-generator/test/generated/**/*.graphql.ts", + "packages/effect/test/fixtures/thrift/gold.json", + "packages/effect/test/fixtures/parquet/gold.json", "!scratchpad/**/*", "**/.*", "!.agents", diff --git a/packages/ai/openai/src/OpenAiClient.ts b/packages/ai/openai/src/OpenAiClient.ts index f4a44fda860..a71ff430dc9 100644 --- a/packages/ai/openai/src/OpenAiClient.ts +++ b/packages/ai/openai/src/OpenAiClient.ts @@ -602,12 +602,12 @@ const makeSocket = Effect.gen(function*() { if (event.type === "error") { const status = "status" in event ? Number(event.status) : NaN const error = ("error" in event ? event.error : event) as { - readonly type?: string - readonly code?: string - readonly message: string + readonly type?: string | null + readonly code?: string | null + readonly message?: string } const errorType = error.code ?? - (error.type === "api_error" && /^gRPC error: Response with id=\S+ not found$/.test(error.message) + (error.type === "api_error" && /^gRPC error: Response with id=\S+ not found$/.test(error.message ?? "") ? "previous_response_not_found" : error.type ?? "unknown") const json = JSON.stringify(error) @@ -736,11 +736,14 @@ const ErrorEvent = Schema.Struct({ ), status: Schema.optionalKey(Schema.Int), // xAI sends `code` in place of `type`, e.g. `previous_response_not_found` - error: Schema.Struct({ - type: Schema.optional(Schema.String), - code: Schema.optional(Schema.String), - message: Schema.String - }) + error: Schema.StructWithRest( + Schema.Struct({ + type: Schema.optionalKey(Schema.NullOr(Schema.String)), + code: Schema.optionalKey(Schema.NullOr(Schema.String)), + message: Schema.optionalKey(Schema.String) + }), + [Schema.Record(Schema.String, Schema.Unknown)] + ) }) const errorTypeToStatus: Record = { diff --git a/packages/ai/openai/test/OpenAiClient.test.ts b/packages/ai/openai/test/OpenAiClient.test.ts index 0912b4863f6..6e7f17435aa 100644 --- a/packages/ai/openai/test/OpenAiClient.test.ts +++ b/packages/ai/openai/test/OpenAiClient.test.ts @@ -545,6 +545,63 @@ describe("OpenAiClient", () => { }) describe("createResponseStream", () => { + it.effect("preserves the HTTP usage-limit error body", () => + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const result = yield* client.createResponseStream({ model: "gpt-4o", input: "test" }).pipe(Effect.flip) + assert.strictEqual(result.reason._tag, "RateLimitError") + if (result.reason._tag !== "RateLimitError") return + assert.deepStrictEqual(JSON.parse(result.reason.http!.body!), { error: usageLimitError }) + }).pipe(Effect.provide(makeTestLayer(undefined, { + _tag: "Json", + status: 429, + body: { error: usageLimitError } + })))) + + it.live.each([ + { name: "explicit status", status: 429, error: usageLimitError, reason: "RateLimitError" }, + { name: "no status", error: usageLimitError, reason: "InternalProviderError" }, + { + name: "no message", + status: 429, + error: { type: "usage_limit_reached", plan_type: "pro" }, + reason: "RateLimitError" + }, + { name: "null code", status: 429, error: { ...usageLimitError, code: null }, reason: "RateLimitError" }, + { name: "null type", status: 429, error: { ...usageLimitError, type: null }, reason: "RateLimitError" }, + { name: "mapped type", error: { ...usageLimitError, type: "rate_limit_error" }, reason: "RateLimitError" }, + { name: "api error without message", error: { type: "api_error" }, reason: "InternalProviderError" } + ])("preserves WebSocket error details with $name", (frame) => + Effect.gen(function*() { + const server = yield* Effect.acquireRelease( + Effect.sync(() => + new WS(`wss://errors.test/${encodeURIComponent(frame.name)}/responses`, { jsonProtocol: true }) + ), + (server) => Effect.sync(() => server.close()) + ) + yield* OpenAiClient.withWebSocketMode( + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const [, stream] = yield* client.createResponseStream({ model: "gpt-4o", input: "test" }) + const [result] = yield* Effect.all([ + Stream.runDrain(stream).pipe(Effect.flip), + nextWebSocketCreate(server).pipe( + Effect.tap(() => Effect.sync(() => server.send({ type: "error", ...frame }))) + ) + ], { concurrency: "unbounded" }) + assert.strictEqual(result._tag, "AiError") + assert.strictEqual(result.reason._tag, frame.reason) + if (result.reason._tag !== "RateLimitError" && result.reason._tag !== "InternalProviderError") return + assert.deepStrictEqual(result.reason.metadata, frame.error) + assert.deepStrictEqual(JSON.parse(result.reason.http!.body!), frame.error) + }) + ).pipe( + Effect.provide(makeTestLayer({ apiUrl: `https://errors.test/${encodeURIComponent(frame.name)}` })), + Effect.provideService(Socket.WebSocketConstructor, (url) => new globalThis.WebSocket(url)), + Effect.timeout("1 second") + ) + })) + it.effect("emits response.completed before [DONE] in the same SSE chunk", () => Effect.gen(function*() { const client = yield* OpenAiClient.OpenAiClient @@ -1376,3 +1433,11 @@ const makeResponse = ( } }) } + +const usageLimitError = { + type: "usage_limit_reached", + message: "Usage limit reached", + resets_at: 1800000000, + plan_type: "pro", + limit_window_minutes: 300 +} diff --git a/packages/effect/src/LayerRef.ts b/packages/effect/src/LayerRef.ts index f850c5e72a8..d795751f160 100644 --- a/packages/effect/src/LayerRef.ts +++ b/packages/effect/src/LayerRef.ts @@ -134,7 +134,7 @@ export interface LayerRef { * @since 4.0.0 */ export const make = Effect.fnUntraced( - function*( + function*( layer: Layer.Layer, options?: { /** @@ -317,7 +317,7 @@ export const Service = () => R, X, const Deps extends ReadonlyArray> = [], - const Preload extends boolean = never, + const Preload extends boolean = false, SE = never, SR = never >( diff --git a/packages/effect/src/encoding/Parquet.ts b/packages/effect/src/encoding/Parquet.ts new file mode 100644 index 00000000000..e35174f1bd5 --- /dev/null +++ b/packages/effect/src/encoding/Parquet.ts @@ -0,0 +1,1887 @@ +/** + * Reads Apache Parquet files: the footer metadata and the values of one column + * chunk at a time, in buffers laid out the way Apache Arrow lays out arrays. + * + * **Mental model** + * + * - A Parquet file holds row groups; each row group holds one column chunk per + * leaf column of the schema. {@link readMetadata} reads the footer, which + * describes the schema, the row groups and where every column chunk is. + * - {@link readColumn} reads one column chunk and returns a {@link Column}: one + * {@link Layer} per schema node on the column's path, with Arrow-style + * validity bitmaps and list offsets, and the leaf's {@link Values} with a slot + * for every null. + * - Both read through a {@link Source}, so the file can be in memory + * ({@link fromBytes}), on disk or behind HTTP range requests. + * - Compressed pages go through a {@link Decompress} function supplied per + * codec in {@link readColumn}'s options. This module has no codecs of its own. + * + * **Example** (Reading a column) + * + * ```ts + * import { Effect } from "effect" + * import { Parquet } from "effect/encoding" + * import { readFile } from "node:fs/promises" + * + * const program = Effect.gen(function*() { + * const source = Parquet.fromBytes(new Uint8Array(yield* Effect.promise(() => readFile("data.parquet")))) + * const metadata = yield* Parquet.readMetadata(source) + * const column = metadata.columns.findIndex((c) => c.path.join(".") === "price") + * const { values, layers } = yield* Parquet.readColumn(source, metadata, { rowGroup: 0, column }) + * // layers[0].validity marks the non-null rows, values.values holds them + * return { values, validity: layers[0].validity } + * }) + * ``` + * + * **Details** + * + * The reader follows + * [apache/parquet-format 2.14.0](https://github.com/apache/parquet-format/tree/apache-parquet-format-2.14.0): + * + * - Physical types: all eight. Values stay in their physical type; logical + * types and converted types are decoded into the schema but not applied + * (a `DATE` column reads as `INT32` days, a `STRING` as UTF-8 bytes). + * - Text: every `STRING`, `ENUM` and `JSON` value, and every value of their + * converted types (`UTF8`, `ENUM`, `JSON`) on a node without a logical + * type, is checked to be UTF-8, and so is every value of such a column's + * dictionary. A value that is not UTF-8 fails with `Malformed`. `JSON` + * values are not parsed. + * - Annotations: a logical type added after 2.14.0, or an annotation the + * spec doesn't define for the physical type it is on (a `STRING` on an + * `INT32`), is left out of the schema, and so is the order of the column's + * statistics. A converted type added after 2.14.0 stays as its numeric + * code, also without the order. When the annotation left out is a logical + * type, the node's converted type is left out too, so only the physical + * type applies. A `DECIMAL` precision or scale and an `INTEGER` bit width + * that the spec forbids fail with `Malformed`. + * - Encodings: `PLAIN`, `PLAIN_DICTIONARY` and `RLE_DICTIONARY` (values + * are looked up in the dictionary), `RLE` (booleans), `DELTA_BINARY_PACKED`, + * `DELTA_LENGTH_BYTE_ARRAY`, `DELTA_BYTE_ARRAY` and `BYTE_STREAM_SPLIT`; + * levels in `RLE` or the deprecated `BIT_PACKED`. + * - Pages: data pages v1 and v2 and dictionary pages; index pages and page + * types added after 2.14.0 are skipped. Every page read to decode the + * column chunk's values, skipped ones included, is checked when its header + * has a CRC-32, and fails with `Malformed` on a mismatch. A page without + * one is not checked. Pages after the last value, and the pages of a chunk + * with no values, are not read and so not checked. + * - Nesting: any depth of optional, required and repeated fields, assembled from + * repetition and definition levels into {@link Layer}s. + * + * Not supported, failing with a {@link ParquetError} of kind `Unsupported`: + * encrypted files and columns, the `ALP` encoding, column chunks in other + * files (`file_path`), `TIME` and `TIMESTAMP` units added after 2.14.0, and + * compressed pages without a matching {@link Decompress}. Not read: page + * indexes, bloom filters, size and geospatial statistics. + * + * Input is untrusted: every offset and length is checked against the bytes + * that are there, and the sizes a file claims are capped by options + * (`maxFooterLength`, `maxDepth`, `maxPageSize`, `maxValues`, + * `maxBinaryLength`) before anything that large is allocated. + * + * Writing Parquet is not supported. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Data from "../Data.ts" +import * as Effect from "../Effect.ts" +import { crc32 } from "../internal/crc32.ts" +import { type Annotation, logicalOf } from "../internal/parquet/annotations.ts" +import * as Encodings from "../internal/parquet/encodings.ts" +import * as Predicate from "../Predicate.ts" +import * as Thrift from "./Thrift.ts" + +const ParquetErrorTypeId = "~effect/encoding/Parquet/ParquetError" + +/** + * Error raised when a Parquet file can't be read. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `Malformed`: the file breaks the Parquet format, for example a bad magic + * number, invalid Thrift metadata, an offset outside the file, pages that + * don't add up to their column chunk, or a page that doesn't match its CRC. + * - `Unsupported`: the file uses a feature this module doesn't read, or a + * compression codec with no {@link Decompress} in the options. + * - `Limit`: the file needs more than an option such as `maxPageSize` allows. + * - `Codec`: a {@link Decompress} function failed; its error is the `cause`. + * - `Usage`: an option or argument is invalid. These are bugs in the caller, + * so they are defects, not failures. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class ParquetError extends Data.TaggedError("ParquetError")<{ + readonly kind: "Malformed" | "Unsupported" | "Limit" | "Codec" | "Usage" + readonly message: string + readonly cause?: unknown +}> { + /** + * Marks this value as a Parquet error for runtime guards. + * + * @stability experimental + * @since 4.1.5 + */ + readonly [ParquetErrorTypeId] = ParquetErrorTypeId +} + +/** + * Checks whether a value is a {@link ParquetError}. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isParquetError = (u: unknown): u is ParquetError => Predicate.hasProperty(u, ParquetErrorTypeId) + +/** + * A Parquet physical type: how values are stored. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type PhysicalType = + | "BOOLEAN" + | "INT32" + | "INT64" + | "INT96" + | "FLOAT" + | "DOUBLE" + | "BYTE_ARRAY" + | "FIXED_LEN_BYTE_ARRAY" + +/** + * Whether a schema field is required, optional (may be null) or repeated (a + * list of zero or more values). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Repetition = "REQUIRED" | "OPTIONAL" | "REPEATED" + +/** + * A Parquet encoding. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Encoding = + | "PLAIN" + | "PLAIN_DICTIONARY" + | "RLE" + | "BIT_PACKED" + | "DELTA_BINARY_PACKED" + | "DELTA_LENGTH_BYTE_ARRAY" + | "DELTA_BYTE_ARRAY" + | "RLE_DICTIONARY" + | "BYTE_STREAM_SPLIT" + | "ALP" + +/** + * A Parquet compression codec. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type CompressionCodec = "UNCOMPRESSED" | "SNAPPY" | "GZIP" | "LZO" | "BROTLI" | "LZ4" | "ZSTD" | "LZ4_RAW" + +/** + * A legacy type annotation, superseded by {@link LogicalType}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type ConvertedType = + | "UTF8" + | "MAP" + | "MAP_KEY_VALUE" + | "LIST" + | "ENUM" + | "DECIMAL" + | "DATE" + | "TIME_MILLIS" + | "TIME_MICROS" + | "TIMESTAMP_MILLIS" + | "TIMESTAMP_MICROS" + | "UINT_8" + | "UINT_16" + | "UINT_32" + | "UINT_64" + | "INT_8" + | "INT_16" + | "INT_32" + | "INT_64" + | "JSON" + | "BSON" + | "INTERVAL" + +/** + * A type annotation that says how to interpret a field's physical values. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type LogicalType = + | { + readonly type: + | "STRING" + | "MAP" + | "LIST" + | "ENUM" + | "DATE" + | "UNKNOWN" + | "JSON" + | "BSON" + | "UUID" + | "FLOAT16" + | "FILE" + } + | { readonly type: "DECIMAL"; readonly scale: number; readonly precision: number } + | { + readonly type: "TIME" | "TIMESTAMP" + readonly isAdjustedToUTC: boolean + readonly unit: "MILLIS" | "MICROS" | "NANOS" + } + | { readonly type: "INTEGER"; readonly bitWidth: number; readonly isSigned: boolean } + | { readonly type: "VARIANT"; readonly specificationVersion: number | undefined } + | { readonly type: "GEOMETRY"; readonly crs: string | undefined } + | { + readonly type: "GEOGRAPHY" + readonly crs: string | undefined + readonly algorithm: "SPHERICAL" | "VINCENTY" | "THOMAS" | "ANDOYER" | "KARNEY" | number | undefined + } + +/** + * The sort order of a column's `minValue` and `maxValue` statistics. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type ColumnOrder = "TYPE_ORDER" | "IEEE_754_TOTAL_ORDER" | "INT96_TIMESTAMP_ORDER" + +/** + * A node of the schema tree: a group with `children`, or a leaf with a + * `physicalType`. + * + * **Details** + * + * Enum-valued fields hold the spec's name, or the numeric code of a value this + * module doesn't know. + * + * `convertedType` and `logicalType` are each `undefined` when the spec doesn't + * define the annotation for this node: for its physical type and + * `typeLength`, or for a group. `convertedType` is also `undefined` when the + * node has a logical type that is ignored, as the spec then says to use only + * the physical type. Otherwise they are not compared with each other. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface SchemaNode { + readonly name: string + /** `undefined` only for the root. */ + readonly repetition: Repetition | undefined + /** `undefined` for groups. */ + readonly physicalType: PhysicalType | undefined + /** The byte width of `FIXED_LEN_BYTE_ARRAY` values. */ + readonly typeLength: number | undefined + readonly convertedType: ConvertedType | number | undefined + /** `undefined` when absent or of a kind this module doesn't know. */ + readonly logicalType: LogicalType | undefined + readonly scale: number | undefined + readonly precision: number | undefined + readonly fieldId: number | undefined + /** `undefined` for leaves. */ + readonly children: ReadonlyArray | undefined +} + +/** + * A leaf column of the schema, in the order of the column chunks of every row + * group. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ColumnDescriptor { + /** The names of {@link nodes}. */ + readonly path: ReadonlyArray + /** The schema nodes from a child of the root down to the leaf. */ + readonly nodes: ReadonlyArray + readonly physicalType: PhysicalType + readonly typeLength: number | undefined + /** The number of optional and repeated nodes on the path. */ + readonly maxDefinitionLevel: number + /** The number of repeated nodes on the path. */ + readonly maxRepetitionLevel: number + /** + * `undefined` when the file has no column orders, or one this module doesn't + * know, or when the leaf has an annotation this module doesn't know or the + * spec doesn't define for its physical type. + */ + readonly columnOrder: ColumnOrder | undefined +} + +/** + * Key/value metadata of a file or column chunk. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface KeyValue { + readonly key: string + readonly value: string | undefined +} + +/** + * Statistics of a column chunk, as written. + * + * **Details** + * + * A count that is `undefined` is unknown, not zero. Min and max values are + * the plain-encoded bytes of a value, ordered by the column's + * {@link ColumnDescriptor.columnOrder}; `min` and `max` are the deprecated + * fields, ordered by signed comparison. + * + * **Gotchas** + * + * - When the column order is `undefined`, `minValue` and `maxValue` have no + * known order. They are returned as written: don't compare values with + * them. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Statistics { + readonly nullCount: number | undefined + readonly distinctCount: number | undefined + readonly nanCount: number | undefined + readonly minValue: Uint8Array | undefined + readonly maxValue: Uint8Array | undefined + readonly isMinValueExact: boolean | undefined + readonly isMaxValueExact: boolean | undefined + readonly min: Uint8Array | undefined + readonly max: Uint8Array | undefined +} + +/** + * Where a column chunk's pages are and how they are written. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ColumnMetaData { + readonly physicalType: PhysicalType + readonly encodings: ReadonlyArray + readonly pathInSchema: ReadonlyArray + readonly codec: CompressionCodec | number + /** The number of values, nulls included. */ + readonly numValues: number + readonly totalUncompressedSize: number + readonly totalCompressedSize: number + readonly keyValueMetadata: ReadonlyArray + readonly dataPageOffset: number + readonly indexPageOffset: number | undefined + readonly dictionaryPageOffset: number | undefined + readonly statistics: Statistics | undefined +} + +/** + * A column chunk of a row group. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ColumnChunk { + /** Set when the chunk is in another file, which this module doesn't read. */ + readonly filePath: string | undefined + /** Whether the chunk is encrypted, which this module doesn't read. */ + readonly encrypted: boolean + /** `undefined` only when the chunk is encrypted. */ + readonly metaData: ColumnMetaData | undefined +} + +/** + * A row group: one column chunk per leaf column. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface RowGroup { + readonly numRows: number + readonly totalByteSize: number + /** In the order of {@link FileMetaData.columns}. */ + readonly columns: ReadonlyArray +} + +/** + * The metadata in a Parquet file's footer. + * + * **Gotchas** + * + * Binary statistics are views of the footer bytes read from the + * {@link Source}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface FileMetaData { + readonly version: number + readonly numRows: number + readonly createdBy: string | undefined + readonly keyValueMetadata: ReadonlyArray + /** The root of the schema tree. */ + readonly schema: SchemaNode + /** The leaf columns, in schema order. */ + readonly columns: ReadonlyArray + readonly rowGroups: ReadonlyArray +} + +/** + * Random access to the bytes of a Parquet file. + * + * **Details** + * + * `read` returns the `length` bytes at `offset`; the reader asks only for + * ranges inside `size`. Returning fewer bytes fails with `Malformed`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Source { + readonly size: number + readonly read: (offset: number, length: number) => Effect.Effect +} + +/** + * Creates a {@link Source} over bytes in memory. + * + * **Gotchas** + * + * Reads return views of `bytes`, so do not mutate them while reading. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromBytes = (bytes: Uint8Array): Source => ({ + size: bytes.length, + read: (offset, length) => Effect.succeed(bytes.subarray(offset, offset + length)) +}) + +/** + * Decompresses one page: the bytes as written, and the size the page header + * says they decompress to. + * + * **Details** + * + * A result of another size fails with `Malformed`; a failure becomes a + * {@link ParquetError} of kind `Codec` with the error as its `cause`. For + * `GZIP`, pages may hold several gzip members. `LZ4` is the deprecated + * Hadoop-framed codec; `LZ4_RAW` is the plain LZ4 block format. + * + * **Gotchas** + * + * Report failures in the error channel: a function that throws is a defect. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Decompress = (data: Uint8Array, uncompressedSize: number) => Effect.Effect + +/** + * The values of a column chunk's leaf, one slot per slot of the leaf's + * {@link Layer}, in Arrow's layout. + * + * **Details** + * + * - `BOOLEAN` values are a bitmap, least significant bit first. + * - `INT96` values are 12 bytes each, `FIXED_LEN_BYTE_ARRAY` values `width` + * bytes each. + * - `BYTE_ARRAY` value `i` is `values.subarray(offsets[i], offsets[i + 1])`. + * - A null slot holds zero, `false`, zero bytes or an empty byte array. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Values = + | { readonly type: "BOOLEAN"; readonly values: Uint8Array } + | { readonly type: "INT32"; readonly values: Int32Array } + | { readonly type: "INT64"; readonly values: BigInt64Array } + | { readonly type: "INT96"; readonly values: Uint8Array } + | { readonly type: "FLOAT"; readonly values: Float32Array } + | { readonly type: "DOUBLE"; readonly values: Float64Array } + | { readonly type: "BYTE_ARRAY"; readonly offsets: Int32Array; readonly values: Uint8Array } + | { readonly type: "FIXED_LEN_BYTE_ARRAY"; readonly width: number; readonly values: Uint8Array } + +/** + * One schema node of a column's path, as an Arrow array without its values. + * + * **Details** + * + * A node that isn't repeated has one slot per slot of its parent (per row for + * a child of the root). A repeated node has one slot per element, and its + * `offsets` give, for slot `i` of the parent, the elements from + * `offsets[i]` to `offsets[i + 1]`. + * + * An optional node has a `validity` bitmap, least significant bit first: a + * set bit is a non-null slot. A slot whose ancestor is null is null too. A + * repeated node has no validity; an empty and a null list are told apart by + * the optional node above it. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Layer { + readonly node: SchemaNode + readonly length: number + /** For optional nodes. */ + readonly validity: Uint8Array | undefined + readonly nullCount: number + /** For repeated nodes: `length` of the parent plus one. */ + readonly offsets: Int32Array | undefined +} + +/** + * A column chunk's values. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Column { + readonly descriptor: ColumnDescriptor + readonly numRows: number + /** One per node of {@link ColumnDescriptor.nodes}. */ + readonly layers: ReadonlyArray + /** One slot per slot of the last layer. */ + readonly values: Values +} + +// --------------------------------------------------------------------------- +// Errors and options +// --------------------------------------------------------------------------- + +const { Failure, fail, u32 } = Encodings + +const error = (kind: ParquetError["kind"], message: string, cause?: unknown): ParquetError => + new ParquetError(cause === undefined ? { kind, message } : { kind, message, cause }) + +const usage = (message: string): never => { + throw error("Usage", message) +} + +const limit = (name: string, value: number | undefined, fallback: number, max: number): number => { + if (value === undefined) return fallback + if (!Number.isSafeInteger(value) || value < 1 || value > max) { + usage(`${name} must be an integer from 1 to ${max}, got ${value}`) + } + return value +} + +const MAX_INT32 = 0x7fffffff + +/** Runs synchronous decoding, turning its failures into {@link ParquetError}s. */ +const attempt = (f: () => A): Effect.Effect => + Effect.suspend(() => { + try { + return Effect.succeed(f()) + } catch (cause) { + if (cause instanceof Failure) return Effect.fail(error(cause.kind, cause.message)) + if (Thrift.isThriftError(cause) && cause.kind !== "Usage") { + return Effect.fail( + error(cause.kind === "Limit" ? "Limit" : "Malformed", `invalid Thrift metadata: ${cause.message}`) + ) + } + return Effect.die(cause) + } + }) + +/** Prefixes an error's message with where in the file it happened. */ +const locate = (where: string) => (e: ParquetError): ParquetError => error(e.kind, `${where}: ${e.message}`, e.cause) + +const readRange = ( + source: Source, + offset: number, + length: number +): Effect.Effect => + Effect.flatMap(source.read(offset, length), (bytes) => + bytes.length === length + ? Effect.succeed(bytes) + : Effect.fail( + error("Malformed", `the source returned ${bytes.length} bytes at offset ${offset} instead of ${length}`) + )) + +// --------------------------------------------------------------------------- +// Thrift metadata +// --------------------------------------------------------------------------- + +const T = Thrift.Type + +const physicalTypes: ReadonlyArray = [ + "BOOLEAN", + "INT32", + "INT64", + "INT96", + "FLOAT", + "DOUBLE", + "BYTE_ARRAY", + "FIXED_LEN_BYTE_ARRAY" +] +const repetitions: ReadonlyArray = ["REQUIRED", "OPTIONAL", "REPEATED"] +const encodings: ReadonlyArray = [ + "PLAIN", + undefined, + "PLAIN_DICTIONARY", + "RLE", + "BIT_PACKED", + "DELTA_BINARY_PACKED", + "DELTA_LENGTH_BYTE_ARRAY", + "DELTA_BYTE_ARRAY", + "RLE_DICTIONARY", + "BYTE_STREAM_SPLIT", + "ALP" +] +const codecs: ReadonlyArray = [ + "UNCOMPRESSED", + "SNAPPY", + "GZIP", + "LZO", + "BROTLI", + "LZ4", + "ZSTD", + "LZ4_RAW" +] +const convertedTypes: ReadonlyArray = [ + "UTF8", + "MAP", + "MAP_KEY_VALUE", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME_MILLIS", + "TIME_MICROS", + "TIMESTAMP_MILLIS", + "TIMESTAMP_MICROS", + "UINT_8", + "UINT_16", + "UINT_32", + "UINT_64", + "INT_8", + "INT_16", + "INT_32", + "INT_64", + "JSON", + "BSON", + "INTERVAL" +] +const edgeAlgorithms = ["SPHERICAL", "VINCENTY", "THOMAS", "ANDOYER", "KARNEY"] as const +const timeUnits = [undefined, "MILLIS", "MICROS", "NANOS"] as const +const columnOrders = [undefined, "TYPE_ORDER", "IEEE_754_TOTAL_ORDER", "INT96_TIMESTAMP_ORDER"] as const + +const name = (names: ReadonlyArray, code: number): A | number => names[code] ?? code + +type Reader = Thrift.Reader + +const struct = (r: Reader, field: (id: number, type: Thrift.Type) => void): void => { + r.readStructBegin() + for (let type = r.readFieldBegin(); type !== T.Stop; type = r.readFieldBegin()) field(r.fieldId, type) + r.readStructEnd() +} + +/** A struct field: its id and how to read it. */ +type Field = readonly [id: number, read: (r: Reader, type: Thrift.Type) => A] + +/** + * Reads a struct into an object with a key per entry of `fields`: the value of + * the field with that id, or `undefined` when it is absent. Other fields are + * skipped. + */ +const readStruct = }>( + r: Reader, + fields: F +): { [K in keyof F]: ReturnType | undefined } => { + const out: Record = {} + const keys: Array = [] + for (const key in fields) { + out[key] = undefined + keys[fields[key][0]] = key + } + struct(r, (id, type) => { + const key = keys[id] + if (key === undefined) r.skip(type) + else out[key] = fields[key][1](r, type) + }) + return out as { [K in keyof F]: ReturnType | undefined } +} + +// Generated Thrift code skips a field of an unexpected type, as these do. +const skip = (r: Reader, type: Thrift.Type): undefined => { + r.skip(type) + return undefined +} +const i32 = (r: Reader, type: Thrift.Type) => type === T.I32 ? r.readI32() : skip(r, type) +const i8 = (r: Reader, type: Thrift.Type) => type === T.I8 ? r.readI8() : skip(r, type) +const i64 = (r: Reader, type: Thrift.Type) => type === T.I64 ? r.readI64Number() : skip(r, type) +const bool = (r: Reader, type: Thrift.Type) => type === T.Bool ? r.readBool() : skip(r, type) +const string = (r: Reader, type: Thrift.Type) => type === T.Binary ? r.readString() : skip(r, type) +const binary = (r: Reader, type: Thrift.Type) => type === T.Binary ? r.readBinary() : skip(r, type) +const structOf = (read: (r: Reader) => A) => (r: Reader, type: Thrift.Type): A | undefined => + type === T.Struct ? read(r) : skip(r, type) +const list = + (element: Thrift.Type, read: (r: Reader) => A) => (r: Reader, type: Thrift.Type): Array | undefined => { + if (type !== T.List) return skip(r, type) + const out: Array = [] + for (let n = r.readListBegin(), elementType = r.elementType; n > 0; n--) { + if (elementType !== element) return fail("Malformed", "a metadata list has elements of the wrong type") + out.push(read(r)) + } + return out + } +/** For a field whose presence is all that matters. */ +const present = (r: Reader, type: Thrift.Type): true => { + r.skip(type) + return true +} + +const required = (value: A | undefined, what: string): A => + value === undefined ? fail("Malformed", `${what} is missing`) : value + +const keyValue = (r: Reader): KeyValue => { + const f = readStruct(r, { key: [1, string], value: [2, string] }) + return { key: required(f.key, "KeyValue.key"), value: f.value } +} + +const statistics = (r: Reader): Statistics => + readStruct(r, { + nullCount: [3, i64], + distinctCount: [4, i64], + nanCount: [9, i64], + minValue: [6, binary], + maxValue: [5, binary], + isMinValueExact: [8, bool], + isMaxValueExact: [7, bool], + min: [2, binary], + max: [1, binary] + }) + +const simpleLogicalTypes = { + 1: "STRING", + 2: "MAP", + 3: "LIST", + 4: "ENUM", + 6: "DATE", + 11: "UNKNOWN", + 12: "JSON", + 13: "BSON", + 14: "UUID", + 15: "FLOAT16", + 19: "FILE" +} as const + +/** + * Reads a union of empty structs, such as TimeUnit or ColumnOrder, as the name + * of its field, or as the id of a field this module doesn't know. + */ +const unionName = (r: Reader, names: ReadonlyArray): A | number | undefined => { + let out: A | number | undefined + struct(r, (id, type) => { + r.skip(type) + const known = names[id] + if (known === undefined) out = id + else if (type === T.Struct) out = known + }) + return out +} + +/** Reads a LogicalType union, or the id of a member this module doesn't know. */ +const logicalType = (r: Reader): LogicalType | number | undefined => { + let out: LogicalType | number | undefined + struct(r, (id, type) => { + if (type !== T.Struct) { + r.skip(type) + out = id + return + } + const simple = simpleLogicalTypes[id as keyof typeof simpleLogicalTypes] + if (simple !== undefined) { + r.skip(type) + out = { type: simple } + return + } + switch (id) { + case 5: { + const f = readStruct(r, { scale: [1, i32], precision: [2, i32] }) + out = { + type: "DECIMAL", + scale: required(f.scale, "DecimalType.scale"), + precision: required(f.precision, "DecimalType.precision") + } + return + } + case 7: + case 8: { + const kind = id === 7 ? "TIME" : "TIMESTAMP" + const what = id === 7 ? "TimeType" : "TimestampType" + const f = readStruct(r, { isAdjustedToUTC: [1, bool], unit: [2, structOf((r) => unionName(r, timeUnits))] }) + const isAdjustedToUTC = required(f.isAdjustedToUTC, `${what}.isAdjustedToUTC`) + const unit = required(f.unit, `${what}.unit`) + if (typeof unit === "number") return fail("Unsupported", `${kind} unit ${unit} is not supported`) + out = { type: kind, isAdjustedToUTC, unit } + return + } + case 10: { + const f = readStruct(r, { bitWidth: [1, i8], isSigned: [2, bool] }) + out = { + type: "INTEGER", + bitWidth: required(f.bitWidth, "IntType.bitWidth"), + isSigned: required(f.isSigned, "IntType.isSigned") + } + return + } + case 16: + out = { type: "VARIANT", specificationVersion: readStruct(r, { version: [1, i8] }).version } + return + case 17: + out = { type: "GEOMETRY", crs: readStruct(r, { crs: [1, string] }).crs } + return + case 18: { + const f = readStruct(r, { crs: [1, string], algorithm: [2, i32] }) + out = { + type: "GEOGRAPHY", + crs: f.crs, + algorithm: f.algorithm === undefined ? undefined : name(edgeAlgorithms, f.algorithm) + } + return + } + default: + r.skip(type) + out = id + } + }) + return out +} + +const schemaElement = (r: Reader) => { + const f = readStruct(r, { + type: [1, i32], + typeLength: [2, i32], + repetition: [3, i32], + name: [4, string], + numChildren: [5, i32], + convertedType: [6, i32], + scale: [7, i32], + precision: [8, i32], + fieldId: [9, i32], + logicalType: [10, structOf(logicalType)] + }) + return { ...f, name: required(f.name, "SchemaElement.name") } +} + +type Element = ReturnType + +const columnMetaData = (r: Reader): ColumnMetaData => { + const f = readStruct(r, { + type: [1, i32], + encodings: [2, list(T.I32, (r) => name(encodings, r.readI32()))], + pathInSchema: [3, list(T.Binary, (r) => r.readString())], + codec: [4, i32], + numValues: [5, i64], + totalUncompressedSize: [6, i64], + totalCompressedSize: [7, i64], + keyValueMetadata: [8, list(T.Struct, keyValue)], + dataPageOffset: [9, i64], + indexPageOffset: [10, i64], + dictionaryPageOffset: [11, i64], + statistics: [12, structOf(statistics)] + }) + const type = required(f.type, "ColumnMetaData.type") + return { + physicalType: physicalTypes[type] ?? fail("Unsupported", `physical type ${type} is not supported`), + encodings: required(f.encodings, "ColumnMetaData.encodings"), + pathInSchema: required(f.pathInSchema, "ColumnMetaData.path_in_schema"), + codec: name(codecs, required(f.codec, "ColumnMetaData.codec")), + numValues: required(f.numValues, "ColumnMetaData.num_values"), + totalUncompressedSize: required(f.totalUncompressedSize, "ColumnMetaData.total_uncompressed_size"), + totalCompressedSize: required(f.totalCompressedSize, "ColumnMetaData.total_compressed_size"), + keyValueMetadata: f.keyValueMetadata ?? [], + dataPageOffset: required(f.dataPageOffset, "ColumnMetaData.data_page_offset"), + indexPageOffset: f.indexPageOffset, + dictionaryPageOffset: f.dictionaryPageOffset, + statistics: f.statistics + } +} + +const columnChunk = (r: Reader): ColumnChunk => { + const f = readStruct(r, { + filePath: [1, string], + metaData: [3, structOf(columnMetaData)], + cryptoMetadata: [8, present], + encryptedColumnMetadata: [9, present] + }) + const encrypted = f.cryptoMetadata === true || f.encryptedColumnMetadata === true + if (!encrypted && f.metaData === undefined) fail("Malformed", "ColumnChunk.meta_data is missing") + return { filePath: f.filePath, encrypted, metaData: f.metaData } +} + +const rowGroup = (r: Reader): RowGroup => { + const f = readStruct(r, { columns: [1, list(T.Struct, columnChunk)], totalByteSize: [2, i64], numRows: [3, i64] }) + return { + columns: required(f.columns, "RowGroup.columns"), + totalByteSize: required(f.totalByteSize, "RowGroup.total_byte_size"), + numRows: rowCount(f.numRows, "RowGroup.num_rows") + } +} + +const rowCount = (n: number | undefined, what: string): number => { + const value = required(n, what) + if (value < 0) fail("Malformed", `${what} is ${value}`) + return value +} + +/** + * The physical type LogicalTypes.md defines each annotation for, `undefined` + * for a group. The annotations that are not here depend on more than that. + */ +const annotated: { + readonly [ + K in Exclude + ]: PhysicalType | undefined +} = { + MAP: undefined, + MAP_KEY_VALUE: undefined, + LIST: undefined, + VARIANT: undefined, + FILE: undefined, + STRING: "BYTE_ARRAY", + ENUM: "BYTE_ARRAY", + JSON: "BYTE_ARRAY", + BSON: "BYTE_ARRAY", + GEOMETRY: "BYTE_ARRAY", + GEOGRAPHY: "BYTE_ARRAY", + DATE: "INT32", + TIMESTAMP: "INT64" +} +const annotatedLengths = { UUID: 16, FLOAT16: 2, INTERVAL: 12 } as const + +/** + * Whether LogicalTypes.md defines `annotation` for the node of schema element + * `e`: a leaf of `physicalType`, or a group. Fails on parameters it forbids + * for every node. + */ +const annotates = (annotation: Annotation, e: Element, physicalType: PhysicalType | undefined): boolean => { + switch (annotation.type) { + case "UNKNOWN": + // "a column that is always null", of any type + return true + case "UUID": + case "FLOAT16": + case "INTERVAL": + return physicalType === "FIXED_LEN_BYTE_ARRAY" && e.typeLength === annotatedLengths[annotation.type] + case "TIME": + return physicalType === (annotation.unit === "MILLIS" ? "INT32" : "INT64") + case "INTEGER": { + const { bitWidth } = annotation + if (bitWidth !== 8 && bitWidth !== 16 && bitWidth !== 32 && bitWidth !== 64) { + fail("Malformed", `schema element "${e.name}" is an INTEGER with a bit width of ${bitWidth}`) + } + return physicalType === (bitWidth === 64 ? "INT64" : "INT32") + } + case "DECIMAL": { + const { precision, scale } = annotation + const invalid = (reason: string): never => + fail("Malformed", `schema element "${e.name}" is a DECIMAL with ${reason}`) + if (precision < 1) invalid(`a precision of ${precision}`) + if (scale < 0 || scale > precision) invalid(`a scale of ${scale} for a precision of ${precision}`) + if (physicalType === "BYTE_ARRAY") return true + const digits = physicalType === "INT32" + ? 9 + : physicalType === "INT64" + ? 18 + : physicalType === "FIXED_LEN_BYTE_ARRAY" + // floor(log10(2^(8n - 1) - 1)), computed without the last - 1, as no power of 2 is a power of 10 + ? Math.floor((8 * e.typeLength! - 1) * Math.log10(2)) + : undefined + if (digits === undefined) return false + if (precision > digits) { + invalid(`a precision of ${precision}, more than the ${digits} digits of its ${physicalType}`) + } + return true + } + default: + return physicalType === annotated[annotation.type] + } +} + +/** + * The annotations of a schema element's node, without those the spec doesn't + * define for it. `known` is whether the node kept all it had: when it didn't, + * the order of its values is unknown too. + */ +const annotations = (e: Element, physicalType: PhysicalType | undefined): { + readonly logicalType: LogicalType | undefined + readonly convertedType: ConvertedType | number | undefined + readonly known: boolean +} => { + const converted = e.convertedType === undefined ? undefined : name(convertedTypes, e.convertedType) + const logicalType = typeof e.logicalType === "object" && annotates(e.logicalType, e, physicalType) + ? e.logicalType + : undefined + // The converted type stands in for a logical type only when there is none. + // An ignored logical type leaves only the physical type (#unsupported-logical-types). + if (e.logicalType !== undefined && logicalType === undefined) { + return { logicalType, convertedType: undefined, known: false } + } + if (typeof converted !== "string") return { logicalType, convertedType: converted, known: converted === undefined } + const annotation: Annotation = converted !== "DECIMAL" + ? logicalOf[converted] + // A DecimalType supersedes the scale and precision of the schema element. + : logicalType?.type === "DECIMAL" + ? logicalType + : { + type: "DECIMAL", + scale: e.scale ?? 0, + precision: required(e.precision, `the precision of DECIMAL schema element "${e.name}"`) + } + return annotates(annotation, e, physicalType) + ? { logicalType, convertedType: converted, known: true } + : { logicalType, convertedType: undefined, known: false } +} + +interface Schema { + readonly root: SchemaNode + readonly columns: Array +} + +/** Rebuilds the schema tree from its depth-first list of elements. */ +const schemaTree = ( + elements: ReadonlyArray, + orders: ReadonlyArray | undefined, + maxDepth: number +): Schema => { + const columns: Array = [] + let index = 0 + const node = (path: ReadonlyArray, definition: number, repetitionLevel: number): SchemaNode => { + if (index >= elements.length) fail("Malformed", "the schema ends inside a group") + const e = elements[index++] + const isRoot = path.length === 0 && index === 1 + let repetition: Repetition | undefined + if (!isRoot) { + if (path.length + 1 > maxDepth) fail("Limit", `the schema nests deeper than maxDepth (${maxDepth})`) + const code = required(e.repetition, `the repetition of schema element "${e.name}"`) + repetition = repetitions[code] ?? fail("Malformed", `schema element "${e.name}" has repetition ${code}`) + if (repetition !== "REQUIRED") definition++ + if (repetition === "REPEATED") repetitionLevel++ + } + const base = { + name: e.name, + repetition, + typeLength: e.typeLength, + scale: e.scale, + precision: e.precision, + fieldId: e.fieldId + } + if (e.type === undefined) { + const count = required(e.numChildren, `the type or children of schema element "${e.name}"`) + if (count < 0) fail("Malformed", `schema element "${e.name}" has ${count} children`) + const children: Array = [] + const { convertedType, logicalType } = annotations(e, undefined) + const self: SchemaNode = { ...base, convertedType, logicalType, physicalType: undefined, children } + const nodes = isRoot ? path : [...path, self] + for (let i = 0; i < count; i++) children.push(node(nodes, definition, repetitionLevel)) + return self + } + if (isRoot) fail("Malformed", "the schema root is not a group") + if (e.numChildren !== undefined && e.numChildren !== 0) { + fail("Malformed", `schema element "${e.name}" has both a type and children`) + } + const physicalType = physicalTypes[e.type] ?? fail("Unsupported", `physical type ${e.type} is not supported`) + if (physicalType === "FIXED_LEN_BYTE_ARRAY" && !(e.typeLength !== undefined && e.typeLength > 0)) { + fail("Malformed", `FIXED_LEN_BYTE_ARRAY column "${e.name}" has no positive type_length`) + } + const { convertedType, known, logicalType } = annotations(e, physicalType) + const self: SchemaNode = { ...base, convertedType, logicalType, physicalType, children: undefined } + const nodes = [...path, self] + const order = orders?.[columns.length] + columns.push({ + path: nodes.map((n) => n.name), + nodes, + physicalType, + typeLength: physicalType === "FIXED_LEN_BYTE_ARRAY" ? e.typeLength : undefined, + maxDefinitionLevel: definition, + maxRepetitionLevel: repetitionLevel, + columnOrder: typeof order === "number" || !known ? undefined : order + }) + return self + } + const root = node([], 0, 0) + if (index !== elements.length) fail("Malformed", `the schema has ${elements.length - index} elements after its root`) + if (orders !== undefined && orders.length !== columns.length) { + fail("Malformed", `the file has ${orders.length} column orders for ${columns.length} columns`) + } + return { root, columns } +} + +const fileMetaData = (footer: Uint8Array, maxDepth: number): FileMetaData => { + const r = Thrift.makeReader(footer) + const f = readStruct(r, { + version: [1, i32], + schema: [2, list(T.Struct, schemaElement)], + numRows: [3, i64], + rowGroups: [4, list(T.Struct, rowGroup)], + keyValueMetadata: [5, list(T.Struct, keyValue)], + createdBy: [6, string], + orders: [7, list(T.Struct, (r) => unionName(r, columnOrders))] + }) + const elements = required(f.schema, "FileMetaData.schema") + if (elements.length === 0) fail("Malformed", "FileMetaData.schema is empty") + const { columns, root } = schemaTree(elements, f.orders, maxDepth) + const rowGroups = required(f.rowGroups, "FileMetaData.row_groups") + for (const group of rowGroups) { + if (group.columns.length !== columns.length) { + fail("Malformed", `a row group has ${group.columns.length} column chunks for ${columns.length} columns`) + } + group.columns.forEach((chunk, i) => { + const meta = chunk.metaData + const column = columns[i] + if (meta === undefined) return + if (meta.physicalType !== column.physicalType) { + fail( + "Malformed", + `column chunk ${i} is ${meta.physicalType}, but column "${column.path.join(".")}" is ${column.physicalType}` + ) + } + if (meta.pathInSchema.join("\u0000") !== column.path.join("\u0000")) { + fail("Malformed", `column chunk ${i} is for "${meta.pathInSchema.join(".")}", not "${column.path.join(".")}"`) + } + }) + } + return { + version: required(f.version, "FileMetaData.version"), + numRows: rowCount(f.numRows, "FileMetaData.num_rows"), + createdBy: f.createdBy, + keyValueMetadata: f.keyValueMetadata ?? [], + schema: root, + columns, + rowGroups + } +} + +const MAGIC = 0x31524150 // "PAR1" little-endian +const MAGIC_ENCRYPTED = 0x45524150 // "PARE" + +/** + * Reads the metadata in the footer of a Parquet file. + * + * **Details** + * + * This reads the last 8 bytes, the first 4 and the footer. `maxFooterLength` + * (default 64 MiB) caps the footer's size and `maxDepth` (default 64, at most + * 255) how deep the schema may nest. + * + * @stability experimental + * @category decoding + * @since 4.1.5 + */ +export const readMetadata = ( + source: Source, + options?: { + readonly maxFooterLength?: number | undefined + readonly maxDepth?: number | undefined + } +): Effect.Effect => + Effect.gen(function*() { + const maxFooterLength = limit("maxFooterLength", options?.maxFooterLength, 64 * 1024 * 1024, 0xffffffff) + const maxDepth = limit("maxDepth", options?.maxDepth, 64, 255) + const size = source.size + if (!Number.isSafeInteger(size) || size < 0) usage(`the source size ${size} is not a non-negative integer`) + if (size < 12) return yield* error("Malformed", `a file of ${size} bytes is too short to be Parquet`) + const tail = yield* readRange(source, size - 8, 8) + const magic = u32(tail, 4) + if (magic === MAGIC_ENCRYPTED) { + return yield* error("Unsupported", "files with an encrypted footer are not supported") + } + if (magic !== MAGIC) return yield* error("Malformed", "the file does not end with the magic number PAR1") + if (u32(yield* readRange(source, 0, 4), 0) !== MAGIC) { + return yield* error("Malformed", "the file does not start with the magic number PAR1") + } + const length = u32(tail, 0) + if (length > size - 12) return yield* error("Malformed", `the footer length ${length} is larger than the file`) + if (length > maxFooterLength) { + return yield* error("Limit", `the footer of ${length} bytes exceeds maxFooterLength (${maxFooterLength})`) + } + const footer = yield* readRange(source, size - 8 - length, length) + return yield* attempt(() => fileMetaData(footer, maxDepth)) + }) + +// --------------------------------------------------------------------------- +// Pages +// --------------------------------------------------------------------------- + +const DATA_PAGE = 0 +const DICTIONARY_PAGE = 2 +const DATA_PAGE_V2 = 3 + +interface PageHeader { + readonly type: number + readonly uncompressedSize: number + readonly compressedSize: number + /** The CRC-32 as the header holds it, a signed i32. */ + readonly crc: number | undefined + readonly numValues: number + readonly encoding: Encoding | number + readonly definitionEncoding: Encoding | number + readonly repetitionEncoding: Encoding | number + readonly numNulls: number + readonly numRows: number + readonly definitionLength: number + readonly repetitionLength: number + readonly compressed: boolean +} + +const pageHeader = (r: Reader): PageHeader => { + const f = readStruct(r, { + type: [1, i32], + uncompressedSize: [2, i32], + compressedSize: [3, i32], + crc: [4, i32], + dataPage: [ + 5, + structOf((r) => + readStruct(r, { + numValues: [1, i32], + encoding: [2, i32], + definitionEncoding: [3, i32], + repetitionEncoding: [4, i32] + }) + ) + ], + dictionaryPage: [7, structOf((r) => readStruct(r, { numValues: [1, i32], encoding: [2, i32] }))], + dataPageV2: [ + 8, + structOf((r) => + readStruct(r, { + numValues: [1, i32], + numNulls: [2, i32], + numRows: [3, i32], + encoding: [4, i32], + definitionLength: [5, i32], + repetitionLength: [6, i32], + compressed: [7, bool] + }) + ) + ] + }) + const type = required(f.type, "PageHeader.type") + const uncompressedSize = required(f.uncompressedSize, "PageHeader.uncompressed_page_size") + const compressedSize = required(f.compressedSize, "PageHeader.compressed_page_size") + if (uncompressedSize < 0 || compressedSize < 0) fail("Malformed", "a page header has a negative size") + const count = (n: number | undefined, what: string): number => { + const value = required(n, what) + if (value < 0) fail("Malformed", `${what} is negative`) + return value + } + const encoding = (code: number | undefined, what: string) => name(encodings, required(code, what)) + const header: PageHeader = { + type, + uncompressedSize, + compressedSize, + crc: f.crc, + numValues: 0, + encoding: "PLAIN", + definitionEncoding: "RLE", + repetitionEncoding: "RLE", + numNulls: 0, + numRows: 0, + definitionLength: 0, + repetitionLength: 0, + compressed: true + } + switch (type) { + case DATA_PAGE: { + const p = required(f.dataPage, "PageHeader.data_page_header") + return { + ...header, + numValues: count(p.numValues, "DataPageHeader.num_values"), + encoding: encoding(p.encoding, "DataPageHeader.encoding"), + definitionEncoding: encoding(p.definitionEncoding, "DataPageHeader.definition_level_encoding"), + repetitionEncoding: encoding(p.repetitionEncoding, "DataPageHeader.repetition_level_encoding") + } + } + case DATA_PAGE_V2: { + const p = required(f.dataPageV2, "PageHeader.data_page_header_v2") + const v2 = { + ...header, + numValues: count(p.numValues, "DataPageHeaderV2.num_values"), + numNulls: count(p.numNulls, "DataPageHeaderV2.num_nulls"), + numRows: count(p.numRows, "DataPageHeaderV2.num_rows"), + encoding: encoding(p.encoding, "DataPageHeaderV2.encoding"), + definitionLength: count(p.definitionLength, "DataPageHeaderV2.definition_levels_byte_length"), + repetitionLength: count(p.repetitionLength, "DataPageHeaderV2.repetition_levels_byte_length"), + compressed: p.compressed !== false + } + const levels = v2.repetitionLength + v2.definitionLength + if (levels > compressedSize || levels > uncompressedSize) { + fail("Malformed", "the levels of a v2 data page are larger than the page") + } + return v2 + } + case DICTIONARY_PAGE: { + const p = required(f.dictionaryPage, "PageHeader.dictionary_page_header") + return { + ...header, + numValues: count(p.numValues, "DictionaryPageHeader.num_values"), + encoding: encoding(p.encoding, "DictionaryPageHeader.encoding") + } + } + default: + return header + } +} + +const kindOf = (column: ColumnDescriptor): Encodings.Kind => { + switch (column.physicalType) { + case "BOOLEAN": + return { tag: "bool" } + case "INT32": + return { tag: "i32", width: 4 } + case "FLOAT": + return { tag: "f32", width: 4 } + case "INT64": + return { tag: "i64", width: 8 } + case "DOUBLE": + return { tag: "f64", width: 8 } + case "INT96": + return { tag: "int96", width: 12 } + case "FIXED_LEN_BYTE_ARRAY": + return { tag: "fixed", width: column.typeLength! } + case "BYTE_ARRAY": + return { tag: "bytes" } + } +} + +/** Decodes RLE/bit-packed hybrid levels from 0 to `max` into `out`, unless the column has none. */ +const decodeLevels = ( + bytes: Uint8Array, + start: number, + end: number, + max: number, + out: Uint8Array | undefined, + what: string +): void => { + if (out !== undefined) Encodings.hybrid(bytes, start, end, Encodings.bitWidth(max), max, out, what) +} + +/** Decodes the pages of one column chunk and assembles its {@link Column}. */ +class ChunkDecoder { + readonly column: ColumnDescriptor + readonly kind: Encodings.Kind + readonly numValues: number + private readonly budget: Encodings.Budget + /** Whether the values are annotated as text, which has to be UTF-8. */ + private readonly utf8: boolean + /** The levels of every value, nulls included; `undefined` when the column has none. */ + private readonly repetitions: Uint8Array | undefined + private readonly definitions: Uint8Array | undefined + private readonly dense: Array = [] + private dictionary: Encodings.Dense | undefined + private seenData = false + /** Levels decoded so far: values, nulls included. */ + count = 0 + + constructor(column: ColumnDescriptor, kind: Encodings.Kind, numValues: number, maxBinaryLength: number) { + this.column = column + this.kind = kind + this.numValues = numValues + this.budget = { left: maxBinaryLength } + // A logical type supersedes the converted type. + const leaf = column.nodes[column.nodes.length - 1] + const annotation = leaf.logicalType?.type ?? leaf.convertedType + this.utf8 = kind.tag === "bytes" && + (annotation === "STRING" || annotation === "UTF8" || annotation === "ENUM" || annotation === "JSON") + // Fixed-width values take their bytes in every slot, nulls included. + if (kind.tag === "fixed" || kind.tag === "int96") Encodings.spend(this.budget, numValues * kind.width) + this.repetitions = column.maxRepetitionLevel === 0 ? undefined : new Uint8Array(numValues) + this.definitions = column.maxDefinitionLevel === 0 ? undefined : new Uint8Array(numValues) + } + + dictionaryPage(header: PageHeader, data: Uint8Array): void { + if (this.dictionary !== undefined || this.seenData) { + fail("Malformed", "a dictionary page must be the first page of a column chunk, and the only one") + } + if (header.encoding !== "PLAIN" && header.encoding !== "PLAIN_DICTIONARY") { + fail("Malformed", `a dictionary page is encoded with ${header.encoding}, not PLAIN`) + } + this.dictionary = Encodings.plain(this.kind, data, 0, data.length, header.numValues, this.budget) + if (this.utf8) Encodings.utf8(this.dictionary as Encodings.Binary, "dictionary value") + } + + /** A v1 data page: repetition levels, definition levels and values, all after decompression. */ + dataPage(header: PageHeader, page: Uint8Array): void { + const [repetition, definition] = this.start(header.numValues) + let pos = 0 + const levels = (out: Uint8Array | undefined, encoding: Encoding | number, max: number, what: string) => { + if (out === undefined) return + if (encoding === "RLE") { + const end = Encodings.lengthPrefix(page, pos, page.length, what) + decodeLevels(page, pos + 4, end, max, out, what) + pos = end + } else if (encoding === "BIT_PACKED") { + pos = Encodings.bitPacked(page, pos, page.length, Encodings.bitWidth(max), max, out, what) + } else { + fail("Unsupported", `${what} encoded with ${encoding} are not supported`) + } + } + levels(repetition, header.repetitionEncoding, this.column.maxRepetitionLevel, "repetition levels") + levels(definition, header.definitionEncoding, this.column.maxDefinitionLevel, "definition levels") + this.values(header.encoding, page, pos, this.tally(header.numValues, repetition, definition).nonNull) + } + + /** A v2 data page: the levels as written and the values after decompression. */ + dataPageV2(header: PageHeader, levels: Uint8Array, values: Uint8Array): void { + const n = header.numValues + const [repetition, definition] = this.start(n) + const split = header.repetitionLength + decodeLevels(levels, 0, split, this.column.maxRepetitionLevel, repetition, "repetition levels") + decodeLevels(levels, split, levels.length, this.column.maxDefinitionLevel, definition, "definition levels") + const { nonNull, rows } = this.tally(n, repetition, definition) + if (n - nonNull !== header.numNulls) { + fail("Malformed", `a v2 data page has ${n - nonNull} nulls, but its header says ${header.numNulls}`) + } + if (repetition !== undefined && n > 0 && repetition[0] !== 0) { + fail("Malformed", "a v2 data page does not start at a row boundary") + } + if (rows !== header.numRows) { + fail("Malformed", `a v2 data page has ${rows} rows, but its header says ${header.numRows}`) + } + this.values(header.encoding, values, 0, nonNull) + } + + /** Starts a data page of `n` values, and returns its parts of the chunk's repetition and definition levels. */ + private start(n: number): readonly [Uint8Array | undefined, Uint8Array | undefined] { + if (n > this.numValues - this.count) fail("Malformed", "the pages hold more values than their column chunk") + this.seenData = true + const end = this.count + n + return [this.repetitions?.subarray(this.count, end), this.definitions?.subarray(this.count, end)] + } + + /** Counts a page's non-null values and rows, and adds its levels to the chunk's. */ + private tally( + n: number, + repetition: Uint8Array | undefined, + definition: Uint8Array | undefined + ): { readonly nonNull: number; readonly rows: number } { + const max = this.column.maxDefinitionLevel + let nonNull = 0 + let rows = 0 + for (let i = 0; i < n; i++) { + if (definition === undefined || definition[i] === max) nonNull++ + if (repetition === undefined || repetition[i] === 0) rows++ + } + this.count += n + return { nonNull, rows } + } + + private values(encoding: Encoding | number, bytes: Uint8Array, pos: number, n: number): void { + const kind = this.kind + const end = bytes.length + const invalid = (): never => + fail("Malformed", `${encoding} is not an encoding for ${this.column.physicalType} values`) + let dense: Encodings.Dense + switch (encoding) { + case "PLAIN": + dense = Encodings.plain(kind, bytes, pos, end, n, this.budget) + break + case "PLAIN_DICTIONARY": + case "RLE_DICTIONARY": { + const dictionary = this.dictionary + if (dictionary === undefined) return fail("Malformed", "a dictionary-encoded page has no dictionary page") + if (n === 0) return + if (pos >= end) fail("Malformed", "dictionary indices are truncated") + const width = bytes[pos] + if (width > 32) fail("Malformed", `dictionary indices have a bit width of ${width}, more than 32`) + const indices = new Int32Array(n) + const max = Encodings.lengthOf(kind, dictionary) - 1 + if (max < 0) fail("Malformed", "a dictionary-encoded page has values but the dictionary is empty") + Encodings.hybrid(bytes, pos + 1, end, width, max, indices, "dictionary indices") + dense = Encodings.gather(kind, dictionary, indices, this.budget) + break + } + case "RLE": { + if (kind.tag !== "bool") return invalid() + const out = new Uint8Array(n) + const stop = Encodings.lengthPrefix(bytes, pos, end, "RLE BOOLEAN values") + Encodings.hybrid(bytes, pos + 4, stop, 1, 1, out, "RLE BOOLEAN values") + dense = out + break + } + case "DELTA_BINARY_PACKED": + if (kind.tag === "i32") dense = Encodings.deltaBinaryPacked(bytes, pos, end, n, false)[0] + else if (kind.tag === "i64") dense = Encodings.deltaBinaryPacked(bytes, pos, end, n, true)[0] + else return invalid() + break + case "DELTA_LENGTH_BYTE_ARRAY": + if (kind.tag !== "bytes") return invalid() + dense = Encodings.deltaLengthByteArray(bytes, pos, end, n, this.budget) + break + case "DELTA_BYTE_ARRAY": + if (kind.tag === "bytes") dense = Encodings.deltaByteArray(bytes, pos, end, n, this.budget) + else if (kind.tag === "fixed") { + dense = Encodings.deltaByteArray(bytes, pos, end, n, this.budget, kind.width).data + } else return invalid() + break + case "BYTE_STREAM_SPLIT": + if (kind.tag === "bool" || kind.tag === "bytes" || kind.tag === "int96") return invalid() + dense = Encodings.byteStreamSplit(kind, bytes, pos, end, n) + break + case "BIT_PACKED": + return invalid() + default: + return fail("Unsupported", `the ${encoding} encoding is not supported`) + } + // The values of a dictionary were checked with its page. + if (this.utf8 && encoding !== "PLAIN_DICTIONARY" && encoding !== "RLE_DICTIONARY") { + Encodings.utf8(dense as Encodings.Binary, "value") + } + this.dense.push(dense) + } + + finish(numRows: number): Column { + if (this.count !== this.numValues) { + fail("Malformed", `the column chunk ends after ${this.count} of its ${this.numValues} values`) + } + const dense = Encodings.concat(this.kind, this.dense) + return assemble(this.column, this.kind, numRows, this.count, this.repetitions, this.definitions, dense) + } +} + +/** + * Turns repetition and definition levels into one {@link Layer} per node of the + * column's path, and spreads the dense values over the leaf's slots. + * + * Each repeated node starts a new "space" of slots (its elements); the other + * nodes share the space of their nearest repeated ancestor, or the rows. A + * level entry (r, d) adds a slot to every space s >= r whose repeated node it + * defines, so to spaces r, r + 1, ... while d reaches the space's definition + * level. + */ +const assemble = ( + column: ColumnDescriptor, + kind: Encodings.Kind, + numRows: number, + n: number, + repetition: Uint8Array | undefined, + definition: Uint8Array | undefined, + dense: Encodings.Dense +): Column => { + const nodes = column.nodes + const maxRepetition = column.maxRepetitionLevel + const maxDefinition = column.maxDefinitionLevel + // spaceDefinition[s]: the definition level at which space s has a slot + const spaceDefinition = [0] + const nodeDefinition: Array = [] + const nodeSpace: Array = [] + const optionalIn: Array> = [[]] + const repeatedOf: Array = [-1] + let d = 0 + for (let i = 0; i < nodes.length; i++) { + const repetitionType = nodes[i].repetition + if (repetitionType !== "REQUIRED") d++ + if (repetitionType === "REPEATED") { + spaceDefinition.push(d) + optionalIn.push([]) + repeatedOf.push(i) + } + nodeDefinition.push(d) + nodeSpace.push(spaceDefinition.length - 1) + if (repetitionType === "OPTIONAL") optionalIn[spaceDefinition.length - 1].push(i) + } + + const counts = new Array(maxRepetition + 1).fill(0) + let open = -1 + for (let e = 0; e < n; e++) { + const r = repetition === undefined ? 0 : repetition[e] + const level = definition === undefined ? maxDefinition : definition[e] + if (r > 0 && (open < r || level < spaceDefinition[r])) { + fail("Malformed", `level ${e} repeats a list that has no elements (repetition ${r}, definition ${level})`) + } + let s = r + while (s <= maxRepetition && level >= spaceDefinition[s]) counts[s++]++ + open = s - 1 + } + if (counts[0] !== numRows) { + fail("Malformed", `the column chunk holds ${counts[0]} rows, but its row group has ${numRows}`) + } + + const validity = nodes.map((node, i) => + node.repetition === "OPTIONAL" ? new Uint8Array((counts[nodeSpace[i]] + 7) >> 3) : undefined + ) + const valid = new Array(nodes.length).fill(0) + const offsets = nodes.map((node, i) => + node.repetition === "REPEATED" ? new Int32Array(counts[nodeSpace[i] - 1] + 1) : undefined + ) + const slots = counts[maxRepetition] + const present = new Uint8Array(slots) + const next = new Array(maxRepetition + 1).fill(0) + for (let e = 0; e < n; e++) { + const r = repetition === undefined ? 0 : repetition[e] + const level = definition === undefined ? maxDefinition : definition[e] + for (let s = r; s <= maxRepetition && level >= spaceDefinition[s]; s++) { + const slot = next[s]++ + if (s < maxRepetition) offsets[repeatedOf[s + 1]]![slot] = next[s + 1] + for (const i of optionalIn[s]) { + if (level >= nodeDefinition[i]) { + validity[i]![slot >> 3] |= 1 << (slot & 7) + valid[i]++ + } + } + if (s === maxRepetition && level === maxDefinition) present[slot] = 1 + } + } + for (let s = 1; s <= maxRepetition; s++) offsets[repeatedOf[s]]![counts[s - 1]] = counts[s] + + return { + descriptor: column, + numRows, + layers: nodes.map((node, i) => { + const length = counts[nodeSpace[i]] + return { + node, + length, + validity: validity[i], + nullCount: validity[i] === undefined ? 0 : length - valid[i], + offsets: offsets[i] + } + }), + values: spread(column, kind, dense, present, slots) + } +} + +/** Gives every leaf slot a value, leaving null slots empty. */ +const spread = ( + column: ColumnDescriptor, + kind: Encodings.Kind, + dense: Encodings.Dense, + present: Uint8Array, + slots: number +): Values => { + const full = Encodings.lengthOf(kind, dense) === slots + switch (kind.tag) { + case "bool": { + const bits = dense as Uint8Array + const out = new Uint8Array((slots + 7) >> 3) + for (let j = 0, k = 0; j < slots; j++) { + if (present[j] === 1 && bits[k++] === 1) out[j >> 3] |= 1 << (j & 7) + } + return { type: "BOOLEAN", values: out } + } + case "bytes": { + const values = dense as Encodings.Binary + let offsets = values.offsets + if (!full) { + offsets = new Int32Array(slots + 1) + for (let j = 0, k = 0; j < slots; j++) { + offsets[j + 1] = offsets[j] + (present[j] === 1 ? values.offsets[k + 1] - values.offsets[k++] : 0) + } + } + return { type: "BYTE_ARRAY", offsets, values: values.data } + } + case "int96": + case "fixed": { + const width = kind.width + let values = dense as Uint8Array + if (!full) { + const from = values + values = new Uint8Array(slots * width) + for (let j = 0, k = 0; j < slots; j++) { + if (present[j] !== 1) continue + for (let at = j * width, stop = at + width; at < stop; at++) values[at] = from[k++] + } + } + return kind.tag === "int96" ? { type: "INT96", values } : { type: "FIXED_LEN_BYTE_ARRAY", width, values } + } + default: { + let values = dense as Int32Array + if (!full) { + const from = values + values = Encodings.alloc(kind, slots) as Int32Array + for (let j = 0, k = 0; j < slots; j++) if (present[j] === 1) values[j] = from[k++] + } + return { type: column.physicalType, values } as Values + } + } +} + +const decompress = ( + codec: CompressionCodec | number, + decompressors: { readonly [codec: string]: Decompress | undefined } | undefined, + data: Uint8Array, + size: number +): Effect.Effect => { + const checked = (out: Uint8Array) => + out.length === size + ? Effect.succeed(out) + : Effect.fail(error("Malformed", `a page decompressed to ${out.length} bytes, but its header says ${size}`)) + if (codec === "UNCOMPRESSED") return checked(data) + const f = typeof codec === "string" ? decompressors?.[codec] : undefined + if (f === undefined) { + return Effect.fail(error("Unsupported", `${codec} pages need a Decompress for ${codec} in the codecs option`)) + } + return Effect.flatMap( + Effect.mapError(f(data, size), (cause) => error("Codec", `${codec} decompression failed`, cause)), + checked + ) +} + +/** + * Reads the values of one column chunk: column `column` of + * `metadata.columns` in row group `rowGroup`. + * + * **Details** + * + * Pages compressed with a codec other than `UNCOMPRESSED` need a + * {@link Decompress} for it in `codecs`. Dictionary-encoded values are looked + * up in the dictionary, so the result never refers to it. + * + * Limits for untrusted files, each at most 2^31 - 1: + * + * - `maxPageSize` (default 256 MiB) caps each page's compressed and + * uncompressed size. + * - `maxValues` (default 2^24) caps the values of the column chunk and of + * each page, nulls included. + * - `maxBinaryLength` (default 1 GiB) caps the total bytes of the chunk's + * `BYTE_ARRAY`, `FIXED_LEN_BYTE_ARRAY` and `INT96` values. Fixed-width + * values are counted for every value, nulls included. + * + * **Gotchas** + * + * - Memory and time grow with the values times the nodes on the column's + * path: every optional node has a validity bitmap. Lower `maxValues` for + * deeply nested columns from untrusted files. + * - `rowGroup` and `column` must be indexes into `metadata`: anything else + * is a defect, not a failure. + * + * @stability experimental + * @category decoding + * @since 4.1.5 + */ +export const readColumn = ( + source: Source, + metadata: FileMetaData, + options: { + readonly rowGroup: number + readonly column: number + readonly codecs?: { readonly [K in Exclude]?: Decompress | undefined } | undefined + readonly maxPageSize?: number | undefined + readonly maxValues?: number | undefined + readonly maxBinaryLength?: number | undefined + } +): Effect.Effect => + Effect.gen(function*() { + const maxPageSize = limit("maxPageSize", options.maxPageSize, 256 * 1024 * 1024, MAX_INT32) + const maxValues = limit("maxValues", options.maxValues, 2 ** 24, MAX_INT32) + const maxBinaryLength = limit("maxBinaryLength", options.maxBinaryLength, 1024 * 1024 * 1024, MAX_INT32) + const group = Number.isInteger(options.rowGroup) ? metadata.rowGroups[options.rowGroup] : undefined + const column = Number.isInteger(options.column) ? metadata.columns[options.column] : undefined + if (group === undefined) return usage(`the file has no row group ${options.rowGroup}`) + if (column === undefined) return usage(`the file has no column ${options.column}`) + const chunk = group.columns[options.column] + const where = `column "${column.path.join(".")}" of row group ${options.rowGroup}` + if (chunk.filePath !== undefined) { + return yield* error("Unsupported", `${where} is in another file, ${chunk.filePath}`) + } + if (chunk.encrypted || chunk.metaData === undefined) { + return yield* error("Unsupported", `${where} is encrypted`) + } + const meta = chunk.metaData + if (meta.numValues < 0) return yield* error("Malformed", `${where} has ${meta.numValues} values`) + if (meta.numValues > maxValues) { + return yield* error("Limit", `${where} has ${meta.numValues} values, more than maxValues (${maxValues})`) + } + const decoder = yield* Effect.mapError( + attempt(() => new ChunkDecoder(column, kindOf(column), meta.numValues, maxBinaryLength)), + locate(where) + ) + // A chunk without values needs no pages; parquet-cpp writes data_page_offset 0 for one. + if (meta.numValues > 0) { + let start = meta.dataPageOffset + const dictionary = meta.dictionaryPageOffset + if (dictionary !== undefined && dictionary > 0 && dictionary < start) start = dictionary + const length = meta.totalCompressedSize + if (start < 4 || length < 0 || length > source.size - start) { + return yield* error("Malformed", `${where} has pages outside the file`) + } + const bytes = yield* readRange(source, start, length) + const thrift = Thrift.makeReader(bytes) + let pos = 0 + while (decoder.count < meta.numValues) { + if (pos >= bytes.length) { + return yield* error("Malformed", `${where} ends after ${decoder.count} of its ${meta.numValues} values`) + } + const headerAt = start + pos + const header = yield* Effect.mapError( + attempt(() => { + thrift.position = pos + return pageHeader(thrift) + }), + locate(`${where}, page header at byte ${headerAt}`) + ) + const body = thrift.position + if (header.compressedSize > bytes.length - body) { + return yield* error("Malformed", `a page of ${where} is larger than its column chunk`) + } + if (header.compressedSize > maxPageSize || header.uncompressedSize > maxPageSize) { + return yield* error("Limit", `a page of ${where} is larger than maxPageSize (${maxPageSize})`) + } + // Caps dictionary pages too, and rejects a page before decompressing it. + if (header.numValues > maxValues) { + return yield* error("Limit", `a page of ${where} has more values than maxValues (${maxValues})`) + } + pos = body + header.compressedSize + const page = bytes.subarray(body, pos) + // The CRC is of the page as it is stored: with its levels, and compressed. + if (header.crc !== undefined && crc32(0, page) !== header.crc >>> 0) { + return yield* error("Malformed", `${where}, page at byte ${headerAt}: the page does not match its CRC`) + } + // Index pages and page types added later are skipped. + if (header.type !== DATA_PAGE && header.type !== DATA_PAGE_V2 && header.type !== DICTIONARY_PAGE) continue + const levels = header.type === DATA_PAGE_V2 ? header.repetitionLength + header.definitionLength : 0 + const codec = header.compressed ? meta.codec : "UNCOMPRESSED" + yield* decompress(codec, options.codecs, page.subarray(levels), header.uncompressedSize - levels).pipe( + Effect.flatMap((data) => + attempt(() => + header.type === DICTIONARY_PAGE + ? decoder.dictionaryPage(header, data) + : header.type === DATA_PAGE + ? decoder.dataPage(header, data) + : decoder.dataPageV2(header, page.subarray(0, levels), data) + ) + ), + Effect.mapError(locate(`${where}, page at byte ${headerAt}`)) + ) + } + } + return yield* Effect.mapError(attempt(() => decoder.finish(group.numRows)), locate(where)) + }) diff --git a/packages/effect/src/encoding/Thrift.ts b/packages/effect/src/encoding/Thrift.ts new file mode 100644 index 00000000000..f424e5cf906 --- /dev/null +++ b/packages/effect/src/encoding/Thrift.ts @@ -0,0 +1,1125 @@ +/** + * Reads and writes the Apache Thrift compact protocol: structs, lists, sets, + * maps and scalars, without an IDL. + * + * **Mental model** + * + * - A Thrift struct is a sequence of fields, each a numbered value of some + * {@link Type}, ended by a stop marker. Lists, sets and maps hold values of + * one element type. Nothing on the wire names a field: the IDL that says + * field 3 is `num_rows` lives with the caller. + * - This module works at the protocol level, as generated Thrift code does. + * A {@link Reader} is a cursor over the bytes: read a field header, look at + * its id and type, read the value or {@link Reader.skip} it. A + * {@link Writer} mirrors those calls. + * - The method names and their meaning follow Thrift's `TProtocol`, so code + * written against another Thrift implementation reads the same: a struct + * is `readStructBegin`, then `readFieldBegin` until it returns + * `Type.Stop`, then `readStructEnd`. + * - {@link Reader.readValue} and {@link Writer.writeValue} read and write any + * value as a {@link Value} tree, for debugging and tests. + * + * **Example** (Writing and reading a struct) + * + * ```ts import.meta.vitest + * import { Thrift } from "effect/encoding" + * import { Result } from "effect" + * + * // struct Person { 1: string name; 2: i64 age; 3: list tags } + * const w = Thrift.makeWriter() + * w.writeStructBegin() + * w.writeFieldBegin(Thrift.Type.Binary, 1) + * w.writeString("Ada") + * w.writeFieldBegin(Thrift.Type.I64, 2) + * w.writeI64Number(36) + * w.writeFieldBegin(Thrift.Type.List, 3) + * w.writeListBegin(Thrift.Type.Binary, 1) + * w.writeString("math") + * w.writeFieldStop() + * w.writeStructEnd() + * const bytes = w.finish() + * + * const person = Thrift.decode(bytes, (r) => { + * let name = "" + * let age = 0 + * const tags: Array = [] + * r.readStructBegin() + * for (let type = r.readFieldBegin(); type !== Thrift.Type.Stop; type = r.readFieldBegin()) { + * const id = r.fieldId + * if (id === 1 && type === Thrift.Type.Binary) name = r.readString() + * else if (id === 2 && type === Thrift.Type.I64) age = r.readI64Number() + * else if (id === 3 && type === Thrift.Type.List) { + * for (let n = r.readListBegin(); n > 0; n--) tags.push(r.readString()) + * } else r.skip(type) // unknown fields, or fields of an unexpected type + * } + * r.readStructEnd() + * return { name, age, tags } + * }) + * + * Result.getOrThrow(person) // => { name: "Ada", age: 36, tags: ["math"] } + * ``` + * + * **Details** + * + * Every read is bounds-checked: malformed or hostile input throws a + * {@link ThriftError} (or makes {@link decode} fail) instead of reading past + * the bytes. A length or container size larger than the bytes that remain + * fails before anything is allocated, varints are capped at the width of + * their type, and nesting is limited by {@link ReaderOptions.maxDepth}. + * Strings must be valid UTF-8. + * + * Binary and `uuid` values, also inside a {@link Value}, are views into the + * input, not copies: they change when the input does and keep its buffer + * alive. Copy one with `slice()` to keep it on its own. + * + * `i64` values read as `bigint` ({@link Reader.readI64}) or, when they are + * known to be safe integers such as offsets and counts, as `number` + * ({@link Reader.readI64Number}). + * + * The binary and JSON protocols and the RPC message envelope are not + * supported. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Data from "../Data.ts" +import * as Utf8 from "../internal/utf8.ts" +import * as Predicate from "../Predicate.ts" +import * as Result from "../Result.ts" + +const ThriftErrorTypeId = "~effect/encoding/Thrift/ThriftError" + +/** + * Error raised for malformed input or misuse of a {@link Reader} or + * {@link Writer}. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `Truncated`: the input ends inside a value, or a length or container + * size is larger than the bytes that remain. + * - `Malformed`: an unknown type code, a varint wider than its type (16 + * bits for `i16` and field ids, 32 for `i32` and sizes, 64 for `i64`), a + * field id over 32767, or a non-empty container of `Stop`. + * - `InvalidUtf8`: a string is not valid UTF-8. + * - `Range`: an `i64` read with {@link Reader.readI64Number} is not a safe + * integer. + * - `Limit`: the input nests deeper than {@link ReaderOptions.maxDepth}, or a + * length or size exceeds {@link ReaderOptions.maxBinaryLength} or + * {@link ReaderOptions.maxContainerSize}. + * - `Usage`: the reader or writer was called out of order or with an + * invalid argument or option, or a written value is out of range for its + * type. {@link decode} rethrows these caller errors. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class ThriftError extends Data.TaggedError("ThriftError")<{ + readonly kind: "Truncated" | "Malformed" | "InvalidUtf8" | "Range" | "Limit" | "Usage" + readonly message: string +}> { + /** + * Marks this value as a Thrift error for runtime guards. + * + * @stability experimental + * @since 4.1.5 + */ + readonly [ThriftErrorTypeId] = ThriftErrorTypeId +} + +/** + * Checks whether a value is a {@link ThriftError}. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isThriftError = (u: unknown): u is ThriftError => Predicate.hasProperty(u, ThriftErrorTypeId) + +const fail = (kind: ThriftError["kind"], message: string): never => { + throw new ThriftError({ kind, message }) +} + +const checkInteger = (what: string, value: number, min: number, max: number): number => { + if (!Number.isInteger(value) || value < min || value > max) { + fail("Usage", `${what} ${value} is not an integer from ${min} to ${max}`) + } + return value +} + +const integerOption = (name: string, value: number | undefined, min: number, max: number, fallback: number): number => + value === undefined ? fallback : checkInteger(name, value, min, max) + +/** + * The type of a field or container element. The codes are Thrift's `TType` + * codes, the same in every protocol; `Stop` ends a struct. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export const Type = { + Stop: 0, + Bool: 2, + I8: 3, + Double: 4, + I16: 6, + I32: 8, + I64: 10, + Binary: 11, + Struct: 12, + Map: 13, + Set: 14, + List: 15, + Uuid: 16 +} as const + +/** + * A {@link Type} code. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Type = typeof Type[keyof typeof Type] + +/** + * The name of a {@link Type}, as used by {@link Value}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TypeName = keyof typeof Type + +/** Compact type nibble → TType. Nibbles 1 and 2 are a bool field's true and false; 14 and 15 are unassigned. */ +const fromCompact: ReadonlyArray = [0, 2, 2, 3, 6, 8, 10, 4, 11, 15, 14, 13, 12, 16] + +/** TType → compact type nibble, with `Bool` as 1. */ +const toCompact: Array = [] +fromCompact.forEach((type, nibble) => { + if (nibble !== 2) toCompact[type] = nibble +}) + +const typeNames: Array = [] +for (const name of Object.keys(Type) as Array) typeNames[Type[name]] = name + +const describe = (type: number): string => typeNames[type] ?? String(type) + +/** + * Any Thrift value, tagged with the name of its {@link Type}: what + * {@link Reader.readValue} returns and {@link Writer.writeValue} writes. + * + * **Details** + * + * The wire does not tell strings from binary, so both read as `Binary`. An + * empty map carries no key and value types on the wire; they read as + * `"Stop"`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Value = + | { readonly _tag: "Bool"; readonly value: boolean } + | { readonly _tag: "I8" | "I16" | "I32" | "Double"; readonly value: number } + | { readonly _tag: "I64"; readonly value: bigint } + | { readonly _tag: "Binary" | "Uuid"; readonly value: Uint8Array } + | { readonly _tag: "Struct"; readonly fields: ReadonlyArray } + | { readonly _tag: "List" | "Set"; readonly elementType: TypeName; readonly elements: ReadonlyArray } + | { + readonly _tag: "Map" + readonly keyType: TypeName + readonly valueType: TypeName + readonly entries: ReadonlyArray + } + +/** + * A field of a `Struct` {@link Value}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Field { + readonly id: number + readonly value: Value +} + +/** + * Limits for reading untrusted input. A value that isn't an integer in range + * throws a {@link ThriftError} of kind `Usage`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface ReaderOptions { + /** + * How deep structs and containers may nest, counting each struct and each + * container entered by {@link Reader.readStructBegin}, + * {@link Reader.skip} or {@link Reader.readValue}. An integer from 1 to + * 1000, which keeps their recursion within the stack; defaults to 64. + */ + readonly maxDepth?: number | undefined + /** The longest binary or string value, in bytes: a non-negative integer. No limit by default. */ + readonly maxBinaryLength?: number | undefined + /** The largest list, set or map: a non-negative integer. No limit by default. */ + readonly maxContainerSize?: number | undefined +} + +const MAX_SAFE = Number.MAX_SAFE_INTEGER +const BIGINT_1 = BigInt(1) +const BIGINT_7 = BigInt(7) +const BIGINT_63 = BigInt(63) +const BIGINT_0x7f = BigInt(0x7f) + +const zigzag = (n: number): number => ((n << 1) ^ (n >> 31)) >>> 0 +const unzigzag = (n: number): number => (n >>> 1) ^ -(n & 1) + +/** + * A cursor over compact-protocol bytes. Each `read*` method reads one value + * at {@link position} and moves past it, or throws a {@link ThriftError} on + * malformed input. + * + * **Gotchas** + * + * - A nested read overwrites {@link fieldId}, {@link elementType}, + * {@link keyType} and {@link valueType}: copy one into a local first. + * - {@link readBinary}, {@link readUuid} and the binary values of + * {@link readValue} are views of {@link bytes}: copy them before mutating + * or reusing the input. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export class Reader { + /** The input. */ + readonly bytes: Uint8Array + /** The id of the field whose header was read last. */ + fieldId = 0 + /** The element type of the list or set whose header was read last. */ + elementType: Type = Type.Stop + /** The key type of the map whose header was read last. */ + keyType: Type = Type.Stop + /** The value type of the map whose header was read last. */ + valueType: Type = Type.Stop + + private pos = 0 + private readonly view: DataView + private readonly end: number + private readonly maxDepth: number + private readonly maxBinaryLength: number + private readonly maxContainerSize: number + private lastFieldId = 0 + private readonly fieldIds: Array = [] + private depth = 0 + /** The compact type of a bool field header not yet read: 1 true, 2 false, 0 none. */ + private bool = 0 + /** Set by {@link varint64}: whether the zigzag value is negative. */ + private negative = false + + constructor(bytes: Uint8Array, options?: ReaderOptions) { + this.bytes = bytes + this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + this.end = bytes.length + this.maxDepth = integerOption("maxDepth", options?.maxDepth, 1, 1000, 64) + this.maxBinaryLength = integerOption("maxBinaryLength", options?.maxBinaryLength, 0, Infinity, Infinity) + this.maxContainerSize = integerOption("maxContainerSize", options?.maxContainerSize, 0, Infinity, Infinity) + } + + /** + * Where the next read starts in {@link bytes}. Set it to read from + * elsewhere; a position outside the input is a `Usage` error, so check + * offsets read from the input first. + */ + get position(): number { + return this.pos + } + set position(position: number) { + this.pos = checkInteger("position", position, 0, this.end) + this.bool = 0 + } + + private truncated(what: string): never { + return fail("Truncated", `the input ends inside ${what} at byte ${this.pos}`) + } + + private byte(what: string): number { + if (this.pos >= this.end) this.truncated(what) + return this.bytes[this.pos++] + } + + private advance(n: number, what: string): number { + const at = this.pos + if (n > this.end - at) this.truncated(what) + this.pos = at + n + return at + } + + /** An unsigned varint of at most 32 bits. */ + private varint32(what: string): number { + const bytes = this.bytes + const end = this.end + let pos = this.pos + if (pos < end) { + let b = bytes[pos++] + if (b < 0x80) { + this.pos = pos + return b + } + let result = b & 0x7f + for (let shift = 7; shift < 28; shift += 7) { + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + result |= (b & 0x7f) << shift + if (b < 0x80) { + this.pos = pos + return result + } + } + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + if (b > 0x0f) return fail("Malformed", `${what} at byte ${this.pos} is longer than 32 bits`) + this.pos = pos + return (result | (b << 28)) >>> 0 + } + return this.truncated(what) + } + + /** A zigzag varint of at most 16 bits. */ + private varint16(what: string): number { + const n = this.varint32(what) + if (n > 0xffff) fail("Malformed", `${what} before byte ${this.pos} is longer than 16 bits`) + return unzigzag(n) + } + + /** + * A zigzag varint of at most 64 bits as a number: returns `v` for v >= 0 or + * `-v - 1` for v < 0 (exact below 2^53) and sets {@link negative}. + */ + private varint64(what: string): number { + const bytes = this.bytes + const end = this.end + const start = this.pos + let pos = start + if (pos >= end) return this.truncated(what) + let b = bytes[pos++] + this.negative = (b & 1) === 1 + let half = (b & 0x7f) >>> 1 + let scale = 64 + while (b >= 0x80) { + if (pos >= end) return this.truncated(what) + b = bytes[pos++] + if (pos - start === 10 && b > 0x01) { + return fail("Malformed", `${what} at byte ${start} is longer than 64 bits`) + } + half += (b & 0x7f) * scale + scale *= 128 + } + this.pos = pos + return half + } + + /** Checks a container size; every element takes at least `bytesPerElement` bytes. */ + private checkSize(size: number, bytesPerElement: number, what: string): void { + if (size > this.maxContainerSize) { + fail("Limit", `${what} of ${size} elements exceeds the limit of ${this.maxContainerSize}`) + } + if (size * bytesPerElement > this.end - this.pos) { + fail("Truncated", `${what} of ${size} elements at byte ${this.pos} is larger than the remaining input`) + } + } + + /** Reads a binary length and moves past the bytes; returns where they start. */ + private binary(what: string): number { + const length = this.varint32(`${what} length`) + if (length > this.maxBinaryLength) { + fail("Limit", `${what} of ${length} bytes exceeds the limit of ${this.maxBinaryLength}`) + } + return this.advance(length, `${what} of ${length} bytes`) + } + + private listHeader(what: string): number { + const header = this.byte(`${what} header`) + let size = header >> 4 + if (size === 15) size = this.varint32(`${what} size`) + this.elementType = this.elementOf(header & 0x0f, size, what) + this.checkSize(size, 1, what) + return size + } + + private enter(): void { + if (++this.depth > this.maxDepth) { + fail("Limit", `the input nests deeper than ${this.maxDepth} at byte ${this.pos}`) + } + } + + private elementOf(nibble: number, size: number, what: string): Type { + const type = fromCompact[nibble] + if (type === undefined || (type === Type.Stop && size > 0)) { + return fail("Malformed", `${what} at byte ${this.pos} has element type code ${nibble}`) + } + return type + } + + /** Starts reading a struct. Read its fields with {@link readFieldBegin}. */ + readStructBegin(): void { + this.enter() + this.fieldIds.push(this.lastFieldId) + this.lastFieldId = 0 + } + + /** Finishes the struct started with {@link readStructBegin}, after its `Stop`. */ + readStructEnd(): void { + if (this.fieldIds.length === 0) fail("Usage", "readStructEnd without readStructBegin") + this.depth-- + this.lastFieldId = this.fieldIds.pop()! + } + + /** + * Reads a field header: returns the field's type and sets {@link fieldId}. + * Returns `Type.Stop` at the end of the struct. + */ + readFieldBegin(): Type { + const header = this.byte("a field header") + this.bool = 0 + const nibble = header & 0x0f + if (nibble === 0) return Type.Stop + const delta = header >> 4 + let id: number + if (delta === 0) { + id = this.varint16("a field id") + } else { + id = this.lastFieldId + delta + if (id > 32767) fail("Malformed", `the field header at byte ${this.pos - 1} has id ${id}, over 32767`) + } + this.lastFieldId = id + this.fieldId = id + const type = fromCompact[nibble] + if (type === undefined) { + return fail("Malformed", `the field header at byte ${this.pos - 1} has type code ${nibble}`) + } + if (nibble <= 2) this.bool = nibble + return type + } + + /** Reads a list header: returns the list's size and sets {@link elementType}. */ + readListBegin(): number { + return this.listHeader("a list") + } + + /** Reads a set header: returns the set's size and sets {@link elementType}. */ + readSetBegin(): number { + return this.listHeader("a set") + } + + /** Reads a map header: returns the map's size and sets {@link keyType} and {@link valueType}. */ + readMapBegin(): number { + const size = this.varint32("a map size") + if (size === 0) { + this.keyType = Type.Stop + this.valueType = Type.Stop + return 0 + } + const types = this.byte("a map header") + this.keyType = this.elementOf(types >> 4, size, "a map") + this.valueType = this.elementOf(types & 0x0f, size, "a map") + this.checkSize(size, 2, "a map") + return size + } + + /** Reads a `bool`: the value of a `Bool` field header just read, or a byte inside a container. */ + readBool(): boolean { + const bool = this.bool + if (bool !== 0) { + this.bool = 0 + return bool === 1 + } + return this.byte("a bool") === 1 + } + + /** Reads an `i8` (`byte`). */ + readI8(): number { + return (this.byte("an i8") << 24) >> 24 + } + + /** Reads an `i16`. */ + readI16(): number { + return this.varint16("an i16") + } + + /** Reads an `i32`. */ + readI32(): number { + return unzigzag(this.varint32("an i32")) + } + + /** Reads an `i64` as a `bigint`. */ + readI64(): bigint { + const start = this.pos + const half = this.varint64("an i64") + if (half < MAX_SAFE) return BigInt(this.negative ? -half - 1 : half) + let u = BigInt(0) + let shift = BigInt(0) + for (let pos = start; pos < this.pos; pos++) { + u |= (BigInt(this.bytes[pos]) & BIGINT_0x7f) << shift + shift += BIGINT_7 + } + u = BigInt.asUintN(64, u) + return (u >> BIGINT_1) ^ -(u & BIGINT_1) + } + + /** + * Reads an `i64` as a `number`. Fails with kind `Range` when the value is + * not a safe integer, leaving {@link position} at the value so that + * {@link readI64} can read it. + */ + readI64Number(): number { + const start = this.pos + const half = this.varint64("an i64") + if (half < MAX_SAFE || (half === MAX_SAFE && !this.negative)) return this.negative ? -half - 1 : half + this.pos = start + return fail("Range", `the i64 at byte ${start} is not a safe integer`) + } + + /** Reads a `double` (little-endian, as the compact protocol stores it). */ + readDouble(): number { + return this.view.getFloat64(this.advance(8, "a double"), true) + } + + /** Reads a `binary` value as a view of {@link bytes}, not a copy. */ + readBinary(): Uint8Array { + return this.bytes.subarray(this.binary("a binary value"), this.pos) + } + + /** Reads a `string`: a `binary` value decoded as UTF-8. */ + readString(): string { + const at = this.binary("a string") + return Utf8.decode(this.bytes, at, this.pos) ?? fail("InvalidUtf8", `the string at byte ${at} is not valid UTF-8`) + } + + /** Reads a `uuid`: 16 bytes, as a view of {@link bytes}. */ + readUuid(): Uint8Array { + const at = this.advance(16, "a uuid") + return this.bytes.subarray(at, at + 16) + } + + /** + * Moves past a value of the given type: the value of a field whose header + * was just read, or a container element. + */ + skip(type: Type): void { + switch (type) { + case Type.Bool: + this.readBool() + return + case Type.I8: + this.advance(1, "an i8") + return + case Type.I16: + this.varint16("an i16") + return + case Type.I32: + this.varint32("an i32") + return + case Type.I64: + this.varint64("an i64") + return + case Type.Double: + this.advance(8, "a double") + return + case Type.Binary: + this.binary("a binary value") + return + case Type.Uuid: + this.advance(16, "a uuid") + return + case Type.Struct: + this.readStructBegin() + for (let t = this.readFieldBegin(); t !== Type.Stop; t = this.readFieldBegin()) this.skip(t) + this.readStructEnd() + return + case Type.List: + case Type.Set: { + this.enter() + const size = type === Type.List ? this.readListBegin() : this.readSetBegin() + const element = this.elementType + for (let i = 0; i < size; i++) this.skip(element) + this.depth-- + return + } + case Type.Map: { + this.enter() + const size = this.readMapBegin() + const key = this.keyType + const value = this.valueType + for (let i = 0; i < size; i++) { + this.skip(key) + this.skip(value) + } + this.depth-- + return + } + default: + fail("Usage", `can't skip a value of type ${describe(type)}`) + } + } + + /** + * Reads a value of the given type as a {@link Value}: the value of a field + * whose header was just read, a container element, or a whole input such + * as a struct. + */ + readValue(type: Type): Value { + switch (type) { + case Type.Bool: + return { _tag: "Bool", value: this.readBool() } + case Type.I8: + return { _tag: "I8", value: this.readI8() } + case Type.I16: + return { _tag: "I16", value: this.readI16() } + case Type.I32: + return { _tag: "I32", value: this.readI32() } + case Type.I64: + return { _tag: "I64", value: this.readI64() } + case Type.Double: + return { _tag: "Double", value: this.readDouble() } + case Type.Binary: + return { _tag: "Binary", value: this.readBinary() } + case Type.Uuid: + return { _tag: "Uuid", value: this.readUuid() } + case Type.Struct: { + const fields: Array = [] + this.readStructBegin() + for (let t = this.readFieldBegin(); t !== Type.Stop; t = this.readFieldBegin()) { + const id = this.fieldId + fields.push({ id, value: this.readValue(t) }) + } + this.readStructEnd() + return { _tag: "Struct", fields } + } + case Type.List: + case Type.Set: { + this.enter() + const size = type === Type.List ? this.readListBegin() : this.readSetBegin() + const element = this.elementType + const elements: Array = [] + for (let i = 0; i < size; i++) elements.push(this.readValue(element)) + this.depth-- + return { _tag: type === Type.List ? "List" : "Set", elementType: typeNames[element], elements } + } + case Type.Map: { + this.enter() + const size = this.readMapBegin() + const key = this.keyType + const value = this.valueType + const entries: Array = [] + for (let i = 0; i < size; i++) { + const k = this.readValue(key) + entries.push([k, this.readValue(value)]) + } + this.depth-- + return { _tag: "Map", keyType: typeNames[key], valueType: typeNames[value], entries } + } + default: + return fail("Usage", `can't read a value of type ${describe(type)}`) + } + } +} + +/** + * Creates a {@link Reader} at the start of `bytes`, which it reads in place. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const makeReader = (bytes: Uint8Array, options?: ReaderOptions): Reader => new Reader(bytes, options) + +/** + * Reads `bytes` with `read`, which gets a {@link Reader} at the start of + * them. Malformed input becomes a failed `Result` instead of an exception. + * + * **Gotchas** + * + * `Usage` errors and errors other than {@link ThriftError} are rethrown: + * they are bugs in `read` or invalid options, not bad input. + * + * @stability experimental + * @category reading + * @since 4.1.5 + */ +export const decode = ( + bytes: Uint8Array, + read: (reader: Reader) => A, + options?: ReaderOptions +): Result.Result => { + try { + return Result.succeed(read(new Reader(bytes, options))) + } catch (cause) { + if (isThriftError(cause) && cause.kind !== "Usage") return Result.fail(cause) + throw cause + } +} + +const encoder = new TextEncoder() + +/** + * Writes compact-protocol bytes into a buffer that grows as needed. The + * calls mirror {@link Reader}'s. + * + * **Details** + * + * Write a struct as {@link writeStructBegin}, then for each field + * {@link writeFieldBegin} and its value, then {@link writeFieldStop} and + * {@link writeStructEnd}. + * + * Containers are a header ({@link writeListBegin}, {@link writeSetBegin}, + * {@link writeMapBegin}) followed by exactly that many elements (for a map, + * key then value). + * + * Integers outside their type's range, values of no {@link Type}, and calls + * out of order throw a {@link ThriftError} of kind `Usage`. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export class Writer { + private bytes: Uint8Array + private view: DataView + private position = 0 + private lastFieldId = 0 + private readonly fieldIds: Array = [] + private boolPending = false + private boolFieldId = 0 + + constructor(options?: { readonly initialSize?: number | undefined }) { + this.bytes = new Uint8Array(Math.max(16, integerOption("initialSize", options?.initialSize, 0, Infinity, 1024))) + this.view = new DataView(this.bytes.buffer) + } + + /** The number of bytes written. */ + get length(): number { + return this.position + } + + private reserve(n: number): void { + if (this.position + n <= this.bytes.length) return + let size = this.bytes.length * 2 + while (size < this.position + n) size *= 2 + const next = new Uint8Array(size) + next.set(this.bytes.subarray(0, this.position)) + this.bytes = next + this.view = new DataView(next.buffer) + } + + private checkNoBool(what: string): void { + if (this.boolPending) fail("Usage", `${what} after writeFieldBegin(Type.Bool) instead of writeBool`) + } + + private writeByte(b: number): void { + this.reserve(1) + this.bytes[this.position++] = b + } + + /** An unsigned varint of at most 32 bits. */ + private varint32(n: number): void { + this.reserve(5) + const bytes = this.bytes + let pos = this.position + while (n > 0x7f) { + bytes[pos++] = (n & 0x7f) | 0x80 + n >>>= 7 + } + bytes[pos++] = n + this.position = pos + } + + private raw(value: Uint8Array): void { + this.reserve(value.length) + this.bytes.set(value, this.position) + this.position += value.length + } + + private integer(value: number, min: number, max: number, what: string): void { + this.checkNoBool(what) + checkInteger(what, value, min, max) + } + + private elementNibble(type: number, size: number, what: string): number { + const nibble = toCompact[type] + if (nibble === undefined || (type === Type.Stop && size > 0)) { + return fail("Usage", `${describe(type)} is not a valid ${what}`) + } + return nibble + } + + private fieldHeader(nibble: number, id: number): void { + const delta = id - this.lastFieldId + if (delta > 0 && delta <= 15) { + this.writeByte((delta << 4) | nibble) + } else { + this.writeByte(nibble) + this.varint32(zigzag(id)) + } + this.lastFieldId = id + } + + /** Starts a struct. Write its fields, then {@link writeFieldStop} and {@link writeStructEnd}. */ + writeStructBegin(): void { + this.checkNoBool("writeStructBegin") + this.fieldIds.push(this.lastFieldId) + this.lastFieldId = 0 + } + + /** Finishes the struct started with {@link writeStructBegin}. */ + writeStructEnd(): void { + this.checkNoBool("writeStructEnd") + if (this.fieldIds.length === 0) fail("Usage", "writeStructEnd without writeStructBegin") + this.lastFieldId = this.fieldIds.pop()! + } + + /** Writes a field header. For a `Bool` field, the header is written by the {@link writeBool} that follows. */ + writeFieldBegin(type: Type, id: number): void { + this.checkNoBool("writeFieldBegin") + if (this.fieldIds.length === 0) fail("Usage", "writeFieldBegin outside writeStructBegin/writeStructEnd") + checkInteger("field id", id, -32768, 32767) + if (type === Type.Bool) { + this.boolPending = true + this.boolFieldId = id + return + } + const nibble = toCompact[type] + if (nibble === undefined || type === Type.Stop) { + fail("Usage", `${describe(type)} is not a field type`) + } + this.fieldHeader(nibble, id) + } + + /** Writes the `Stop` that ends a struct's fields. */ + writeFieldStop(): void { + this.checkNoBool("writeFieldStop") + this.writeByte(0) + } + + private listHeader(elementType: Type, size: number, method: string): void { + this.checkNoBool(method) + checkInteger("container size", size, 0, 0xffffffff) + const nibble = this.elementNibble(elementType, size, "element type") + if (size < 15) { + this.writeByte((size << 4) | nibble) + } else { + this.writeByte(0xf0 | nibble) + this.varint32(size) + } + } + + /** Writes a list header; write `size` elements of `elementType` next. */ + writeListBegin(elementType: Type, size: number): void { + this.listHeader(elementType, size, "writeListBegin") + } + + /** Writes a set header; write `size` elements of `elementType` next. */ + writeSetBegin(elementType: Type, size: number): void { + this.listHeader(elementType, size, "writeSetBegin") + } + + /** Writes a map header; write `size` key-value pairs next. */ + writeMapBegin(keyType: Type, valueType: Type, size: number): void { + this.checkNoBool("writeMapBegin") + checkInteger("container size", size, 0, 0xffffffff) + const key = this.elementNibble(keyType, size, "map key type") + const value = this.elementNibble(valueType, size, "map value type") + if (size === 0) { + this.writeByte(0) + } else { + this.varint32(size) + this.writeByte((key << 4) | value) + } + } + + /** Writes a `bool`: the header of the `Bool` field just begun, or a byte inside a container. */ + writeBool(value: boolean): void { + if (this.boolPending) { + this.boolPending = false + this.fieldHeader(value ? 1 : 2, this.boolFieldId) + } else { + this.writeByte(value ? 1 : 2) + } + } + + /** Writes an `i8` (`byte`). */ + writeI8(value: number): void { + this.integer(value, -128, 127, "i8") + this.writeByte(value & 0xff) + } + + /** Writes an `i16`. */ + writeI16(value: number): void { + this.integer(value, -32768, 32767, "i16") + this.varint32(zigzag(value)) + } + + /** Writes an `i32`. */ + writeI32(value: number): void { + this.integer(value, -2147483648, 2147483647, "i32") + this.varint32(zigzag(value)) + } + + /** Writes an `i64` given as a safe-integer `number`. */ + writeI64Number(value: number): void { + this.integer(value, -MAX_SAFE, MAX_SAFE, "i64") + const negative = value < 0 + let half = negative ? -value - 1 : value + this.reserve(10) + const bytes = this.bytes + let pos = this.position + // The first byte holds the sign bit and the low six bits of the magnitude. + let b = ((half % 64) * 2) | (negative ? 1 : 0) + half = Math.floor(half / 64) + while (half > 0) { + bytes[pos++] = b | 0x80 + b = half % 128 + half = Math.floor(half / 128) + } + bytes[pos++] = b + this.position = pos + } + + /** Writes an `i64`. */ + writeI64(value: bigint): void { + this.checkNoBool("writeI64") + if (value >= -MAX_SAFE && value <= MAX_SAFE) return this.writeI64Number(Number(value)) + if (BigInt.asIntN(64, value) !== value) fail("Usage", `i64 ${value} is out of range`) + let u = BigInt.asUintN(64, (value << BIGINT_1) ^ (value >> BIGINT_63)) + this.reserve(10) + while (u > BIGINT_0x7f) { + this.bytes[this.position++] = Number(u & BIGINT_0x7f) | 0x80 + u >>= BIGINT_7 + } + this.bytes[this.position++] = Number(u) + } + + /** Writes a `double` (little-endian). */ + writeDouble(value: number): void { + this.checkNoBool("writeDouble") + this.reserve(8) + this.view.setFloat64(this.position, value, true) + this.position += 8 + } + + /** Writes a `binary` value. */ + writeBinary(value: Uint8Array): void { + this.checkNoBool("writeBinary") + this.varint32(value.length) + this.raw(value) + } + + /** Writes a `string` as UTF-8 `binary`. Lone surrogates, which UTF-8 can't encode, become U+FFFD. */ + writeString(value: string): void { + this.checkNoBool("writeString") + const length = value.length + if (length < 128) { + // One length byte; ASCII strings are written directly. + this.reserve(1 + length) + const bytes = this.bytes + const start = this.position + 1 + let i = 0 + for (; i < length; i++) { + const c = value.charCodeAt(i) + if (c >= 0x80) break + bytes[start + i] = c + } + if (i === length) { + bytes[this.position] = length + this.position = start + length + return + } + } + this.writeBinary(encoder.encode(value)) + } + + /** Writes a `uuid`: exactly 16 bytes. */ + writeUuid(value: Uint8Array): void { + this.checkNoBool("writeUuid") + if (value.length !== 16) fail("Usage", `a uuid is 16 bytes, got ${value.length}`) + this.raw(value) + } + + /** Writes a {@link Value}: a field's value after its header, a container element, or a whole value. */ + writeValue(value: Value): void { + switch (value._tag) { + case "Bool": + return this.writeBool(value.value) + case "I8": + return this.writeI8(value.value) + case "I16": + return this.writeI16(value.value) + case "I32": + return this.writeI32(value.value) + case "I64": + return this.writeI64(value.value) + case "Double": + return this.writeDouble(value.value) + case "Binary": + return this.writeBinary(value.value) + case "Uuid": + return this.writeUuid(value.value) + case "Struct": + this.writeStructBegin() + for (const field of value.fields) { + this.writeFieldBegin(Type[field.value._tag], field.id) + this.writeValue(field.value) + } + this.writeFieldStop() + return this.writeStructEnd() + case "List": + case "Set": { + const type = Type[value.elementType] + if (value._tag === "List") this.writeListBegin(type, value.elements.length) + else this.writeSetBegin(type, value.elements.length) + for (const element of value.elements) { + if (element._tag !== value.elementType) { + fail("Usage", `a ${value._tag} of ${value.elementType} contains a ${element._tag}`) + } + this.writeValue(element) + } + return + } + case "Map": { + this.writeMapBegin(Type[value.keyType], Type[value.valueType], value.entries.length) + for (const [k, v] of value.entries) { + if (k._tag !== value.keyType || v._tag !== value.valueType) { + fail("Usage", `a Map of ${value.keyType} to ${value.valueType} contains ${k._tag} to ${v._tag}`) + } + this.writeValue(k) + this.writeValue(v) + } + return + } + default: + fail("Usage", `${String((value as { readonly _tag: unknown })._tag)} is not a Thrift value`) + } + } + + /** A copy of the bytes written so far. Fails when a struct or `Bool` field is unfinished. */ + finish(): Uint8Array { + this.checkNoBool("finish") + if (this.fieldIds.length > 0) fail("Usage", `finish with ${this.fieldIds.length} unfinished structs`) + return this.bytes.slice(0, this.position) + } +} + +/** + * Creates a {@link Writer}. `initialSize` is the first buffer size in bytes, + * a non-negative integer (default 1024); the buffer grows as needed. + * + * @stability experimental + * @category writing + * @since 4.1.5 + */ +export const makeWriter = (options?: { readonly initialSize?: number | undefined }): Writer => new Writer(options) diff --git a/packages/effect/src/encoding/Zip.ts b/packages/effect/src/encoding/Zip.ts index cb9eadd459e..3be685aa28f 100644 --- a/packages/effect/src/encoding/Zip.ts +++ b/packages/effect/src/encoding/Zip.ts @@ -27,6 +27,7 @@ import * as Channel from "../Channel.ts" import * as Data from "../Data.ts" import * as Effect from "../Effect.ts" import * as Exit from "../Exit.ts" +import { crc32 } from "../internal/crc32.ts" import { ByteQueue, InflateError, InflateMemory, Inflater, NeedInput } from "../internal/inflate.ts" import * as Predicate from "../Predicate.ts" import * as Pull from "../Pull.ts" @@ -837,62 +838,6 @@ const decodeName = (bytes: Uint8Array, utf8: boolean): string => { } } -// CRC-32 (IEEE) lookup tables for sixteen bytes at a time -const CRC_TABLE = (() => { - const table = new Int32Array(16 * 256) - for (let n = 0; n < 256; n++) { - let c = n - for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 - table[n] = c - } - for (let n = 0; n < 256; n++) { - let c = table[n] - for (let k = 1; k < 16; k++) { - c = table[c & 0xff] ^ (c >>> 8) - table[k * 256 + n] = c - } - } - return table -})() - -const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1 - -const crc32 = (crc: number, bytes: Uint8Array): number => { - const t = CRC_TABLE - let c = ~crc - let i = 0 - const length = bytes.length - // Reading whole words is faster than combining bytes, but needs aligned, - // little-endian words - if (littleEndian && length >= 64) { - for (; ((bytes.byteOffset + i) & 3) !== 0; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) - const count = ((length - i) >>> 4) << 2 - const words = new Int32Array(bytes.buffer, bytes.byteOffset + i, count) - for (let j = 0; j < count; j += 4) { - const a = c ^ words[j] - const b = words[j + 1] - const d = words[j + 2] - const e = words[j + 3] - c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ - t[3072 + (a >>> 24)] ^ t[2816 + (b & 0xff)] ^ t[2560 + ((b >>> 8) & 0xff)] ^ - t[2304 + ((b >>> 16) & 0xff)] ^ t[2048 + (b >>> 24)] ^ t[1792 + (d & 0xff)] ^ - t[1536 + ((d >>> 8) & 0xff)] ^ t[1280 + ((d >>> 16) & 0xff)] ^ t[1024 + (d >>> 24)] ^ - t[768 + (e & 0xff)] ^ t[512 + ((e >>> 8) & 0xff)] ^ t[256 + ((e >>> 16) & 0xff)] ^ t[e >>> 24] - } - i += count * 4 - } - for (; i + 16 <= length; i += 16) { - const a = c ^ (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24)) - c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ - t[3072 + (a >>> 24)] ^ t[2816 + bytes[i + 4]] ^ t[2560 + bytes[i + 5]] ^ t[2304 + bytes[i + 6]] ^ - t[2048 + bytes[i + 7]] ^ t[1792 + bytes[i + 8]] ^ t[1536 + bytes[i + 9]] ^ t[1280 + bytes[i + 10]] ^ - t[1024 + bytes[i + 11]] ^ t[768 + bytes[i + 12]] ^ t[512 + bytes[i + 13]] ^ t[256 + bytes[i + 14]] ^ - t[bytes[i + 15]] - } - for (; i < length; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) - return ~c >>> 0 -} - const validate = (header: Header): void => { const { method, mode, mtime, path, size, type } = header if (path === "") throw invalidHeader("path is empty") diff --git a/packages/effect/src/encoding/index.ts b/packages/effect/src/encoding/index.ts index 24b5f934af2..054e0e66906 100644 --- a/packages/effect/src/encoding/index.ts +++ b/packages/effect/src/encoding/index.ts @@ -71,6 +71,12 @@ export * as Jsonc from "./Jsonc.ts" */ export * as Ndjson from "./Ndjson.ts" +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Parquet from "./Parquet.ts" + /** * @stability unstable * @since 4.0.0 @@ -89,6 +95,12 @@ export * as Sse from "./Sse.ts" */ export * as Tar from "./Tar.ts" +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Thrift from "./Thrift.ts" + /** * @stability unstable * @since 4.0.0 diff --git a/packages/effect/src/internal/crc32.ts b/packages/effect/src/internal/crc32.ts new file mode 100644 index 00000000000..d6ededffad9 --- /dev/null +++ b/packages/effect/src/internal/crc32.ts @@ -0,0 +1,64 @@ +// CRC-32 shared by `effect/encoding/Zip` and the gzip decoder of +// `effect/encoding/Compression`. + +// CRC-32 (IEEE) lookup tables for sixteen bytes at a time +const CRC_TABLE = (() => { + const table = new Int32Array(16 * 256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + table[n] = c + } + for (let n = 0; n < 256; n++) { + let c = table[n] + for (let k = 1; k < 16; k++) { + c = table[c & 0xff] ^ (c >>> 8) + table[k * 256 + n] = c + } + } + return table +})() + +const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1 + +/** + * Continues the CRC-32 (IEEE, as in zip and gzip) `crc` over `bytes`; start + * from 0. + * + * @internal + */ +export const crc32 = (crc: number, bytes: Uint8Array): number => { + const t = CRC_TABLE + let c = ~crc + let i = 0 + const length = bytes.length + // Reading whole words is faster than combining bytes, but needs aligned, + // little-endian words + if (littleEndian && length >= 64) { + for (; ((bytes.byteOffset + i) & 3) !== 0; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) + const count = ((length - i) >>> 4) << 2 + const words = new Int32Array(bytes.buffer, bytes.byteOffset + i, count) + for (let j = 0; j < count; j += 4) { + const a = c ^ words[j] + const b = words[j + 1] + const d = words[j + 2] + const e = words[j + 3] + c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ + t[3072 + (a >>> 24)] ^ t[2816 + (b & 0xff)] ^ t[2560 + ((b >>> 8) & 0xff)] ^ + t[2304 + ((b >>> 16) & 0xff)] ^ t[2048 + (b >>> 24)] ^ t[1792 + (d & 0xff)] ^ + t[1536 + ((d >>> 8) & 0xff)] ^ t[1280 + ((d >>> 16) & 0xff)] ^ t[1024 + (d >>> 24)] ^ + t[768 + (e & 0xff)] ^ t[512 + ((e >>> 8) & 0xff)] ^ t[256 + ((e >>> 16) & 0xff)] ^ t[e >>> 24] + } + i += count * 4 + } + for (; i + 16 <= length; i += 16) { + const a = c ^ (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24)) + c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ + t[3072 + (a >>> 24)] ^ t[2816 + bytes[i + 4]] ^ t[2560 + bytes[i + 5]] ^ t[2304 + bytes[i + 6]] ^ + t[2048 + bytes[i + 7]] ^ t[1792 + bytes[i + 8]] ^ t[1536 + bytes[i + 9]] ^ t[1280 + bytes[i + 10]] ^ + t[1024 + bytes[i + 11]] ^ t[768 + bytes[i + 12]] ^ t[512 + bytes[i + 13]] ^ t[256 + bytes[i + 14]] ^ + t[bytes[i + 15]] + } + for (; i < length; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) + return ~c >>> 0 +} diff --git a/packages/effect/src/internal/parquet/annotations.ts b/packages/effect/src/internal/parquet/annotations.ts new file mode 100644 index 00000000000..ba9fdcba963 --- /dev/null +++ b/packages/effect/src/internal/parquet/annotations.ts @@ -0,0 +1,52 @@ +/** + * The logical type that LogicalTypes.md gives for each converted type + * (https://github.com/apache/parquet-format/blob/apache-parquet-format-2.14.0/LogicalTypes.md). + * + * @internal + */ +import type { ConvertedType, LogicalType } from "../../encoding/Parquet.ts" + +/** + * A logical type, or a converted type as the logical type that replaced it. + * `INTERVAL` and `MAP_KEY_VALUE` have no logical type, so they keep their names. + * + * @internal + */ +export type Annotation = LogicalType | { readonly type: "INTERVAL" | "MAP_KEY_VALUE" } + +const utc = (type: "TIME" | "TIMESTAMP", unit: "MILLIS" | "MICROS"): Annotation => ({ + type, + isAdjustedToUTC: true, + unit +}) + +const integer = (bitWidth: number, isSigned: boolean): Annotation => ({ type: "INTEGER", bitWidth, isSigned }) + +/** + * `DECIMAL` is not here: its scale and precision come from the schema element. + * + * @internal + */ +export const logicalOf: { readonly [K in Exclude]: Annotation } = { + UTF8: { type: "STRING" }, + MAP: { type: "MAP" }, + MAP_KEY_VALUE: { type: "MAP_KEY_VALUE" }, + LIST: { type: "LIST" }, + ENUM: { type: "ENUM" }, + DATE: { type: "DATE" }, + TIME_MILLIS: utc("TIME", "MILLIS"), + TIME_MICROS: utc("TIME", "MICROS"), + TIMESTAMP_MILLIS: utc("TIMESTAMP", "MILLIS"), + TIMESTAMP_MICROS: utc("TIMESTAMP", "MICROS"), + UINT_8: integer(8, false), + UINT_16: integer(16, false), + UINT_32: integer(32, false), + UINT_64: integer(64, false), + INT_8: integer(8, true), + INT_16: integer(16, true), + INT_32: integer(32, true), + INT_64: integer(64, true), + JSON: { type: "JSON" }, + BSON: { type: "BSON" }, + INTERVAL: { type: "INTERVAL" } +} diff --git a/packages/effect/src/internal/parquet/encodings.ts b/packages/effect/src/internal/parquet/encodings.ts new file mode 100644 index 00000000000..5e3bc0b8d07 --- /dev/null +++ b/packages/effect/src/internal/parquet/encodings.ts @@ -0,0 +1,629 @@ +/** + * Value and level decoders for the Parquet encodings + * (https://github.com/apache/parquet-format/blob/apache-parquet-format-2.14.0/Encodings.md). + * + * Every decoder reads from `bytes[pos, end)` and throws a {@link Failure} instead of + * reading past `end`. Values come out dense: one entry per non-null value. + * + * @internal + */ +import * as Bytes from "../../Bytes.ts" + +/** @internal */ +export class Failure { + readonly kind: "Malformed" | "Unsupported" | "Limit" + readonly message: string + constructor(kind: "Malformed" | "Unsupported" | "Limit", message: string) { + this.kind = kind + this.message = message + } +} + +/** @internal */ +export const fail = (kind: Failure["kind"], message: string): never => { + throw new Failure(kind, message) +} + +/** + * How a physical type's values are held while decoding: `bool` one byte (0 or 1) per + * value, `int96` 12 bytes and `fixed` `width` bytes per value (FIXED_LEN_BYTE_ARRAY), + * `bytes` offsets into concatenated data (BYTE_ARRAY). + * + * @internal + */ +export type Kind = + | { readonly tag: "bool" } + | { readonly tag: "i32" | "f32"; readonly width: 4 } + | { readonly tag: "i64" | "f64"; readonly width: 8 } + | { readonly tag: "int96"; readonly width: 12 } + | { readonly tag: "fixed"; readonly width: number } + | { readonly tag: "bytes" } + +type Fixed = Extract + +/** @internal */ +export interface Binary { + readonly offsets: Int32Array + readonly data: Uint8Array +} + +/** @internal */ +export type Dense = Uint8Array | Int32Array | BigInt64Array | Float32Array | Float64Array | Binary + +/** Bytes of binary values (BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY and INT96) a column chunk may still produce. @internal */ +export interface Budget { + left: number +} + +/** Takes `n` bytes from `budget`. @internal */ +export const spend = (budget: Budget, n: number): void => { + if (n > budget.left) fail("Limit", "the column chunk's binary values exceed maxBinaryLength") + budget.left -= n +} + +const POW2: Array = [] +for (let i = 0; i <= 64; i++) POW2.push(2 ** i) + +const littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1 + +/** The 4-byte little-endian unsigned integer at `at`. @internal */ +export const u32 = (bytes: Uint8Array, at: number): number => + (bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16) | (bytes[at + 3] << 24)) >>> 0 + +/** Views `n` little-endian values of `width` bytes in `owned`, a fresh copy, as a typed array. */ +const fixedLE = ( + owned: Uint8Array, + n: number, + width: number, + make: (buffer: ArrayBuffer, length: number) => A, + get: (view: DataView, at: number) => number | bigint +): A => { + const out = make(owned.buffer, n) + if (!littleEndian) { + const view = new DataView(owned.buffer) + const target = out as unknown as Array + for (let i = 0; i < n; i++) target[i] = get(view, i * width) + } + return out +} + +/** The `n` little-endian values in `owned`, a fresh copy that the result may take over. */ +const fromLE = (kind: Fixed, owned: Uint8Array, n: number): Dense => { + switch (kind.tag) { + case "i32": + return fixedLE(owned, n, 4, (b, l) => new Int32Array(b, 0, l), (v, at) => v.getInt32(at, true)) + case "f32": + return fixedLE(owned, n, 4, (b, l) => new Float32Array(b, 0, l), (v, at) => v.getFloat32(at, true)) + case "i64": + return fixedLE(owned, n, 8, (b, l) => new BigInt64Array(b, 0, l), (v, at) => v.getBigInt64(at, true)) + case "f64": + return fixedLE(owned, n, 8, (b, l) => new Float64Array(b, 0, l), (v, at) => v.getFloat64(at, true)) + default: + return owned + } +} + +/** + * Reads values of up to 32 bits packed from the least significant bit of each byte, as + * the RLE/bit-packed hybrid and DELTA_BINARY_PACKED store them. The caller checks that + * the bytes are there. + */ +class BitReader { + private buffer = 0 + private bits = 0 + private readonly bytes: Uint8Array + private pos: number + constructor(bytes: Uint8Array, pos: number) { + this.bytes = bytes + this.pos = pos + } + read(width: number): number { + while (this.bits < width) { + this.buffer += this.bytes[this.pos++] * POW2[this.bits] + this.bits += 8 + } + const value = this.buffer % POW2[width] + this.buffer = (this.buffer - value) / POW2[width] + this.bits -= width + return value + } +} + +/** An unsigned LEB128 varint of at most 32 bits, and where it ends. */ +const uleb32 = (bytes: Uint8Array, pos: number, end: number, what: string): readonly [number, number] => { + let result = 0 + for (let shift = 0; shift < 35; shift += 7) { + if (pos >= end) return fail("Malformed", `${what} is truncated`) + const b = bytes[pos++] + result += (b & 0x7f) * POW2[shift] + if (b < 0x80) { + if (result > 0xffffffff) break + return [result, pos] + } + } + return fail("Malformed", `${what} is longer than 32 bits`) +} + +const BIG_7 = BigInt(7) +const BIG_1 = BigInt(1) +const BIG_32 = BigInt(32) +const BIG_0x7f = BigInt(0x7f) + +/** A zigzag ULEB128 varint of at most 64 bits, and where it ends. */ +const zigzag64 = (bytes: Uint8Array, pos: number, end: number, what: string): readonly [bigint, number] => { + let result = BigInt(0) + for (let i = 0; i < 10; i++) { + if (pos >= end) return fail("Malformed", `${what} is truncated`) + const b = bytes[pos++] + if (i === 9 && b > 1) break + result |= (BigInt(b) & BIG_0x7f) << (BIG_7 * BigInt(i)) + if (b < 0x80) return [(result >> BIG_1) ^ -(result & BIG_1), pos] + } + return fail("Malformed", `${what} is longer than 64 bits`) +} + +/** The bit width of values from 0 to `max`. @internal */ +export const bitWidth = (max: number): number => max === 0 ? 0 : 32 - Math.clz32(max) + +const outOfRange = (what: string, value: number, max: number): never => + fail("Malformed", `${what} include ${value}, out of range for a maximum of ${max}`) + +/** + * Decodes `out.length` values from 0 to `max` of the RLE/bit-packed hybrid (no length + * prefix) into `out`. + * + * @internal + */ +export const hybrid = ( + bytes: Uint8Array, + pos: number, + end: number, + width: number, + max: number, + out: Uint8Array | Int32Array, + what: string +): void => { + const n = out.length + const valueBytes = (width + 7) >> 3 + let i = 0 + while (i < n) { + if (pos >= end) fail("Malformed", `${what} end after ${i} of ${n} values`) + const [header, start] = uleb32(bytes, pos, end, `${what} run header`) + const runLength = Math.floor(header / 2) + if (runLength === 0) fail("Malformed", `${what} has a run of length 0`) + if (header % 2 === 1) { + // runLength groups of 8 values, each group `width` bytes + const take = Math.min(runLength * 8, n - i) + if (Math.ceil(take * width / 8) > end - start) fail("Malformed", `${what} has a truncated bit-packed run`) + const reader = new BitReader(bytes, start) + for (let k = 0; k < take; k++) { + const value = reader.read(width) + if (value > max) outOfRange(what, value, max) + out[i++] = value + } + pos = Math.min(start + runLength * width, end) + } else { + if (valueBytes > end - start) fail("Malformed", `${what} has a truncated RLE run`) + let value = 0 + for (let k = 0; k < valueBytes; k++) value += bytes[start + k] * POW2[8 * k] + if (value >= POW2[width]) fail("Malformed", `${what} has an RLE value wider than ${width} bits`) + if (value > max) outOfRange(what, value, max) + const stop = Math.min(i + runLength, n) + out.fill(value, i, stop) + i = stop + pos = start + valueBytes + } + } +} + +/** + * Decodes `out.length` values from 0 to `max` of the deprecated BIT_PACKED level + * encoding (most significant bit first, no runs) into `out`, and returns where they end. + * + * @internal + */ +export const bitPacked = ( + bytes: Uint8Array, + pos: number, + end: number, + width: number, + max: number, + out: Uint8Array, + what: string +): number => { + const n = out.length + const size = Math.ceil(n * width / 8) + if (size > end - pos) fail("Malformed", `${what} need ${size} bytes`) + let at = pos + let bit = 0 + for (let i = 0; i < n; i++) { + let value = 0 + for (let k = 0; k < width; k++) { + value = (value << 1) | ((bytes[at] >> (7 - bit)) & 1) + if (++bit === 8) { + bit = 0 + at++ + } + } + if (value > max) outOfRange(what, value, max) + out[i] = value + } + return pos + size +} + +/** Reads the 4-byte little-endian length that prefixes some hybrid runs, and returns where the runs end. @internal */ +export const lengthPrefix = (bytes: Uint8Array, pos: number, end: number, what: string): number => { + if (end - pos < 4) fail("Malformed", `${what} length is truncated`) + const length = u32(bytes, pos) + if (length > end - pos - 4) fail("Malformed", `${what} of ${length} bytes is longer than the page`) + return pos + 4 + length +} + +/** + * Decodes `n` PLAIN values. + * + * @internal + */ +export const plain = (kind: Kind, bytes: Uint8Array, pos: number, end: number, n: number, budget: Budget): Dense => { + switch (kind.tag) { + case "bool": { + if (Math.ceil(n / 8) > end - pos) fail("Malformed", `PLAIN BOOLEAN values need ${Math.ceil(n / 8)} bytes`) + const out = new Uint8Array(n) + for (let i = 0; i < n; i++) out[i] = (bytes[pos + (i >> 3)] >> (i & 7)) & 1 + return out + } + case "bytes": { + let total = 0 + let p = pos + for (let i = 0; i < n; i++) { + if (end - p < 4) fail("Malformed", `PLAIN BYTE_ARRAY value ${i} of ${n} is truncated`) + const length = u32(bytes, p) + p += 4 + if (length > end - p) fail("Malformed", `PLAIN BYTE_ARRAY value ${i} of ${length} bytes is truncated`) + p += length + total += length + } + spend(budget, total) + const offsets = new Int32Array(n + 1) + const data = new Uint8Array(total) + p = pos + for (let i = 0, at = 0; i < n; i++) { + const length = u32(bytes, p) + data.set(bytes.subarray(p + 4, p + 4 + length), at) + p += 4 + length + offsets[i + 1] = at += length + } + return { offsets, data } + } + default: { + const size = n * kind.width + if (size > end - pos) fail("Malformed", `PLAIN values need ${size} bytes`) + return fromLE(kind, bytes.slice(pos, pos + size), n) + } + } +} + +/** + * Decodes `n` BYTE_STREAM_SPLIT values, which fill the rest of the page. + * + * @internal + */ +export const byteStreamSplit = (kind: Fixed, bytes: Uint8Array, pos: number, end: number, n: number): Dense => { + const width = kind.width + if (end - pos !== n * width) { + fail("Malformed", `BYTE_STREAM_SPLIT data of ${end - pos} bytes is not ${n} values of ${width} bytes`) + } + const joined = new Uint8Array(n * width) + for (let k = 0; k < width; k++) { + const stream = pos + k * n + for (let i = 0; i < n; i++) joined[i * width + k] = bytes[stream + i] + } + return fromLE(kind, joined, n) +} + +/** + * Decodes a DELTA_BINARY_PACKED stream that must hold exactly `n` values, and returns + * them with where the stream ends. + * + * @internal + */ +export const deltaBinaryPacked: { + (bytes: Uint8Array, pos: number, end: number, n: number, wide: false): readonly [Int32Array, number] + (bytes: Uint8Array, pos: number, end: number, n: number, wide: true): readonly [BigInt64Array, number] +} = ((bytes: Uint8Array, pos: number, end: number, n: number, wide: boolean) => { + const what = "DELTA_BINARY_PACKED header" + const [blockSize, p1] = uleb32(bytes, pos, end, what) + const [miniblocks, p2] = uleb32(bytes, p1, end, what) + const [total, p3] = uleb32(bytes, p2, end, what) + const [first, p4] = zigzag64(bytes, p3, end, what) + if (blockSize === 0 || blockSize % 128 !== 0) { + fail("Malformed", `DELTA_BINARY_PACKED block size ${blockSize} is not a multiple of 128`) + } + if (miniblocks === 0 || blockSize % miniblocks !== 0 || (blockSize / miniblocks) % 32 !== 0) { + fail( + "Malformed", + `DELTA_BINARY_PACKED blocks of ${blockSize} values can't hold ${miniblocks} miniblocks of a multiple of 32 values` + ) + } + if (total !== n) fail("Malformed", `DELTA_BINARY_PACKED holds ${total} values, expected ${n}`) + const perMiniblock = blockSize / miniblocks + const maxWidth = wide ? 64 : 32 + const out = wide ? new BigInt64Array(n) : new Int32Array(n) + pos = p4 + if (n === 0) return [out, pos] + let last32 = Number(BigInt.asIntN(32, first)) + let last64 = BigInt.asIntN(64, first) + if (wide) (out as BigInt64Array)[0] = last64 + else (out as Int32Array)[0] = last32 + let i = 1 + while (i < n) { + const [minDelta, widths] = zigzag64(bytes, pos, end, "DELTA_BINARY_PACKED block") + if (miniblocks > end - widths) fail("Malformed", "DELTA_BINARY_PACKED miniblock bit widths are truncated") + pos = widths + miniblocks + const minDelta32 = Number(BigInt.asIntN(32, minDelta)) + for (let m = 0; m < miniblocks && i < n; m++) { + const width = bytes[widths + m] + if (width > maxWidth) { + fail("Malformed", `DELTA_BINARY_PACKED miniblock bit width ${width} is wider than ${maxWidth}`) + } + const size = perMiniblock * width / 8 + if (size > end - pos) fail("Malformed", "DELTA_BINARY_PACKED miniblock is truncated") + const take = Math.min(perMiniblock, n - i) + if (width === 0) { + // every delta is minDelta + if (wide) { + const target = out as BigInt64Array + for (let k = 0; k < take; k++) target[i++] = last64 = BigInt.asIntN(64, last64 + minDelta) + } else { + const target = out as Int32Array + for (let k = 0; k < take; k++) target[i++] = last32 = (last32 + minDelta32) | 0 + } + continue + } + const reader = new BitReader(bytes, pos) + if (wide) { + const target = out as BigInt64Array + for (let k = 0; k < take; k++) { + const delta = width <= 32 + ? BigInt(reader.read(width)) + : BigInt(reader.read(32)) | (BigInt(reader.read(width - 32)) << BIG_32) + target[i++] = last64 = BigInt.asIntN(64, last64 + minDelta + delta) + } + } else { + const target = out as Int32Array + for (let k = 0; k < take; k++) target[i++] = last32 = (last32 + minDelta32 + reader.read(width)) | 0 + } + pos += size + } + } + return [out, pos] +}) as never + +const lengths = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + what: string +): readonly [Int32Array, number] => { + const [values, next] = deltaBinaryPacked(bytes, pos, end, n, false) + for (let i = 0; i < n; i++) if (values[i] < 0) fail("Malformed", `${what} ${values[i]} is negative`) + return [values, next] +} + +/** + * Reads the lengths of `n` DELTA_LENGTH_BYTE_ARRAY values, checks that their bytes are + * there, and returns their offsets with where the bytes start. + */ +const lengthOffsets = (bytes: Uint8Array, pos: number, end: number, n: number): readonly [Int32Array, number] => { + const [sizes, start] = lengths(bytes, pos, end, n, "DELTA_LENGTH_BYTE_ARRAY length") + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + total += sizes[i] + if (total > end - start) fail("Malformed", "DELTA_LENGTH_BYTE_ARRAY data is truncated") + offsets[i + 1] = total + } + return [offsets, start] +} + +/** + * Decodes `n` DELTA_LENGTH_BYTE_ARRAY values. + * + * @internal + */ +export const deltaLengthByteArray = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + budget: Budget +): Binary => { + const [offsets, start] = lengthOffsets(bytes, pos, end, n) + spend(budget, offsets[n]) + return { offsets, data: bytes.slice(start, start + offsets[n]) } +} + +/** + * Decodes `n` DELTA_BYTE_ARRAY values. + * + * With a `width`, every value must be `width` bytes long (FIXED_LEN_BYTE_ARRAY), and the + * values are not taken from `budget`: the caller has already counted fixed-width values. + * + * @internal + */ +export const deltaByteArray = ( + bytes: Uint8Array, + pos: number, + end: number, + n: number, + budget: Budget, + width?: number +): Binary => { + const [prefixes, next] = lengths(bytes, pos, end, n, "DELTA_BYTE_ARRAY prefix length") + const [suffixes, start] = lengthOffsets(bytes, next, end, n) + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + const prefix = prefixes[i] + if (i === 0 ? prefix !== 0 : prefix > offsets[i] - offsets[i - 1]) { + fail("Malformed", `DELTA_BYTE_ARRAY value ${i} has a prefix of ${prefix} bytes, longer than the value before it`) + } + const length = prefix + suffixes[i + 1] - suffixes[i] + if (width !== undefined && length !== width) { + fail("Malformed", `DELTA_BYTE_ARRAY value ${i} has ${length} bytes, not ${width}`) + } + offsets[i + 1] = total += length + } + if (width === undefined) spend(budget, total) + const out = new Uint8Array(total) + for (let i = 0; i < n; i++) { + const at = offsets[i] + const prefix = prefixes[i] + if (prefix > 0) out.copyWithin(at, offsets[i - 1], offsets[i - 1] + prefix) + out.set(bytes.subarray(start + suffixes[i], start + suffixes[i + 1]), at + prefix) + } + return { offsets, data: out } +} + +/** + * Whether `data[pos, end)` is well-formed UTF-8 (the Unicode Standard, table + * 3-7): no overlong forms, no surrogates, nothing after U+10FFFF. + */ +const wellFormed = (data: Uint8Array, pos: number, end: number): boolean => { + while (pos < end) { + const lead = data[pos++] + if (lead < 0x80) continue + if (lead < 0xc2 || lead > 0xf4) return false + // The second byte's range is what excludes the forms above. + let more = 1 + let low = 0x80 + let high = 0xbf + if (lead >= 0xf0) { + more = 3 + if (lead === 0xf0) low = 0x90 + else if (lead === 0xf4) high = 0x8f + } else if (lead >= 0xe0) { + more = 2 + if (lead === 0xe0) low = 0xa0 + else if (lead === 0xed) high = 0x9f + } + if (more > end - pos) return false + const second = data[pos++] + if (second < low || second > high) return false + while (--more > 0) if ((data[pos++] & 0xc0) !== 0x80) return false + } + return true +} + +/** + * Fails unless every one of `values` is UTF-8 on its own. No strings are made. + * + * @internal + */ +export const utf8 = ({ data, offsets }: Binary, what: string): void => { + for (let i = 0, n = offsets.length - 1; i < n; i++) { + if (!wellFormed(data, offsets[i], offsets[i + 1])) fail("Malformed", `${what} ${i} is not UTF-8`) + } +} + +/** The number of values in `dense`. @internal */ +export const lengthOf = (kind: Kind, dense: Dense): number => + kind.tag === "bytes" + ? (dense as Binary).offsets.length - 1 + : kind.tag === "fixed" || kind.tag === "int96" + ? (dense as Uint8Array).length / kind.width + : (dense as Uint8Array).length + +/** `n` values of `kind`: zeros, or empty byte arrays. @internal */ +export const alloc = (kind: Kind, n: number): Dense => { + switch (kind.tag) { + case "bool": + return new Uint8Array(n) + case "i32": + return new Int32Array(n) + case "i64": + return new BigInt64Array(n) + case "f32": + return new Float32Array(n) + case "f64": + return new Float64Array(n) + case "int96": + case "fixed": + return new Uint8Array(n * kind.width) + case "bytes": + return { offsets: new Int32Array(n + 1), data: new Uint8Array(0) } + } +} + +/** + * Looks up dictionary `indices` in `dictionary`. The caller checks that they are in + * range. + * + * @internal + */ +export const gather = (kind: Kind, dictionary: Dense, indices: Int32Array, budget: Budget): Dense => { + const n = indices.length + switch (kind.tag) { + case "bytes": { + const dict = dictionary as Binary + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) offsets[i + 1] = total += dict.offsets[indices[i] + 1] - dict.offsets[indices[i]] + spend(budget, total) + const data = new Uint8Array(total) + for (let i = 0; i < n; i++) { + data.set(dict.data.subarray(dict.offsets[indices[i]], dict.offsets[indices[i] + 1]), offsets[i]) + } + return { offsets, data } + } + case "int96": + case "fixed": { + const dict = dictionary as Uint8Array + const width = kind.width + const out = new Uint8Array(n * width) + for (let i = 0, at = 0; i < n; i++) { + for (let from = indices[i] * width, stop = from + width; from < stop; from++) out[at++] = dict[from] + } + return out + } + default: { + const dict = dictionary as Int32Array + const out = alloc(kind, n) as Int32Array + for (let i = 0; i < n; i++) out[i] = dict[indices[i]] + return out + } + } +} + +/** + * Concatenates dense values decoded page by page. + * + * @internal + */ +export const concat = (kind: Kind, chunks: ReadonlyArray): Dense => { + if (chunks.length === 1) return chunks[0] + if (kind.tag === "bytes") { + const parts = chunks as ReadonlyArray + let count = 0 + for (const part of parts) count += part.offsets.length - 1 + const offsets = new Int32Array(count + 1) + let i = 0 + let at = 0 + for (const part of parts) { + for (let k = 1; k < part.offsets.length; k++) offsets[++i] = at + part.offsets[k] + at += part.data.length + } + return { offsets, data: Bytes.concat(parts.map((part) => part.data)) } + } + const parts = chunks as ReadonlyArray + let total = 0 + for (const part of parts) total += lengthOf(kind, part) + const out = alloc(kind, total) as Int32Array + let at = 0 + for (const part of parts) { + out.set(part, at) + at += part.length + } + return out +} diff --git a/packages/effect/test/encoding/Parquet.test.ts b/packages/effect/test/encoding/Parquet.test.ts new file mode 100644 index 00000000000..32f49de94e1 --- /dev/null +++ b/packages/effect/test/encoding/Parquet.test.ts @@ -0,0 +1,1896 @@ +import { assert, describe, it } from "@effect/vitest" +import { Bytes, Cause, Effect, Exit } from "effect" +import * as Hex from "effect/encoding/Hex" +import * as Parquet from "effect/encoding/Parquet" +import * as Thrift from "effect/encoding/Thrift" +import { readFileSync } from "node:fs" +import { crc32, gunzipSync, gzipSync } from "node:zlib" + +// Two kinds of evidence: +// - "pyarrow fixtures": files written by pyarrow, read back and compared with +// gold.json. Interoperability evidence; see fixtures/parquet/README.md. +// - "conformance": one test per assertion of apache-parquet-format 2.14.0, +// named after its id. fixtures/parquet/CONFORMANCE.md maps every assertion to +// its test, or says why it has none. +type Gold = null | boolean | number | string | ReadonlyArray +interface GoldFile { + readonly name: string + readonly numRows: number + readonly rowGroups: ReadonlyArray<{ + readonly numRows: number + readonly columns: ReadonlyArray<{ + readonly codec: Parquet.CompressionCodec + readonly encodings: ReadonlyArray + }> + }> + readonly columns: ReadonlyArray<{ + readonly path: ReadonlyArray + readonly physicalType: string + readonly typeLength: number | null + readonly maxDefinitionLevel: number + readonly maxRepetitionLevel: number + readonly convertedType: string | null + readonly logicalType: Parquet.LogicalType | null + readonly values: ReadonlyArray + }> +} + +const gold: { readonly writer: string; readonly files: ReadonlyArray } = JSON.parse( + readFileSync(new URL("../fixtures/parquet/gold.json", import.meta.url), "utf8") +) +const fixture = (name: string) => + Parquet.fromBytes(new Uint8Array(readFileSync(new URL("../fixtures/parquet/" + name, import.meta.url)))) + +const gzip: Parquet.Decompress = (data) => Effect.sync(() => new Uint8Array(gunzipSync(data))) + +/** Turns a column's layers and values back into one nested value per row, like gold.json. */ +const rows = (column: Parquet.Column): Array => { + const { layers, values } = column + const bit = (bits: Uint8Array, i: number) => (bits[i >> 3] >> (i & 7) & 1) === 1 + const leaf = (j: number): Gold => { + switch (values.type) { + case "BOOLEAN": + return bit(values.values, j) + case "INT64": + return String(values.values[j]) + case "INT32": + case "FLOAT": + case "DOUBLE": + return values.values[j] + case "BYTE_ARRAY": + return Hex.encode(values.values.subarray(values.offsets[j], values.offsets[j + 1])) + case "INT96": + return Hex.encode(values.values.subarray(j * 12, j * 12 + 12)) + case "FIXED_LEN_BYTE_ARRAY": + return Hex.encode(values.values.subarray(j * values.width, (j + 1) * values.width)) + } + } + // The value below node i, at slot j of node i's slots (i = -1 for the rows) + const below = (i: number, j: number): Gold => { + if (i === layers.length - 1) return leaf(j) + const next = layers[i + 1] + if (next.offsets === undefined) return at(i + 1, j) + const out: Array = [] + for (let k = next.offsets[j]; k < next.offsets[j + 1]; k++) out.push(below(i + 1, k)) + return out + } + const at = (i: number, j: number): Gold => { + const validity = layers[i].validity + return validity !== undefined && !bit(validity, j) ? null : below(i, j) + } + return Array.from({ length: column.numRows }, (_, row) => below(-1, row)) +} + +// --------------------------------------------------------------------------- +// Synthetic files, written with the Thrift writer +// --------------------------------------------------------------------------- + +const text = (s: string) => new TextEncoder().encode(s) + +type V = Thrift.Value +const I8 = (value: number): V => ({ _tag: "I8", value }) +const I32 = (value: number): V => ({ _tag: "I32", value }) +const I64 = (value: number): V => ({ _tag: "I64", value: BigInt(value) }) +const Bool = (value: boolean): V => ({ _tag: "Bool", value }) +const Str = (value: string): V => ({ _tag: "Binary", value: text(value) }) +const List = (elementType: Thrift.TypeName, elements: ReadonlyArray): V => ({ _tag: "List", elementType, elements }) +const Struct = (fields: { readonly [id: number]: V | undefined }): V => ({ + _tag: "Struct", + fields: Object.entries(fields).flatMap(([id, value]) => value === undefined ? [] : [{ id: Number(id), value }]) +}) +const thrift = (value: V): Uint8Array => { + const writer = Thrift.makeWriter() + writer.writeValue(value) + return writer.finish() +} +const bytes = (...parts: ReadonlyArray>): Uint8Array => + Bytes.concat(parts.map((part) => Uint8Array.from(part))) +const u32 = (n: number) => [n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, n >>> 24] +const uleb = (n: number): Array => n < 0x80 ? [n] : [(n & 0x7f) | 0x80, ...uleb(Math.floor(n / 0x80))] +const int32s = (...values: ReadonlyArray) => new Uint8Array(Int32Array.from(values).buffer) +const plainBinary = (...values: ReadonlyArray) => + bytes(...values.flatMap((v) => [u32(text(v).length), text(v)])) +/** RLE/bit-packed hybrid runs of repeated values: [value, count] pairs. */ +const runs = (width: number, ...pairs: ReadonlyArray) => + bytes(...pairs.map(([value, count]) => [...uleb(count * 2), ...u32(value).slice(0, (width + 7) >> 3)])) +const lengthPrefixed = (data: Uint8Array) => bytes(u32(data.length), data) + +const Type = { BOOLEAN: 0, INT32: 1, INT64: 2, INT96: 3, FLOAT: 4, DOUBLE: 5, BYTE_ARRAY: 6, FIXED_LEN_BYTE_ARRAY: 7 } +const REQUIRED = 0 +const OPTIONAL = 1 +const REPEATED = 2 +const PLAIN = 0 +const PLAIN_DICTIONARY = 2 +const RLE = 3 +const BIT_PACKED = 4 +const DELTA_BINARY_PACKED = 5 +const DELTA_LENGTH_BYTE_ARRAY = 6 +const DELTA_BYTE_ARRAY = 7 +const RLE_DICTIONARY = 8 +const BYTE_STREAM_SPLIT = 9 +const ALP = 10 +const GZIP = 2 + +const element = (name: string, options: { + readonly type?: number | undefined + readonly typeLength?: number | undefined + readonly repetition?: number | undefined + readonly children?: number | undefined + readonly convertedType?: number | undefined + readonly scale?: number | undefined + readonly precision?: number | undefined + /** A LogicalType union: a struct with one field. */ + readonly logicalType?: V | undefined +}): V => + Struct({ + 1: options.type === undefined ? undefined : I32(options.type), + 2: options.typeLength === undefined ? undefined : I32(options.typeLength), + 3: options.repetition === undefined ? undefined : I32(options.repetition), + 4: Str(name), + 5: options.children === undefined ? undefined : I32(options.children), + 6: options.convertedType === undefined ? undefined : I32(options.convertedType), + 7: options.scale === undefined ? undefined : I32(options.scale), + 8: options.precision === undefined ? undefined : I32(options.precision), + 10: options.logicalType + }) +const root = (children: number) => element("schema", { children }) + +/** `crc` is the unsigned CRC-32, which the header holds as a signed i32. */ +const page = ( + type: number, + header: { readonly [id: number]: V }, + body: Uint8Array, + uncompressedSize = body.length, + crc?: number +) => + bytes( + thrift(Struct({ + 1: I32(type), + 2: I32(uncompressedSize), + 3: I32(body.length), + 4: crc === undefined ? undefined : I32(crc | 0), + ...header + })), + body + ) +const dataPage = (numValues: number, body: Uint8Array, options?: { + readonly encoding?: number | undefined + readonly levels?: number | undefined + readonly uncompressedSize?: number | undefined + readonly crc?: number | undefined +}) => + page( + 0, + { + 5: Struct({ + 1: I32(numValues), + 2: I32(options?.encoding ?? PLAIN), + 3: I32(options?.levels ?? RLE), + 4: I32(options?.levels ?? RLE) + }) + }, + body, + options?.uncompressedSize, + options?.crc + ) +const dictionaryPage = ( + numValues: number, + body: Uint8Array, + encoding = PLAIN, + options?: { readonly uncompressedSize?: number | undefined; readonly crc?: number | undefined } +) => page(2, { 7: Struct({ 1: I32(numValues), 2: I32(encoding) }) }, body, options?.uncompressedSize, options?.crc) +const dataPageV2 = (options: { + readonly numValues: number + readonly numNulls: number + readonly numRows: number + readonly encoding?: number | undefined + readonly repetition?: Uint8Array + readonly definition?: Uint8Array + readonly values: Uint8Array + readonly compressed?: boolean | undefined + readonly uncompressedSize?: number | undefined + /** Defaults to the length of `definition`. */ + readonly definitionLength?: number | undefined + /** Defaults to the length of `repetition`. */ + readonly repetitionLength?: number | undefined + /** Computes the CRC from the page's bytes: levels, then values. */ + readonly crc?: ((body: Uint8Array) => number) | undefined +}) => { + const repetition = options.repetition ?? new Uint8Array(0) + const definition = options.definition ?? new Uint8Array(0) + const body = bytes(repetition, definition, options.values) + return page( + 3, + { + 8: Struct({ + 1: I32(options.numValues), + 2: I32(options.numNulls), + 3: I32(options.numRows), + 4: I32(options.encoding ?? PLAIN), + 5: I32(options.definitionLength ?? definition.length), + 6: I32(options.repetitionLength ?? repetition.length), + 7: options.compressed === undefined ? undefined : Bool(options.compressed) + }) + }, + body, + options.uncompressedSize, + options.crc?.(body) + ) +} + +interface Chunk { + readonly type: number + readonly path: ReadonlyArray + readonly pages: ReadonlyArray + readonly numValues: number + readonly codec?: number | undefined + /** Whether the first page is a dictionary page. */ + readonly dictionary?: boolean | undefined + readonly statistics?: V | undefined + readonly chunk?: { readonly [id: number]: V } | undefined +} + +/** Writes a Parquet file with one row group. */ +const file = (options: { + readonly schema: ReadonlyArray + readonly numRows: number + readonly chunks: ReadonlyArray + readonly metadata?: { readonly [id: number]: V } | undefined +}): Uint8Array => { + const body: Array = [] + let offset = 4 + const columns = options.chunks.map((chunk) => { + const data = bytes(...chunk.pages) + const meta = Struct({ + 1: I32(chunk.type), + 2: List("I32", [I32(PLAIN), I32(RLE)]), + 3: List("Binary", chunk.path.map(Str)), + 4: I32(chunk.codec ?? 0), + 5: I64(chunk.numValues), + 6: I64(data.length), + 7: I64(data.length), + 9: I64(chunk.dictionary === true ? offset + chunk.pages[0].length : offset), + 11: chunk.dictionary === true ? I64(offset) : undefined, + 12: chunk.statistics + }) + body.push(data) + const column = Struct({ 2: I64(offset), 3: meta, ...chunk.chunk }) + offset += data.length + return column + }) + const footer = thrift(Struct({ + 1: I32(1), + 2: List("Struct", options.schema), + 3: I64(options.numRows), + 4: List("Struct", [Struct({ 1: List("Struct", columns), 2: I64(offset - 4), 3: I64(options.numRows) })]), + ...options.metadata + })) + return bytes(text("PAR1"), ...body, footer, u32(footer.length), text("PAR1")) +} + +type Annotation = Omit[1], "type" | "repetition" | "children"> + +/** A file with one top-level column. */ +const flat = (options: { + readonly type: number + readonly repetition: number + readonly numRows: number + /** Defaults to `numRows`. */ + readonly numValues?: number | undefined + readonly pages: ReadonlyArray + readonly typeLength?: number | undefined + readonly codec?: number | undefined + readonly dictionary?: boolean | undefined + readonly statistics?: V | undefined + readonly chunk?: { readonly [id: number]: V } | undefined + readonly metadata?: { readonly [id: number]: V } | undefined + readonly annotation?: Annotation | undefined +}) => + file({ + schema: [ + root(1), + element("x", { + type: options.type, + repetition: options.repetition, + typeLength: options.typeLength, + ...options.annotation + }) + ], + numRows: options.numRows, + chunks: [{ ...options, path: ["x"], numValues: options.numValues ?? options.numRows }], + metadata: options.metadata + }) + +/** A file with one required top-level column of `n` values in one data page. */ +const requiredColumn = (type: number, n: number, data: Uint8Array, options?: { + readonly encoding?: number | undefined + readonly typeLength?: number | undefined + readonly codec?: number | undefined + readonly uncompressedSize?: number | undefined + readonly annotation?: Annotation | undefined + readonly crc?: number | undefined +}) => + flat({ + type, + repetition: REQUIRED, + numRows: n, + typeLength: options?.typeLength, + codec: options?.codec, + annotation: options?.annotation, + pages: [dataPage(n, data, options)] + }) + +const int32s3 = requiredColumn(Type.INT32, 3, int32s(1, 2, 3)) + +/** DELTA_BINARY_PACKED for at most 128 small values: one block of four miniblocks. */ +const delta = (values: ReadonlyArray): Uint8Array => { + const zigzag = (n: number) => n < 0 ? -2 * n - 1 : 2 * n + const header = [...uleb(128), 4, ...uleb(values.length), ...uleb(zigzag(values[0] ?? 0))] + if (values.length <= 1) return new Uint8Array(header) + const deltas = values.slice(1).map((v, i) => v - values[i]) + const min = Math.min(...deltas) + const relative = deltas.map((d) => d - min) + const width = 32 - Math.clz32(Math.max(...relative)) + const miniblocks = Math.ceil(relative.length / 32) + const packed = new Uint8Array(miniblocks * 32 * width / 8) + relative.forEach((value, i) => { + for (let b = 0; b < width; b++) if ((value >> b) & 1) packed[(i * width + b) >> 3] |= 1 << ((i * width + b) & 7) + }) + return bytes(header, uleb(zigzag(min)), [0, 1, 2, 3].map((m) => m < miniblocks ? width : 0), packed) +} + +const metadataOf = (bytes: Uint8Array) => Parquet.readMetadata(Parquet.fromBytes(bytes)) +type ReadOptions = Omit[2], "rowGroup" | "column"> +const read = Effect.fnUntraced(function*(bytes: Uint8Array, options?: ReadOptions) { + const source = Parquet.fromBytes(bytes) + const metadata = yield* Parquet.readMetadata(source) + return yield* Parquet.readColumn(source, metadata, { rowGroup: 0, column: 0, ...options }) +}) +const values = (bytes: Uint8Array, options?: ReadOptions) => Effect.map(read(bytes, options), rows) +const failsWith = Effect.fnUntraced(function*( + effect: Effect.Effect, + kind: Parquet.ParquetError["kind"], + message?: RegExp +) { + const error = yield* Effect.flip(effect) + assert.isTrue(Parquet.isParquetError(error)) + assert.strictEqual(error.kind, kind, error.message) + if (message !== undefined) assert.match(error.message, message) +}) + +describe("Parquet", () => { + describe("pyarrow " + gold.writer + " fixtures", () => { + for (const file of gold.files) { + it.effect(file.name, () => + Effect.gen(function*() { + const source = fixture(file.name) + const metadata = yield* Parquet.readMetadata(source) + assert.strictEqual(metadata.numRows, file.numRows) + assert.strictEqual(metadata.createdBy, "parquet-cpp-arrow version " + gold.writer) + assert.deepStrictEqual( + metadata.rowGroups.map((group) => ({ + numRows: group.numRows, + columns: group.columns.map((chunk) => ({ + codec: chunk.metaData!.codec, + encodings: [...chunk.metaData!.encodings].sort() + })) + })), + file.rowGroups.map((group) => ({ + numRows: group.numRows, + columns: group.columns.map((c) => ({ codec: c.codec, encodings: [...c.encodings].sort() })) + })) + ) + assert.strictEqual(metadata.columns.length, file.columns.length) + for (let c = 0; c < file.columns.length; c++) { + const expected = file.columns[c] + const descriptor = metadata.columns[c] + const leaf = descriptor.nodes[descriptor.nodes.length - 1] + assert.deepStrictEqual({ + path: descriptor.path, + physicalType: descriptor.physicalType, + typeLength: descriptor.typeLength ?? null, + maxDefinitionLevel: descriptor.maxDefinitionLevel, + maxRepetitionLevel: descriptor.maxRepetitionLevel, + convertedType: leaf.convertedType ?? null, + logicalType: leaf.logicalType ?? null + }, { + path: expected.path, + physicalType: expected.physicalType, + typeLength: expected.typeLength, + maxDefinitionLevel: expected.maxDefinitionLevel, + maxRepetitionLevel: expected.maxRepetitionLevel, + convertedType: expected.convertedType, + logicalType: expected.logicalType + }) + const values: Array = [] + for (let g = 0; g < metadata.rowGroups.length; g++) { + const column = yield* Parquet.readColumn(source, metadata, { + rowGroup: g, + column: c, + codecs: { GZIP: gzip } + }) + values.push(...rows(column)) + } + assert.deepStrictEqual(values, expected.values, expected.path.join(".")) + } + })) + } + }) + describe("conformance (apache-parquet-format-2.14.0)", () => { + const optionalInt32 = (pages: ReadonlyArray, numRows: number, codec?: number) => + flat({ type: Type.INT32, repetition: OPTIONAL, numRows, pages, codec }) + const repeatedInt32 = (pages: ReadonlyArray, numRows: number, numValues: number) => + flat({ type: Type.INT32, repetition: REPEATED, numRows, numValues, pages }) + // def levels 1, 0, 1 as RLE runs, for a column with one optional node + const someNull = runs(1, [1, 1], [0, 1], [1, 1]) + /** + * A file with one required column `x` that has statistics in TYPE_ORDER, and no rows. The annotation is on `x`, + * or without a `type` on a group around it. + */ + const annotated = (type: number | undefined, annotation: Annotation) => { + const leaf = type ?? Type.BYTE_ARRAY + return file({ + schema: type === undefined + ? [ + root(1), + element("g", { repetition: REQUIRED, children: 1, ...annotation }), + element("x", { type: leaf, repetition: REQUIRED }) + ] + : [root(1), element("x", { type, repetition: REQUIRED, ...annotation })], + numRows: 0, + chunks: [{ type: leaf, path: type === undefined ? ["g", "x"] : ["x"], numValues: 0, pages: [] }], + metadata: { 7: List("Struct", [Struct({ 1: Struct({}) })]) } + }) + } + /** What the reader says about the annotated node and the order of column `x`. */ + const annotationOf = (bytes: Uint8Array) => + Effect.map(metadataOf(bytes), ({ columns: [column] }) => ({ + logicalType: column.nodes[0].logicalType, + convertedType: column.nodes[0].convertedType, + columnOrder: column.columnOrder + })) + const empty = Struct({}) + const TIME = 7 + const TIMESTAMP = 8 + /** A TimeType or TimestampType with a TimeUnit union, or without its unit. */ + const temporal = (member: number, unit: V | undefined) => Struct({ [member]: Struct({ 1: Bool(true), 2: unit }) }) + + describe("README.md#file-format", () => { + it.effect("PQ-FILE-1 MUST start and end with the magic number PAR1", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(int32s3), [1, 2, 3]) + const badTrailer = int32s3.slice() + badTrailer[badTrailer.length - 1] = 0x30 + yield* failsWith(metadataOf(badTrailer), "Malformed", /end with the magic number/) + const badHeader = int32s3.slice() + badHeader[3] = 0x30 + yield* failsWith(metadataOf(badHeader), "Malformed", /start with the magic number/) + })) + + it.effect("PQ-FILE-2 MUST find the metadata from the 4-byte little-endian length before the final magic", () => + Effect.gen(function*() { + const n = int32s3.length + const metadata = yield* metadataOf(int32s3) + assert.strictEqual(metadata.rowGroups[0].columns[0].metaData?.dataPageOffset, 4) + const tooLong = int32s3.slice() + tooLong.set(u32(n - 11), n - 8) + yield* failsWith(metadataOf(tooLong), "Malformed", /footer length/) + const garbage = bytes(text("PAR1"), [0x19, 0xfc], u32(2), text("PAR1")) + yield* failsWith(metadataOf(garbage), "Malformed", /invalid Thrift metadata/) + })) + }) + + describe("parquet.thrift FileMetaData", () => { + it.effect("PQ-META-1 SHOULD accept versions 1 and 2 interchangeably (L1411-1413)", () => + Effect.gen(function*() { + const v2 = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [dataPage(3, int32s(1, 2, 3))], + metadata: { 1: I32(2) } + }) + assert.strictEqual((yield* metadataOf(v2)).version, 2) + assert.deepStrictEqual(yield* values(v2), yield* values(int32s3)) + })) + + it.effect("PQ-META-2 MUST give a column order per leaf column, in schema order (L1440-1455)", () => + Effect.gen(function*() { + const make = (orders: ReadonlyArray) => + file({ + schema: [ + root(2), + element("a", { type: Type.INT32, repetition: REQUIRED }), + element("b", { type: Type.DOUBLE, repetition: REQUIRED }) + ], + numRows: 0, + chunks: [ + { type: Type.INT32, path: ["a"], numValues: 0, pages: [] }, + { type: Type.DOUBLE, path: ["b"], numValues: 0, pages: [] } + ], + metadata: { 7: List("Struct", orders) } + }) + const metadata = yield* metadataOf(make([Struct({ 1: Struct({}) }), Struct({ 2: Struct({}) })])) + assert.deepStrictEqual(metadata.columns.map((c) => c.columnOrder), ["TYPE_ORDER", "IEEE_754_TOTAL_ORDER"]) + yield* failsWith(metadataOf(make([Struct({ 1: Struct({}) })])), "Malformed", /column orders/) + })) + + it.effect("PQ-META-3 MUST reject a negative num_rows in the file or a row group (L1058-1059, L1426-1427)", () => + Effect.gen(function*() { + const make = (file_: number, rowGroup: number) => + metadataOf(file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: rowGroup, + chunks: [{ type: Type.INT32, path: ["x"], numValues: 0, pages: [] }], + metadata: { 3: I64(file_) } + })) + const metadata = yield* make(0, 0) + assert.deepStrictEqual([metadata.numRows, metadata.rowGroups[0].numRows], [0, 0]) + yield* failsWith(make(0, -1), "Malformed", /RowGroup\.num_rows is -1/) + yield* failsWith(make(-1, 0), "Malformed", /FileMetaData\.num_rows is -1/) + })) + }) + + describe("parquet.thrift SchemaElement", () => { + it.effect("PQ-SCHEMA-1 MUST give every node but the root a repetition (L535-537)", () => + failsWith( + metadataOf(file({ + schema: [root(1), element("x", { type: Type.INT32 })], + numRows: 0, + chunks: [{ type: Type.INT32, path: ["x"], numValues: 0, pages: [] }] + })), + "Malformed", + /repetition/ + )) + + it.effect("PQ-SCHEMA-2 MUST rebuild the schema tree from its depth-first elements and num_children (L518-523, L542-547)", () => + Effect.gen(function*() { + const schema = [ + root(2), + element("a", { repetition: OPTIONAL, children: 2 }), + element("b", { type: Type.INT32, repetition: REPEATED }), + element("c", { type: Type.BYTE_ARRAY, repetition: OPTIONAL }), + element("d", { type: Type.FIXED_LEN_BYTE_ARRAY, typeLength: 16, repetition: REQUIRED }) + ] + const chunks = [ + { type: Type.INT32, path: ["a", "b"], numValues: 0, pages: [] }, + { type: Type.BYTE_ARRAY, path: ["a", "c"], numValues: 0, pages: [] }, + { type: Type.FIXED_LEN_BYTE_ARRAY, path: ["d"], numValues: 0, pages: [] } + ] + const metadata = yield* metadataOf(file({ schema, numRows: 0, chunks })) + assert.deepStrictEqual(metadata.schema.children?.map((n) => n.name), ["a", "d"]) + assert.deepStrictEqual(metadata.schema.children?.[0].children?.map((n) => n.name), ["b", "c"]) + assert.deepStrictEqual(metadata.columns.map((c) => [c.path, c.physicalType, c.typeLength]), [ + [["a", "b"], "INT32", undefined], + [["a", "c"], "BYTE_ARRAY", undefined], + [["d"], "FIXED_LEN_BYTE_ARRAY", 16] + ]) + yield* failsWith( + metadataOf(file({ schema: schema.slice(0, 4), numRows: 0, chunks })), + "Malformed", + /ends inside a group/ + ) + })) + }) + + describe("LogicalTypes.md", () => { + it.effect("PQ-LT-1 MUST handle an unknown TIMESTAMP unit as an unsupported feature (#timestamp)", () => + Effect.gen(function*() { + const TIMESTAMP_MICROS = 10 + const timestamp = (unit: number) => + annotated(Type.INT64, { + logicalType: temporal(TIMESTAMP, Struct({ [unit]: empty })), + convertedType: TIMESTAMP_MICROS + }) + assert.deepStrictEqual((yield* annotationOf(timestamp(3))).logicalType, { + type: "TIMESTAMP", + isAdjustedToUTC: true, + unit: "NANOS" + }) + // a fourth unit, which must not read as the legacy TIMESTAMP_MICROS + yield* failsWith(metadataOf(timestamp(4)), "Unsupported", /TIMESTAMP unit 4/) + })) + + it.effect("PQ-LT-2 MUST reject a TIME or TIMESTAMP without its unit (parquet.thrift L370-373, L380-383)", () => + Effect.gen(function*() { + const make = (member: number, unit: V | undefined) => + metadataOf(annotated(Type.INT64, { logicalType: temporal(member, unit) })) + yield* failsWith(make(TIMESTAMP, undefined), "Malformed", /TimestampType\.unit is missing/) + yield* failsWith(make(TIME, undefined), "Malformed", /TimeType\.unit is missing/) + // a TimeUnit union with no member + yield* failsWith(make(TIMESTAMP, empty), "Malformed", /TimestampType\.unit is missing/) + })) + + it.effect("PQ-LT-3 SHOULD ignore an unrecognized logical type and the column's order (#unsupported-logical-types)", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT64, {})), { + logicalType: undefined, + convertedType: undefined, + columnOrder: "TYPE_ORDER" + }) + // member 99 of the LogicalType union, which 2.14.0 doesn't have + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT64, { logicalType: Struct({ 99: empty }) })), { + logicalType: undefined, + convertedType: undefined, + columnOrder: undefined + }) + // the converted type doesn't stand in for a logical type that is there but ignored (#convertedtype) + for (const type of [Type.BYTE_ARRAY, undefined]) { + const annotation = { logicalType: Struct({ 99: empty }), convertedType: type === undefined ? 3 : 0 } + assert.deepStrictEqual(yield* annotationOf(annotated(type, annotation)), { + logicalType: undefined, + convertedType: undefined, + columnOrder: type === undefined ? "TYPE_ORDER" : undefined + }) + } + })) + + const GROUP = undefined + const FIXED = Type.FIXED_LEN_BYTE_ARRAY + const logical = (member: number, fields: { readonly [id: number]: V } = {}): Annotation => ({ + logicalType: Struct({ [member]: Struct(fields) }) + }) + const integer = (bitWidth: number, isSigned: boolean) => logical(10, { 1: I8(bitWidth), 2: Bool(isSigned) }) + const time = (unit: number) => ({ logicalType: temporal(TIME, Struct({ [unit]: empty })) }) + const utc = { isAdjustedToUTC: true } + + it.effect("PQ-LT-4 SHOULD ignore an annotation on a physical type it is not defined for, and the column's order (#unsupported-logical-types)", () => + Effect.gen(function*() { + type Node = readonly [type: number | undefined, typeLength?: number] + // the annotation as written and as read, a node the spec defines it for, and one it doesn't + const cases: ReadonlyArray< + readonly [Annotation, Parquet.LogicalType | Parquet.ConvertedType, defined: Node, other: Node] + > = [ + [logical(1), { type: "STRING" }, [Type.BYTE_ARRAY], [Type.INT32]], + [logical(2), { type: "MAP" }, [GROUP], [Type.BYTE_ARRAY]], + [logical(3), { type: "LIST" }, [GROUP], [Type.INT32]], + [logical(4), { type: "ENUM" }, [Type.BYTE_ARRAY], [FIXED, 4]], + [logical(5, { 1: I32(2), 2: I32(9) }), { type: "DECIMAL", scale: 2, precision: 9 }, [Type.INT32], [ + Type.DOUBLE + ]], + [logical(6), { type: "DATE" }, [Type.INT32], [Type.INT64]], + [time(1), { type: "TIME", ...utc, unit: "MILLIS" }, [Type.INT32], [Type.INT64]], + [time(2), { type: "TIME", ...utc, unit: "MICROS" }, [Type.INT64], [Type.INT32]], + [time(3), { type: "TIME", ...utc, unit: "NANOS" }, [Type.INT64], [Type.INT32]], + [ + { logicalType: temporal(TIMESTAMP, Struct({ 1: empty })) }, + { type: "TIMESTAMP", ...utc, unit: "MILLIS" }, + [ + Type.INT64 + ], + [Type.INT96] + ], + [integer(8, true), { type: "INTEGER", bitWidth: 8, isSigned: true }, [Type.INT32], [Type.INT64]], + [integer(16, false), { type: "INTEGER", bitWidth: 16, isSigned: false }, [Type.INT32], [Type.INT64]], + [integer(32, true), { type: "INTEGER", bitWidth: 32, isSigned: true }, [Type.INT32], [Type.INT64]], + [integer(64, false), { type: "INTEGER", bitWidth: 64, isSigned: false }, [Type.INT64], [Type.INT32]], + [logical(12), { type: "JSON" }, [Type.BYTE_ARRAY], [Type.INT64]], + [logical(13), { type: "BSON" }, [Type.BYTE_ARRAY], [Type.INT32]], + [logical(14), { type: "UUID" }, [FIXED, 16], [FIXED, 4]], + [logical(15), { type: "FLOAT16" }, [FIXED, 2], [Type.FLOAT]], + [logical(16), { type: "VARIANT", specificationVersion: undefined }, [GROUP], [Type.BYTE_ARRAY]], + [logical(17), { type: "GEOMETRY", crs: undefined }, [Type.BYTE_ARRAY], [GROUP]], + [logical(18), { type: "GEOGRAPHY", crs: undefined, algorithm: undefined }, [Type.BYTE_ARRAY], [FIXED, 21]], + [logical(19), { type: "FILE" }, [GROUP], [Type.BYTE_ARRAY]], + // converted types, by their code in parquet.thrift + [{ convertedType: 0 }, "UTF8", [Type.BYTE_ARRAY], [Type.INT32]], + [{ convertedType: 1 }, "MAP", [GROUP], [Type.BYTE_ARRAY]], + [{ convertedType: 2 }, "MAP_KEY_VALUE", [GROUP], [Type.BYTE_ARRAY]], + [{ convertedType: 3 }, "LIST", [GROUP], [Type.INT32]], + [{ convertedType: 4 }, "ENUM", [Type.BYTE_ARRAY], [Type.INT32]], + [{ convertedType: 5, scale: 2, precision: 9 }, "DECIMAL", [Type.INT32], [Type.FLOAT]], + [{ convertedType: 6 }, "DATE", [Type.INT32], [Type.INT64]], + [{ convertedType: 7 }, "TIME_MILLIS", [Type.INT32], [Type.INT64]], + [{ convertedType: 8 }, "TIME_MICROS", [Type.INT64], [Type.INT32]], + [{ convertedType: 9 }, "TIMESTAMP_MILLIS", [Type.INT64], [Type.INT32]], + [{ convertedType: 10 }, "TIMESTAMP_MICROS", [Type.INT64], [Type.INT96]], + [{ convertedType: 11 }, "UINT_8", [Type.INT32], [Type.INT64]], + [{ convertedType: 12 }, "UINT_16", [Type.INT32], [Type.INT64]], + [{ convertedType: 13 }, "UINT_32", [Type.INT32], [Type.INT64]], + [{ convertedType: 14 }, "UINT_64", [Type.INT64], [Type.INT32]], + [{ convertedType: 15 }, "INT_8", [Type.INT32], [Type.INT64]], + [{ convertedType: 16 }, "INT_16", [Type.INT32], [Type.INT64]], + [{ convertedType: 17 }, "INT_32", [Type.INT32], [Type.INT64]], + [{ convertedType: 18 }, "INT_64", [Type.INT64], [Type.INT32]], + [{ convertedType: 19 }, "JSON", [Type.BYTE_ARRAY], [Type.INT32]], + [{ convertedType: 20 }, "BSON", [Type.BYTE_ARRAY], [GROUP]], + [{ convertedType: 21 }, "INTERVAL", [FIXED, 12], [FIXED, 4]] + ] + for (const [annotation, expected, [type, typeLength], [otherType, otherLength]] of cases) { + const label = JSON.stringify(expected) + const kept = yield* annotationOf(annotated(type, { ...annotation, typeLength })) + assert.deepStrictEqual( + kept, + typeof expected === "string" + ? { logicalType: undefined, convertedType: expected, columnOrder: "TYPE_ORDER" } + : { logicalType: expected, convertedType: undefined, columnOrder: "TYPE_ORDER" }, + label + ) + const ignored = yield* annotationOf(annotated(otherType, { ...annotation, typeLength: otherLength })) + assert.deepStrictEqual(ignored, { + logicalType: undefined, + convertedType: undefined, + // the order belongs to the leaf, so it stays when the annotation was on the group around it + columnOrder: otherType === GROUP ? "TYPE_ORDER" : undefined + }, label) + } + // UNKNOWN is for "a column that is always null", whatever its type + for (const type of [Type.BOOLEAN, Type.INT32, Type.BYTE_ARRAY]) { + assert.deepStrictEqual(yield* annotationOf(annotated(type, logical(11))), { + logicalType: { type: "UNKNOWN" }, + convertedType: undefined, + columnOrder: "TYPE_ORDER" + }) + } + })) + + it.effect("PQ-LT-5 reads a node's logical type and converted type independently of each other, unless the logical type is ignored", () => + Effect.gen(function*() { + // both defined for INT32, though DATE corresponds to the converted type DATE + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { ...logical(6), convertedType: 7 })), { + logicalType: { type: "DATE" }, + convertedType: "TIME_MILLIS", + columnOrder: "TYPE_ORDER" + }) + // STRING is not defined for INT32, so it is ignored, and so is INT_32, which only stands in for a + // logical type when there is none (#convertedtype, #unsupported-logical-types) + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { ...logical(1), convertedType: 17 })), { + logicalType: undefined, + convertedType: undefined, + columnOrder: undefined + }) + // the DecimalType supersedes the scale and precision of the schema element (parquet.thrift L557-564) + assert.deepStrictEqual( + yield* annotationOf(annotated(Type.INT32, { ...logical(5, { 1: I32(2), 2: I32(9) }), convertedType: 5 })), + { + logicalType: { type: "DECIMAL", scale: 2, precision: 9 }, + convertedType: "DECIMAL", + columnOrder: "TYPE_ORDER" + } + ) + // a converted type 2.14.0 doesn't have + assert.deepStrictEqual(yield* annotationOf(annotated(Type.INT32, { convertedType: 99 })), { + logicalType: undefined, + convertedType: 99, + columnOrder: undefined + }) + })) + + it.effect("PQ-LT-6 MUST reject a DECIMAL whose precision or scale breaks the rules (#decimal)", () => + Effect.gen(function*() { + type Case = readonly [ + type: number, + scale: number | undefined, + precision: number | undefined, + typeLength?: number + ] + const forms = ( + [type, scale, precision, typeLength]: Case + ) => [ + // the converted type takes its parameters from the schema element, where the scale defaults to 0 + annotated(type, { convertedType: 5, scale, precision, typeLength }), + ...(scale === undefined || precision === undefined + ? [] + : [annotated(type, { ...logical(5, { 1: I32(scale), 2: I32(precision) }), typeLength })]) + ] + const valid: ReadonlyArray = [ + [Type.INT32, 0, 1], + [Type.INT32, 9, 9], + [Type.INT32, undefined, 9], + [Type.INT64, 2, 18], + // floor(log10(2^(8n - 1) - 1)) digits: 2^7 - 1 = 127, 2^31 - 1 = 2147483647, 2^127 - 1 is about 1.7e38 + [FIXED, 0, 2, 1], + [FIXED, 0, 9, 4], + [FIXED, 10, 38, 16], + [Type.BYTE_ARRAY, 0, 1000] + ] + for (const case_ of valid) { + for (const bytes_ of forms(case_)) { + assert.strictEqual((yield* annotationOf(bytes_)).columnOrder, "TYPE_ORDER", JSON.stringify(case_)) + } + } + const invalid: ReadonlyArray = [ + [[Type.INT32, 5, 2], /scale/], + [[Type.INT32, -1, 2], /scale/], + [[Type.INT32, 0, 0], /precision/], + [[Type.BYTE_ARRAY, 0, -1], /precision/], + [[Type.BYTE_ARRAY, 0, undefined], /precision/], + [[Type.INT32, 0, 10], /precision/], + [[Type.INT64, 0, 19], /precision/], + [[FIXED, 0, 3, 1], /precision/], + [[FIXED, 0, 10, 4], /precision/], + [[FIXED, 0, 39, 16], /precision/] + ] + for (const [case_, message] of invalid) { + for (const bytes_ of forms(case_)) yield* failsWith(metadataOf(bytes_), "Malformed", message) + } + })) + + it.effect("PQ-LT-7 MUST reject an INTEGER whose bit width is not 8, 16, 32 or 64 (#signed-integers)", () => + Effect.gen(function*() { + for (const bitWidth of [0, 24, 65, -8]) { + yield* failsWith(metadataOf(annotated(Type.INT32, integer(bitWidth, true))), "Malformed", /bit width/) + } + })) + type Binaries = ReadonlyArray> + const forms = ["PLAIN", "DELTA_LENGTH_BYTE_ARRAY", "DELTA_BYTE_ARRAY", "dictionary"] as const + /** A required BYTE_ARRAY column of `values`, each of which is its own dictionary entry in the last form. */ + const binaries = (form: typeof forms[number], values: Binaries, annotation?: Annotation) => { + const n = values.length + const lengths = values.map((value) => value.length) + const plain = bytes(...values.map((value) => lengthPrefixed(Uint8Array.from(value)))) + switch (form) { + case "PLAIN": + return requiredColumn(Type.BYTE_ARRAY, n, plain, { annotation }) + case "DELTA_LENGTH_BYTE_ARRAY": + return requiredColumn(Type.BYTE_ARRAY, n, bytes(delta(lengths), ...values), { + encoding: DELTA_LENGTH_BYTE_ARRAY, + annotation + }) + case "DELTA_BYTE_ARRAY": + return requiredColumn(Type.BYTE_ARRAY, n, bytes(delta(lengths.map(() => 0)), delta(lengths), ...values), { + encoding: DELTA_BYTE_ARRAY, + annotation + }) + case "dictionary": + return flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: n, + dictionary: true, + annotation, + pages: [ + dictionaryPage(n, plain), + dataPage(n, bytes([8], runs(8, ...values.map((_, i) => [i, 1] as const))), { encoding: RLE_DICTIONARY }) + ] + }) + } + } + const hex = (values: Binaries) => values.map((value) => Hex.encode(Uint8Array.from(value))) + const STRING = logical(1) + const utf8 = [ + ["STRING", STRING], + ["ENUM", logical(4)], + ["JSON", logical(12)], + ["UTF8", { convertedType: 0 }], + ["ENUM (converted)", { convertedType: 4 }], + ["JSON (converted)", { convertedType: 19 }] + ] as const + // "é" is C3 A9 + const split: Binaries = [[0x61, 0xc3], [0xa9]] + + it.effect("PQ-LT-8 MUST reject a STRING, ENUM or JSON value that is not UTF-8 (#string, #enum, #json)", () => + Effect.gen(function*() { + const valid: Binaries = [[], [0x61], [0xc3, 0xa9], [], [0xe2, 0x82, 0xac], [0xf0, 0x9f, 0x98, 0x80]] + for (const [label, annotation] of utf8) { + for (const form of forms) { + const where = `${label} ${form}` + assert.deepStrictEqual(yield* values(binaries(form, valid, annotation)), hex(valid), where) + yield* failsWith(values(binaries(form, [[0x61], [0xff]], annotation)), "Malformed", /value 1 .*UTF-8/) + // every value is UTF-8 on its own: here the two are only when joined + yield* failsWith(values(binaries(form, split, annotation)), "Malformed", /value 0 .*UTF-8/) + } + } + // v2 data pages + const v2 = (value: ReadonlyArray) => + flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: 1, + annotation: STRING, + pages: [dataPageV2({ + numValues: 1, + numNulls: 0, + numRows: 1, + values: lengthPrefixed(Uint8Array.from(value)) + })] + }) + assert.deepStrictEqual(yield* values(v2([0xc3, 0xa9])), ["c3a9"]) + yield* failsWith(values(v2([0xc3])), "Malformed", /value 0 .*UTF-8/) + })) + + it.effect("PQ-LT-9 MUST reject every sequence of bytes that is not well-formed UTF-8 (#string)", () => + Effect.gen(function*() { + const string = (value: ReadonlyArray) => values(binaries("PLAIN", [value], STRING)) + // the first and last code points of every length, and those around the surrogates + const valid: Binaries = [ + [0x00], + [0x7f], + [0xc2, 0x80], + [0xdf, 0xbf], + [0xe0, 0xa0, 0x80], + [0xed, 0x9f, 0xbf], + [0xee, 0x80, 0x80], + [0xef, 0xbf, 0xbf], + [0xf0, 0x90, 0x80, 0x80], + [0xf4, 0x8f, 0xbf, 0xbf], + // a byte order mark is a character like any other + [0xef, 0xbb, 0xbf, 0x61] + ] + for (const value of valid) assert.deepStrictEqual(yield* string(value), hex([value])) + const invalid: Binaries = [ + // a continuation byte first, and bytes that are never in UTF-8 + [0x80], + [0xbf], + [0xc0, 0x80], + [0xc1, 0xbf], + [0xf5, 0x80, 0x80, 0x80], + [0xff], + // sequences that end early + [0xc2], + [0xe2, 0x82], + [0xf0, 0x9f, 0x98], + [0xc2, 0x41], + [0xe2, 0x82, 0x41], + [0xf0, 0x9f, 0x98, 0x41], + // overlong forms + [0xe0, 0x9f, 0xbf], + [0xf0, 0x8f, 0xbf, 0xbf], + // surrogates, and code points after U+10FFFF + [0xed, 0xa0, 0x80], + [0xed, 0xbf, 0xbf], + [0xf4, 0x90, 0x80, 0x80] + ] + const strict = new TextDecoder("utf-8", { fatal: true }) + for (const value of invalid) { + yield* failsWith(string(value), "Malformed", /value 0 is not UTF-8/) + assert.throws(() => strict.decode(Uint8Array.from(value)), TypeError, undefined, hex([value])[0]) + } + })) + + it.effect("PQ-LT-10 checks DELTA_BYTE_ARRAY values after their prefixes are added (Encodings.md#delta-strings-delta_byte_array--7)", () => + Effect.gen(function*() { + const make = (prefixes: ReadonlyArray, suffixes: Binaries) => + values(requiredColumn( + Type.BYTE_ARRAY, + suffixes.length, + bytes(delta(prefixes), delta(suffixes.map((suffix) => suffix.length)), ...suffixes), + { encoding: DELTA_BYTE_ARRAY, annotation: STRING } + )) + // the suffix A9 is not UTF-8, the value C3 A9 is + assert.deepStrictEqual(yield* make([0, 1], [[0xc3, 0xa9], [0xa9]]), ["c3a9", "c3a9"]) + // the suffix "(" is UTF-8, the value C3 28 is not + yield* failsWith(make([0, 1], [[0xc3, 0xa9], [0x28]]), "Malformed", /value 1 .*UTF-8/) + })) + + it.effect("PQ-LT-11 checks a dictionary once, with the values no page refers to", () => + Effect.gen(function*() { + const make = (dictionary: Binaries) => + values(flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows: 2, + dictionary: true, + annotation: STRING, + pages: [ + dictionaryPage( + dictionary.length, + bytes(...dictionary.map((value) => lengthPrefixed(Uint8Array.from(value)))) + ), + dataPage(2, bytes([1], runs(1, [0, 2])), { encoding: RLE_DICTIONARY }) + ] + })) + assert.deepStrictEqual(yield* make([[0x61], [0x62]]), ["61", "61"]) + yield* failsWith(make([[0x61], [0xff]]), "Malformed", /dictionary value 1 .*UTF-8/) + })) + + it.effect("PQ-LT-12 doesn't check the bytes of other BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns", () => + Effect.gen(function*() { + const binary: Binaries = [[0xff], ...split] + const annotations: ReadonlyArray = [ + undefined, + logical(13), + { convertedType: 20 }, + logical(17), + // a logical type supersedes UTF8, also when it is ignored for being unknown or not for BYTE_ARRAY + // (#convertedtype, #unsupported-logical-types); without one, UTF8 is checked (PQ-LT-8) + { ...logical(13), convertedType: 0 }, + { logicalType: Struct({ 99: empty }), convertedType: 0 }, + { ...logical(6), convertedType: 0 } + ] + for (const annotation of annotations) { + for (const form of forms) { + assert.deepStrictEqual(yield* values(binaries(form, binary, annotation)), hex(binary)) + } + } + const fixed = requiredColumn(FIXED, 2, new Uint8Array([0xff, 0xc3, 0xa9, 0xff]), { typeLength: 2 }) + assert.deepStrictEqual(yield* values(fixed), ["ffc3", "a9ff"]) + })) + }) + + describe("README.md#nested-encoding", () => { + it.effect("PQ-NEST-1 MUST compute the maximum levels from the optional and repeated nodes of the path", () => + Effect.gen(function*() { + const metadata = yield* metadataOf(file({ + schema: [ + root(1), + element("a", { repetition: OPTIONAL, children: 1 }), + element("list", { repetition: REPEATED, children: 1 }), + element("b", { repetition: REQUIRED, children: 1 }), + element("c", { type: Type.INT32, repetition: REPEATED }) + ], + numRows: 0, + chunks: [{ type: Type.INT32, path: ["a", "list", "b", "c"], numValues: 0, pages: [] }] + })) + assert.strictEqual(metadata.columns[0].maxDefinitionLevel, 3) + assert.strictEqual(metadata.columns[0].maxRepetitionLevel, 2) + })) + + it.effect("PQ-NEST-2 MUST assemble lists from repetition and definition levels", () => + Effect.gen(function*() { + // rows [1, 2], [], [3]: levels (r, d) = (0, 1) (1, 1) (0, 0) (0, 1) + const page = dataPage( + 4, + bytes( + lengthPrefixed(runs(1, [0, 1], [1, 1], [0, 2])), + lengthPrefixed(runs(1, [1, 2], [0, 1], [1, 1])), + int32s(1, 2, 3) + ) + ) + const column = yield* read(repeatedInt32([page], 3, 4)) + assert.deepStrictEqual(rows(column), [[1, 2], [], [3]]) + assert.deepStrictEqual(Array.from(column.layers[0].offsets!), [0, 2, 2, 3]) + })) + }) + + describe("README.md#nulls", () => { + it.effect("PQ-NULL-1 MUST read nulls from the definition levels only, with no values in the data", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(optionalInt32([dataPage(3, lengthPrefixed(runs(1, [0, 3])))], 3)), [ + null, + null, + null + ]) + const column = yield* read(optionalInt32([dataPage(3, bytes(lengthPrefixed(someNull), int32s(5, 7)))], 3)) + assert.deepStrictEqual(rows(column), [5, null, 7]) + assert.strictEqual(column.layers[0].nullCount, 1) + })) + }) + + describe("README.md#data-pages", () => { + it.effect("PQ-PAGE-1 MUST read a required top-level column's page as values only", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(int32s3), [1, 2, 3]) + })) + + it.effect("PQ-PAGE-2 MUST read uncompressed_page_size as the size of levels and values together", () => + Effect.gen(function*() { + const body = bytes(lengthPrefixed(someNull), int32s(5, 7)) + const make = (uncompressedSize: number) => + optionalInt32([dataPage(3, gzipSync(body), { uncompressedSize })], 3, GZIP) + assert.deepStrictEqual(yield* values(make(body.length), { codecs: { GZIP: gzip } }), [5, null, 7]) + yield* failsWith(values(make(body.length + 1), { codecs: { GZIP: gzip } }), "Malformed", /decompressed to/) + })) + + it.effect("PQ-PAGE-3 MUST reject a negative count or size in a page header (parquet.thrift PageHeader)", () => + Effect.gen(function*() { + const present = runs(1, [1, 1]) + const column = (repetition: number, ...pages: ReadonlyArray) => + flat({ type: Type.INT32, repetition, numRows: 1, pages }) + const v2 = (options: Partial[0]>) => + dataPageV2({ numValues: 1, numNulls: 0, numRows: 1, values: int32s(7), ...options }) + const rejects = (bytes_: Uint8Array, message: RegExp) => failsWith(values(bytes_), "Malformed", message) + // with and without levels + yield* rejects( + column(OPTIONAL, dataPage(-1, bytes(lengthPrefixed(present), int32s(7)))), + /DataPageHeader\.num_values is negative/ + ) + yield* rejects(column(REQUIRED, dataPage(-1, int32s(7))), /DataPageHeader\.num_values is negative/) + yield* rejects( + column(OPTIONAL, v2({ numValues: -1, definition: present })), + /DataPageHeaderV2\.num_values is negative/ + ) + yield* rejects(column(REQUIRED, v2({ numValues: -1 })), /DataPageHeaderV2\.num_values is negative/) + yield* rejects(column(REQUIRED, v2({ numNulls: -1 })), /num_nulls is negative/) + yield* rejects(column(REQUIRED, v2({ numRows: -1 })), /num_rows is negative/) + yield* rejects(column(REQUIRED, v2({ definitionLength: -1 })), /definition_levels_byte_length is negative/) + yield* rejects(column(REQUIRED, v2({ repetitionLength: -1 })), /repetition_levels_byte_length is negative/) + yield* rejects( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + dictionary: true, + pages: [ + dictionaryPage(-1, int32s(7)), + dataPage(1, bytes([1], runs(1, [0, 1])), { encoding: RLE_DICTIONARY }) + ] + }), + /DictionaryPageHeader\.num_values is negative/ + ) + yield* rejects(column(REQUIRED, dataPage(1, int32s(7), { uncompressedSize: -1 })), /negative size/) + const header = Struct({ 1: I32(1), 2: I32(PLAIN), 3: I32(RLE), 4: I32(RLE) }) + yield* rejects( + column(REQUIRED, bytes(thrift(Struct({ 1: I32(0), 2: I32(4), 3: I32(-1), 5: header })), int32s(7))), + /negative size/ + ) + })) + }) + + describe("README.md#column-chunks", () => { + const dictionary = dictionaryPage(2, int32s(10, 20)) + const indices = dataPage(3, bytes([1], runs(1, [0, 1], [1, 2])), { encoding: RLE_DICTIONARY }) + const make = (pages: ReadonlyArray, numRows = 3) => + flat({ type: Type.INT32, repetition: REQUIRED, numRows, pages, dictionary: true }) + + it.effect("PQ-CHUNK-1 MUST read the dictionary page first, and at most one (parquet.thrift L727-729)", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(make([dictionary, indices])), [10, 20, 20]) + yield* failsWith(values(make([dictionary, dictionary, indices])), "Malformed", /dictionary page/) + yield* failsWith(values(make([dictionary, indices, dictionary, indices], 6)), "Malformed", /dictionary page/) + })) + + it.effect("PQ-CHUNK-2 MAY skip pages it does not read: index pages and later page types", () => + Effect.gen(function*() { + const index = page(1, { 6: Struct({}) }, new Uint8Array([1, 2, 3])) + const future = page(42, {}, new Uint8Array([4, 5])) + assert.deepStrictEqual( + yield* values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [index, future, dataPage(3, int32s(1, 2, 3))] + }) + ), + [1, 2, 3] + ) + })) + + it.effect("PQ-CHUNK-3 MUST reject a column chunk whose num_values is negative (parquet.thrift L922-923)", () => + Effect.gen(function*() { + const levels = lengthPrefixed(runs(1, [0, 1])) + // with repetition and definition levels, and without any + yield* failsWith( + values(repeatedInt32([dataPage(1, bytes(levels, levels))], 1, -1)), + "Malformed", + /has -1 values/ + ) + yield* failsWith( + values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + numValues: -1, + pages: [dataPage(1, int32s(7))] + }) + ), + "Malformed", + /has -1 values/ + ) + })) + }) + + describe("parquet.thrift DataPageHeaderV2", () => { + it.effect("PQ-V2-1 MUST begin every v2 page at a row boundary (L759-761)", () => + Effect.gen(function*() { + const part = (repetition: number, value: number, numRows: number) => + dataPageV2({ + numValues: 1, + numNulls: 0, + numRows, + repetition: runs(1, [repetition, 1]), + definition: runs(1, [1, 1]), + values: int32s(value) + }) + assert.deepStrictEqual(yield* values(repeatedInt32([part(0, 1, 1), part(0, 2, 1)], 2, 2)), [[1], [2]]) + yield* failsWith(values(repeatedInt32([part(0, 1, 1), part(1, 2, 0)], 1, 2)), "Malformed", /row boundary/) + })) + + it.effect("PQ-V2-2 MUST read v2 levels uncompressed and without a length prefix (L744-745, L767)", () => + Effect.gen(function*() { + const values_ = int32s(5, 7) + const decompress: Parquet.Decompress = (data, size) => { + assert.deepStrictEqual(gunzipSync(data), Buffer.from(values_)) + assert.strictEqual(size, 8) + return gzip(data, size) + } + const v2 = dataPageV2({ + numValues: 3, + numNulls: 1, + numRows: 3, + definition: someNull, + values: gzipSync(values_), + uncompressedSize: someNull.length + 8 + }) + assert.deepStrictEqual(yield* values(optionalInt32([v2], 3, GZIP), { codecs: { GZIP: decompress } }), [ + 5, + null, + 7 + ]) + })) + + it.effect("PQ-V2-3 MUST NOT decompress the values of a v2 page whose is_compressed is false (L745, L779)", () => + Effect.gen(function*() { + const v2 = dataPageV2({ + numValues: 3, + numNulls: 1, + numRows: 3, + definition: someNull, + values: int32s(5, 7), + compressed: false + }) + assert.deepStrictEqual(yield* values(optionalInt32([v2], 3, GZIP)), [5, null, 7]) + })) + + it.effect("PQ-V2-4 MUST hold num_values - num_nulls values in the data section (L755-756)", () => + Effect.gen(function*() { + const v2 = dataPageV2({ numValues: 3, numNulls: 0, numRows: 3, definition: someNull, values: int32s(5, 7) }) + yield* failsWith(values(optionalInt32([v2], 3)), "Malformed", /nulls/) + })) + }) + + describe("parquet.thrift Statistics", () => { + const withStatistics = (statistics: V) => + flat({ type: Type.DOUBLE, repetition: REQUIRED, numRows: 0, pages: [], statistics }) + const statisticsOf = (bytes: Uint8Array) => + Effect.map(metadataOf(bytes), (m) => m.rowGroups[0].columns[0].metaData?.statistics) + + it.effect("PQ-STATS-1 MUST distinguish an absent null_count from a null_count of 0 (L286-289)", () => + Effect.gen(function*() { + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({ 3: I64(0) }))))?.nullCount, 0) + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({}))))?.nullCount, undefined) + })) + + it.effect("PQ-STATS-2 MUST NOT assume nan_count is 0 when it is absent (L315-316)", () => + Effect.gen(function*() { + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({ 9: I64(0) }))))?.nanCount, 0) + assert.strictEqual((yield* statisticsOf(withStatistics(Struct({}))))?.nanCount, undefined) + })) + }) + + describe("Encodings.md#plain-plain--0", () => { + it.effect("PQ-PLAIN-1 MUST read PLAIN values of every physical type", () => + Effect.gen(function*() { + const cases: ReadonlyArray, number?]> = [ + // BOOLEAN: bit-packed, least significant bit first + [Type.BOOLEAN, 10, new Uint8Array([0b10001101, 0b01]), [ + true, + false, + true, + true, + false, + false, + false, + true, + true, + false + ]], + [Type.INT32, 2, int32s(-1, 0x7fffffff), [-1, 0x7fffffff]], + [Type.INT64, 2, new Uint8Array(BigInt64Array.from([-(2n ** 63n), 5n]).buffer), [String(-(2n ** 63n)), "5"]], + [Type.INT96, 1, new Uint8Array(12).fill(7), ["070707070707070707070707"]], + [Type.FLOAT, 2, new Uint8Array(Float32Array.from([1.5, -0]).buffer), [1.5, -0]], + [Type.DOUBLE, 1, new Uint8Array(Float64Array.from([Math.PI]).buffer), [Math.PI]], + [Type.BYTE_ARRAY, 3, plainBinary("", "ab", "é"), ["", "6162", "c3a9"]], + [Type.FIXED_LEN_BYTE_ARRAY, 2, new Uint8Array([1, 2, 3, 4]), ["0102", "0304"], 2] + ] + for (const [type, n, data, expected, typeLength] of cases) { + assert.deepStrictEqual(yield* values(requiredColumn(type, n, data, { typeLength })), expected) + } + })) + }) + + describe("Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8", () => { + const make = (dictionaryEncoding: number, encoding: number, indices: Uint8Array, numRows = 3) => + flat({ + type: Type.BYTE_ARRAY, + repetition: REQUIRED, + numRows, + dictionary: true, + pages: [ + dictionaryPage(2, plainBinary("a", "b"), dictionaryEncoding), + dataPage(numRows, indices, { encoding }) + ] + }) + + it.effect("PQ-DICT-1 MUST read indices as a bit-width byte followed by RLE/bit-packed runs (L84-85)", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(make(PLAIN, RLE_DICTIONARY, bytes([1], runs(1, [1, 2], [0, 1])))), [ + "62", + "62", + "61" + ]) + yield* failsWith(values(make(PLAIN, RLE_DICTIONARY, bytes([33], runs(33, [1, 3])))), "Malformed", /bit width/) + })) + + it.effect("PQ-DICT-2 SHOULD read the deprecated PLAIN_DICTIONARY in dictionary and data pages (L87-88)", () => + Effect.gen(function*() { + const indices = bytes([1], runs(1, [0, 3])) + assert.deepStrictEqual(yield* values(make(PLAIN_DICTIONARY, PLAIN_DICTIONARY, indices)), ["61", "61", "61"]) + })) + }) + + describe("Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3", () => { + it.effect("PQ-RLE-1 MUST unpack bit-packed runs from the least significant bit (note 1)", () => + Effect.gen(function*() { + // the spec's example: 0 to 7 with bit width 3 is 10001000 11000110 11111010 + const bytes_ = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 8, + dictionary: true, + pages: [ + dictionaryPage(8, int32s(10, 11, 12, 13, 14, 15, 16, 17)), + dataPage(8, new Uint8Array([3, 0b11, 0b10001000, 0b11000110, 0b11111010]), { encoding: RLE_DICTIONARY }) + ] + }) + assert.deepStrictEqual(yield* values(bytes_), [10, 11, 12, 13, 14, 15, 16, 17]) + })) + + it.effect("PQ-RLE-2 MUST reject runs shorter than 1 (note 3, L141)", () => + failsWith( + values(optionalInt32([dataPage(1, bytes(lengthPrefixed(new Uint8Array([0, 1])), int32s(1)))], 1)), + "Malformed", + /run of length 0/ + )) + + it.effect("PQ-RLE-3 MUST read RLE booleans with a 4-byte length in v1 and v2 pages (L155-170)", () => + Effect.gen(function*() { + const data = lengthPrefixed(runs(1, [1, 3], [0, 2])) + const expected = [true, true, true, false, false] + const v1 = dataPage(5, data, { encoding: RLE }) + const v2 = dataPageV2({ numValues: 5, numNulls: 0, numRows: 5, encoding: RLE, values: data }) + for (const page of [v1, v2]) { + assert.deepStrictEqual( + yield* values(flat({ type: Type.BOOLEAN, repetition: REQUIRED, numRows: 5, pages: [page] })), + expected + ) + } + })) + }) + + describe("Encodings.md#bit-packed-deprecated-bit_packed--4", () => { + // seven nested optional nodes: definition levels 0 to 7 need 3 bits + const schema = [ + root(1), + ...Array.from({ length: 6 }, (_, i) => element("g" + i, { repetition: OPTIONAL, children: 1 })), + element("x", { type: Type.INT32, repetition: OPTIONAL }) + ] + const make = (page: Uint8Array) => + file({ + schema, + numRows: 8, + chunks: [{ type: Type.INT32, path: ["g0", "g1", "g2", "g3", "g4", "g5", "x"], numValues: 8, pages: [page] }] + }) + + it.effect("PQ-BP-1 MUST unpack BIT_PACKED levels from the most significant bit (L182-195)", () => + Effect.gen(function*() { + // the spec's example: 0 to 7 with bit width 3 is 00000101 00111001 01110111 + const page = dataPage(8, bytes([0b00000101, 0b00111001, 0b01110111], int32s(42)), { levels: BIT_PACKED }) + const column = yield* read(make(page)) + assert.deepStrictEqual(rows(column), [null, null, null, null, null, null, null, 42]) + assert.deepStrictEqual(column.layers.map((l) => l.nullCount), [1, 2, 3, 4, 5, 6, 7]) + })) + + it.effect("PQ-BP-2 MUST NOT read BIT_PACKED as a values encoding (L197-198)", () => + failsWith( + values(requiredColumn(Type.INT32, 1, int32s(1), { encoding: BIT_PACKED })), + "Malformed", + /not an encoding/ + )) + }) + + describe("Encodings.md#delta-encoding-delta_binary_packed--5", () => { + const int32 = (data: Uint8Array, n: number) => + values(requiredColumn(Type.INT32, n, data, { encoding: DELTA_BINARY_PACKED })) + // Example 2 (7, 5, 3, 1, 2, 3, 4, 5), with a valid block size of 128 and four miniblocks of 32 + const example2 = (widths: ReadonlyArray, packed: ReadonlyArray) => + bytes([0x80, 0x01, 4, 8, 14], [3], widths, packed) + + it.effect("PQ-DELTA-1 MUST read the header and blocks of bit-packed miniblocks (examples 1 and 2)", () => + Effect.gen(function*() { + // Example 1: 1, 2, 3, 4, 5 is a min delta of 1 and bit width 0 + assert.deepStrictEqual(yield* int32(new Uint8Array([0x80, 0x01, 4, 5, 2, 2, 0, 0, 0, 0]), 5), [1, 2, 3, 4, 5]) + // relative deltas 0, 0, 0, 3, 3, 3, 3 on 2 bits, padded to 32 values + const packed = [0b11000000, 0b00111111, 0, 0, 0, 0, 0, 0] + assert.deepStrictEqual(yield* int32(example2([2, 0, 0, 0], packed), 8), [7, 5, 3, 1, 2, 3, 4, 5]) + })) + + it.effect("PQ-DELTA-2 MUST accept arbitrary padding bits in the last miniblock (L257-260)", () => + Effect.gen(function*() { + const packed = [0b11000000, 0b11111111, 0xff, 0xaa, 0x55, 0xff, 0xff, 0xff] + assert.deepStrictEqual(yield* int32(example2([2, 0, 0, 0], packed), 8), [7, 5, 3, 1, 2, 3, 4, 5]) + })) + + it.effect("PQ-DELTA-3 MUST accept arbitrary bit widths for unneeded miniblocks (L262-268)", () => + Effect.gen(function*() { + const packed = [0b11000000, 0b00111111, 0, 0, 0, 0, 0, 0] + assert.deepStrictEqual(yield* int32(example2([2, 0xff, 0x41, 7], packed), 8), [7, 5, 3, 1, 2, 3, 4, 5]) + })) + + it.effect("PQ-DELTA-4 MUST wrap deltas around in two's complement (L270-273)", () => + Effect.gen(function*() { + // INT32: 2^31 - 1, -2^31, 2^31 - 1 are deltas 1 and -1 after wrapping + const i32 = bytes([0x80, 0x01, 4, 3], uleb(0xfffffffe), [1], [2, 0, 0, 0], [0b0010, 0, 0, 0, 0, 0, 0, 0]) + assert.deepStrictEqual(yield* int32(i32, 3), [0x7fffffff, -0x80000000, 0x7fffffff]) + // INT64: 2^63 - 1, -2^63 is a delta of 1 after wrapping + const max64 = [...Array(9).fill(0xff).map((b, i) => i === 0 ? 0xfe : b), 0x01] + const i64 = bytes([0x80, 0x01, 4, 2], max64, [2], [0, 0, 0, 0]) + assert.deepStrictEqual( + yield* values(requiredColumn(Type.INT64, 2, i64, { encoding: DELTA_BINARY_PACKED })), + [String(2n ** 63n - 1n), String(-(2n ** 63n))] + ) + })) + + it.effect("PQ-DELTA-5 MUST reject miniblocks wider than the physical type (L274-276)", () => + failsWith( + int32(bytes([0x80, 0x01, 4, 2, 0], [0], [33, 0, 0, 0], new Uint8Array(132)), 2), + "Malformed", + /bit width 33/ + )) + }) + + describe("Encodings.md#delta-length-byte-array-delta_length_byte_array--6", () => { + it.effect("PQ-DLBA-1 MUST read delta-encoded lengths followed by the concatenated bytes (example)", () => + Effect.gen(function*() { + const data = bytes(delta([5, 5, 6, 6]), text("HelloWorldFoobarABCDEF")) + assert.deepStrictEqual( + yield* values(requiredColumn(Type.BYTE_ARRAY, 4, data, { encoding: DELTA_LENGTH_BYTE_ARRAY })), + ["Hello", "World", "Foobar", "ABCDEF"].map((s) => Hex.encode(text(s))) + ) + })) + }) + + describe("Encodings.md#delta-strings-delta_byte_array--7", () => { + it.effect("PQ-DBA-1 MUST read prefix lengths then suffixes as DELTA_LENGTH_BYTE_ARRAY (example)", () => + Effect.gen(function*() { + const data = bytes(delta([0, 2, 0, 3]), delta([4, 2, 6, 5]), text("axislebabbleyhood")) + assert.deepStrictEqual( + yield* values(requiredColumn(Type.BYTE_ARRAY, 4, data, { encoding: DELTA_BYTE_ARRAY })), + ["axis", "axle", "babble", "babyhood"].map((s) => Hex.encode(text(s))) + ) + })) + + it.effect("PQ-DBA-2 MUST read FIXED_LEN_BYTE_ARRAY values with all their lengths encoded", () => + Effect.gen(function*() { + const data = bytes(delta([0, 2, 1]), delta([4, 2, 3]), text("axislebcd")) + const file_ = requiredColumn(Type.FIXED_LEN_BYTE_ARRAY, 3, data, { + encoding: DELTA_BYTE_ARRAY, + typeLength: 4 + }) + assert.deepStrictEqual(yield* values(file_), ["axis", "axle", "abcd"].map((s) => Hex.encode(text(s)))) + })) + }) + + describe("Encodings.md#byte-stream-split-byte_stream_split--9", () => { + const split = new Uint8Array([0xaa, 0x00, 0xa3, 0xbb, 0x11, 0xb4, 0xcc, 0x22, 0xc5, 0xdd, 0x33, 0xd6]) + const make = (data: Uint8Array) => requiredColumn(Type.FLOAT, 3, data, { encoding: BYTE_STREAM_SPLIT }) + + it.effect("PQ-BSS-1 MUST gather byte k of every value from stream k (example)", () => + Effect.gen(function*() { + const raw = new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd, 0x00, 0x11, 0x22, 0x33, 0xa3, 0xb4, 0xc5, 0xd6]) + assert.deepStrictEqual(yield* values(make(split)), Array.from(new Float32Array(raw.buffer))) + })) + + it.effect("PQ-BSS-2 MUST hold exactly K * N bytes, with no padding", () => + failsWith(values(make(bytes(split, [0]))), "Malformed", /BYTE_STREAM_SPLIT/)) + }) + + describe("Compression.md", () => { + const body = int32s(1, 2, 3) + const make = (data: Uint8Array, codec: number) => + requiredColumn(Type.INT32, 3, data, { codec, uncompressedSize: 12 }) + + it.effect("PQ-COMP-1 MUST decompress each page with the column chunk's codec", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* values(make(gzipSync(body), GZIP), { codecs: { GZIP: gzip } }), [1, 2, 3]) + yield* failsWith(values(make(gzipSync(body), GZIP)), "Unsupported", /GZIP/) + const failing: Parquet.Decompress = () => Effect.fail("corrupt") + const error = yield* Effect.flip(values(make(gzipSync(body), GZIP), { codecs: { GZIP: failing } })) + assert.strictEqual(error.kind, "Codec") + assert.strictEqual(error.cause, "corrupt") + assert.match(error.message, /^column "x" of row group 0, page at byte 4: GZIP decompression failed/) + })) + + it.effect("PQ-COMP-2 SHOULD read GZIP pages holding several members (L61), given a Decompress that does", () => + Effect.gen(function*() { + const members = bytes(gzipSync(body.subarray(0, 4)), gzipSync(body.subarray(4))) + assert.deepStrictEqual(yield* values(make(members, GZIP), { codecs: { GZIP: gzip } }), [1, 2, 3]) + })) + }) + + describe("README.md#checksumming", () => { + const options = { codecs: { GZIP: gzip } } + const wrong = (body: Uint8Array) => (crc32(body) ^ 1) >>> 0 + + it.effect("PQ-CRC-1 MUST check a v1 data page against its CRC, of the bytes after the header (parquet.thrift L840-852)", () => + Effect.gen(function*() { + const seven = int32s(7) + const eight = int32s(8) + const make = (body: Uint8Array, crc?: number) => values(requiredColumn(Type.INT32, 1, body, { crc })) + assert.deepStrictEqual(yield* make(seven, crc32(seven)), [7]) + // the value changed and the CRC didn't + yield* failsWith(make(eight, crc32(seven)), "Malformed", /CRC/) + // a page without a CRC is not checked + assert.deepStrictEqual(yield* make(eight), [8]) + // of the compressed bytes, with the levels + const body = bytes(lengthPrefixed(someNull), int32s(5, 7)) + const compressed = gzipSync(body) + const gzipped = (crc: number) => + values( + optionalInt32([dataPage(3, compressed, { uncompressedSize: body.length, crc })], 3, GZIP), + options + ) + assert.deepStrictEqual(yield* gzipped(crc32(compressed)), [5, null, 7]) + yield* failsWith(gzipped(crc32(body)), "Malformed", /CRC/) + })) + + it.effect("PQ-CRC-2 MUST check a v2 data page against its CRC, of the levels and the stored values (parquet.thrift L840-852)", () => + Effect.gen(function*() { + const values_ = int32s(5, 7) + const make = (compressed: boolean, crc: (body: Uint8Array) => number) => + values( + optionalInt32( + [dataPageV2({ + numValues: 3, + numNulls: 1, + numRows: 3, + definition: someNull, + values: compressed ? gzipSync(values_) : values_, + compressed, + uncompressedSize: someNull.length + values_.length, + crc + })], + 3, + GZIP + ), + options + ) + for (const compressed of [false, true]) { + assert.deepStrictEqual(yield* make(compressed, crc32), [5, null, 7]) + yield* failsWith(make(compressed, wrong), "Malformed", /CRC/) + // not of the values alone + yield* failsWith( + make(compressed, (body) => crc32(body.subarray(someNull.length))), + "Malformed", + /CRC/ + ) + } + })) + + it.effect("PQ-CRC-3 MUST check a dictionary page against its CRC (parquet.thrift L840-852)", () => + Effect.gen(function*() { + const dictionary = int32s(10, 11) + const make = (compressed: boolean, crc: (body: Uint8Array) => number) => { + const stored = (body: Uint8Array) => compressed ? gzipSync(body) : body + const indices = bytes([1], runs(1, [1, 2], [0, 1])) + return values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + dictionary: true, + codec: compressed ? GZIP : undefined, + pages: [ + dictionaryPage(2, stored(dictionary), PLAIN, { + uncompressedSize: dictionary.length, + crc: crc(stored(dictionary)) + }), + dataPage(3, stored(indices), { encoding: RLE_DICTIONARY, uncompressedSize: indices.length }) + ] + }), + options + ) + } + for (const compressed of [false, true]) { + assert.deepStrictEqual(yield* make(compressed, crc32), [11, 11, 10]) + yield* failsWith(make(compressed, wrong), "Malformed", /CRC/) + } + })) + + it.effect("PQ-CRC-4 MUST compute the standard CRC-32 of GZIP, as an unsigned number (#checksumming)", () => + Effect.gen(function*() { + // the check value of CRC-32, which is negative as the i32 of the page header + const check = 0xcbf43926 + const body = text("123456789") + const make = (crc: number) => + values(requiredColumn(Type.FIXED_LEN_BYTE_ARRAY, 1, body, { typeLength: 9, crc })) + assert.strictEqual(check | 0, -873187034) + assert.deepStrictEqual(yield* make(check), [Hex.encode(body)]) + yield* failsWith(make(check ^ 0x80000000), "Malformed", /CRC/) + // CRC-32C, of the same bytes + yield* failsWith(make(0xe3069283), "Malformed", /CRC/) + })) + + it.effect("PQ-CRC-5 checks the CRC of the pages it skips", () => + Effect.gen(function*() { + const index = text("index") + const make = (crc: number) => + values( + flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [page(1, { 6: Struct({}) }, index, index.length, crc), dataPage(3, int32s(1, 2, 3))] + }) + ) + assert.deepStrictEqual(yield* make(crc32(index)), [1, 2, 3]) + yield* failsWith(make(wrong(index)), "Malformed", /CRC/) + })) + }) + + describe("unsupported features", () => { + it.effect("PQ-UNSUP-1 fails with Unsupported on an encrypted footer (PARE)", () => + Effect.gen(function*() { + const encrypted = int32s3.slice() + encrypted.set(text("PARE"), encrypted.length - 4) + yield* failsWith(metadataOf(encrypted), "Unsupported", /encrypted footer/) + })) + + it.effect("PQ-UNSUP-2 fails with Unsupported on an encrypted column chunk", () => + failsWith( + read(flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [dataPage(3, int32s(1, 2, 3))], + chunk: { 8: Struct({ 1: Struct({}) }) } + })), + "Unsupported", + /encrypted/ + )) + + it.effect("PQ-UNSUP-3 fails with Unsupported on the ALP encoding", () => + failsWith( + values(requiredColumn(Type.DOUBLE, 1, new Uint8Array(8), { encoding: ALP })), + "Unsupported", + /ALP/ + )) + + it.effect("PQ-UNSUP-4 fails with Unsupported on a column chunk in another file", () => + failsWith( + read(flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + pages: [dataPage(3, int32s(1, 2, 3))], + chunk: { 1: Str("other.parquet") } + })), + "Unsupported", + /other\.parquet/ + )) + + it.effect("PQ-UNSUP-5 fails with Unsupported on a codec without a Decompress, or unknown", () => + Effect.gen(function*() { + for (const codec of [1, 3, 4, 5, 6, 7, 99]) { + yield* failsWith(values(requiredColumn(Type.INT32, 3, int32s(1, 2, 3), { codec })), "Unsupported") + } + })) + + it.effect("PQ-UNSUP-6 fails with Unsupported on a TIME unit added after 2.14.0", () => + Effect.gen(function*() { + const time = (unit: number) => + annotated(Type.INT64, { logicalType: temporal(TIME, Struct({ [unit]: empty })) }) + assert.deepStrictEqual((yield* annotationOf(time(2))).logicalType, { + type: "TIME", + isAdjustedToUTC: true, + unit: "MICROS" + }) + yield* failsWith(metadataOf(time(4)), "Unsupported", /TIME unit 4/) + })) + }) + }) + + describe("hostile input", () => { + const strings = requiredColumn(Type.BYTE_ARRAY, 2, plainBinary("abc", "de")) + + it.effect("enforces maxFooterLength, maxDepth, maxPageSize, maxValues and maxBinaryLength", () => + Effect.gen(function*() { + yield* failsWith( + Parquet.readMetadata(Parquet.fromBytes(int32s3), { maxFooterLength: 10 }), + "Limit", + /maxFooterLength/ + ) + const deep = file({ + schema: [ + root(1), + element("a", { repetition: OPTIONAL, children: 1 }), + element("b", { type: Type.INT32, repetition: OPTIONAL }) + ], + numRows: 0, + chunks: [{ type: Type.INT32, path: ["a", "b"], numValues: 0, pages: [] }] + }) + assert.strictEqual((yield* Parquet.readMetadata(Parquet.fromBytes(deep), { maxDepth: 2 })).columns.length, 1) + yield* failsWith(Parquet.readMetadata(Parquet.fromBytes(deep), { maxDepth: 1 }), "Limit", /maxDepth/) + yield* failsWith(read(int32s3, { maxPageSize: 11 }), "Limit", /maxPageSize/) + assert.deepStrictEqual(rows(yield* read(int32s3, { maxPageSize: 12, maxValues: 3 })), [1, 2, 3]) + yield* failsWith(read(int32s3, { maxValues: 2 }), "Limit", /maxValues/) + assert.strictEqual(rows(yield* read(strings, { maxBinaryLength: 5 })).length, 2) + yield* failsWith(read(strings, { maxBinaryLength: 4 }), "Limit", /maxBinaryLength/) + })) + + it.effect("caps fixed-width values and per-page value counts", () => + Effect.gen(function*() { + const wide = flat({ + type: Type.FIXED_LEN_BYTE_ARRAY, + typeLength: 0x7fffffff, + repetition: OPTIONAL, + numRows: 1000, + pages: [dataPage(1000, lengthPrefixed(runs(1, [0, 1000])))] + }) + yield* failsWith(read(wide), "Limit", /maxBinaryLength/) + const dictionary = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 3, + dictionary: true, + pages: [ + dictionaryPage(8, int32s(0, 1, 2, 3, 4, 5, 6, 7)), + dataPage(3, bytes([3], runs(3, [7, 3])), { encoding: RLE_DICTIONARY }) + ] + }) + assert.deepStrictEqual(rows(yield* read(dictionary)), [7, 7, 7]) + yield* failsWith(read(dictionary, { maxValues: 5 }), "Limit", /maxValues/) + })) + + it.effect("checks page sizes before calling a codec", () => + Effect.gen(function*() { + const calls: Array = [] + const codecs = { + GZIP: (data: Uint8Array, size: number) => { + calls.push(size) + return gzip(data, size) + } + } + const v2 = (definitionLength: number, uncompressedSize: number) => + dataPageV2({ + numValues: 1, + numNulls: 0, + numRows: 1, + values: new Uint8Array(4), + definitionLength, + uncompressedSize + }) + const make = (page_: Uint8Array) => + flat({ type: Type.INT32, repetition: OPTIONAL, numRows: 1, codec: GZIP, pages: [page_] }) + yield* failsWith( + read(make(dataPage(1, new Uint8Array(4), { uncompressedSize: -1 })), { codecs }), + "Malformed", + /negative size/ + ) + yield* failsWith(read(make(v2(5, 8)), { codecs }), "Malformed", /levels of a v2 data page/) + yield* failsWith(read(make(v2(3, 2)), { codecs }), "Malformed", /levels of a v2 data page/) + yield* failsWith(read(make(v2(-1, 8)), { codecs }), "Malformed", /negative/) + assert.deepStrictEqual(calls, []) + })) + + it.effect("dies on invalid options and indexes", () => + Effect.gen(function*() { + const dies = (effect: Effect.Effect) => + Effect.map(Effect.exit(effect), (exit) => assert.isTrue(Exit.isFailure(exit) && Cause.hasDies(exit.cause))) + yield* dies(Parquet.readMetadata(Parquet.fromBytes(int32s3), { maxDepth: 0 })) + yield* dies(read(int32s3, { maxPageSize: 1.5 })) + const source = Parquet.fromBytes(int32s3) + const metadata = yield* Parquet.readMetadata(source) + yield* dies(Parquet.readColumn(source, metadata, { rowGroup: 1, column: 0 })) + yield* dies(Parquet.readColumn(source, metadata, { rowGroup: 0, column: -1 })) + })) + + it.effect("rejects sources, chunks and pages that don't add up", () => + Effect.gen(function*() { + const short: Parquet.Source = { + size: int32s3.length, + read: (offset, length) => Effect.succeed(int32s3.subarray(offset, offset + length - 1)) + } + yield* failsWith(Parquet.readMetadata(short), "Malformed", /source returned/) + yield* failsWith(metadataOf(new Uint8Array(11)), "Malformed", /too short/) + const make = (pages: ReadonlyArray, numRows = 3) => + flat({ type: Type.INT32, repetition: REQUIRED, numRows, pages }) + yield* failsWith(read(make([dataPage(4, int32s(1, 2, 3, 4))])), "Malformed", /more values/) + yield* failsWith(read(make([dataPage(2, int32s(1, 2))])), "Malformed", /ends after 2/) + yield* failsWith(read(make([dataPage(3, int32s(1, 2))])), "Malformed", /PLAIN values need/) + const truncated = make([dataPage(3, int32s(1, 2, 3))]) + yield* failsWith(read(truncated.subarray(0, 20)), "Malformed") + // a page header claiming more bytes than the chunk + yield* failsWith( + read( + make([ + page(0, { 5: Struct({ 1: I32(3), 2: I32(0), 3: I32(3), 4: I32(3) }) }, int32s(1, 2, 3), 12).subarray( + 0, + 20 + ) + ]) + ), + "Malformed" + ) + })) + + it.effect("rejects levels and dictionary indices out of range", () => + Effect.gen(function*() { + const optional = (data: Uint8Array) => + flat({ type: Type.INT32, repetition: OPTIONAL, numRows: 1, pages: [dataPage(1, data)] }) + yield* failsWith( + read(optional(bytes(lengthPrefixed(runs(1, [3, 1])), int32s(1)))), + "Malformed", + /RLE value wider/ + ) + yield* failsWith(read(optional(bytes(lengthPrefixed(runs(2, [2, 1])), int32s(1)))), "Malformed") + const dictionary = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + dictionary: true, + pages: [dictionaryPage(1, int32s(9)), dataPage(1, bytes([1], runs(1, [1, 1])), { encoding: RLE_DICTIONARY })] + }) + yield* failsWith(read(dictionary), "Malformed", /out of range/) + const noDictionary = flat({ + type: Type.INT32, + repetition: REQUIRED, + numRows: 1, + pages: [dataPage(1, bytes([1], runs(1, [0, 1])), { encoding: RLE_DICTIONARY })] + }) + yield* failsWith(read(noDictionary), "Malformed", /no dictionary page/) + })) + + it.effect("rejects levels that repeat a missing list or disagree with the row count", () => + Effect.gen(function*() { + const repeated = ( + rep: ReadonlyArray, + def: ReadonlyArray, + n: number, + numRows: number + ) => + flat({ + type: Type.INT32, + repetition: REPEATED, + numRows, + numValues: n, + pages: [ + dataPage( + n, + bytes(lengthPrefixed(runs(1, ...rep)), lengthPrefixed(runs(1, ...def)), int32s(...Array(n).fill(1))) + ) + ] + }) + // the first level repeats a list that was never started + yield* failsWith(read(repeated([[1, 1]], [[1, 1]], 1, 0)), "Malformed", /repeats a list/) + // an empty list (d = 0) followed by a repetition into it + yield* failsWith(read(repeated([[0, 1], [1, 1]], [[0, 1], [1, 1]], 2, 1)), "Malformed", /repeats a list/) + yield* failsWith(read(repeated([[0, 2]], [[1, 2]], 2, 3)), "Malformed", /holds 2 rows/) + })) + + it.effect("rejects metadata that disagrees with the schema", () => + Effect.gen(function*() { + const make = (type: number, path: ReadonlyArray) => + metadataOf(file({ + schema: [root(1), element("x", { type: Type.INT32, repetition: REQUIRED })], + numRows: 0, + chunks: [{ type, path, numValues: 0, pages: [] }] + })) + yield* failsWith(make(Type.INT64, ["x"]), "Malformed", /is INT64/) + yield* failsWith(make(Type.INT32, ["y"]), "Malformed", /is for "y"/) + yield* failsWith( + metadataOf( + file({ + schema: [root(1), element("x", { type: Type.FIXED_LEN_BYTE_ARRAY, repetition: REQUIRED })], + numRows: 0, + chunks: [] + }) + ), + "Malformed", + /type_length/ + ) + })) + }) +}) diff --git a/packages/effect/test/encoding/Thrift.test.ts b/packages/effect/test/encoding/Thrift.test.ts new file mode 100644 index 00000000000..a50da6b98bb --- /dev/null +++ b/packages/effect/test/encoding/Thrift.test.ts @@ -0,0 +1,418 @@ +import { assert, describe, it } from "@effect/vitest" +import { Result, Schema } from "effect" +import * as Hex from "effect/encoding/Hex" +import * as Thrift from "effect/encoding/Thrift" +import { readFileSync } from "node:fs" + +// Expected bytes and trees come from the official Apache Thrift Python +// library; see fixtures/thrift/README.md +type Json = + | { readonly _tag: "Bool"; readonly value: boolean } + | { readonly _tag: "I8" | "I16" | "I32"; readonly value: number } + | { readonly _tag: "I64"; readonly value: string } + | { readonly _tag: "Double"; readonly value: number | string } + | { readonly _tag: "Binary" | "Uuid"; readonly value: string } + | { readonly _tag: "Struct"; readonly fields: ReadonlyArray<{ readonly id: number; readonly value: Json }> } + | { readonly _tag: "List" | "Set"; readonly elementType: Thrift.TypeName; readonly elements: ReadonlyArray } + | { + readonly _tag: "Map" + readonly keyType: Thrift.TypeName + readonly valueType: Thrift.TypeName + readonly entries: ReadonlyArray + } +interface Case { + readonly id: string + readonly type: Thrift.TypeName + readonly value: Json + readonly bytes: string +} + +const gold: { readonly cases: ReadonlyArray; readonly parquet: ReadonlyArray> } = JSON + .parse(readFileSync(new URL("../fixtures/thrift/gold.json", import.meta.url), "utf8")) +const corpus: ReadonlyArray = [...gold.cases, ...gold.parquet.map((c) => ({ ...c, type: "Struct" as const }))] +const parquet = (id: string) => fromHex(gold.parquet.find((c) => c.id === id)!.bytes) + +const fromHex = (hex: string) => Result.getOrThrow(Hex.decode(hex)) + +const fromJson = (v: Json): Thrift.Value => { + switch (v._tag) { + case "I64": + return { _tag: "I64", value: BigInt(v.value) } + case "Double": + return { _tag: "Double", value: Number(v.value) } + case "Binary": + case "Uuid": + return { _tag: v._tag, value: fromHex(v.value) } + case "Struct": + return { _tag: "Struct", fields: v.fields.map((f) => ({ id: f.id, value: fromJson(f.value) })) } + case "List": + case "Set": + return { ...v, elements: v.elements.map(fromJson) } + case "Map": + return { ...v, entries: v.entries.map(([k, x]) => [fromJson(k), fromJson(x)] as const) } + default: + return v + } +} + +const readAll = (bytes: Uint8Array, type: Thrift.TypeName, options?: Thrift.ReaderOptions) => + Thrift.decode(bytes, (r) => { + const value = r.readValue(Thrift.Type[type]) + assert.strictEqual(r.position, bytes.length) + return value + }, options) + +const write = (f: (w: Thrift.Writer) => void): Uint8Array => { + const w = Thrift.makeWriter() + f(w) + return w.finish() +} + +const kindOf = (result: Result.Result): Thrift.ThriftError["kind"] | undefined => + Result.isFailure(result) ? result.failure.kind : undefined + +const thrown = (f: () => unknown): Thrift.ThriftError["kind"] | undefined => { + try { + f() + } catch (error) { + assert.isTrue(Thrift.isThriftError(error), String(error)) + return (error as Thrift.ThriftError).kind + } + return undefined +} + +describe("Thrift", () => { + describe("official library", () => { + it("reads, skips and writes every gold case as the official library does", () => { + for (const c of corpus) { + const bytes = fromHex(c.bytes) + const value = fromJson(c.value) + assert.deepStrictEqual(Result.getOrThrow(readAll(bytes, c.type)), value, c.id) + const r = Thrift.makeReader(bytes) + r.skip(Thrift.Type[c.type]) + assert.strictEqual(r.position, bytes.length, c.id) + assert.strictEqual(Hex.encode(write((w) => w.writeValue(value))), c.bytes, c.id) + } + }) + + it("reads a Parquet footer field by field", () => { + // The fields of parquet.thrift's FileMetaData that pyarrow reports, read + // the way a typed reader does: by field id, skipping the rest. + const struct = (r: Thrift.Reader, field: (id: number, type: Thrift.Type) => void) => { + r.readStructBegin() + for (let t = r.readFieldBegin(); t !== Thrift.Type.Stop; t = r.readFieldBegin()) field(r.fieldId, t) + r.readStructEnd() + } + const list = (r: Thrift.Reader, element: (index: number) => void) => { + for (let n = r.readListBegin(), i = 0; i < n; i++) element(i) + } + const footer = (bytes: Uint8Array) => + Result.getOrThrow(Thrift.decode(bytes, (r) => { + const out = { numRows: 0, columns: [] as Array, createdBy: "", dataPageOffsets: [] as Array } + // ColumnChunk → 3: ColumnMetaData → 9: data_page_offset + const columnChunk = () => + struct(r, (id, t) => { + if (id !== 3) return r.skip(t) + struct(r, (id, t) => id === 9 ? out.dataPageOffsets.push(r.readI64Number()) : r.skip(t)) + }) + // RowGroup → 1: list + const rowGroup = () => struct(r, (id, t) => id === 1 ? list(r, columnChunk) : r.skip(t)) + // SchemaElement → 4: name; element 0 is the root + const schemaElement = (i: number) => + struct(r, (id, t) => id === 4 && i > 0 ? out.columns.push(r.readString()) : r.skip(t)) + struct(r, (id, t) => { + if (id === 2) list(r, schemaElement) + else if (id === 3) out.numRows = r.readI64Number() + else if (id === 4) list(r, rowGroup) + else if (id === 6) out.createdBy = r.readString() + else r.skip(t) + }) + assert.strictEqual(r.position, bytes.length) + return out + })) + const bloom = footer(parquet("data_index_bloom_encoding_stats.parquet/footer")) + assert.strictEqual(bloom.numRows, 14) + assert.deepStrictEqual(bloom.columns, ["String"]) + assert.deepStrictEqual(bloom.dataPageOffsets, [4]) + assert.match(bloom.createdBy, /^parquet-mr version 1\.13\.0-SNAPSHOT/) + const tiny = footer(parquet("alltypes_tiny_pages.parquet/footer")) + assert.strictEqual(tiny.numRows, 7300) + assert.deepStrictEqual(tiny.columns, [ + "id", + "bool_col", + "tinyint_col", + "smallint_col", + "int_col", + "bigint_col", + "float_col", + "double_col", + "date_string_col", + "string_col", + "timestamp_col", + "year", + "month" + ]) + assert.strictEqual(tiny.dataPageOffsets[0], 4) + }) + }) + + describe("Reader", () => { + const roundTrip = (writeOne: (w: Thrift.Writer) => void, readOne: (r: Thrift.Reader) => A): A => { + const bytes = write(writeOne) + const r = Thrift.makeReader(bytes) + const back = readOne(r) + assert.strictEqual(r.position, bytes.length) + return back + } + + it.prop( + "i64 round-trips as bigint", + [Schema.BigInt.check( + Schema.isBetweenBigInt({ minimum: -(BigInt(1) << BigInt(63)), maximum: (BigInt(1) << BigInt(63)) - BigInt(1) }) + )], + ([n]) => { + assert.strictEqual(roundTrip((w) => w.writeI64(n), (r) => r.readI64()), n) + } + ) + + it.prop("i64 round-trips as number", [Schema.Int], ([n]) => { + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64Number()), n) + assert.strictEqual(roundTrip((w) => w.writeI64Number(n), (r) => r.readI64()), BigInt(n)) + }) + + it.prop("string round-trips through UTF-8", [Schema.String], ([s]) => { + // Lone surrogates have no UTF-8 encoding and become U+FFFD. + const expected = new TextDecoder("utf-8", { ignoreBOM: true }).decode(new TextEncoder().encode(s)) + assert.strictEqual(roundTrip((w) => w.writeString(s), (r) => r.readString()), expected) + }) + + it("keeps a leading byte order mark in a string", () => { + assert.strictEqual(roundTrip((w) => w.writeString("\uFEFFa"), (r) => r.readString()), "\uFEFFa") + }) + + it("reads i64 as a number only when it is a safe integer", () => { + const max = BigInt(Number.MAX_SAFE_INTEGER) + const asNumber = (n: bigint) => Thrift.decode(write((w) => w.writeI64(n)), (r) => r.readI64Number()) + assert.strictEqual(Result.getOrThrow(asNumber(max)), Number.MAX_SAFE_INTEGER) + assert.strictEqual(Result.getOrThrow(asNumber(-max)), -Number.MAX_SAFE_INTEGER) + assert.strictEqual(kindOf(asNumber(max + BigInt(1))), "Range") + assert.strictEqual(kindOf(asNumber(-max - BigInt(1))), "Range") + }) + + it("reads from a position set by the caller", () => { + // A Parquet page header sits at an offset inside the file. + const r = Thrift.makeReader(Uint8Array.of(0xff, 0xff, 0x15, 0x02, 0x00)) + r.position = 2 + assert.deepStrictEqual(r.readValue(Thrift.Type.Struct), { + _tag: "Struct", + fields: [{ id: 1, value: { _tag: "I32", value: 1 } }] + }) + assert.strictEqual(r.position, 5) + }) + + it("forgets a bool field header when the position moves", () => { + const r = Thrift.makeReader(Uint8Array.of(0x11, 0x02)) + r.readStructBegin() + assert.strictEqual(r.readFieldBegin(), Thrift.Type.Bool) + r.position = 1 + assert.isFalse(r.readBool()) + assert.strictEqual(r.position, 2) + }) + + it("reads binary as a view of the input", () => { + const bytes = Uint8Array.of(2, 7, 8) + const view = Thrift.makeReader(bytes).readBinary() + assert.strictEqual(view.buffer, bytes.buffer) + assert.deepStrictEqual(Array.from(view), [7, 8]) + }) + + it("reads a bool list element type written as 2", () => { + // Some writers use the FALSE code for the element type of list. + const r = Thrift.makeReader(Uint8Array.of(0x12, 0x01)) + assert.strictEqual(r.readListBegin(), 1) + assert.strictEqual(r.elementType, Thrift.Type.Bool) + assert.isTrue(r.readBool()) + }) + }) + + describe("hostile input", () => { + const struct = (r: Thrift.Reader) => r.readValue(Thrift.Type.Struct) + const table: ReadonlyArray< + readonly [string, string, (r: Thrift.Reader) => unknown, Thrift.ThriftError["kind"], Thrift.ReaderOptions?] + > = [ + ["empty input", "", struct, "Truncated"], + ["a struct without its stop", "1502", struct, "Truncated"], + ["an i32 varint over 5 bytes", "808080808001", (r) => r.readI32(), "Malformed"], + ["an i32 varint over 32 bits", "ffffffff1f", (r) => r.readI32(), "Malformed"], + ["an i64 varint over 10 bytes", "ffffffffffffffffff8101", (r) => r.readI64(), "Malformed"], + ["an i64 varint over 64 bits", "ffffffffffffffffff02", (r) => r.readI64(), "Malformed"], + // Other readers wrap these to 16 bits or keep them, so fields would alias. + ["an i16 over 16 bits", "808004", (r) => r.readI16(), "Malformed"], + ["a field id over 16 bits", "0580800400", struct, "Malformed"], + ["a field id delta past 32767", "05feff0300f50000", struct, "Malformed"], + ["an unknown field type", "1e", struct, "Malformed"], + ["an unknown list element type", "1e", (r) => r.readListBegin(), "Malformed"], + ["a non-empty list of Stop", "10", (r) => r.readListBegin(), "Malformed"], + ["a list larger than the input", "f5ffff03", (r) => r.readListBegin(), "Truncated"], + ["a map larger than the input", "0355020406", (r) => r.readMapBegin(), "Truncated"], + ["a binary longer than the input", "0561", (r) => r.readBinary(), "Truncated"], + ["a string that isn't UTF-8", "02c328", (r) => r.readString(), "InvalidUtf8"], + ["a binary over maxBinaryLength", "03616263", (r) => r.readBinary(), "Limit", { maxBinaryLength: 2 }], + ["a skipped binary over maxBinaryLength", "1803616263", (r) => r.skip(Thrift.Type.Struct), "Limit", { + maxBinaryLength: 2 + }], + ["a list over maxContainerSize", "35", (r) => r.readListBegin(), "Limit", { maxContainerSize: 2 }], + ["a struct over maxDepth", "1c1c000000", struct, "Limit", { maxDepth: 2 }], + ["lists over maxDepth", "1919191500", (r) => r.skip(Thrift.Type.List), "Limit", { maxDepth: 2 }] + ] + for (const [name, hex, read, kind, options] of table) { + it(`fails on ${name}`, () => assert.strictEqual(kindOf(Thrift.decode(fromHex(hex), read, options)), kind)) + } + + it("limits nesting to 64 by default and 1000 at most", () => { + const nest = (depth: number): Thrift.Value => { + let v: Thrift.Value = { _tag: "Struct", fields: [] } + for (let i = 1; i < depth; i++) v = { _tag: "Struct", fields: [{ id: 1, value: v }] } + return v + } + const bytesOf = (depth: number) => write((w) => w.writeValue(nest(depth))) + assert.isTrue(Result.isSuccess(readAll(bytesOf(64), "Struct"))) + assert.strictEqual(kindOf(readAll(bytesOf(65), "Struct")), "Limit") + assert.strictEqual(kindOf(Thrift.decode(bytesOf(65), (r) => r.skip(Thrift.Type.Struct))), "Limit") + assert.strictEqual(kindOf(readAll(bytesOf(1001), "Struct", { maxDepth: 1000 })), "Limit") + }) + + it("fails every truncated gold case with Truncated", () => { + for (const c of corpus) { + const bytes = fromHex(c.bytes) + for (let n = 0; n < bytes.length; n += Math.max(1, Math.floor(bytes.length / 16))) { + assert.strictEqual(kindOf(readAll(bytes.subarray(0, n), c.type)), "Truncated", `${c.id} at ${n}`) + } + } + }) + + it("only ever fails with ThriftError on corrupted input", () => { + // Deterministic xorshift so failures reproduce. + let seed = 0x5eed + const random = () => { + seed ^= seed << 13 + seed ^= seed >>> 17 + seed ^= seed << 5 + return (seed >>> 0) / 0x100000000 + } + let failures = 0 + for (const c of corpus) { + const original = fromHex(c.bytes) + const type = Thrift.Type[c.type] + for (let round = 0; round < 50; round++) { + const bytes = original.slice() + const flips = 1 + Math.floor(random() * 4) + for (let i = 0; i < flips && bytes.length > 0; i++) { + bytes[Math.floor(random() * bytes.length)] = Math.floor(random() * 256) + } + if (thrown(() => Thrift.makeReader(bytes).readValue(type)) !== undefined) failures++ + if (thrown(() => Thrift.makeReader(bytes).skip(type)) !== undefined) failures++ + } + } + assert.isAbove(failures, 0) + }) + + it("rejects limits that aren't integers in range", () => { + // NaN compares false with everything, so it would switch a limit off. + for ( + const options of [ + { maxDepth: NaN }, + { maxDepth: 0 }, + { maxDepth: 1.5 }, + { maxDepth: 1001 }, + { maxBinaryLength: -1 }, + { maxBinaryLength: NaN }, + { maxContainerSize: Infinity } + ] + ) { + assert.strictEqual( + thrown(() => Thrift.makeReader(new Uint8Array(0), options)), + "Usage", + Object.entries(options).join() + ) + } + }) + + it("decode rethrows errors that aren't ThriftErrors", () => { + assert.throws(() => + Thrift.decode(Uint8Array.of(0), () => { + throw new Error("boom") + }), "boom") + }) + + it("decode rethrows Usage errors, which are the caller's", () => { + const misuse: ReadonlyArray<(r: Thrift.Reader) => unknown> = [ + (r) => r.readStructEnd(), + (r) => r.position = -1, + (r) => r.position = 2, + (r) => r.position = 0.5, + (r) => r.skip(Thrift.Type.Stop), + (r) => r.readValue(7 as Thrift.Type) + ] + for (const f of misuse) { + assert.strictEqual(thrown(() => Thrift.decode(Uint8Array.of(0), f)), "Usage", String(f)) + } + }) + }) + + describe("Writer", () => { + const usage: ReadonlyArray void]> = [ + ["an i8 out of range", (w) => w.writeI8(128)], + ["a fractional i16", (w) => w.writeI16(1.5)], + ["an i32 out of range", (w) => w.writeI32(2 ** 31)], + ["an unsafe i64 number", (w) => w.writeI64Number(2 ** 53)], + ["an i64 out of range", (w) => w.writeI64(BigInt(1) << BigInt(63))], + ["a uuid that isn't 16 bytes", (w) => w.writeUuid(new Uint8Array(15))], + ["a field outside a struct", (w) => w.writeFieldBegin(Thrift.Type.I32, 1)], + ["a field id out of range", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.I32, 40000) + }], + ["a Stop field", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.Stop, 1) + }], + ["a bool field without writeBool", (w) => { + w.writeStructBegin() + w.writeFieldBegin(Thrift.Type.Bool, 1) + w.writeI32(1) + }], + ["writeStructEnd without writeStructBegin", (w) => w.writeStructEnd()], + ["finish inside a struct", (w) => { + w.writeStructBegin() + w.finish() + }], + ["a non-empty list of Stop", (w) => w.writeListBegin(Thrift.Type.Stop, 1)], + ["a negative list size", (w) => w.writeListBegin(Thrift.Type.I32, -1)], + [ + "a list element of another type", + (w) => w.writeValue({ _tag: "List", elementType: "I32", elements: [{ _tag: "I8", value: 1 }] }) + ], + ["a value of no Thrift type", (w) => w.writeValue({ _tag: "Text", value: "a" } as unknown as Thrift.Value)] + ] + for (const [name, f] of usage) { + it(`rejects ${name}`, () => assert.strictEqual(thrown(() => f(Thrift.makeWriter())), "Usage")) + } + + it("rejects an initialSize that isn't a non-negative integer", () => { + // A NaN size would never satisfy the loop that grows the buffer. + for (const initialSize of [NaN, Infinity, -1, 1.5]) { + assert.strictEqual(thrown(() => Thrift.makeWriter({ initialSize })), "Usage", String(initialSize)) + } + }) + + it("grows from a small buffer and returns a copy", () => { + const bytes = parquet("alltypes_tiny_pages.parquet/footer") + const w = Thrift.makeWriter({ initialSize: 1 }) + w.writeValue(Result.getOrThrow(readAll(bytes, "Struct"))) + const out = w.finish() + assert.deepStrictEqual(out, bytes) + assert.strictEqual(out.buffer.byteLength, out.length) + }) + }) +}) diff --git a/packages/effect/test/fixtures/parquet/CONFORMANCE.md b/packages/effect/test/fixtures/parquet/CONFORMANCE.md new file mode 100644 index 00000000000..2cf93f888bb --- /dev/null +++ b/packages/effect/test/fixtures/parquet/CONFORMANCE.md @@ -0,0 +1,115 @@ +# Parquet conformance + +The reader in `effect/encoding/Parquet` targets +[apache/parquet-format `apache-parquet-format-2.14.0`](https://github.com/apache/parquet-format/tree/apache-parquet-format-2.14.0). +References are section anchors of the Markdown specs and line numbers of +`src/main/thrift/parquet.thrift` at that tag, so they don't move. + +Every applicable assertion has an id and one test in +`packages/effect/test/encoding/Parquet.test.ts`, under +`conformance (apache-parquet-format-2.14.0)`, whose name starts with the id. +The pyarrow fixtures in this directory are interoperability evidence on top of +these tests, not a replacement for them. + +This is not a claim of full conformance: the sections below list what the +reader rejects, what it doesn't read yet, and what applies only to writers. + +## Tested + +| Id | Reference | Assertion | +| ----------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| PQ-FILE-1 | README.md#file-format | A file starts and ends with the magic number `PAR1` | +| PQ-FILE-2 | README.md#file-format | The metadata is found from the 4-byte little-endian length before the final magic | +| PQ-META-1 | parquet.thrift L1411-1413 | Readers should accept versions 1 and 2 interchangeably | +| PQ-META-2 | parquet.thrift L1440-1455 | `column_orders` has one entry per leaf column, in schema order | +| PQ-META-3 | parquet.thrift L1058-1059, L1426-1427 | `num_rows` of the file and of a row group is not negative (rejected when it is) | +| PQ-SCHEMA-1 | parquet.thrift L535-537 | Every schema node but the root has a repetition | +| PQ-SCHEMA-2 | parquet.thrift L518-523, L542-547 | The schema tree is the depth-first list of elements, groups giving `num_children` | +| PQ-LT-1 | LogicalTypes.md#timestamp | Unknown `TIMESTAMP` units are unsupported features, not errors in the file | +| PQ-LT-2 | parquet.thrift L370-373, L380-383 | `TimestampType.unit` and `TimeType.unit` are required (rejected when missing) | +| PQ-LT-3 | LogicalTypes.md#unsupported-logical-types | An unrecognized logical type is ignored, with its column's order and its node's converted type | +| PQ-LT-4 | LogicalTypes.md#unsupported-logical-types | An annotation on a physical type it isn't defined for is ignored, with the column order | +| PQ-LT-5 | LogicalTypes.md#convertedtype | Local policy: both are checked independently; an ignored logical type drops the converted type | +| PQ-LT-6 | LogicalTypes.md#decimal | `DECIMAL` precision and scale fit the rules and the physical type (rejected otherwise) | +| PQ-LT-7 | LogicalTypes.md#signed-integers, parquet.thrift L388 | `INTEGER` bit widths are 8, 16, 32 or 64 (rejected otherwise) | +| PQ-LT-8 | LogicalTypes.md#string, #enum, #json, parquet.thrift L50, L322, L326 | `STRING`, `ENUM` and `JSON` values, and those of their converted types, are UTF-8 (rejected otherwise) | +| PQ-LT-9 | LogicalTypes.md#string | Local policy: UTF-8 is the well-formed sequences of the Unicode Standard, which the spec doesn't define | +| PQ-LT-10 | Encodings.md#delta-strings-delta_byte_array--7 | `DELTA_BYTE_ARRAY` values are checked whole, after their prefix is added | +| PQ-LT-11 | LogicalTypes.md#string | Local policy: a dictionary is checked once with its page, values no page refers to included | +| PQ-LT-12 | LogicalTypes.md#bson, README.md#types | `BYTE_ARRAY` values with no text annotation in effect and `FIXED_LEN_BYTE_ARRAY` values are any bytes | +| PQ-NEST-1 | README.md#nested-encoding | Maximum levels follow from the optional and repeated nodes of the path | +| PQ-NEST-2 | README.md#nested-encoding | Repetition and definition levels encode lists (Dremel) | +| PQ-NULL-1 | README.md#nulls | Nulls are encoded in the definition levels only, not in the data | +| PQ-PAGE-1 | README.md#data-pages | A required, non-nested column's page holds only the values | +| PQ-PAGE-2 | README.md#data-pages | `uncompressed_page_size` covers levels and values together | +| PQ-PAGE-3 | parquet.thrift L707, L733, L754-772, L835-838 | A page header's counts and sizes are not negative (rejected when they are) | +| PQ-CHUNK-1 | README.md#column-chunks, parquet.thrift L727-729 | The dictionary page comes first, and there is at most one | +| PQ-CHUNK-2 | README.md#column-chunks | Readers can skip pages they don't read (index pages, later page types) | +| PQ-CHUNK-3 | parquet.thrift L922-923 | A column chunk's `num_values` is not negative (rejected when it is) | +| PQ-V2-1 | parquet.thrift L759-761 | Every v2 data page begins at a row boundary | +| PQ-V2-2 | parquet.thrift L744-745, L767 | v2 levels are uncompressed RLE without a length prefix | +| PQ-V2-3 | parquet.thrift L745, L779 | v2 values are compressed only when `is_compressed` is true | +| PQ-V2-4 | parquet.thrift L755-756 | A v2 page holds `num_values - num_nulls` values | +| PQ-STATS-1 | parquet.thrift L286-289 | Readers must distinguish an absent `null_count` from 0 | +| PQ-STATS-2 | parquet.thrift L315-316 | Readers must not assume `nan_count` is 0 when absent | +| PQ-PLAIN-1 | Encodings.md#plain-plain--0 | PLAIN encoding of every physical type | +| PQ-DICT-1 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | Indices are a bit-width byte (at most 32) and RLE/bit-packed runs | +| PQ-DICT-2 | Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 | The deprecated `PLAIN_DICTIONARY` is still read | +| PQ-RLE-1 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 1 | Bit-packed runs are packed from the least significant bit | +| PQ-RLE-2 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3 | Run lengths are at least 1 | +| PQ-RLE-3 | Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, length table | RLE booleans carry a 4-byte length in v1 and v2 pages | +| PQ-BP-1 | Encodings.md#bit-packed-deprecated-bit_packed--4, L182-195 | `BIT_PACKED` levels are packed from the most significant bit | +| PQ-BP-2 | Encodings.md#bit-packed-deprecated-bit_packed--4, L197-198 | `BIT_PACKED` is only for levels | +| PQ-DELTA-1 | Encodings.md#delta-encoding-delta_binary_packed--5, examples 1 and 2 | Header, blocks and miniblocks | +| PQ-DELTA-2 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary padding bits | +| PQ-DELTA-3 | Encodings.md#delta-encoding-delta_binary_packed--5 | Readers must accept arbitrary bit widths of unneeded miniblocks | +| PQ-DELTA-4 | Encodings.md#delta-encoding-delta_binary_packed--5 | Deltas wrap around in two's complement | +| PQ-DELTA-5 | Encodings.md#delta-encoding-delta_binary_packed--5 | Miniblocks are no wider than the physical type (rejected when they are) | +| PQ-DLBA-1 | Encodings.md#delta-length-byte-array-delta_length_byte_array--6 | Delta-encoded lengths, then the concatenated bytes | +| PQ-DBA-1 | Encodings.md#delta-strings-delta_byte_array--7 | Prefix lengths, then suffixes as `DELTA_LENGTH_BYTE_ARRAY` | +| PQ-DBA-2 | Encodings.md#delta-strings-delta_byte_array--7 | `FIXED_LEN_BYTE_ARRAY` values encode all their lengths | +| PQ-BSS-1 | Encodings.md#byte-stream-split-byte_stream_split--9 | Byte k of every value is in stream k | +| PQ-BSS-2 | Encodings.md#byte-stream-split-byte_stream_split--9 | The streams are exactly K * N bytes, with no padding | +| PQ-COMP-1 | Compression.md#codecs | Pages are decompressed with the column chunk's codec | +| PQ-COMP-2 | Compression.md#gzip | Readers should read GZIP pages with several members (through the supplied `Decompress`) | +| PQ-CRC-1 | README.md#checksumming, parquet.thrift L840-852 | A v1 data page's `crc` is of its stored bytes after the header (rejected on a mismatch) | +| PQ-CRC-2 | parquet.thrift L840-852 | A v2 data page's `crc` is of its levels and its values as stored | +| PQ-CRC-3 | parquet.thrift L840-852 | A dictionary page's `crc` is checked the same way | +| PQ-CRC-4 | README.md#checksumming, parquet.thrift L842-843 | The CRC is the standard CRC-32 of GZIP; the i32 of the header is read as unsigned | +| PQ-CRC-5 | parquet.thrift L844-845 | Local policy: the `crc` of a skipped page before the chunk's last value is checked too | + +## Unsupported: fails with `Unsupported` + +| Id | Reference | Feature | +| ---------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| PQ-UNSUP-1 | Encryption.md | Files with an encrypted footer (`PARE`) | +| PQ-UNSUP-2 | Encryption.md, parquet.thrift ColumnChunk fields 8-9 | Encrypted column chunks | +| PQ-UNSUP-3 | Encodings.md#adaptive-lossless-floating-point-alp--10 | The `ALP` encoding (in preview at 2.14.0) | +| PQ-UNSUP-4 | parquet.thrift ColumnChunk.file_path | Column chunks in another file | +| PQ-UNSUP-5 | Compression.md#codecs | Every codec but `UNCOMPRESSED` without a `Decompress`, and unknown codecs. The module has no codecs; the shared ones are planned in `effect/encoding/Compression` | +| PQ-UNSUP-6 | LogicalTypes.md#time | A `TIME` unit added after 2.14.0. Local policy: the spec states the rule for `TIMESTAMP` only (PQ-LT-1) | + +## Deferred: not read yet + +- Page index: `ColumnIndex` and `OffsetIndex` (PageIndex.md, parquet.thrift L1235-1373). The reader reads every page in order. +- Bloom filters (BloomFilter.md). +- Statistics semantics (parquet.thrift L1100-1219): sort orders, NaN rules and exactness are exposed as written, not interpreted. Size and geospatial statistics are skipped. +- Logical types (LogicalTypes.md): annotations are decoded into the schema and checked against the node they are on (PQ-LT-1 to PQ-LT-7). Values stay in their physical type. Apart from the UTF-8 of text (PQ-LT-8) they are not checked against the annotation: not that `JSON` parses, nor the range of a narrow `INTEGER` (which the spec leaves to the implementation), the digits of a `DECIMAL` or the nulls of `UNKNOWN`. The structure of `LIST`, `MAP`, `VARIANT` and `FILE` groups is not checked or interpreted, which includes the LIST and MAP backward-compatibility rules. Variant and Geometry values are not read. +- Error recovery (README.md#error-recovery): each `readColumn` call is independent, so a corrupt column chunk doesn't affect the others, but nothing inside a chunk is recovered. +- Binary protocol extensions (BinaryProtocolExtensions.md). + +## Not applicable: writer obligations + +- Writing in general: this module only reads. +- parquet.thrift L286 and L1338: writers should always write `null_count` and `null_counts`. +- parquet.thrift L1411-1413: writers should write version 1. +- Encodings.md#delta-encoding-delta_binary_packed--5: padding and unused bit widths should be zero; the reader side is PQ-DELTA-2 and PQ-DELTA-3. +- Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3, note 3: writers must not emit runs longer than 2^31 - 1. The reader keeps the stricter 32-bit run header limit. +- README.md#configurations, and the dictionary fallback in Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8. +- parquet.thrift L36: new writers should not write INT96. The reader still reads it (pyarrow fixture `int96.parquet`). + +## Resource limits + +The limits (`maxFooterLength`, `maxDepth`, `maxPageSize`, `maxValues`, +`maxBinaryLength`) aren't spec requirements. Their tests are under +`hostile input`, with the other rejections of malformed input. diff --git a/packages/effect/test/fixtures/parquet/README.md b/packages/effect/test/fixtures/parquet/README.md new file mode 100644 index 00000000000..c3743a9061f --- /dev/null +++ b/packages/effect/test/fixtures/parquet/README.md @@ -0,0 +1,31 @@ +# Parquet fixtures + +`generate.py` writes these files with pyarrow 26.0.1 (parquet-cpp, `created_by` +`parquet-cpp-arrow version 26.0.1`) and `gold.json`, which records for every +file its row groups (row count, codec and encodings of each column chunk) and +its leaf columns (path, physical type, levels, converted and logical type, and +the stored value of every row). The gold values are computed from the source +data, not read back from the files; each file is also read back with pyarrow +and compared to its source table. Small pages (64 or 128 bytes) give most +column chunks several pages. + +| File | Covers | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `plain.parquet` | Data page v1, PLAIN, 2 row groups: bool, required int32, int64, float, double, string, binary, fixed_size_binary(4), int8, uint16, date32, timestamp[ms, UTC], time64[us], decimal128(9, 2) | +| `int96.parquet` | A nullable INT96 timestamp column. `use_deprecated_int96_timestamps` applies to every timestamp in a file, so it has its own file | +| `dictionary.parquet` | Data page v2, dictionary encoding for int32, int64, double, string and fixed_size_binary(4) | +| `encodings.parquet` | Data page v1: DELTA_BINARY_PACKED (int32, int64, with wrapping deltas between min and max and two blocks per page), DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY (string with shared prefixes, fixed_size_binary(3)), BYTE_STREAM_SPLIT (float, double, int32, int64, fixed_size_binary(3)), RLE booleans | +| `encodings_v2.parquet` | The same table and encodings with data page v2 | +| `nested.parquet` | Data page v1, 2 row groups: list with null, empty and null-element lists, list>, nullable struct>, map, required struct, list> | +| `nested_v2.parquet` | The same table with data page v2 | +| `gzip.parquet` | GZIP, data page v2, dictionary: int64, string, list | +| `empty.parquet` | An int32 and a string column, zero rows | + +All files are uncompressed except `gzip.parquet`. pyarrow accepted every +requested encoding combination. + +Regenerate from `packages/effect`; the output is byte for byte the same: + +```sh +uv run --no-project --with pyarrow==26.0.1 python test/fixtures/parquet/generate.py +``` diff --git a/packages/effect/test/fixtures/parquet/dictionary.parquet b/packages/effect/test/fixtures/parquet/dictionary.parquet new file mode 100644 index 00000000000..cae0d3555d8 Binary files /dev/null and b/packages/effect/test/fixtures/parquet/dictionary.parquet differ diff --git a/packages/effect/test/fixtures/parquet/empty.parquet b/packages/effect/test/fixtures/parquet/empty.parquet new file mode 100644 index 00000000000..be96537f87d Binary files /dev/null and b/packages/effect/test/fixtures/parquet/empty.parquet differ diff --git a/packages/effect/test/fixtures/parquet/encodings.parquet b/packages/effect/test/fixtures/parquet/encodings.parquet new file mode 100644 index 00000000000..9e2fa62a59a Binary files /dev/null and b/packages/effect/test/fixtures/parquet/encodings.parquet differ diff --git a/packages/effect/test/fixtures/parquet/encodings_v2.parquet b/packages/effect/test/fixtures/parquet/encodings_v2.parquet new file mode 100644 index 00000000000..daaf007dcb3 Binary files /dev/null and b/packages/effect/test/fixtures/parquet/encodings_v2.parquet differ diff --git a/packages/effect/test/fixtures/parquet/generate.py b/packages/effect/test/fixtures/parquet/generate.py new file mode 100644 index 00000000000..bfae0379d62 --- /dev/null +++ b/packages/effect/test/fixtures/parquet/generate.py @@ -0,0 +1,319 @@ +""" +Generates the Parquet fixtures read by the effect Parquet reader tests, and +gold.json: for every file, its row groups (row count, codec and encodings of +each column chunk) and its leaf columns (schema facts and the stored value of +every row). README.md describes the files and how to regenerate them. + +A leaf value is the row's value projected onto the leaf path: a struct takes +one path component, a list or map two ("list", "element" / "key_value", +"key" | "value") and maps over its items, and a null anywhere is null. Leaf +scalars are encoded by Parquet physical type: + BOOLEAN true / false + INT32, FLOAT, DOUBLE number + INT64 decimal string + BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY hex of the stored bytes + INT96 hex of the 12 stored bytes +Gold values come from the source data below, not from reading the files back; +every file is also read back with pyarrow and compared to its source table. +""" + +import json +import random +import struct +from decimal import Decimal +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +HERE = Path(__file__).parent +I32_MIN, I32_MAX = -(2**31), 2**31 - 1 +I64_MIN, I64_MAX = -(2**63), 2**63 - 1 +NS_PER_DAY = 86_400 * 10**9 +JULIAN_UNIX_EPOCH = 2_440_588 + +rng = random.Random(0x9A4E) + + +def nulls(values, every, offset=0): + return [None if (i + offset) % every == 0 else v for i, v in enumerate(values)] + + +def f32(x): + return struct.unpack(" python values).""" + table = pa.table({f.name: pa.array(data[f.name], f.type) for f in schema}, schema=schema) + path = HERE / name + pq.write_table(table, path, write_batch_size=8, **options) + assert pq.read_table(path).equals(table), name + + meta = pq.ParquetFile(path).metadata + row_groups = [] + for i in range(meta.num_row_groups): + rg = meta.row_group(i) + row_groups.append({ + "numRows": rg.num_rows, + "columns": [ + {"codec": rg.column(j).compression, "encodings": sorted(set(rg.column(j).encodings))} + for j in range(rg.num_columns) + ], + }) + columns = [] + for i in range(meta.num_columns): + column = meta.schema.column(i) + top, *rest = column.path.split(".") + t = schema.field(top).type + columns.append({ + "path": [top, *rest], + "physicalType": column.physical_type, + "typeLength": column.length if column.physical_type == "FIXED_LEN_BYTE_ARRAY" else None, + "maxDefinitionLevel": column.max_definition_level, + "maxRepetitionLevel": column.max_repetition_level, + "convertedType": None if column.converted_type == "NONE" else column.converted_type, + "logicalType": logical_type(column), + "values": [project(v, t, rest, column) for v in data[top]], + }) + files.append({"name": name, "numRows": meta.num_rows, "rowGroups": row_groups, "columns": columns}) + + +# --- plain.parquet: one flat column per physical / logical type --------------- + +N = 40 +STRINGS = ["", "a", "hello", "héllo wörld", "日本語", "🌍", "parquet", "x" * 20] +plain = { + "bool": nulls([i % 3 == 0 for i in range(N)], 7), + "int32": [I32_MIN, I32_MAX, 0, -1] + [rng.randint(-10**6, 10**6) for _ in range(N - 4)], + "int64": nulls([I64_MIN, I64_MAX, 0, -1] + [rng.randint(-2**40, 2**40) for _ in range(N - 4)], 9, 5), + "float32": nulls([f32(rng.uniform(-1e3, 1e3)) for _ in range(N)], 8, 3), + "float64": nulls([rng.uniform(-1e9, 1e9) for _ in range(N - 2)] + [5e-324, 1.7976931348623157e308], 6, 2), + "string": nulls([STRINGS[i % len(STRINGS)] for i in range(N)], 11, 10), + "binary": nulls([bytes(rng.getrandbits(8) for _ in range(i % 6)) for i in range(N)], 5, 4), + "fixed_size_binary": nulls([struct.pack(">I", rng.getrandbits(32)) for _ in range(N)], 7, 1), + "int8": nulls([(-128, 127, 0)[i % 3] if i < 3 else rng.randint(-128, 127) for i in range(N)], 10, 6), + "uint16": nulls([65535 if i == 0 else rng.randint(0, 65535) for i in range(N)], 9), + "date32": nulls([rng.randint(-50_000, 50_000) for _ in range(N)], 8, 4), + "timestamp_ms_utc": nulls([rng.randint(-2**41, 2**41) for _ in range(N)], 7, 2), + "time64_us": nulls([rng.randint(0, 86_400 * 10**6 - 1) for _ in range(N)], 6), + "decimal": nulls([Decimal(rng.randint(-(10**9) + 1, 10**9 - 1)).scaleb(-2) for _ in range(N)], 5, 3), +} +PLAIN_SCHEMA = pa.schema([ + ("bool", pa.bool_()), + pa.field("int32", pa.int32(), nullable=False), + ("int64", pa.int64()), + ("float32", pa.float32()), + ("float64", pa.float64()), + ("string", pa.string()), + ("binary", pa.binary()), + ("fixed_size_binary", pa.binary(4)), + ("int8", pa.int8()), + ("uint16", pa.uint16()), + ("date32", pa.date32()), + ("timestamp_ms_utc", pa.timestamp("ms", tz="UTC")), + ("time64_us", pa.time64("us")), + ("decimal", pa.decimal128(9, 2)), +]) +FLAT = dict( + compression="NONE", data_page_version="1.0", use_dictionary=False, row_group_size=25, data_page_size=64 +) +write("plain.parquet", plain, PLAIN_SCHEMA, **FLAT) + +# use_deprecated_int96_timestamps applies to every timestamp column of a file. +int96 = {"timestamp_ns": nulls([I64_MIN + 1, I64_MAX, 0, -1, NS_PER_DAY - 1, -NS_PER_DAY] + + [rng.randint(-2**62, 2**62) for _ in range(N - 6)], 9, 7)} +write("int96.parquet", int96, pa.schema([("timestamp_ns", pa.timestamp("ns"))]), + use_deprecated_int96_timestamps=True, **FLAT) + +# --- dictionary.parquet ------------------------------------------------------- + +dictionary = { + "int32": nulls([(7, -3, I32_MAX)[i % 3] for i in range(N)], 5), + "int64": nulls([(I64_MIN, 42)[i % 4 // 3] for i in range(N)], 6, 1), + "float64": nulls([(0.5, -1.25, 3e10)[rng.randint(0, 2)] for _ in range(N)], 7, 2), + "string": nulls([("red", "green", "blue", "")[rng.randint(0, 3)] for _ in range(N)], 4, 3), + "fixed_size_binary": nulls([(b"\x00\x01\x02\x03", b"abcd")[i % 2] for i in range(N)], 8, 4), +} +write("dictionary.parquet", dictionary, pa.schema([ + ("int32", pa.int32()), + ("int64", pa.int64()), + ("float64", pa.float64()), + ("string", pa.string()), + ("fixed_size_binary", pa.binary(4)), +]), compression="NONE", data_page_version="2.0", use_dictionary=True, row_group_size=25, data_page_size=128) + +# --- encodings.parquet / encodings_v2.parquet --------------------------------- + +# Delta pages are cut only between 128-value blocks, so each delta column is +# one page of 160 values: a full block and a partial one. +M = 160 +WORDS = ["axis", "axle", "axolotl", "babble", "baby", "babyhood", "back", "backbone", "bad", "cab", "cabin"] + + +def walk(extremes): + values = [rng.randint(-1000, 1000)] + while len(values) < M: + values.append(values[-1] + rng.randint(-50, 60)) + values[100:100] = extremes # adjacent extremes give wrapping deltas + return values[:M] + + +encodings = { + "delta_int32": walk([I32_MIN, I32_MAX, I32_MIN, I32_MAX, 0]), + "delta_int64": nulls(walk([I64_MAX, I64_MIN, I64_MAX, 0, I64_MIN]), 13, 5), + "delta_length_string": nulls(["".join(rng.choice("abcé日 ") for _ in range(rng.randint(0, 5))) + for _ in range(M)], 17), + "delta_string": nulls([WORDS[i * 3 % len(WORDS)] + ("" if i % 4 else str(i)) for i in range(M)], 19, 3), + "delta_fixed_size_binary": [b"ab" + bytes([i % 7]) if i % 3 else bytes([i % 256, 0, 1]) for i in range(M)], + "bss_float32": nulls([rng.randint(-10**4, 10**4) / 16 for _ in range(M)], 11, 2), + "bss_float64": nulls([rng.randint(-10**6, 10**6) / 1024 for _ in range(M)], 7, 3), + "bss_int32": [I32_MIN, I32_MAX] + [rng.randint(-10**5, 10**5) for _ in range(M - 2)], + "bss_int64": nulls([I64_MIN, I64_MAX] + [rng.randint(-2**40, 2**40) for _ in range(M - 2)], 23, 9), + "bss_fixed_size_binary": nulls([bytes(rng.getrandbits(8) for _ in range(3)) for _ in range(M)], 29, 4), + "rle_bool": nulls([i % 10 < 7 for i in range(M)], 31, 6), +} +ENCODINGS_SCHEMA = pa.schema([ + pa.field("delta_int32", pa.int32(), nullable=False), + ("delta_int64", pa.int64()), + ("delta_length_string", pa.string()), + ("delta_string", pa.string()), + ("delta_fixed_size_binary", pa.binary(3)), + ("bss_float32", pa.float32()), + ("bss_float64", pa.float64()), + ("bss_int32", pa.int32()), + ("bss_int64", pa.int64()), + ("bss_fixed_size_binary", pa.binary(3)), + ("rle_bool", pa.bool_()), +]) +COLUMN_ENCODING = { + "delta_int32": "DELTA_BINARY_PACKED", + "delta_int64": "DELTA_BINARY_PACKED", + "delta_length_string": "DELTA_LENGTH_BYTE_ARRAY", + "delta_string": "DELTA_BYTE_ARRAY", + "delta_fixed_size_binary": "DELTA_BYTE_ARRAY", + "bss_float32": "BYTE_STREAM_SPLIT", + "bss_float64": "BYTE_STREAM_SPLIT", + "bss_int32": "BYTE_STREAM_SPLIT", + "bss_int64": "BYTE_STREAM_SPLIT", + "bss_fixed_size_binary": "BYTE_STREAM_SPLIT", + "rle_bool": "RLE", +} +for name, version in [("encodings.parquet", "1.0"), ("encodings_v2.parquet", "2.0")]: + write(name, encodings, ENCODINGS_SCHEMA, compression="NONE", data_page_version=version, use_dictionary=False, + column_encoding=COLUMN_ENCODING, data_page_size=1024) + +# --- nested.parquet / nested_v2.parquet --------------------------------------- + + +def maybe(i, every, value): + return None if i % every == 0 else value + + +nested = { + "ints": [[None, [], [None], [1, None, 3]][i] if i < 4 else + maybe(i, 9, [rng.randint(-100, 100) for _ in range(i % 4)]) for i in range(N)], + "nested_strings": [maybe(i, 7, [maybe(j + i, 5, [STRINGS[(i + j + k) % len(STRINGS)] for k in range(j)]) + for j in range(i % 3)]) for i in range(N)], + "struct": [maybe(i, 6, {"a": i * 1000 - 20_000, "b": maybe(i, 4, [maybe(k, 3, i * k - 2**40) for k in range(i % 5)])}) + for i in range(N)], + "map": [maybe(i, 8, [(f"k{j}", maybe(i + j, 4, i * j)) for j in range(i % 4)]) for i in range(N)], + "required_struct": [{"x": i * 0.5 - 7} for i in range(N)], + "structs": [maybe(i, 10, [maybe(i + j, 6, {"s": maybe(j, 4, STRINGS[(i * j) % len(STRINGS)]), "t": maybe(j, 3, i % 2 == 0)}) + for j in range(i % 4)]) for i in range(N)], +} +NESTED_SCHEMA = pa.schema([ + ("ints", pa.list_(pa.int32())), + ("nested_strings", pa.list_(pa.list_(pa.string()))), + ("struct", pa.struct([pa.field("a", pa.int32(), nullable=False), ("b", pa.list_(pa.int64()))])), + ("map", pa.map_(pa.string(), pa.int32())), + pa.field("required_struct", pa.struct([("x", pa.float64())]), nullable=False), + ("structs", pa.list_(pa.struct([("s", pa.string()), ("t", pa.bool_())]))), +]) +for name, version in [("nested.parquet", "1.0"), ("nested_v2.parquet", "2.0")]: + write(name, nested, NESTED_SCHEMA, compression="NONE", data_page_version=version, row_group_size=20, + data_page_size=128) + +# --- gzip.parquet / empty.parquet --------------------------------------------- + +gzip = { + "int64": nulls([rng.choice([5, -5, 2**50, I64_MIN]) for _ in range(N)], 7), + "string": nulls([STRINGS[i % 5] * (i % 3) for i in range(N)], 6, 2), + "ints": [maybe(i, 9, [i % 5, -(i % 3)] * (i % 3)) for i in range(N)], +} +write("gzip.parquet", gzip, pa.schema([("int64", pa.int64()), ("string", pa.string()), ("ints", pa.list_(pa.int32()))]), + compression="gzip", data_page_version="2.0", use_dictionary=True, row_group_size=25, data_page_size=128) + +write("empty.parquet", {"int32": [], "string": []}, pa.schema([("int32", pa.int32()), ("string", pa.string())]), + compression="NONE") + +(HERE / "gold.json").write_text(json.dumps({"writer": pa.__version__, "files": files}, separators=(",", ":")) + "\n") +print(", ".join(f"{f['name']} ({f['numRows']} rows)" for f in files)) diff --git a/packages/effect/test/fixtures/parquet/gold.json b/packages/effect/test/fixtures/parquet/gold.json new file mode 100644 index 00000000000..fdf2421640b --- /dev/null +++ b/packages/effect/test/fixtures/parquet/gold.json @@ -0,0 +1 @@ +{"writer":"26.0.1","files":[{"name":"plain.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]},{"numRows":15,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["bool"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,false,false,true,false,false,true,null,false,true,false,false,true,false,null,true,false,false,true,false,false,null,false,false,true,false,false,true,null,false,true,false,false,true,false,null,true,false,false,true]},{"path":["int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":0,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-2147483648,2147483647,0,-1,-2485,240748,289867,486454,-750849,-383303,507566,375400,941788,518714,-236855,259349,-852255,-744797,132897,-6338,-879326,775747,-937588,-650874,165936,462545,799320,294874,619374,742750,-907447,143913,-903073,-280663,813649,-693188,-809534,460108,-603483,545663]},{"path":["int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","9223372036854775807","0","-1",null,"-612517975526","1014778902415","311431409545","-202245119271","983832560846","-31291490818","837871651048","970588578440",null,"-235619389794","-721672261917","364739787284","229872738901","23654086610","-694745346433","-229461817390","702863645291",null,"-984134634385","816973333979","-1040843898097","351167913739","478024230911","941786591876","-636946374567","912800268601",null,"1056608115412","-202405809974","-827900779241","538401484164","822152175252","628224872673","297712530435","-866090127401"]},{"path":["float32"],"physicalType":"FLOAT","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-450.6831359863281,-299.3732604980469,600.4390869140625,-47.3121223449707,-141.6328125,null,538.801025390625,648.2783813476562,108.50069427490234,762.208984375,-365.00225830078125,-812.13525390625,113.71562194824219,null,-201.44508361816406,-50.45414733886719,-265.34210205078125,-3.591360092163086,874.5276489257812,-340.8632507324219,747.6163330078125,null,950.2943725585938,-831.8798217773438,-511.589599609375,-825.4498291015625,703.2380981445312,-100.73992919921875,-114.27555084228516,null,-923.2208862304688,481.6390380859375,-699.5462646484375,732.0736694335938,-855.2393188476562,-537.63330078125,-510.3900146484375,null,626.786865234375,-586.336669921875]},{"path":["float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-361948578.8402506,-39844526.547680736,462198887.2298949,-351572414.94397473,null,-540291037.2727698,-842337550.5208129,542572955.2318871,788407864.287821,167400813.1610577,null,-530703108.8940255,975408029.9216423,-977593887.954108,216775576.54046702,940647964.6998701,null,-758723214.8041923,309062622.1151371,683942435.2329497,767721777.7373397,38824400.26790285,null,588883746.7285874,-572492743.6244751,946169165.0968335,-625056550.9666383,-637809239.9030551,null,894357105.4446449,-399041737.85538054,785407526.5399792,-57431156.704161644,781951290.5817332,null,512244004.9062264,-637118366.8924098,274101705.2052095,5e-324,1.7976931348623157e+308]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["",null,"68656c6c6f","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574","7878787878787878787878787878787878787878","","61","68656c6c6f","68c3a96c6c6f2077c3b6726c64",null,"f09f8c8d","70617271756574","7878787878787878787878787878787878787878","","61","68656c6c6f","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574",null,"","61","68656c6c6f","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574","7878787878787878787878787878787878787878","","61",null,"68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9e","f09f8c8d","70617271756574","7878787878787878787878787878787878787878"]},{"path":["binary"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["",null,"8531","328f2f","54d89635","132f7fc867",null,"8f","99b6","5806e2","4db42857",null,"","3c","1444","f2351c",null,"d2785dadef","","3a","bee2",null,"5ecc8687","6d325a5425","","c5",null,"534126","746cd2ba","56a07ecf0b","",null,"bcd4","29097d","3b4e7065","80db33b220",null,"cc","c2ef","b11652"]},{"path":["fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":4,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["be685304","2aeb3486","d46ddd41","a298ecdc","52e48c4d","024a48b2",null,"30df6448","1edd9397","332374af","e055f523","557e69aa","7c0bd114",null,"84cb4198","f3e69f34","92799511","81685de3","73b1ecc8","ca0f16c5",null,"3edeabef","c280635b","e2a57a95","7fcaf6e9","452e8b90","620d6b3a",null,"48de7b54","2976b061","3d8abd9e","29bdc553","92b8f589","2fe96d19",null,"0545516d","1895cb96","abe13fa9","1de21521","57a6237d"]},{"path":["int8"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"INT_8","logicalType":{"type":"INTEGER","bitWidth":8,"isSigned":true},"values":[-128,127,0,-65,null,-91,4,-17,49,-51,49,64,111,84,null,-65,0,-122,-57,-63,-102,-32,3,-86,null,-5,-42,116,-111,-32,-118,-111,77,67,null,-32,-52,7,96,13]},{"path":["uint16"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UINT_16","logicalType":{"type":"INTEGER","bitWidth":16,"isSigned":false},"values":[null,46435,43512,58879,34506,62158,57263,34426,19311,null,50138,13517,28563,52386,43984,62883,22592,1375,null,12247,34577,8629,45224,24098,23030,19699,3810,null,22697,6343,43936,35597,16595,55979,48814,25057,null,58807,44144,12998]},{"path":["date32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"DATE","logicalType":{"type":"DATE"},"values":[2011,-528,2237,-30841,null,16171,37455,29178,-44273,-23872,-9776,-11965,null,-14072,-47147,1770,10891,4855,-1796,-42665,null,-9862,-47911,-8056,36150,-34656,-20640,-13481,null,-3350,30965,-5844,-25409,35294,13387,21373,null,-26135,-10208,8331]},{"path":["timestamp_ms_utc"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"TIMESTAMP_MILLIS","logicalType":{"type":"TIMESTAMP","isAdjustedToUTC":true,"unit":"MILLIS"},"values":["-887934534497","-179353769780","-1745779504790","2101404986895","-1701787662337",null,"-1771210649084","-893324018340","2129597418792","-1532568076546","-1062417404954","-1531271410110",null,"-1030157107266","801765703572","-1772851035818","1806680405144","-369682922021","-96310095459",null,"-1565897530671","-2021567124130","-835828118937","-516351822760","678737883977","-1248421768682",null,"772843221303","1865999106498","-2088949775672","867424331385","23918359260","-564009072615",null,"-1263572046450","1463908715091","1346894376114","1611319502689","1680974754938","1857540507649"]},{"path":["time64_us"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":{"type":"TIME","isAdjustedToUTC":false,"unit":"MICROS"},"values":[null,"7658039589","55294262265","22172311358","50783133622","74027678352",null,"33811403730","22709832814","35332610090","80892681096","57056123678",null,"18690423441","35307860611","54300400780","27254171160","62724332812",null,"10448801251","28714564282","49131855093","78812007313","20133576206",null,"9680375702","885974325","35538329740","22092052588","26420901929",null,"24415876618","46612475354","34170021051","33122380013","21050186942",null,"72181531332","52804541502","70749633083"]},{"path":["decimal"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":4,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"DECIMAL","logicalType":{"type":"DECIMAL","scale":2,"precision":9},"values":["2f60c000","f0f3e297",null,"1fa02eb5","c88542a0","d0f13757","2ab91f51",null,"e6f4ae37","cbc68f2e","f41ce627","e1ab0a41",null,"03fe6e19","23fa4acb","0eb51767","06b45c7d",null,"2ccc5af0","0ea713c8","06ae9b0e","fd867600",null,"f3570659","f699f594","c4c92630","f482e0e4",null,"2196d40a","33cb2887","0a9be157","15f9c56a",null,"366507df","d6fd8482","00977fc7","e7770008",null,"07af039a","019c6c3c"]}]},{"name":"int96.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]},{"numRows":15,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["timestamp_ns"],"physicalType":"INT96","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["010000afb10000008c9c2300","ffff4ee2e24d00008bde2600",null,"ffff4e91944e00008b3d2500","ffff4e91944e00008c3d2500","00000000000000008b3d2500","27011d89872500008b602500","c29993cbcf3f0000d8dd2500","0f20813ac83f0000c94c2500","fba507097a4e00000ebb2500","385cd982a30c00005d822500",null,"5f3ba79f144100008f782500","33300b966f1b00000e892500","8111f96d972e0000cab92400","e868847c022d000059482500","d1b2337bbb2f000002702400","80dd67c2a8490000f4db2400","9c85791c3801000051782500","6cb80a75313800004e282500",null,"8cab2ff5711a000034fb2400","41654702fb18000009e12400","ae1b1276e107000006c72400","9e050cd9872d0000d7ec2500","01918e857f170000727c2500","e4ac16e4ce460000e6c62400","4231f8b1fd3b00005fbc2500","7da49dd10a460000f6082500",null,"27bdb965cf270000f2d82500","f3906b4280250000709a2400","fe37af707e33000029a02500","70fdb2322b1300003bf92400","eaa52d66c842000059dd2400","d39be29f2c23000093172500","dc2b142ac71d00002b5d2500","4787e840a91d00006dd02400",null,"c6b98d605d31000036aa2500"]}]},{"name":"dictionary.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]},{"numRows":15,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]}],"columns":[{"path":["int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,-3,2147483647,7,-3,null,7,-3,2147483647,7,null,2147483647,7,-3,2147483647,null,-3,2147483647,7,-3,null,7,-3,2147483647,7,null,2147483647,7,-3,2147483647,null,-3,2147483647,7,-3,null,7,-3,2147483647,7]},{"path":["int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","-9223372036854775808","-9223372036854775808","42","-9223372036854775808",null,"-9223372036854775808","42","-9223372036854775808","-9223372036854775808","-9223372036854775808",null,"-9223372036854775808","-9223372036854775808","-9223372036854775808","42","-9223372036854775808",null,"-9223372036854775808","42","-9223372036854775808","-9223372036854775808","-9223372036854775808",null,"-9223372036854775808","-9223372036854775808","-9223372036854775808","42","-9223372036854775808",null,"-9223372036854775808","42","-9223372036854775808","-9223372036854775808","-9223372036854775808",null,"-9223372036854775808","-9223372036854775808","-9223372036854775808","42"]},{"path":["float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[30000000000.0,-1.25,30000000000.0,0.5,0.5,null,-1.25,-1.25,0.5,30000000000.0,-1.25,-1.25,null,30000000000.0,30000000000.0,0.5,0.5,0.5,-1.25,null,30000000000.0,-1.25,0.5,30000000000.0,30000000000.0,30000000000.0,null,30000000000.0,0.5,0.5,-1.25,30000000000.0,0.5,null,30000000000.0,30000000000.0,0.5,0.5,30000000000.0,0.5]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["726564",null,"726564","726564","",null,"","726564","726564",null,"726564","726564","726564",null,"726564","626c7565","",null,"","677265656e","626c7565",null,"626c7565","626c7565","726564",null,"","726564","726564",null,"677265656e","626c7565","626c7565",null,"677265656e","","726564",null,"","726564"]},{"path":["fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":4,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364","00010203","61626364","00010203","61626364",null,"61626364","00010203","61626364"]}]},{"name":"encodings.parquet","numRows":160,"rowGroups":[{"numRows":160,"columns":[{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_LENGTH_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]}],"columns":[{"path":["delta_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":0,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[346,352,338,311,324,355,366,426,451,420,399,351,308,362,343,307,357,362,387,389,421,431,469,499,451,402,413,373,405,395,374,390,450,413,410,365,420,421,432,392,369,381,374,407,414,385,405,439,413,419,391,422,450,480,465,505,551,510,543,544,554,559,548,606,564,589,605,623,605,625,685,645,601,551,611,651,707,765,790,808,774,818,801,769,722,733,723,678,693,701,723,677,730,714,723,705,740,693,674,660,-2147483648,2147483647,-2147483648,2147483647,0,705,716,705,719,757,760,792,760,791,815,869,845,848,886,894,844,855,831,871,858,827,822,774,758,750,768,780,751,702,677,646,690,748,712,747,777,754,799,845,814,868,905,895,882,836,818,850,885,836,791,847,859,868,819,863]},{"path":["delta_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["995","1000","1056","1014","1015","994","996","992",null,"985","982","1033","1012","1063","1058","1112","1113","1110","1063","1065","1123",null,"1069","1062","1035","1076","1086","1061","1083","1067","1025","1020","973","937",null,"873","904","942","995","1018","1020","1048","1061","1070","1092","1129","1174",null,"1279","1308","1293","1310","1288","1258","1227","1187","1233","1185","1160","1120",null,"1121","1083","1123","1168","1147","1128","1129","1126","1077","1036","1070","1085",null,"1144","1156","1110","1082","1120","1082","1032","1075","1048","1034","998","1058",null,"1016","1017","1042","1037","1047","1101","1158","1194","1204","1241","1221","1274",null,"9223372036854775807","-9223372036854775808","9223372036854775807","0","-9223372036854775808","1309","1259","1274","1310","1328","1322","1377",null,"1391","1361","1360","1352","1308","1355","1327","1352","1406","1410","1365","1326",null,"1412","1373","1331","1314","1277","1240","1204","1176","1235","1211","1226","1258",null,"1272","1257","1284","1254","1254","1243","1233","1276","1256","1228","1249","1229",null,"1248","1262","1254","1278","1250","1277","1295","1279"]},{"path":["delta_length_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,"e697a5e697a5","61626362","2062c3a9","e697a563e697a5","e697a56163","62c3a920","61c3a9","20","","6161e697a5","6162","61e697a563","e697a5c3a961","61","c3a9e697a563e697a5","63c3a92020",null,"61e697a5","63626120e697a5","6320622061","6261","c3a920","e697a562e697a5","c3a9e697a5","6262c3a9","6320612063","e697a5","e697a561616263","636263c3a9e697a5","63c3a920e697a563","e697a5c3a9","e697a520e697a56363","2061",null,"e697a5","e697a5c3a920","","e697a5c3a920","6162","20206120","20","62e697a5","e697a56361","636162","20c3a9","61c3a961","e697a562616162","62622061","2061c3a96162","61",null,"20c3a961","c3a9","632061","c3a9","6320e697a562","62206120e697a5","e697a5","","e697a5","c3a9","","6120","2062e697a56261","c3a9","61","c3a9",null,"6161","","","","61e697a5","c3a96262","","","63626361","","c3a96161","6320","c3a9c3a9636163","2063","62c3a96162","63",null,"20e697a5","6261c3a9","2020206262","20616263","20e697a520c3a9e697a5","c3a961c3a963e697a5","","616161","202062e697a5","20e697a5","","61","c3a961e697a520","6362","20","",null,"","c3a9e697a520c3a9","20206261c3a9","c3a961","6261e697a563","e697a5c3a9c3a961","62c3a9e697a563","6263e697a5","","6120c3a9","61c3a962c3a963","62636363e697a5","61","e697a5","63c3a9e697a5","e697a5e697a5636120",null,"62","e697a562","62","62c3a9636120","626262e697a5e697a5","6263","62c3a962e697a520","62","e697a5c3a962","62e697a5","62","e697a520616161","6361e697a561","61","e697a520","c3a962c3a9",null,"c3a9206363e697a5","e697a56162","6262e697a5","2062c3a963c3a9","616261e697a5e697a5","616362e697a5","c3a96162e697a5","61e697a5c3a9","6361","63c3a9626262","e697a5c3a9c3a9","e697a52062","20","","6120c3a92020","20e697a52061c3a9",null,"e697a561","636361","6362c3a9","e697a5c3a92061","","626262e697a562"]},{"path":["delta_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["6178697330","626162626c65","6261636b","636162","61786c6534","62616279","6261636b626f6e65","636162696e","61786f6c6f746c38","62616279686f6f64","626164","61786973","626162626c653132","6261636b","636162","61786c65",null,"6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643230","626164","61786973","626162626c65","6261636b3234","636162","61786c65","62616279","6261636b626f6e653238","636162696e","61786f6c6f746c","62616279686f6f64","6261643332","61786973","626162626c65",null,"6361623336","61786c65","62616279","6261636b626f6e65","636162696e3430","61786f6c6f746c","62616279686f6f64","626164","617869733434","626162626c65","6261636b","636162","61786c653438","62616279","6261636b626f6e65","636162696e","61786f6c6f746c3532","62616279686f6f64",null,"61786973","626162626c653536","6261636b","636162","61786c65","626162793630","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643634","626164","61786973","626162626c65","6261636b3638","636162","61786c65","62616279","6261636b626f6e653732",null,"61786f6c6f746c","62616279686f6f64","6261643736","61786973","626162626c65","6261636b","6361623830","61786c65","62616279","6261636b626f6e65","636162696e3834","61786f6c6f746c","62616279686f6f64","626164","617869733838","626162626c65","6261636b","636162",null,"62616279","6261636b626f6e65","636162696e","61786f6c6f746c3936","62616279686f6f64","626164","61786973","626162626c65313030","6261636b","636162","61786c65","62616279313034","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f64313038","626164","61786973",null,"6261636b313132","636162","61786c65","62616279","6261636b626f6e65313136","636162696e","61786f6c6f746c","62616279686f6f64","626164313230","61786973","626162626c65","6261636b","636162313234","61786c65","62616279","6261636b626f6e65","636162696e313238","61786f6c6f746c",null,"626164","61786973313332","626162626c65","6261636b","636162","61786c65313336","62616279","6261636b626f6e65","636162696e","61786f6c6f746c313430","62616279686f6f64","626164","61786973","626162626c65313434","6261636b","636162","61786c65","62616279313438",null,"636162696e","61786f6c6f746c","62616279686f6f64313532","626164","61786973","626162626c65","6261636b313536","636162","61786c65","62616279"]},{"path":["delta_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["000001","616201","616202","030001","616204","616205","060001","616200","616201","090001","616203","616204","0c0001","616206","616200","0f0001","616202","616203","120001","616205","616206","150001","616201","616202","180001","616204","616205","1b0001","616200","616201","1e0001","616203","616204","210001","616206","616200","240001","616202","616203","270001","616205","616206","2a0001","616201","616202","2d0001","616204","616205","300001","616200","616201","330001","616203","616204","360001","616206","616200","390001","616202","616203","3c0001","616205","616206","3f0001","616201","616202","420001","616204","616205","450001","616200","616201","480001","616203","616204","4b0001","616206","616200","4e0001","616202","616203","510001","616205","616206","540001","616201","616202","570001","616204","616205","5a0001","616200","616201","5d0001","616203","616204","600001","616206","616200","630001","616202","616203","660001","616205","616206","690001","616201","616202","6c0001","616204","616205","6f0001","616200","616201","720001","616203","616204","750001","616206","616200","780001","616202","616203","7b0001","616205","616206","7e0001","616201","616202","810001","616204","616205","840001","616200","616201","870001","616203","616204","8a0001","616206","616200","8d0001","616202","616203","900001","616205","616206","930001","616201","616202","960001","616204","616205","990001","616200","616201","9c0001","616203","616204","9f0001"]},{"path":["bss_float32"],"physicalType":"FLOAT","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-43.0,43.3125,-22.0625,316.5,-439.9375,-127.3125,18.8125,598.625,623.0,null,420.375,-139.0625,-436.5625,-220.0625,463.9375,586.4375,66.125,414.5625,-337.3125,-129.3125,null,-156.5,-192.625,-19.0625,-254.125,-182.625,512.5625,15.375,227.5625,-61.8125,46.6875,null,365.3125,-375.25,-34.8125,-98.0,596.5625,-375.8125,593.8125,-59.625,-31.1875,366.3125,null,-175.0625,-503.375,70.0625,360.375,307.5625,347.0625,-342.5,-552.125,624.4375,376.125,null,-90.6875,356.5,605.6875,331.3125,555.4375,-249.0,-436.75,-149.6875,193.0,521.5,null,-210.4375,-8.75,75.25,-306.25,599.4375,618.125,482.6875,-42.5625,235.0,-507.625,null,-431.4375,-389.0,-447.0,-393.625,425.0,-388.125,-427.0,435.4375,-455.125,-316.5625,null,-193.9375,-515.125,27.5625,-197.1875,116.3125,-19.6875,487.375,58.0,6.875,-391.5625,null,-214.0625,493.625,265.375,410.5,96.1875,-123.125,-372.8125,-331.0,182.9375,-223.0625,null,4.5625,218.625,422.3125,-617.8125,235.5625,-302.8125,-575.5,-621.125,-270.5625,-470.875,null,212.375,125.6875,5.8125,409.5625,243.1875,-339.25,-40.0,544.6875,340.625,-455.8125,null,372.5,-107.6875,-326.8125,-421.125,-260.0625,-382.6875,526.9375,365.375,-102.125,455.6875,null,329.125,526.625,-33.8125,-126.125,157.25,-252.8125,504.0625,-193.8125,457.0625,-393.875,null,-187.5,-386.9375,-306.875,-81.9375,-591.6875,100.375,-242.4375]},{"path":["bss_float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[367.146484375,216.3037109375,-45.404296875,-246.9521484375,null,-939.69140625,-200.38671875,-704.7138671875,775.865234375,922.1630859375,-521.6357421875,null,602.3759765625,-648.45703125,-219.9970703125,59.5732421875,-777.11328125,888.3837890625,null,-743.2919921875,-618.986328125,-459.5986328125,-37.6669921875,-508.603515625,251.0556640625,null,-224.1376953125,972.560546875,-296.939453125,-637.6865234375,-690.7109375,118.5244140625,null,-668.4931640625,18.4228515625,628.2099609375,387.2919921875,-560.029296875,-882.931640625,null,531.5068359375,-386.9130859375,-201.7763671875,268.939453125,-682.5419921875,-142.603515625,null,370.99609375,-362.736328125,42.3154296875,-254.3662109375,655.669921875,694.8515625,null,-101.5029296875,533.9765625,813.228515625,166.6044921875,191.0478515625,-95.1650390625,null,937.26953125,-246.8505859375,-612.4248046875,99.2333984375,-497.138671875,-349.3037109375,null,-349.6103515625,349.333984375,705.724609375,482.576171875,-271.2900390625,-764.9677734375,null,-678.041015625,-10.9296875,342.4619140625,959.15234375,607.236328125,-445.1103515625,null,711.8271484375,-740.19921875,35.1142578125,-123.166015625,9.7666015625,-555.3564453125,null,769.6357421875,394.380859375,-317.14453125,-705.2509765625,815.4677734375,-97.205078125,null,-555.1708984375,529.1474609375,244.80078125,-148.810546875,837.091796875,358.9521484375,null,-511.701171875,194.2822265625,277.2734375,327.8388671875,-315.7138671875,-464.7490234375,null,-614.7822265625,606.330078125,476.701171875,-336.1572265625,-378.59765625,515.888671875,null,-813.3203125,-59.9111328125,583.8701171875,-812.3291015625,217.8701171875,876.6845703125,null,-42.65625,851.103515625,-369.6923828125,-623.7890625,99.3505859375,720.28515625,null,-828.4794921875,90.01953125,-371.2607421875,10.7314453125,405.2333984375,237.5390625,null,284.154296875,90.720703125,-89.6220703125,-377.3857421875,676.6484375,199.1337890625,null,-255.5400390625,888.41796875,-268.052734375,12.4375,581.1142578125,341.697265625,null,755.982421875,813.00390625,-767.6181640625,-130.4541015625,476.11328125,-39.353515625,null,803.1201171875]},{"path":["bss_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-2147483648,2147483647,24630,-8252,-88741,279,-80859,-79846,-88224,11214,-72621,41673,94160,-72079,-75456,49714,-28221,-69862,35645,9817,15468,70413,76991,89996,-77851,-19432,36584,-36803,-18873,-27782,4874,16048,88154,-57213,-15930,97013,-64599,63937,-10116,31204,41596,-58577,8126,-65323,-19876,-37092,-43312,-89958,-19789,27103,22296,19318,69678,-10861,39994,-8405,-27595,37714,78587,27400,88771,22216,83967,51193,-12492,11652,47228,84674,42964,-8228,73119,-64033,-99646,90841,23044,-90804,79377,-44780,19843,18044,-74864,21640,-57604,-40662,-54977,-98373,-18308,67031,30243,97549,97982,51741,95370,-75576,80731,80124,55178,89130,-30615,-2998,52618,70509,-20366,-96531,33925,80541,-76437,28102,47292,68212,13253,-28233,50059,10424,67564,-65300,-25483,94973,83740,-58425,-57885,-32337,50174,-70539,-93538,-74075,73099,-51392,-55340,-70481,-61805,9271,-70489,-88830,23499,-49353,-46996,-50008,-16437,21629,26720,89281,71447,21080,1327,-12061,-56619,-13475,1719,95956,77982,-18198,87923,53004,-44195,-69179,36258,-27110,18966,-55393]},{"path":["bss_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","9223372036854775807","987863054100","-296709105316","797163188552","283198447560","210302952758","-242326649351","682448063443","-133756023825","1087324371926","992378598252","202450873419","-163663339987",null,"-851188577698","-175190867480","-568420662460","888396909884","-967582777543","-438169682332","464692519730","867134292246","278899198129","253340965962","-802603576546","-81558278880","356243668616","617686406039","219022649198","-990766883952","-32296480474","358083665127","-971286852589","-378225370839","797453355920","-807104191328",null,"-390682297362","-1092497215362","-978571312995","-91580169782","1066730189811","430481217755","307658968460","-302016529785","910846036120","-262418637231","-410538560962","695107625814","540223280584","337319050436","-997323342453","230232436340","-1062128698264","-298926060898","-410931378914","-736778008989","-65991568603","851673167648",null,"234018773466","235019331533","605069254338","-1014013373950","628920372329","-687878596681","-708899400047","-85900646935","312486659162","-567271953017","1019981739437","693375127142","-948621157977","-1073149451028","755298532586","-1081886191699","963462205821","-24009810685","847648551381","-527998211714","-762169547994","319743072226",null,"-755692925920","121169648679","-1040928921954","251238298346","1015370757681","-194980851552","822769963093","-133418024571","-564926299881","-660269138406","1056900410002","239753541378","-92796144166","-687408872136","-403448570799","987893209335","102662608696","528580901162","957056644830","-254655530643","-21421591238","366790029881",null,"-710728631032","483739032797","-225612438291","992590595094","-391556708723","441239395898","227561934972","420505691518","-219258864319","-836798736275","-263566729195","593315416161","127055598463","510154441724","-326140058306","-379289775153","1079181312388","118834101207","-1010855571923","-960810927306","-271852908247","-871532118498",null,"-609178482600","-145211802073","-407971053503","974973273544","-418132649076","-845833554356","-907607164810","-821233710697","-611538191768","-59725661798","-188190488612","1051278023789","-486014714752","-420053141405","670152470078","-878859760470","-522447514654","681864633933","562381935015","138329833268","875971386725","-967975647744",null,"-469328883599","701452300781","-1016314675471","-428111787833","1091174619886","-501130950046","514645087743"]},{"path":["bss_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["aacfc4","414c98","04274c","20cea8","aaf446","bff03e","ad6cc2","0f32b5","69f5c6","21b4c7","8c9325","99905f","8ec640","a2d58f","9753b6","053825","f72a1a","83416b","3e04ff","42cc67","082117","e64bc7","e6ca6f","a553de","592678",null,"6b1331","64733d","c9a7c8","79f0ba","26675f","10ecdb","b36d78","6c54c5","14ebce","2426c3","a84823","20fd34","ce3fc9","4ec436","2758c3","ad640f","de41e5","1931fb","66f81c","eb1465","48ed00","383de9","58d935","0996a6","df2d17","73bd80","e2bce5","699c6a",null,"844b90","935ff9","facec5","46dadc","57a0ca","e55153","a685f3","c842b0","e42886","8ca306","11dede","f7cef8","725728","cd2dbe","424d36","8f619b","aadc45","474b82","2a4c0f","7b3470","b1a09b","02ecf2","10f84b","94c9a6","438720","dac3ba","597fdd","590000",null,"2a2b18","2e2164","b2c489","9722ed","ff67ba","9c9f7a","7697c6","a1f865","abba76","c44f0c","9f887b","cb87d8","77f255","1b9d85","b16bf6","bc1f67","4ad820","aa0ecd","667475","094bb5","e053b7","15a6d9","298b89","632c49","974fdd","86e4a8","87f8e7","865966",null,"6e1ce3","9b829d","71d715","989afc","12c8c3","d0500e","706596","bc6b3e","9d1fbf","7ca7e8","9109f7","4d0ad6","ebf77b","d9ba59","46237b","f7a01c","0948e5","8b1398","d4c9c8","9dbc3a","36b566","efc0dd","e564db","6d7268","2362a3","ae5f34","05d20f","8f5d1d",null,"c77a35","3180d2","39e592","2d7a7c","0555cd","3db2ea","ea2b83","348b82","4f7a35","d21428","5ea75b","6c29a1","9b474f","3c98bb","62f80e","1a4a5b","cb50f7","d35d35"]},{"path":["rle_bool"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,null,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,null,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,null,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,null,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,null,true,true,true,true,true,true,true,false,false,false]}]},{"name":"encodings_v2.parquet","numRows":160,"rowGroups":[{"numRows":160,"columns":[{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_LENGTH_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["BYTE_STREAM_SPLIT","RLE"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]}],"columns":[{"path":["delta_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":0,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[346,352,338,311,324,355,366,426,451,420,399,351,308,362,343,307,357,362,387,389,421,431,469,499,451,402,413,373,405,395,374,390,450,413,410,365,420,421,432,392,369,381,374,407,414,385,405,439,413,419,391,422,450,480,465,505,551,510,543,544,554,559,548,606,564,589,605,623,605,625,685,645,601,551,611,651,707,765,790,808,774,818,801,769,722,733,723,678,693,701,723,677,730,714,723,705,740,693,674,660,-2147483648,2147483647,-2147483648,2147483647,0,705,716,705,719,757,760,792,760,791,815,869,845,848,886,894,844,855,831,871,858,827,822,774,758,750,768,780,751,702,677,646,690,748,712,747,777,754,799,845,814,868,905,895,882,836,818,850,885,836,791,847,859,868,819,863]},{"path":["delta_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["995","1000","1056","1014","1015","994","996","992",null,"985","982","1033","1012","1063","1058","1112","1113","1110","1063","1065","1123",null,"1069","1062","1035","1076","1086","1061","1083","1067","1025","1020","973","937",null,"873","904","942","995","1018","1020","1048","1061","1070","1092","1129","1174",null,"1279","1308","1293","1310","1288","1258","1227","1187","1233","1185","1160","1120",null,"1121","1083","1123","1168","1147","1128","1129","1126","1077","1036","1070","1085",null,"1144","1156","1110","1082","1120","1082","1032","1075","1048","1034","998","1058",null,"1016","1017","1042","1037","1047","1101","1158","1194","1204","1241","1221","1274",null,"9223372036854775807","-9223372036854775808","9223372036854775807","0","-9223372036854775808","1309","1259","1274","1310","1328","1322","1377",null,"1391","1361","1360","1352","1308","1355","1327","1352","1406","1410","1365","1326",null,"1412","1373","1331","1314","1277","1240","1204","1176","1235","1211","1226","1258",null,"1272","1257","1284","1254","1254","1243","1233","1276","1256","1228","1249","1229",null,"1248","1262","1254","1278","1250","1277","1295","1279"]},{"path":["delta_length_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,"e697a5e697a5","61626362","2062c3a9","e697a563e697a5","e697a56163","62c3a920","61c3a9","20","","6161e697a5","6162","61e697a563","e697a5c3a961","61","c3a9e697a563e697a5","63c3a92020",null,"61e697a5","63626120e697a5","6320622061","6261","c3a920","e697a562e697a5","c3a9e697a5","6262c3a9","6320612063","e697a5","e697a561616263","636263c3a9e697a5","63c3a920e697a563","e697a5c3a9","e697a520e697a56363","2061",null,"e697a5","e697a5c3a920","","e697a5c3a920","6162","20206120","20","62e697a5","e697a56361","636162","20c3a9","61c3a961","e697a562616162","62622061","2061c3a96162","61",null,"20c3a961","c3a9","632061","c3a9","6320e697a562","62206120e697a5","e697a5","","e697a5","c3a9","","6120","2062e697a56261","c3a9","61","c3a9",null,"6161","","","","61e697a5","c3a96262","","","63626361","","c3a96161","6320","c3a9c3a9636163","2063","62c3a96162","63",null,"20e697a5","6261c3a9","2020206262","20616263","20e697a520c3a9e697a5","c3a961c3a963e697a5","","616161","202062e697a5","20e697a5","","61","c3a961e697a520","6362","20","",null,"","c3a9e697a520c3a9","20206261c3a9","c3a961","6261e697a563","e697a5c3a9c3a961","62c3a9e697a563","6263e697a5","","6120c3a9","61c3a962c3a963","62636363e697a5","61","e697a5","63c3a9e697a5","e697a5e697a5636120",null,"62","e697a562","62","62c3a9636120","626262e697a5e697a5","6263","62c3a962e697a520","62","e697a5c3a962","62e697a5","62","e697a520616161","6361e697a561","61","e697a520","c3a962c3a9",null,"c3a9206363e697a5","e697a56162","6262e697a5","2062c3a963c3a9","616261e697a5e697a5","616362e697a5","c3a96162e697a5","61e697a5c3a9","6361","63c3a9626262","e697a5c3a9c3a9","e697a52062","20","","6120c3a92020","20e697a52061c3a9",null,"e697a561","636361","6362c3a9","e697a5c3a92061","","626262e697a562"]},{"path":["delta_string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["6178697330","626162626c65","6261636b","636162","61786c6534","62616279","6261636b626f6e65","636162696e","61786f6c6f746c38","62616279686f6f64","626164","61786973","626162626c653132","6261636b","636162","61786c65",null,"6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643230","626164","61786973","626162626c65","6261636b3234","636162","61786c65","62616279","6261636b626f6e653238","636162696e","61786f6c6f746c","62616279686f6f64","6261643332","61786973","626162626c65",null,"6361623336","61786c65","62616279","6261636b626f6e65","636162696e3430","61786f6c6f746c","62616279686f6f64","626164","617869733434","626162626c65","6261636b","636162","61786c653438","62616279","6261636b626f6e65","636162696e","61786f6c6f746c3532","62616279686f6f64",null,"61786973","626162626c653536","6261636b","636162","61786c65","626162793630","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f643634","626164","61786973","626162626c65","6261636b3638","636162","61786c65","62616279","6261636b626f6e653732",null,"61786f6c6f746c","62616279686f6f64","6261643736","61786973","626162626c65","6261636b","6361623830","61786c65","62616279","6261636b626f6e65","636162696e3834","61786f6c6f746c","62616279686f6f64","626164","617869733838","626162626c65","6261636b","636162",null,"62616279","6261636b626f6e65","636162696e","61786f6c6f746c3936","62616279686f6f64","626164","61786973","626162626c65313030","6261636b","636162","61786c65","62616279313034","6261636b626f6e65","636162696e","61786f6c6f746c","62616279686f6f64313038","626164","61786973",null,"6261636b313132","636162","61786c65","62616279","6261636b626f6e65313136","636162696e","61786f6c6f746c","62616279686f6f64","626164313230","61786973","626162626c65","6261636b","636162313234","61786c65","62616279","6261636b626f6e65","636162696e313238","61786f6c6f746c",null,"626164","61786973313332","626162626c65","6261636b","636162","61786c65313336","62616279","6261636b626f6e65","636162696e","61786f6c6f746c313430","62616279686f6f64","626164","61786973","626162626c65313434","6261636b","636162","61786c65","62616279313438",null,"636162696e","61786f6c6f746c","62616279686f6f64313532","626164","61786973","626162626c65","6261636b313536","636162","61786c65","62616279"]},{"path":["delta_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["000001","616201","616202","030001","616204","616205","060001","616200","616201","090001","616203","616204","0c0001","616206","616200","0f0001","616202","616203","120001","616205","616206","150001","616201","616202","180001","616204","616205","1b0001","616200","616201","1e0001","616203","616204","210001","616206","616200","240001","616202","616203","270001","616205","616206","2a0001","616201","616202","2d0001","616204","616205","300001","616200","616201","330001","616203","616204","360001","616206","616200","390001","616202","616203","3c0001","616205","616206","3f0001","616201","616202","420001","616204","616205","450001","616200","616201","480001","616203","616204","4b0001","616206","616200","4e0001","616202","616203","510001","616205","616206","540001","616201","616202","570001","616204","616205","5a0001","616200","616201","5d0001","616203","616204","600001","616206","616200","630001","616202","616203","660001","616205","616206","690001","616201","616202","6c0001","616204","616205","6f0001","616200","616201","720001","616203","616204","750001","616206","616200","780001","616202","616203","7b0001","616205","616206","7e0001","616201","616202","810001","616204","616205","840001","616200","616201","870001","616203","616204","8a0001","616206","616200","8d0001","616202","616203","900001","616205","616206","930001","616201","616202","960001","616204","616205","990001","616200","616201","9c0001","616203","616204","9f0001"]},{"path":["bss_float32"],"physicalType":"FLOAT","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-43.0,43.3125,-22.0625,316.5,-439.9375,-127.3125,18.8125,598.625,623.0,null,420.375,-139.0625,-436.5625,-220.0625,463.9375,586.4375,66.125,414.5625,-337.3125,-129.3125,null,-156.5,-192.625,-19.0625,-254.125,-182.625,512.5625,15.375,227.5625,-61.8125,46.6875,null,365.3125,-375.25,-34.8125,-98.0,596.5625,-375.8125,593.8125,-59.625,-31.1875,366.3125,null,-175.0625,-503.375,70.0625,360.375,307.5625,347.0625,-342.5,-552.125,624.4375,376.125,null,-90.6875,356.5,605.6875,331.3125,555.4375,-249.0,-436.75,-149.6875,193.0,521.5,null,-210.4375,-8.75,75.25,-306.25,599.4375,618.125,482.6875,-42.5625,235.0,-507.625,null,-431.4375,-389.0,-447.0,-393.625,425.0,-388.125,-427.0,435.4375,-455.125,-316.5625,null,-193.9375,-515.125,27.5625,-197.1875,116.3125,-19.6875,487.375,58.0,6.875,-391.5625,null,-214.0625,493.625,265.375,410.5,96.1875,-123.125,-372.8125,-331.0,182.9375,-223.0625,null,4.5625,218.625,422.3125,-617.8125,235.5625,-302.8125,-575.5,-621.125,-270.5625,-470.875,null,212.375,125.6875,5.8125,409.5625,243.1875,-339.25,-40.0,544.6875,340.625,-455.8125,null,372.5,-107.6875,-326.8125,-421.125,-260.0625,-382.6875,526.9375,365.375,-102.125,455.6875,null,329.125,526.625,-33.8125,-126.125,157.25,-252.8125,504.0625,-193.8125,457.0625,-393.875,null,-187.5,-386.9375,-306.875,-81.9375,-591.6875,100.375,-242.4375]},{"path":["bss_float64"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[367.146484375,216.3037109375,-45.404296875,-246.9521484375,null,-939.69140625,-200.38671875,-704.7138671875,775.865234375,922.1630859375,-521.6357421875,null,602.3759765625,-648.45703125,-219.9970703125,59.5732421875,-777.11328125,888.3837890625,null,-743.2919921875,-618.986328125,-459.5986328125,-37.6669921875,-508.603515625,251.0556640625,null,-224.1376953125,972.560546875,-296.939453125,-637.6865234375,-690.7109375,118.5244140625,null,-668.4931640625,18.4228515625,628.2099609375,387.2919921875,-560.029296875,-882.931640625,null,531.5068359375,-386.9130859375,-201.7763671875,268.939453125,-682.5419921875,-142.603515625,null,370.99609375,-362.736328125,42.3154296875,-254.3662109375,655.669921875,694.8515625,null,-101.5029296875,533.9765625,813.228515625,166.6044921875,191.0478515625,-95.1650390625,null,937.26953125,-246.8505859375,-612.4248046875,99.2333984375,-497.138671875,-349.3037109375,null,-349.6103515625,349.333984375,705.724609375,482.576171875,-271.2900390625,-764.9677734375,null,-678.041015625,-10.9296875,342.4619140625,959.15234375,607.236328125,-445.1103515625,null,711.8271484375,-740.19921875,35.1142578125,-123.166015625,9.7666015625,-555.3564453125,null,769.6357421875,394.380859375,-317.14453125,-705.2509765625,815.4677734375,-97.205078125,null,-555.1708984375,529.1474609375,244.80078125,-148.810546875,837.091796875,358.9521484375,null,-511.701171875,194.2822265625,277.2734375,327.8388671875,-315.7138671875,-464.7490234375,null,-614.7822265625,606.330078125,476.701171875,-336.1572265625,-378.59765625,515.888671875,null,-813.3203125,-59.9111328125,583.8701171875,-812.3291015625,217.8701171875,876.6845703125,null,-42.65625,851.103515625,-369.6923828125,-623.7890625,99.3505859375,720.28515625,null,-828.4794921875,90.01953125,-371.2607421875,10.7314453125,405.2333984375,237.5390625,null,284.154296875,90.720703125,-89.6220703125,-377.3857421875,676.6484375,199.1337890625,null,-255.5400390625,888.41796875,-268.052734375,12.4375,581.1142578125,341.697265625,null,755.982421875,813.00390625,-767.6181640625,-130.4541015625,476.11328125,-39.353515625,null,803.1201171875]},{"path":["bss_int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-2147483648,2147483647,24630,-8252,-88741,279,-80859,-79846,-88224,11214,-72621,41673,94160,-72079,-75456,49714,-28221,-69862,35645,9817,15468,70413,76991,89996,-77851,-19432,36584,-36803,-18873,-27782,4874,16048,88154,-57213,-15930,97013,-64599,63937,-10116,31204,41596,-58577,8126,-65323,-19876,-37092,-43312,-89958,-19789,27103,22296,19318,69678,-10861,39994,-8405,-27595,37714,78587,27400,88771,22216,83967,51193,-12492,11652,47228,84674,42964,-8228,73119,-64033,-99646,90841,23044,-90804,79377,-44780,19843,18044,-74864,21640,-57604,-40662,-54977,-98373,-18308,67031,30243,97549,97982,51741,95370,-75576,80731,80124,55178,89130,-30615,-2998,52618,70509,-20366,-96531,33925,80541,-76437,28102,47292,68212,13253,-28233,50059,10424,67564,-65300,-25483,94973,83740,-58425,-57885,-32337,50174,-70539,-93538,-74075,73099,-51392,-55340,-70481,-61805,9271,-70489,-88830,23499,-49353,-46996,-50008,-16437,21629,26720,89281,71447,21080,1327,-12061,-56619,-13475,1719,95956,77982,-18198,87923,53004,-44195,-69179,36258,-27110,18966,-55393]},{"path":["bss_int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["-9223372036854775808","9223372036854775807","987863054100","-296709105316","797163188552","283198447560","210302952758","-242326649351","682448063443","-133756023825","1087324371926","992378598252","202450873419","-163663339987",null,"-851188577698","-175190867480","-568420662460","888396909884","-967582777543","-438169682332","464692519730","867134292246","278899198129","253340965962","-802603576546","-81558278880","356243668616","617686406039","219022649198","-990766883952","-32296480474","358083665127","-971286852589","-378225370839","797453355920","-807104191328",null,"-390682297362","-1092497215362","-978571312995","-91580169782","1066730189811","430481217755","307658968460","-302016529785","910846036120","-262418637231","-410538560962","695107625814","540223280584","337319050436","-997323342453","230232436340","-1062128698264","-298926060898","-410931378914","-736778008989","-65991568603","851673167648",null,"234018773466","235019331533","605069254338","-1014013373950","628920372329","-687878596681","-708899400047","-85900646935","312486659162","-567271953017","1019981739437","693375127142","-948621157977","-1073149451028","755298532586","-1081886191699","963462205821","-24009810685","847648551381","-527998211714","-762169547994","319743072226",null,"-755692925920","121169648679","-1040928921954","251238298346","1015370757681","-194980851552","822769963093","-133418024571","-564926299881","-660269138406","1056900410002","239753541378","-92796144166","-687408872136","-403448570799","987893209335","102662608696","528580901162","957056644830","-254655530643","-21421591238","366790029881",null,"-710728631032","483739032797","-225612438291","992590595094","-391556708723","441239395898","227561934972","420505691518","-219258864319","-836798736275","-263566729195","593315416161","127055598463","510154441724","-326140058306","-379289775153","1079181312388","118834101207","-1010855571923","-960810927306","-271852908247","-871532118498",null,"-609178482600","-145211802073","-407971053503","974973273544","-418132649076","-845833554356","-907607164810","-821233710697","-611538191768","-59725661798","-188190488612","1051278023789","-486014714752","-420053141405","670152470078","-878859760470","-522447514654","681864633933","562381935015","138329833268","875971386725","-967975647744",null,"-469328883599","701452300781","-1016314675471","-428111787833","1091174619886","-501130950046","514645087743"]},{"path":["bss_fixed_size_binary"],"physicalType":"FIXED_LEN_BYTE_ARRAY","typeLength":3,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":["aacfc4","414c98","04274c","20cea8","aaf446","bff03e","ad6cc2","0f32b5","69f5c6","21b4c7","8c9325","99905f","8ec640","a2d58f","9753b6","053825","f72a1a","83416b","3e04ff","42cc67","082117","e64bc7","e6ca6f","a553de","592678",null,"6b1331","64733d","c9a7c8","79f0ba","26675f","10ecdb","b36d78","6c54c5","14ebce","2426c3","a84823","20fd34","ce3fc9","4ec436","2758c3","ad640f","de41e5","1931fb","66f81c","eb1465","48ed00","383de9","58d935","0996a6","df2d17","73bd80","e2bce5","699c6a",null,"844b90","935ff9","facec5","46dadc","57a0ca","e55153","a685f3","c842b0","e42886","8ca306","11dede","f7cef8","725728","cd2dbe","424d36","8f619b","aadc45","474b82","2a4c0f","7b3470","b1a09b","02ecf2","10f84b","94c9a6","438720","dac3ba","597fdd","590000",null,"2a2b18","2e2164","b2c489","9722ed","ff67ba","9c9f7a","7697c6","a1f865","abba76","c44f0c","9f887b","cb87d8","77f255","1b9d85","b16bf6","bc1f67","4ad820","aa0ecd","667475","094bb5","e053b7","15a6d9","298b89","632c49","974fdd","86e4a8","87f8e7","865966",null,"6e1ce3","9b829d","71d715","989afc","12c8c3","d0500e","706596","bc6b3e","9d1fbf","7ca7e8","9109f7","4d0ad6","ebf77b","d9ba59","46237b","f7a01c","0948e5","8b1398","d4c9c8","9dbc3a","36b566","efc0dd","e564db","6d7268","2362a3","ae5f34","05d20f","8f5d1d",null,"c77a35","3180d2","39e592","2d7a7c","0555cd","3db2ea","ea2b83","348b82","4f7a35","d21428","5ea75b","6c29a1","9b474f","3c98bb","62f80e","1a4a5b","cb50f7","d35d35"]},{"path":["rle_bool"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,null,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,null,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,null,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,null,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,true,true,true,true,true,false,false,null,true,true,true,true,true,true,true,false,false,false]}]},{"name":"nested.parquet","numRows":40,"rowGroups":[{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]},{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["ints","list","element"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[],[null],[1,null,3],[],[-63],[85,65],[66,-11,7],[],null,[-2,16],[-37,49,-92],[],[78],[-55,-28],[97,-7,-78],[],[37],null,[95,-78,2],[],[-36],[-63,10],[-24,-68,58],[],[62],[44,96],null,[],[12],[91,58],[64,-18,-59],[],[98],[44,100],[28,8,28],null,[-53],[-86,-49],[13,-52,50]]},{"path":["nested_strings","list","element","list","element"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":5,"maxRepetitionLevel":2,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[[]],[[],["68c3a96c6c6f2077c3b6726c64"]],[],[[]],[null,["70617271756574"]],[],null,[[],["61"]],[],[null],[[],["e697a5e69cace8aa9e"]],[],[[]],null,[],[[]],[[],["68656c6c6f"]],[],[[]],[null,["f09f8c8d"]],null,[[]],[[],[""]],[],[null],[[],["68c3a96c6c6f2077c3b6726c64"]],[],null,[[],null],[],[[]],[[],["61"]],[],[[]],null,[],[[]],[[],["7878787878787878787878787878787878787878"]],[]]},{"path":["struct","a"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,-19000,-18000,-17000,-16000,-15000,null,-13000,-12000,-11000,-10000,-9000,null,-7000,-6000,-5000,-4000,-3000,null,-1000,0,1000,2000,3000,null,5000,6000,7000,8000,9000,null,11000,12000,13000,14000,15000,null,17000,18000,19000]},{"path":["struct","b","list","element"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,"-1099511627774"],[null,"-1099511627773","-1099511627770"],null,[],null,[null,"-1099511627769"],null,[null,"-1099511627767","-1099511627758",null],[],[null],null,[null,"-1099511627763","-1099511627750"],[null,"-1099511627762","-1099511627748",null],[],null,[null,"-1099511627759"],null,[null,"-1099511627757","-1099511627738",null],null,[null],[null,"-1099511627754"],[null,"-1099511627753","-1099511627730"],null,[],[null],[null,"-1099511627749"],null,[null,"-1099511627747","-1099511627718",null],null,[null],null,[null,"-1099511627743","-1099511627710"],[null,"-1099511627742","-1099511627708",null],[],null,[null,"-1099511627739"],[null,"-1099511627738","-1099511627700"],[null,"-1099511627737","-1099511627698",null]]},{"path":["map","key_value","key"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":2,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"]]},{"path":["map","key_value","value"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[0],[0,2],[0,null,6],[],[0],[0,6],[0,null,14],null,[0],[0,10],[0,null,22],[],[0],[0,14],[0,null,30],null,[0],[0,18],[0,null,38],[],[0],[0,22],[0,null,46],null,[0],[0,26],[0,null,54],[],[0],[0,30],[0,null,62],null,[0],[0,34],[0,null,70],[],[0],[0,38],[0,null,78]]},{"path":["required_struct","x"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-7.0,-6.5,-6.0,-5.5,-5.0,-4.5,-4.0,-3.5,-3.0,-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0,12.5]},{"path":["structs","list","element","s"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],null,[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],null,[null],[null,"70617271756574"],[null,null,"70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],null,[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"]]},{"path":["structs","list","element","t"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,null,false],[],[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],null,[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,false,false],[],[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false]]}]},{"name":"nested_v2.parquet","numRows":40,"rowGroups":[{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]},{"numRows":20,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"UNCOMPRESSED","encodings":["RLE"]}]}],"columns":[{"path":["ints","list","element"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[],[null],[1,null,3],[],[-63],[85,65],[66,-11,7],[],null,[-2,16],[-37,49,-92],[],[78],[-55,-28],[97,-7,-78],[],[37],null,[95,-78,2],[],[-36],[-63,10],[-24,-68,58],[],[62],[44,96],null,[],[12],[91,58],[64,-18,-59],[],[98],[44,100],[28,8,28],null,[-53],[-86,-49],[13,-52,50]]},{"path":["nested_strings","list","element","list","element"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":5,"maxRepetitionLevel":2,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[[]],[[],["68c3a96c6c6f2077c3b6726c64"]],[],[[]],[null,["70617271756574"]],[],null,[[],["61"]],[],[null],[[],["e697a5e69cace8aa9e"]],[],[[]],null,[],[[]],[[],["68656c6c6f"]],[],[[]],[null,["f09f8c8d"]],null,[[]],[[],[""]],[],[null],[[],["68c3a96c6c6f2077c3b6726c64"]],[],null,[[],null],[],[[]],[[],["61"]],[],[[]],null,[],[[]],[[],["7878787878787878787878787878787878787878"]],[]]},{"path":["struct","a"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,-19000,-18000,-17000,-16000,-15000,null,-13000,-12000,-11000,-10000,-9000,null,-7000,-6000,-5000,-4000,-3000,null,-1000,0,1000,2000,3000,null,5000,6000,7000,8000,9000,null,11000,12000,13000,14000,15000,null,17000,18000,19000]},{"path":["struct","b","list","element"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,"-1099511627774"],[null,"-1099511627773","-1099511627770"],null,[],null,[null,"-1099511627769"],null,[null,"-1099511627767","-1099511627758",null],[],[null],null,[null,"-1099511627763","-1099511627750"],[null,"-1099511627762","-1099511627748",null],[],null,[null,"-1099511627759"],null,[null,"-1099511627757","-1099511627738",null],null,[null],[null,"-1099511627754"],[null,"-1099511627753","-1099511627730"],null,[],[null],[null,"-1099511627749"],null,[null,"-1099511627747","-1099511627718",null],null,[null],null,[null,"-1099511627743","-1099511627710"],[null,"-1099511627742","-1099511627708",null],[],null,[null,"-1099511627739"],[null,"-1099511627738","-1099511627700"],[null,"-1099511627737","-1099511627698",null]]},{"path":["map","key_value","key"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":2,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],null,["6b30"],["6b30","6b31"],["6b30","6b31","6b32"],[],["6b30"],["6b30","6b31"],["6b30","6b31","6b32"]]},{"path":["map","key_value","value"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[0],[0,2],[0,null,6],[],[0],[0,6],[0,null,14],null,[0],[0,10],[0,null,22],[],[0],[0,14],[0,null,30],null,[0],[0,18],[0,null,38],[],[0],[0,22],[0,null,46],null,[0],[0,26],[0,null,54],[],[0],[0,30],[0,null,62],null,[0],[0,34],[0,null,70],[],[0],[0,38],[0,null,78]]},{"path":["required_struct","x"],"physicalType":"DOUBLE","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[-7.0,-6.5,-6.0,-5.5,-5.0,-4.5,-4.0,-3.5,-3.0,-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0,12.5]},{"path":["structs","list","element","s"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[null,[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],null,[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],null,[null],[null,"70617271756574"],[null,null,"70617271756574"],[],[null],[null,"68656c6c6f"],[null,"68c3a96c6c6f2077c3b6726c64","70617271756574"],[],[null],null,[null,"7878787878787878787878787878787878787878","70617271756574"],[],[null],[null,"68656c6c6f"],[null,null,"70617271756574"],[],[null],[null,"70617271756574"],[null,"7878787878787878787878787878787878787878","70617271756574"]]},{"path":["structs","list","element","t"],"physicalType":"BOOLEAN","typeLength":null,"maxDefinitionLevel":4,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,null,false],[],[null],[null,true],[null,false,false],[],[null],[null,true],[null,false,false],null,[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false],[],[null],null,[null,false,false],[],[null],[null,true],[null,null,false],[],[null],[null,true],[null,false,false]]}]},{"name":"gzip.parquet","numRows":40,"rowGroups":[{"numRows":25,"columns":[{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]},{"numRows":15,"columns":[{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]},{"codec":"GZIP","encodings":["PLAIN","RLE","RLE_DICTIONARY"]}]}],"columns":[{"path":["int64"],"physicalType":"INT64","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[null,"-5","-5","-9223372036854775808","-9223372036854775808","-9223372036854775808","5",null,"1125899906842624","-9223372036854775808","1125899906842624","1125899906842624","1125899906842624","-9223372036854775808",null,"5","-9223372036854775808","-9223372036854775808","-9223372036854775808","-9223372036854775808","1125899906842624",null,"-5","-5","-9223372036854775808","-9223372036854775808","1125899906842624","-9223372036854775808",null,"5","-5","-9223372036854775808","-5","-5","1125899906842624",null,"-5","5","-9223372036854775808","-5"]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":["","61","68656c6c6f68656c6c6f","",null,"","","68656c6c6f","68c3a96c6c6f2077c3b6726c6468c3a96c6c6f2077c3b6726c64","",null,"6161","","68c3a96c6c6f2077c3b6726c64","e697a5e69cace8aa9ee697a5e69cace8aa9e","",null,"68656c6c6f68656c6c6f","","e697a5e69cace8aa9e","","",null,"68c3a96c6c6f2077c3b6726c6468c3a96c6c6f2077c3b6726c64","","","6161","",null,"e697a5e69cace8aa9ee697a5e69cace8aa9e","","61","68656c6c6f68656c6c6f","",null,"","","68656c6c6f","68c3a96c6c6f2077c3b6726c6468c3a96c6c6f2077c3b6726c64",""]},{"path":["ints","list","element"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":3,"maxRepetitionLevel":1,"convertedType":null,"logicalType":null,"values":[null,[1,-1],[2,-2,2,-2],[],[4,-1],[0,-2,0,-2],[],[2,-1],[3,-2,3,-2],null,[0,-1],[1,-2,1,-2],[],[3,-1],[4,-2,4,-2],[],[1,-1],[2,-2,2,-2],null,[4,-1],[0,-2,0,-2],[],[2,-1],[3,-2,3,-2],[],[0,-1],[1,-2,1,-2],null,[3,-1],[4,-2,4,-2],[],[1,-1],[2,-2,2,-2],[],[4,-1],[0,-2,0,-2],null,[2,-1],[3,-2,3,-2],[]]}]},{"name":"empty.parquet","numRows":0,"rowGroups":[{"numRows":0,"columns":[{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]},{"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"]}]}],"columns":[{"path":["int32"],"physicalType":"INT32","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":null,"logicalType":null,"values":[]},{"path":["string"],"physicalType":"BYTE_ARRAY","typeLength":null,"maxDefinitionLevel":1,"maxRepetitionLevel":0,"convertedType":"UTF8","logicalType":{"type":"STRING"},"values":[]}]}]} diff --git a/packages/effect/test/fixtures/parquet/gzip.parquet b/packages/effect/test/fixtures/parquet/gzip.parquet new file mode 100644 index 00000000000..7d148b5d440 Binary files /dev/null and b/packages/effect/test/fixtures/parquet/gzip.parquet differ diff --git a/packages/effect/test/fixtures/parquet/int96.parquet b/packages/effect/test/fixtures/parquet/int96.parquet new file mode 100644 index 00000000000..e5fb4484b61 Binary files /dev/null and b/packages/effect/test/fixtures/parquet/int96.parquet differ diff --git a/packages/effect/test/fixtures/parquet/nested.parquet b/packages/effect/test/fixtures/parquet/nested.parquet new file mode 100644 index 00000000000..a93f81da53d Binary files /dev/null and b/packages/effect/test/fixtures/parquet/nested.parquet differ diff --git a/packages/effect/test/fixtures/parquet/nested_v2.parquet b/packages/effect/test/fixtures/parquet/nested_v2.parquet new file mode 100644 index 00000000000..71c94692452 Binary files /dev/null and b/packages/effect/test/fixtures/parquet/nested_v2.parquet differ diff --git a/packages/effect/test/fixtures/parquet/plain.parquet b/packages/effect/test/fixtures/parquet/plain.parquet new file mode 100644 index 00000000000..136896d67b5 Binary files /dev/null and b/packages/effect/test/fixtures/parquet/plain.parquet differ diff --git a/packages/effect/test/fixtures/thrift/README.md b/packages/effect/test/fixtures/thrift/README.md new file mode 100644 index 00000000000..5be9093390d --- /dev/null +++ b/packages/effect/test/fixtures/thrift/README.md @@ -0,0 +1,24 @@ +# Thrift fixtures + +`gold.json` is generated by `generate.py` and read by +`test/encoding/Thrift.test.ts`. Each case is a value tree and the bytes the +official Apache Thrift Python library (`thrift` 0.25.0 from PyPI, +`TCompactProtocol`) writes for it: one hand-written case per rule of the +compact protocol, then 20 small seeded random structs. + +The `parquet` section holds compact-protocol structs cut from two files of +[apache/parquet-testing](https://github.com/apache/parquet-testing) (commit +`bcefdfd`): the `FileMetaData` footers of `data_index_bloom_encoding_stats.parquet` +and `alltypes_tiny_pages.parquet`, a page header from each, and the bloom +filter header, offset index and column index of the first. Each comes with the +tree the Python reader reads from it. The row counts, column names and +offsets the test checks are the ones pyarrow reports for those files. + +Regenerate from `packages/effect`; the output is byte for byte the same: + +```sh +PARQUET_TESTING=/path/to/parquet-testing \ + uv run --with thrift==0.25.0 python test/fixtures/thrift/generate.py +``` + +`PARQUET_TESTING` needs only the two files above, under `data/`. diff --git a/packages/effect/test/fixtures/thrift/generate.py b/packages/effect/test/fixtures/thrift/generate.py new file mode 100644 index 00000000000..b0ce460fb88 --- /dev/null +++ b/packages/effect/test/fixtures/thrift/generate.py @@ -0,0 +1,460 @@ +""" +Generates gold.json, the gold corpus for effect/encoding/Thrift: values and +the bytes the official Apache Thrift Python library's TCompactProtocol writes +for them. README.md describes the corpus and how to regenerate it. + +Values are JSON trees shaped like Thrift.Value: + {"_tag": "Bool", "value": true} + {"_tag": "I8" | "I16" | "I32", "value": 5} + {"_tag": "I64", "value": "-5"} decimal string + {"_tag": "Double", "value": 1.5} or "NaN", "Infinity", "-Infinity", "-0" + {"_tag": "Binary" | "Uuid", "value": "00ff"} hex + {"_tag": "Struct", "fields": [{"id": 1, "value": ...}]} + {"_tag": "List" | "Set", "elementType": "I32", "elements": [...]} + {"_tag": "Map", "keyType": "I32", "valueType": "Bool", "entries": [[k, v]]} +A case is {"id", "type", "value", "bytes"}: `type` is the Type name of the +top-level value. Every case is also read back here with the Python reader to +check the generator. The "parquet" cases are structs and carry no `type`. +""" + +import json +import math +import os +import random +import struct +import uuid +from pathlib import Path + +from thrift.protocol.TCompactProtocol import TCompactProtocol +from thrift.Thrift import TType +from thrift.transport.TTransport import TMemoryBuffer + +HERE = Path(__file__).parent + +TTYPES = { + "Bool": TType.BOOL, + "I8": TType.BYTE, + "I16": TType.I16, + "I32": TType.I32, + "I64": TType.I64, + "Double": TType.DOUBLE, + "Binary": TType.STRING, + "Uuid": TType.UUID, + "Struct": TType.STRUCT, + "List": TType.LIST, + "Set": TType.SET, + "Map": TType.MAP, + "Stop": TType.STOP, +} +NAMES = {v: k for k, v in TTYPES.items()} + +# TCompactProtocol only writes scalars inside a struct or container; these +# are its internal states for "a value comes next". +VALUE_WRITE = 2 +VALUE_READ = 7 + + +def double_json(d): + if math.isnan(d): + return "NaN" + if math.isinf(d): + return "Infinity" if d > 0 else "-Infinity" + if d == 0 and math.copysign(1, d) < 0: + return "-0" + return d + + +def write(p, v): + tag = v["_tag"] + if tag == "Bool": + p.writeBool(v["value"]) + elif tag == "I8": + p.writeByte(v["value"]) + elif tag == "I16": + p.writeI16(v["value"]) + elif tag == "I32": + p.writeI32(v["value"]) + elif tag == "I64": + p.writeI64(int(v["value"])) + elif tag == "Double": + p.writeDouble(float(v["value"])) + elif tag == "Binary": + p.writeBinary(bytes.fromhex(v["value"])) + elif tag == "Uuid": + p.writeUuid(uuid.UUID(bytes=bytes.fromhex(v["value"]))) + elif tag == "Struct": + p.writeStructBegin(None) + for f in v["fields"]: + p.writeFieldBegin(None, TTYPES[f["value"]["_tag"]], f["id"]) + write(p, f["value"]) + p.writeFieldEnd() + p.writeFieldStop() + p.writeStructEnd() + elif tag in ("List", "Set"): + begin = p.writeListBegin if tag == "List" else p.writeSetBegin + begin(TTYPES[v["elementType"]], len(v["elements"])) + for e in v["elements"]: + write(p, e) + p.writeListEnd() + elif tag == "Map": + p.writeMapBegin(TTYPES[v["keyType"]], TTYPES[v["valueType"]], len(v["entries"])) + for k, x in v["entries"]: + write(p, k) + write(p, x) + p.writeMapEnd() + else: + raise ValueError(tag) + + +def read(p, t): + name = NAMES[t] + if name == "Bool": + return {"_tag": name, "value": p.readBool()} + if name == "I8": + return {"_tag": name, "value": p.readByte()} + if name == "I16": + return {"_tag": name, "value": p.readI16()} + if name == "I32": + return {"_tag": name, "value": p.readI32()} + if name == "I64": + return {"_tag": name, "value": str(p.readI64())} + if name == "Double": + return {"_tag": name, "value": double_json(p.readDouble())} + if name == "Binary": + return {"_tag": name, "value": p.readBinary().hex()} + if name == "Uuid": + return {"_tag": name, "value": p.readUuid().bytes.hex()} + if name == "Struct": + fields = [] + p.readStructBegin() + while True: + _, ft, fid = p.readFieldBegin() + if ft == TType.STOP: + break + fields.append({"id": fid, "value": read(p, ft)}) + p.readFieldEnd() + p.readStructEnd() + return {"_tag": name, "fields": fields} + if name in ("List", "Set"): + et, n = p.readListBegin() + elements = [read(p, et) for _ in range(n)] + p.readListEnd() + return {"_tag": name, "elementType": NAMES[et], "elements": elements} + if name == "Map": + kt, vt, n = p.readMapBegin() + entries = [] + for _ in range(n): + k = read(p, kt) + entries.append([k, read(p, vt)]) + p.readMapEnd() + return {"_tag": name, "keyType": NAMES[kt], "valueType": NAMES[vt], "entries": entries} + raise ValueError(name) + + +def encode(v): + buf = TMemoryBuffer() + p = TCompactProtocol(buf) + if v["_tag"] != "Struct": + p.state = VALUE_WRITE + write(p, v) + return buf.getvalue() + + +def decode(data, type_name, offset=0): + """Reads one value at `offset`; returns it and the number of bytes read.""" + buf = TMemoryBuffer(data[offset:]) + p = TCompactProtocol(buf) + if type_name != "Struct": + p.state = VALUE_READ + v = read(p, TTYPES[type_name]) + return v, buf._buffer.tell() + + +cases = [] + + +def case(id, v): + data = encode(v) + back, n = decode(data, v["_tag"]) + assert back == v, (id, back, v) + assert n == len(data), id + cases.append({"id": id, "type": v["_tag"], "value": v, "bytes": data.hex()}) + + +# --- value constructors ------------------------------------------------------- + + +def b(x): + return {"_tag": "Bool", "value": x} + + +def i8(x): + return {"_tag": "I8", "value": x} + + +def i16(x): + return {"_tag": "I16", "value": x} + + +def i32(x): + return {"_tag": "I32", "value": x} + + +def i64(x): + return {"_tag": "I64", "value": str(x)} + + +def dbl(x): + return {"_tag": "Double", "value": double_json(x)} + + +def binary(x): + return {"_tag": "Binary", "value": bytes(x).hex()} + + +def text(s): + return binary(s.encode("utf-8")) + + +def uid(x): + return {"_tag": "Uuid", "value": bytes(x).hex()} + + +def struct_(*fields): + return {"_tag": "Struct", "fields": [{"id": i, "value": v} for i, v in fields]} + + +def lst(t, *es): + return {"_tag": "List", "elementType": t, "elements": list(es)} + + +def st(t, *es): + return {"_tag": "Set", "elementType": t, "elements": list(es)} + + +def mp(k, v, *entries): + if not entries: + k = v = "Stop" + return {"_tag": "Map", "keyType": k, "valueType": v, "entries": [list(e) for e in entries]} + + +# --- hand-written cases ------------------------------------------------------- + +case("struct/empty", struct_()) +case("bool/fields", struct_((1, b(True)), (2, b(False)), (3, b(True)))) +case("bool/long-form", struct_((20, b(True)), (-1, b(False)))) +case("field/delta-1", struct_((1, i32(1)), (2, i32(2)))) +case("field/delta-15", struct_((15, i32(1)), (30, i32(2)))) +case("field/delta-16", struct_((16, i32(1)), (32, i32(2)))) +case("field/id-0", struct_((0, i32(1)))) +case("field/decreasing", struct_((5, i32(1)), (3, i32(2)), (4, i32(3)))) +case("field/repeated", struct_((5, i32(1)), (5, i32(2)))) +case("field/negative", struct_((-1, i32(1)), (-32768, i32(2)), (1, i32(3)))) +case("field/max", struct_((32767, i32(1)))) +case("field/nested-resets", struct_((5, struct_((1, i8(1)), (2, i8(2)))), (6, i8(3)))) +case("i8/bounds", struct_((1, i8(-128)), (2, i8(127)), (3, i8(0)), (4, i8(-1)))) +case("i16/bounds", struct_((1, i16(-32768)), (2, i16(32767)), (3, i16(0)), (4, i16(-1)), (5, i16(64)), (6, i16(-65)))) +case( + "i32/bounds", + struct_( + (1, i32(-(2**31))), (2, i32(2**31 - 1)), (3, i32(0)), (4, i32(-1)), (5, i32(63)), (6, i32(64)), + (7, i32(8191)), (8, i32(8192)), (9, i32(-8193)) + ), +) +case( + "i64/bounds", + struct_(*enumerate([ + i64(0), i64(-1), i64(1), i64(2**31), i64(-(2**31) - 1), i64(2**53 - 1), i64(-(2**53 - 1)), i64(2**53), + i64(-(2**53)), i64(2**53 + 1), i64(-(2**53) - 1), i64(2**62), i64(2**63 - 1), i64(-(2**63)), + i64(2**56 - 1), i64(1 << 49) + ], 1)), +) +case( + "double/values", + struct_(*enumerate([ + dbl(0.0), dbl(-0.0), dbl(1.5), dbl(float("nan")), dbl(float("inf")), dbl(float("-inf")), dbl(5e-324), + dbl(1.7976931348623157e308), dbl(math.pi), dbl(-2.5) + ], 1)), +) +case("binary/lengths", struct_((1, binary(b"")), (2, binary(b"\x00")), (3, binary(bytes(range(128)))))) +case("binary/utf8", struct_((1, text("héllo 🌍")), (2, text("ascii")))) +case("uuid", struct_((1, uid(range(16))), (2, uid(bytes(16))))) +case("list/empty", struct_((1, lst("I32")), (2, lst("Struct")), (3, lst("Bool")))) +case("list/size-14", struct_((1, lst("I32", *[i32(i) for i in range(14)])))) +case("list/size-15", struct_((1, lst("I32", *[i32(i) for i in range(15)])))) +case("list/bool", struct_((1, lst("Bool", b(True), b(False), b(True))))) +case("list/i8", struct_((1, lst("I8", i8(-1), i8(0), i8(127))))) +case("list/i64", struct_((1, lst("I64", i64(-(2**63)), i64(0), i64(2**63 - 1))))) +case("list/double", struct_((1, lst("Double", dbl(1.0), dbl(float("nan")))))) +case("list/binary", struct_((1, lst("Binary", text("a"), text(""), text("bc"))))) +case("list/uuid", struct_((1, lst("Uuid", uid(range(16)), uid(range(16, 32)))))) +case("list/struct", struct_((1, lst("Struct", struct_((1, b(True))), struct_(), struct_((2, i32(7))))))) +case("list/list", struct_((1, lst("List", lst("I32", i32(1)), lst("I32"), lst("Bool", b(False)))))) +case("list/map", struct_((1, lst("Map", mp("I32", "I32", (i32(1), i32(2))), mp("I32", "I32"))))) +case("set/values", struct_((1, st("I32", i32(3), i32(1))), (2, st("Binary")), (3, st("Bool", b(True))))) +case("set/size-15", struct_((1, st("I8", *[i8(i) for i in range(15)])))) +case("map/empty", struct_((1, mp("I32", "I32")))) +case("map/one", struct_((1, mp("Binary", "I64", (text("k"), i64(-5)))))) +case("map/bools", struct_((1, mp("Bool", "Bool", (b(True), b(False)), (b(False), b(True)))))) +case("map/i32-struct", struct_((1, mp("I32", "Struct", (i32(1), struct_((1, b(True)))), (i32(2), struct_()))))) +case("map/nested", struct_((1, mp("I32", "List", (i32(1), lst("I32", i32(2), i32(3))))))) +case("map/uuid", struct_((1, mp("Uuid", "Uuid", (uid(range(16)), uid(bytes(16))))))) + + +def nest(depth): + v = struct_((1, i32(depth))) + for d in range(depth): + v = struct_((1, v), (2, b(d % 2 == 0))) + return v + + +case("struct/deep-5", nest(5)) +case("top/list", lst("I32", i32(1), i32(-1))) +case("top/i64", i64(-(2**40))) +case("top/binary", text("top")) +case("top/map", mp("Binary", "Binary", (text("a"), text("b")))) +case("top/set", st("Struct", struct_((1, i8(1))))) +case("top/double", dbl(0.1)) + +# --- seeded random structs ---------------------------------------------------- + +rng = random.Random(0x7487) +SCALARS = ["Bool", "I8", "I16", "I32", "I64", "Double", "Binary", "Uuid"] +ALL = SCALARS + ["Struct", "List", "Set", "Map"] + + +def rand_int(bits): + r = rng.random() + if r < 0.3: + return rng.randint(-64, 64) + if r < 0.4: + return rng.choice([-(2 ** (bits - 1)), 2 ** (bits - 1) - 1, 0, -1]) + w = rng.randint(1, bits - 1) + return rng.randint(-(2**w), 2**w - 1) + + +def rand_double(): + r = rng.random() + if r < 0.1: + return rng.choice([0.0, -0.0, float("inf"), float("-inf"), float("nan"), 5e-324]) + if r < 0.4: + return float(rng.randint(-1000, 1000)) + return struct.unpack("= 3 or rng.random() < 0.6 else ALL) + + +def rand_struct(depth): + n = rng.randint(0, 3 if depth > 2 else 8) + ids = [] + last = 0 + for _ in range(n): + r = rng.random() + if r < 0.6: + last += rng.randint(1, 15) + elif r < 0.8: + last += rng.randint(16, 300) + elif r < 0.9: + last = rng.randint(-32768, 32767) + else: + last = rng.randint(-5, 5) + last = max(-32768, min(32767, last)) + ids.append(last) + return struct_(*[(i, rand_value(rand_type(depth), depth)) for i in ids]) + + +# Small ones only: they mix every type, size form and id form at low cost. +n = 0 +while n < 20: + v = rand_struct(0) + if len(encode(v)) <= 128: + case(f"random/{n}", v) + n += 1 + +# --- Parquet structs ---------------------------------------------------------- + + +def field(v, fid): + for f in v["fields"]: + if f["id"] == fid: + return f["value"] + return None + + +parquet = [] + + +def cut(file, data, offset, what): + v, n = decode(data, "Struct", offset) + parquet.append({ + "id": f"{file}/{what}", + "value": v, + "bytes": data[offset:offset + n].hex(), + }) + return v + + +root = os.environ.get("PARQUET_TESTING") +assert root, "set PARQUET_TESTING to a checkout of https://github.com/apache/parquet-testing" +# The tiny-pages file's indexes have thousands of entries; its footer is enough. +for file, indexes in [("data_index_bloom_encoding_stats.parquet", True), ("alltypes_tiny_pages.parquet", False)]: + data = (Path(root) / "data" / file).read_bytes() + assert data[-4:] == b"PAR1" + length = struct.unpack(" + +it("make only exposes build errors when preloading", () => { + expect(LayerRef.make(failing)).type.toBe, never, Scope.Scope>>() + expect(LayerRef.make(failing, { idleTimeToLive: "Infinity" })).type.toBe< + Effect.Effect, never, Scope.Scope> + >() + expect(LayerRef.make(failing, { preload: false })).type.toBe< + Effect.Effect, never, Scope.Scope> + >() + expect(LayerRef.make(failing, { preload: true })).type.toBe< + Effect.Effect, "BuildError", Scope.Scope> + >() +}) + +it("Service only exposes build errors in its layer when preloading", () => { + class Lazy extends LayerRef.Service()("Lazy", { layer: failing }) {} + class ExplicitLazy extends LayerRef.Service()("ExplicitLazy", { layer: failing, preload: false }) {} + class Preloaded extends LayerRef.Service()("Preloaded", { layer: failing, preload: true }) {} + + expect(Lazy.layer).type.toBe>() + expect(Lazy.layerNoDeps).type.toBe>() + expect(Lazy.get).type.toBe>() + expect(ExplicitLazy.layer).type.toBe>() + expect(Preloaded.layer).type.toBe>() +})