diff --git a/.changeset/dataframe.md b/.changeset/dataframe.md new file mode 100644 index 00000000000..bc332451c31 --- /dev/null +++ b/.changeset/dataframe.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add the experimental `effect/dataframe` modules for typed, columnar data frames on `effect/arrow`, with lazy, type-checked queries that run in memory or on PostgreSQL. Tables are declared by an Effect Schema or an Arrow schema. diff --git a/dprint.json b/dprint.json index 0622d2e2642..41ad1584bb4 100644 --- a/dprint.json +++ b/dprint.json @@ -21,6 +21,7 @@ "**/*.tsbuildinfo", "**/CHANGELOG.md", "packages/effect/src/StandardSchema.ts", + "packages/effect/test/fixtures/dataframe/gold/*.json", "packages/tools/graphql-generator/test/generated/**/*.graphql.ts", "!scratchpad/**/*", "**/.*", diff --git a/packages/effect/benchmark/dataframe/columnar.ts b/packages/effect/benchmark/dataframe/columnar.ts new file mode 100644 index 00000000000..ce023371bd5 --- /dev/null +++ b/packages/effect/benchmark/dataframe/columnar.ts @@ -0,0 +1,146 @@ +/** + * effect/dataframe on timestamp, duration and decimal columns, which frames + * hold as integers. Only rows turn them into `DateTime`, `Duration` and + * `BigDecimal` values, so the cases compare work that stays in columns with + * the same input read as rows. + * + * node [--expose-gc] benchmark/dataframe/columnar.ts [--rows=1000000] [--reps=5] [--only=] + * [--out=] [--gc] + * + * The data is generated in memory: a timestamp and a duration in + * microseconds, a decimal(18, 2), a float64 and a string key of 100 values, + * without nulls, encoded once as an Arrow IPC stream. Times are the best of + * `--reps` runs, in milliseconds. + * + * `--gc` (with `node --expose-gc`) collects garbage before every timed run, + * and again after it while the case's result is still referenced. The memory + * columns are the growth between the two, in MiB, in the last run: `heap` is + * `heapUsed`, the JS objects the result keeps alive, and `buffers` is + * `arrayBuffers`, the memory of its typed arrays. Neither counts what a run + * allocated and released. + */ +import { BigDecimal, DateTime, Effect } from "effect" +import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Result from "effect/Result" +import * as Fs from "node:fs" + +const arg = (name: string) => process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3) +const rows = Number(arg("rows") ?? 1_000_000) +const reps = Number(arg("reps") ?? 5) +const only = arg("only") +const out = arg("out") +const gc = process.argv.includes("--gc") ? (globalThis as { gc?: () => void }).gc : undefined + +let state = 0x9e3779b9 +const next = (bound: number) => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return Math.floor(((state >>> 0) / 4294967296) * bound) +} +/** 2024-01-01T00:00:00Z, in milliseconds. */ +const start = 1_704_067_200_000 +const bytes = Result.getOrThrow(ArrowIpc.encodeStream(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make([ + ArrowSchema.field("at", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("took", ArrowSchema.duration("microsecond")), + ArrowSchema.field("amount", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("v", ArrowSchema.float64), + ArrowSchema.field("key", ArrowSchema.utf8) + ]), + [ + // One row a second, up to a second late. + Array.from({ length: rows }, (_, i) => BigInt(start + i * 1000) * 1000n + BigInt(next(1_000_000))), + Array.from({ length: rows }, () => BigInt(next(3_600_000_000))), + Array.from({ length: rows }, () => BigDecimal.make(BigInt(next(2_000_000_000) - 1_000_000_000), 2)), + Array.from({ length: rows }, () => next(1_000_000) / 1000), + Array.from({ length: rows }, () => `key${next(100)}`) + ] +)))) + +const decode = () => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) as DataFrame.DataFrame +const encode = (df: DataFrame.DataFrame) => Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df))) +/** The later half of the rows. */ +const since = DateTime.makeUnsafe(start + (rows / 2) * 1000) +const input = decode() + +const cases: Record { readonly result: unknown; readonly rows: number }> = { + "a decode, filter, group, encode": () => { + const result = Effect.runSync( + Query.from(decode()).pipe( + Query.filter((c) => Expr.gt(c.at, since)), + Query.groupBy("key"), + Query.agg((c) => ({ + amount: Expr.sum(c.amount), + took: Expr.mean(c.took), + last: Expr.max(c.at), + v: Expr.sum(c.v) + })) + ) + ) + return { result: encode(result), rows: result.length } + }, + "b decode, rows": () => { + const result = DataFrame.toRows(decode()) + return { result, rows: result.length } + }, + "c cast decimal(18, 2) to (18, 1)": () => { + const result = Effect.runSync( + Query.from(input).pipe(Query.select((c) => ({ amount: Expr.castDecimal(c.amount, 18, 1) }))) + ) + // Forces the column without converting values to JS. + return { result: DataFrame.toArrowTable(result), rows: result.length } + } +} + +const results: Record = {} +/** References the result of the last run, so that collecting garbage keeps it. */ +const held: { result?: unknown } = {} +/** Runs a case in a call of its own, so that `held` has the only reference to its result afterwards. */ +const time = (run: () => { readonly result: unknown; readonly rows: number }) => { + const start = performance.now() + const { result, rows } = run() + const ms = performance.now() - start + held.result = result + return { ms, rows } +} +const fmt = (n: number | undefined) => (n === undefined ? "—" : n.toFixed(1)).padStart(9) +const mib = (n: number) => n / 1024 / 1024 + +console.log( + `effect/dataframe columnar path, ${rows} rows (${mib(bytes.length).toFixed(1)} MiB of IPC), best of ${reps}` +) +console.log( + `${"case".padEnd(34)} ${"best ms".padStart(9)} ${"first ms".padStart(9)} ${"heap MiB".padStart(9)} ${ + "buffers".padStart(9) + } rows` +) +for (const [name, run] of Object.entries(cases)) { + if (only !== undefined && !name.includes(only)) continue + let best = Infinity + let first = 0 + let count = 0 + let heap: number | undefined + let buffers: number | undefined + for (let r = 0; r < reps; r++) { + held.result = undefined + gc?.() + const before = process.memoryUsage() + const { ms, rows } = time(run) + if (r === 0) first = ms + best = Math.min(best, ms) + count = rows + if (gc !== undefined) { + gc() + const after = process.memoryUsage() + heap = mib(after.heapUsed - before.heapUsed) + buffers = mib(after.arrayBuffers - before.arrayBuffers) + } + } + console.log(`${name.padEnd(34)} ${fmt(best)} ${fmt(first)} ${fmt(heap)} ${fmt(buffers)} ${count}`) + results[name] = { ms: best, first, heapMiB: heap, arrayBuffersMiB: buffers, rows: count } +} + +if (out !== undefined) Fs.writeFileSync(out, JSON.stringify({ rows, reps, results }, null, 2) + "\n") diff --git a/packages/effect/benchmark/dataframe/h2o.py b/packages/effect/benchmark/dataframe/h2o.py new file mode 100644 index 00000000000..77da7859445 --- /dev/null +++ b/packages/effect/benchmark/dataframe/h2o.py @@ -0,0 +1,222 @@ +""" +Performance baseline for effect/dataframe: H2O.ai db-benchmark style group-by +and join queries, run with Polars and pandas on the same data the TypeScript +benchmark reads (benchmark/dataframe/h2o.ts). + + python h2o.py generate --rows 1000000 [--data DIR] # writes Arrow IPC files + python h2o.py run [--data DIR] [--reps 3] # writes DIR/baseline-.json + +`run` times every query with Polars (all threads and one thread, each in its +own process) and pandas, and records a checksum of every result so the +TypeScript side can check it computed the same thing before comparing times. + +Group-by data follows db-benchmark's groupby-datagen (K = 100 groups for +id1/id2/id4/id5, N/K for id3/id6). Integer columns are Int32 on every side. +The join queries are db-benchmark-style (small/medium/big right tables) on +synthetic keys; see JOIN_QUERIES. +""" + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).parent +DEFAULT_DATA = HERE / ".data" +# Table keys of the queries, and their file names (as -.arrow). +TABLES = {"g": "groupby", "x": "join-x", "small": "join-small", "medium": "join-medium", "big": "join-big"} + + +def generate(rows, data): + import numpy as np + import pyarrow as pa + + rng = np.random.default_rng(20261114) + n = rows + k = 100 + data.mkdir(parents=True, exist_ok=True) + + def ids(fmt, count, size): + pool = np.array([fmt % (i + 1) for i in range(count)], dtype=object) + return pa.array(pool[rng.integers(0, count, size)], type=pa.string()) + + v3_pool = np.round(rng.uniform(0, 100, 100), 6) + groupby = pa.table({ + "id1": ids("id%03d", k, n), + "id2": ids("id%03d", k, n), + "id3": ids("id%010d", n // k, n), + "id4": pa.array(rng.integers(1, k + 1, n), type=pa.int32()), + "id5": pa.array(rng.integers(1, k + 1, n), type=pa.int32()), + "id6": pa.array(rng.integers(1, n // k + 1, n), type=pa.int32()), + "v1": pa.array(rng.integers(1, 6, n), type=pa.int32()), + "v2": pa.array(rng.integers(1, 16, n), type=pa.int32()), + "v3": pa.array(v3_pool[rng.integers(0, 100, n)], type=pa.float64()), + }) + write(groupby, data / f"groupby-{rows}.arrow") + + x_id2 = rng.integers(1, 10_001, n) + x = pa.table({ + "id1": pa.array(rng.integers(1, 11, n), type=pa.int32()), + "id2": pa.array(x_id2, type=pa.int32()), + "id3": pa.array(rng.integers(1, n + 1, n), type=pa.int32()), + "id5": pa.array(np.array([f"id{i}" for i in range(10_001)], dtype=object)[x_id2], type=pa.string()), + "v1": pa.array(np.round(rng.uniform(0, 100, n), 6), type=pa.float64()), + }) + small = pa.table({"id1": pa.array(np.arange(1, 11), type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, 10))}) + medium_ids = np.arange(1, 10_001) + medium = pa.table({ + "id2": pa.array(medium_ids, type=pa.int32()), + "id5": pa.array([f"id{i}" for i in medium_ids], type=pa.string()), + "v2": pa.array(rng.uniform(0, 100, 10_000)), + }) + big = pa.table({"id3": pa.array(rng.permutation(n) + 1, type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, n))}) + for name, t in [("x", x), ("small", small), ("medium", medium), ("big", big)]: + write(t, data / f"join-{name}-{rows}.arrow") + print(f"wrote {rows} rows to {data}") + + +def write(table, path): + import pyarrow.ipc as ipc + + with ipc.new_file(str(path), table.schema) as w: + w.write_table(table) + + +# --------------------------------------------------------------------------- +# Queries (Polars) +# --------------------------------------------------------------------------- + + +def polars_queries(pl): + c = pl.col + gb = lambda df, keys, *aggs: df.group_by(keys, maintain_order=True).agg(*aggs) + return { + "q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], c("v1").sum()), + "q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], c("v1").sum()), + "q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], c("v1").sum(), c("v3").mean()), + "q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], c("v1").mean(), c("v2").mean(), c("v3").mean()), + "q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], c("v1").sum(), c("v2").sum(), c("v3").sum()), + "q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], c("v3").median().alias("median"), c("v3").std().alias("sd")), + "q7 max v1 - min v2 by id3": lambda t: gb(t["g"], ["id3"], (c("v1").max() - c("v2").min()).alias("range")), + "q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], c("v3").sum(), pl.len()), + "filter v3 > 50, sum v3": lambda t: t["g"].filter(c("v3") > 50).select(c("v3").sum()), + "sort by v3 desc, id6": lambda t: t["g"].sort(["v3", "id6"], descending=[True, False], maintain_order=True), + "top 10 by v3 desc, id6": lambda t: t["g"].lazy().sort(["v3", "id6"], descending=[True, False], maintain_order=True).head(10).collect(), + "j1 inner small on id1": lambda t: t["x"].join(t["small"], on="id1", how="inner", maintain_order="left_right"), + "j2 inner medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="inner", maintain_order="left_right"), + "j3 left medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="left", maintain_order="left_right"), + "j4 inner medium on id5 (string)": lambda t: t["x"].drop("id2").join(t["medium"].drop("id2"), on="id5", how="inner", maintain_order="left_right"), + "j5 inner big on id3": lambda t: t["x"].join(t["big"], on="id3", how="inner", maintain_order="left_right"), + } + + +def pandas_queries(pd): + gb = lambda df, keys, spec: df.groupby(keys, sort=False, observed=True).agg(**spec).reset_index() + return { + "q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], {"v1": ("v1", "sum")}), + "q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], {"v1": ("v1", "sum")}), + "q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], {"v1": ("v1", "sum"), "v3": ("v3", "mean")}), + "q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], {"v1": ("v1", "mean"), "v2": ("v2", "mean"), "v3": ("v3", "mean")}), + "q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], {"v1": ("v1", "sum"), "v2": ("v2", "sum"), "v3": ("v3", "sum")}), + "q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], {"median": ("v3", "median"), "sd": ("v3", "std")}), + "q7 max v1 - min v2 by id3": lambda t: (lambda r: r.assign(range=r["v1"] - r["v2"])[["id3", "range"]])( + gb(t["g"], ["id3"], {"v1": ("v1", "max"), "v2": ("v2", "min")})), + "q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], {"v3": ("v3", "sum"), "len": ("v1", "size")}), + "filter v3 > 50, sum v3": lambda t: t["g"]["v3"][t["g"]["v3"] > 50].sum(), + "sort by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable"), + "top 10 by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable").head(10), + "j1 inner small on id1": lambda t: t["x"].merge(t["small"], on="id1", how="inner"), + "j2 inner medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="inner"), + "j3 left medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="left"), + "j4 inner medium on id5 (string)": lambda t: t["x"].drop(columns="id2").merge(t["medium"].drop(columns="id2"), on="id5", how="inner"), + "j5 inner big on id3": lambda t: t["x"].merge(t["big"], on="id3", how="inner"), + } + + +def checksum_polars(df): + import polars as pl + + total = 0.0 + for name, dtype in df.schema.items(): + if dtype.is_numeric(): + s = df.get_column(name).cast(pl.Float64).fill_nan(None).sum() + total += s or 0.0 + return {"rows": df.height, "sum": total} + + +def load(read, data, rows): + return {key: read(data / f"{name}-{rows}.arrow") for key, name in TABLES.items()} + + +def time_best(f, reps): + best = float("inf") + result = None + for _ in range(reps): + start = time.perf_counter() + result = f() + best = min(best, time.perf_counter() - start) + return best * 1000, result + + +def run_polars(data, rows, reps): + import polars as pl + + tables = load(pl.read_ipc, data, rows) + out = {"version": pl.__version__, "threads": pl.thread_pool_size(), "queries": {}} + for name, q in polars_queries(pl).items(): + ms, result = time_best(lambda: q(tables), reps) + out["queries"][name] = {"ms": ms, "checksum": checksum_polars(result)} + return out + + +def run_pandas(data, rows, reps): + import pandas as pd + import pyarrow.ipc as ipc + + tables = load(lambda path: ipc.open_file(str(path)).read_all().to_pandas(), data, rows) + out = {"version": pd.__version__, "queries": {}} + for name, q in pandas_queries(pd).items(): + ms, _ = time_best(lambda: q(tables), reps) + out["queries"][name] = {"ms": ms} + return out + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=["generate", "run", "polars", "pandas"]) + parser.add_argument("--rows", type=int, default=1_000_000) + parser.add_argument("--data", type=Path, default=DEFAULT_DATA) + parser.add_argument("--reps", type=int, default=3) + args = parser.parse_args() + if args.command == "generate": + generate(args.rows, args.data) + elif args.command in ("polars", "pandas"): + result = run_polars(args.data, args.rows, args.reps) if args.command == "polars" else run_pandas(args.data, args.rows, args.reps) + print(json.dumps(result)) + else: + def child(command, env=None): + out = subprocess.run( + [sys.executable, __file__, command, "--rows", str(args.rows), "--data", str(args.data), "--reps", str(args.reps)], + check=True, capture_output=True, text=True, env={**os.environ, **(env or {})}, + ) + return json.loads(out.stdout) + + baseline = { + "rows": args.rows, + "machine": {"cpus": os.cpu_count(), "platform": sys.platform}, + "polars": child("polars"), + "polars1": child("polars", {"POLARS_MAX_THREADS": "1"}), + "pandas": child("pandas"), + } + path = args.data / f"baseline-{args.rows}.json" + path.write_text(json.dumps(baseline, indent=2) + "\n") + print(f"wrote {path}") + for name, q in baseline["polars"]["queries"].items(): + print(f"{name:36} polars {q['ms']:9.1f} ms polars(1t) {baseline['polars1']['queries'][name]['ms']:9.1f} ms pandas {baseline['pandas']['queries'][name]['ms']:9.1f} ms") + + +if __name__ == "__main__": + main() diff --git a/packages/effect/benchmark/dataframe/h2o.ts b/packages/effect/benchmark/dataframe/h2o.ts new file mode 100644 index 00000000000..bdd0ace4cff --- /dev/null +++ b/packages/effect/benchmark/dataframe/h2o.ts @@ -0,0 +1,170 @@ +/** + * effect/dataframe against Polars and pandas on H2O db-benchmark style queries. + * + * python benchmark/dataframe/h2o.py generate --rows 1000000 + * python benchmark/dataframe/h2o.py run --rows 1000000 + * node [--expose-gc] benchmark/dataframe/h2o.ts [--rows=1000000] [--reps=3] [--only=] + * [--out=] [--data=] [--gc] + * + * Every query reads the same Arrow IPC files as the Python baseline and runs + * as a lazy query, so the time includes optimising the plan. A result's + * checksum (row count and the sum of its numeric columns) must match Polars + * before its time counts. Times are the best of `--reps` runs, in + * milliseconds. `--gc` (with `node --expose-gc`) collects garbage before + * every timed run. + */ +import { Effect } from "effect" +import { ArrowIpc } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Result from "effect/Result" +import * as Fs from "node:fs" +import * as Path from "node:path" + +const arg = (name: string) => process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3) +const rows = Number(arg("rows") ?? 1_000_000) +const reps = Number(arg("reps") ?? 3) +const only = arg("only") +const out = arg("out") +const data = arg("data") ?? Path.join(import.meta.dirname, ".data") +const gc = process.argv.includes("--gc") ? (globalThis as { gc?: () => void }).gc : undefined + +const load = (name: string) => { + const start = performance.now() + const bytes = new Uint8Array(Fs.readFileSync(Path.join(data, `${name}-${rows}.arrow`))) + const table = Result.getOrThrow(ArrowIpc.decodeFile(bytes)) + const df = DataFrame.fromArrowTable(table) + return { df: df as DataFrame.DataFrame, ms: performance.now() - start } +} + +const loads = { + g: load("groupby"), + x: load("join-x"), + small: load("join-small"), + medium: load("join-medium"), + big: load("join-big") +} +const t = Object.fromEntries(Object.entries(loads).map(([k, v]) => [k, Query.from(v.df)])) as Record< + keyof typeof loads, + Query.Query +> + +/** Untyped frames: joins are built as plan data. */ +const join = (left: Query.Query, right: Query.Query, on: string, how: Query.JoinHow = "inner") => + Query.fromPlan({ _tag: "Join", left: left.plan, right: right.plan, on: [on], how }) + +const queries: Record Query.Query> = { + "q1 sum v1 by id1": () => t.g.pipe(Query.groupBy("id1"), Query.agg((c) => ({ v1: Expr.sum(c.v1) }))), + "q2 sum v1 by id1:id2": () => t.g.pipe(Query.groupBy("id1", "id2"), Query.agg((c) => ({ v1: Expr.sum(c.v1) }))), + "q3 sum v1, mean v3 by id3": () => + t.g.pipe(Query.groupBy("id3"), Query.agg((c) => ({ v1: Expr.sum(c.v1), v3: Expr.mean(c.v3) }))), + "q4 mean v1:v3 by id4": () => + t.g.pipe( + Query.groupBy("id4"), + Query.agg((c) => ({ v1: Expr.mean(c.v1), v2: Expr.mean(c.v2), v3: Expr.mean(c.v3) })) + ), + "q5 sum v1:v3 by id6": () => + t.g.pipe( + Query.groupBy("id6"), + Query.agg((c) => ({ v1: Expr.sum(c.v1), v2: Expr.sum(c.v2), v3: Expr.sum(c.v3) })) + ), + "q6 median, sd v3 by id4:id5": () => + t.g.pipe( + Query.groupBy("id4", "id5"), + Query.agg((c) => ({ median: Expr.median(c.v3), sd: Expr.std(c.v3) })) + ), + "q7 max v1 - min v2 by id3": () => + t.g.pipe( + Query.groupBy("id3"), + Query.agg((c) => ({ v1: Expr.max(c.v1), v2: Expr.min(c.v2) })), + Query.select((c) => ({ id3: c.id3, range: Expr.subtract(c.v1, c.v2) })) + ), + "q10 sum v3, count by id1:id6": () => + t.g.pipe( + Query.groupBy("id1", "id2", "id3", "id4", "id5", "id6"), + Query.agg((c) => ({ v3: Expr.sum(c.v3), len: Expr.count() })) + ), + "filter v3 > 50, sum v3": () => + t.g.pipe(Query.filter((c) => Expr.gt(c.v3, 50)), Query.aggregate((c) => ({ v3: Expr.sum(c.v3) }))), + "sort by v3 desc, id6": () => t.g.pipe(Query.sort((c) => [Expr.desc(c.v3), c.id6])), + "top 10 by v3 desc, id6": () => t.g.pipe(Query.sort((c) => [Expr.desc(c.v3), c.id6]), Query.head(10)), + "j1 inner small on id1": () => join(t.x, t.small, "id1"), + "j2 inner medium on id2": () => join(t.x, t.medium.pipe(Query.omit("id5")), "id2"), + "j3 left medium on id2": () => join(t.x, t.medium.pipe(Query.omit("id5")), "id2", "left"), + "j4 inner medium on id5 (string)": () => join(t.x.pipe(Query.omit("id2")), t.medium.pipe(Query.omit("id2")), "id5"), + "j5 inner big on id3": () => join(t.x, t.big, "id3") +} + +/** + * Row count and the sum of every numeric column. Reading the columns also + * forces the late-materialised columns of a result, so the timed work + * includes producing every value, as it does for Polars. + */ +const checksum = (df: DataFrame.DataFrame) => { + let sum = 0 + for (const field of df.schema.fields) { + const t = field.type._tag + if (t !== "Int" && t !== "Float") continue + for (const v of DataFrame.toArray(df, field.name)) { + if (typeof v === "number" && !Number.isNaN(v)) sum += v + else if (typeof v === "bigint") sum += Number(v) + } + } + return { rows: df.length, sum } +} + +const baselinePath = Path.join(data, `baseline-${rows}.json`) +const baseline = Fs.existsSync(baselinePath) ? JSON.parse(Fs.readFileSync(baselinePath, "utf8")) : undefined +if (baseline === undefined) console.warn(`no baseline at ${baselinePath}: run h2o.py run --rows ${rows}`) + +const results: Record = {} +const fmt = (ms: number | undefined) => (ms === undefined ? "—" : ms.toFixed(1)).padStart(9) +const ratio = (a: number, b: number | undefined) => (b === undefined ? "" : `${(a / b).toFixed(2)}x`).padStart(7) + +console.log( + `effect/dataframe vs Polars ${baseline?.polars.version ?? "?"} (${ + baseline?.polars.threads ?? "?" + } threads) and pandas ${baseline?.pandas.version ?? "?"}, ${rows} rows, best of ${reps}` +) +console.log( + `load (IPC decode + DataFrame): ${Object.entries(loads).map(([k, v]) => `${k} ${v.ms.toFixed(1)} ms`).join(", ")}` +) +console.log( + `${"query".padEnd(34)} ${"effect".padStart(9)} ${"polars".padStart(9)} ${"pl 1t".padStart(9)} ${ + "pandas".padStart(9) + } ${"vs pl".padStart(7)} ${"vs pl1t".padStart(7)} ${"vs pd".padStart(7)} check` +) +for (const [name, query] of Object.entries(queries)) { + if (only !== undefined && !name.includes(only)) continue + let best = Infinity + let first = 0 + let result: DataFrame.DataFrame | undefined + for (let r = 0; r < reps; r++) { + result = undefined + gc?.() + const start = performance.now() + result = Effect.runSync(query()) + // Forces every column of the result without converting values to JS. + DataFrame.toArrowTable(result) + const ms = performance.now() - start + if (r === 0) first = ms + best = Math.min(best, ms) + } + const sum = checksum(result!) + const expected = baseline?.polars.queries[name]?.checksum + const ok = expected === undefined + ? "?" + : expected.rows === sum.rows && Math.abs(expected.sum - sum.sum) <= 1e-6 * Math.max(1, Math.abs(expected.sum)) + ? "ok" + : `MISMATCH rows ${sum.rows}/${expected.rows} sum ${sum.sum}/${expected.sum}` + const pl = baseline?.polars.queries[name]?.ms + const pl1 = baseline?.polars1.queries[name]?.ms + const pd = baseline?.pandas.queries[name]?.ms + console.log( + `${name.padEnd(34)} ${fmt(best)} ${fmt(pl)} ${fmt(pl1)} ${fmt(pd)} ${ratio(best, pl)} ${ratio(best, pl1)} ${ + ratio(best, pd) + } ${ok}` + ) + results[name] = { ms: best, first, checksum: sum, ok, polars: pl, polars1: pl1, pandas: pd } +} + +if (out !== undefined) Fs.writeFileSync(out, JSON.stringify({ rows, reps, results }, null, 2) + "\n") diff --git a/packages/effect/package.json b/packages/effect/package.json index fbc71f36176..32e50df3031 100644 --- a/packages/effect/package.json +++ b/packages/effect/package.json @@ -35,6 +35,7 @@ "./testing": "./src/testing/index.ts", "./ai": "./src/ai/index.ts", "./arrow": "./src/arrow/index.ts", + "./dataframe": "./src/dataframe/index.ts", "./cli": "./src/cli/index.ts", "./cluster": "./src/cluster/index.ts", "./crypto": "./src/crypto/index.ts", @@ -60,6 +61,7 @@ "./*": "./src/*.ts", "./ai/internal/*": null, "./arrow/internal/*": null, + "./dataframe/internal/*": null, "./cli/internal/*": null, "./cluster/internal/*": null, "./crypto/internal/*": null, @@ -97,6 +99,7 @@ "./testing": "./dist/testing/index.js", "./ai": "./dist/ai/index.js", "./arrow": "./dist/arrow/index.js", + "./dataframe": "./dist/dataframe/index.js", "./cli": "./dist/cli/index.js", "./cluster": "./dist/cluster/index.js", "./crypto": "./dist/crypto/index.js", @@ -126,6 +129,7 @@ "./crypto/internal/*": null, "./ai/internal/*": null, "./arrow/internal/*": null, + "./dataframe/internal/*": null, "./eventlog/internal/*": null, "./http/internal/*": null, "./http/FindMyWay/internal/*": null, diff --git a/packages/effect/src/dataframe/Catalog.ts b/packages/effect/src/dataframe/Catalog.ts new file mode 100644 index 00000000000..afc346e6021 --- /dev/null +++ b/packages/effect/src/dataframe/Catalog.ts @@ -0,0 +1,166 @@ +/** + * The tables a query can read by name. + * + * **Mental model** + * + * - `Query.table("trips", Trip)` (or a `Table` node in a plan) refers to a + * table by name. When the query runs, the {@link Catalog} in the context + * provides it. + * - A table is a frame, an effect producing one (read a file, call a + * service, fetch over Arrow Flight), or a stream of frames (batches). Each + * run of a query reads each table it names once. + * - A query over a streamed table runs batch by batch: row-wise operations + * per batch, grouping and aggregation folded into partial aggregates, so a + * stream larger than memory can be filtered and summarised. The batches, + * as the query reads them (converted to its declared columns, if any), + * must have the column names and types of the first, as the batches of an + * Arrow stream share its schema. + * - The default catalog is empty; provide one with {@link layer} or + * `Effect.provideService(Catalog, Catalog.make(...))`. + * + * **Example** (Providing tables) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" + * + * const Trip = Schema.Struct({ city: Schema.String, km: Schema.Number }) + * + * const longTrips = Query.table("trips", Trip).pipe( + * Query.filter((c) => Expr.gt(c.km, 10)), + * Query.pick("city") + * ) + * + * const trips = DataFrame.fromRows(Trip, [ + * { city: "Vienna", km: 4 }, + * { city: "Berlin", km: 31 } + * ]) + * + * const rows = longTrips.pipe( + * Effect.provide(Catalog.layer({ trips })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ city: "Berlin" }] + * ``` + * + * @stability experimental + * @since 4.1.5 + */ +import * as Context from "../Context.ts" +import * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Stream from "../Stream.ts" +import { type DataFrame, DataFrameError, isDataFrame } from "./DataFrame.ts" +import * as F from "./internal/frame.ts" + +/** + * Provides tables by name to running queries. Defaults to an empty catalog. + * + * @stability experimental + * @category services + * @since 4.1.5 + */ +export class Catalog extends Context.Reference("effect/dataframe/Catalog", { + defaultValue: () => empty +}) {} + +/** + * Types of the {@link Catalog} service. + * + * @stability experimental + * @since 4.1.5 + */ +export declare namespace Catalog { + /** + * The shape of a catalog. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ + export interface Service { + /** The table called `name`, as one frame; fails when there is none. */ + readonly table: (name: string) => Effect.Effect, DataFrameError> + /** The table called `name`, as batches. */ + readonly stream: (name: string) => Stream.Stream, DataFrameError> + /** Whether the table called `name` is streamed (queries over it run batch by batch). */ + readonly streaming: (name: string) => boolean + } +} + +/** + * A table: a frame, or an effect producing one. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Source = + | DataFrame + | Effect.Effect, DataFrameError> + | Stream.Stream, DataFrameError> + +const unknownTable = (name: string) => + Effect.fail(new DataFrameError({ message: `unknown table ${JSON.stringify(name)}` })) + +const concatenated = (frames: ReadonlyArray>): Effect.Effect, DataFrameError> => + frames.length === 1 ? Effect.succeed(frames[0]) : Effect.try({ + try: () => F.concat(frames as unknown as ReadonlyArray) as unknown as DataFrame, + catch: (cause) => new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) + }) + +/** + * A catalog of the given tables. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const make = (tables: { readonly [name: string]: Source }): Catalog.Service => { + const entries = new Map(Object.entries(tables)) + return { + table: (name) => { + const source = entries.get(name) + if (source === undefined) return unknownTable(name) + if (isDataFrame(source)) return Effect.succeed(source) + if (Stream.isStream(source)) { + return Effect.flatMap( + Stream.runCollect(source as Stream.Stream, DataFrameError>), + (frames) => + frames.length === 0 + ? Effect.fail(new DataFrameError({ message: `table ${JSON.stringify(name)} has no batches` })) + : concatenated(frames) + ) + } + return source as Effect.Effect, DataFrameError> + }, + stream: (name) => { + const source = entries.get(name) + if (source === undefined) return Stream.fromEffect(unknownTable(name)) + if (isDataFrame(source)) return Stream.succeed(source) + if (Stream.isStream(source)) return source as Stream.Stream, DataFrameError> + return Stream.fromEffect(source as Effect.Effect, DataFrameError>) + }, + streaming: (name) => Stream.isStream(entries.get(name)) + } +} + +/** + * A catalog without tables. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const empty: Catalog.Service = make({}) + +/** + * A layer providing a catalog of the given tables. + * + * @stability experimental + * @category layers + * @since 4.1.5 + */ +export const layer = (tables: { readonly [name: string]: Source }): Layer.Layer => + Layer.succeed(Catalog, make(tables)) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts new file mode 100644 index 00000000000..1be386fa3f5 --- /dev/null +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -0,0 +1,625 @@ +/** + * Typed, columnar tables: the values queries read and produce. + * + * **Mental model** + * + * - A `DataFrame` is an immutable table whose rows have type `Row`, + * stored column by column in Arrow layouts (typed arrays, UTF-8 bytes). + * Frames convert to and from `effect/arrow` tables without copying where + * the layouts agree. + * - Frames are built from rows, columns or Arrow tables, inspected, compared + * and converted back to rows. Transforming them is the job of `Query`: + * `Query.from(frame)` starts a lazy query, and running it produces a new + * frame. + * - Missing values are `null`. A nullable column's row type includes `null`. + * + * **Example** (Building and reading a frame) + * + * ```ts import.meta.vitest + * import { Schema } from "effect" + * import { DataFrame } from "effect/dataframe" + * + * const Reading = Schema.Struct({ city: Schema.String, temp: Schema.NullOr(Schema.Number) }) + * const readings = DataFrame.fromRows(Reading, [ + * { city: "Vienna", temp: 21.5 }, + * { city: "Berlin", temp: null } + * ]) + * + * readings.length // => 2 + * DataFrame.columnNames(readings) // => ["city", "temp"] + * DataFrame.toArray(readings, "temp") // => [21.5, null] + * ``` + * + * **Gotchas** + * + * - Row objects ({@link toRows}, iteration) map values as `effect/arrow` + * does: 64-bit integers are `bigint`, timestamps `DateTime`. + * + * @stability experimental + * @since 4.1.5 + */ +import type * as ArrowColumn from "../arrow/ArrowColumn.ts" +import * as ArrowData from "../arrow/ArrowData.ts" +import * as ArrowIpc from "../arrow/ArrowIpc.ts" +import * as ArrowSchema from "../arrow/ArrowSchema.ts" +import * as ArrowTable from "../arrow/ArrowTable.ts" +import type * as BigDecimal from "../BigDecimal.ts" +import * as Data from "../Data.ts" +import * as DateTime from "../DateTime.ts" +import * as Duration from "../Duration.ts" +import * as Effect from "../Effect.ts" +import type * as Equal from "../Equal.ts" +import { dual } from "../Function.ts" +import type { Inspectable } from "../Inspectable.ts" +import type { Pipeable } from "../Pipeable.ts" +import * as Result from "../Result.ts" +import * as Stream from "../Stream.ts" +import type * as Types from "../Types.ts" +import * as F from "./internal/frame.ts" +import * as Schema_ from "./internal/schema.ts" +import * as T from "./internal/table.ts" +import * as V from "./internal/vector.ts" + +// ----------------------------------------------------------------------------- +// Models +// ----------------------------------------------------------------------------- + +/** + * Identifies values of this module's main type. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const TypeId: "~effect/dataframe/DataFrame" = F.TypeId + +/** + * A table whose rows have type `Row`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface DataFrame extends Pipeable, Inspectable, Equal.Equal, Iterable { + readonly [TypeId]: { + readonly _Row: Types.Covariant + } + /** The number of rows. */ + readonly length: number + /** The column names and their Arrow types. */ + readonly schema: ArrowSchema.Schema +} + +/** + * The row type of a frame. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Row = F extends DataFrame ? R : never + +/** + * The rows a frame holds for rows of type `T`: optional fields become + * required and `undefined` becomes `null`, in nested structs and lists + * too, as frames store a missing value as `null`. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Normalize = { + readonly [K in keyof T]-?: NormalizeValue | ({} extends Pick ? null : never) +} + +type Simplify = { readonly [K in keyof A]: A[K] } extends infer B ? B : never + +type StructSchema = { readonly Type: object; readonly ast: any } + +type TypeOptions = { + readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined +} + +type NormalizeValue = A extends undefined ? null + : A extends BigDecimal.BigDecimal | DateTime.DateTime | Duration.Duration | Uint8Array ? A + : A extends Array ? Array> + : A extends ReadonlyArray ? ReadonlyArray> + : A extends object ? Normalize + : A + +/** + * Raised for invalid input that the types can't rule out (a row that doesn't + * match its schema, an Arrow table whose columns don't match a schema, a + * column type frames can't hold), and by queries that fail when they run. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class DataFrameError extends Data.TaggedError("DataFrameError")<{ + readonly message: string + readonly cause?: unknown +}> {} + +const fromDefect = (cause: unknown): DataFrameError => + new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) + +const unsafe = (f: () => A): A => { + try { + return f() + } catch (cause) { + throw cause instanceof V.DataFrameDefect ? fromDefect(cause) : cause + } +} + +const frame = (self: DataFrame): F.Frame => self as unknown as F.Frame +const wrap = (f: F.Frame): DataFrame => f as unknown as DataFrame + +/** + * Returns `true` for data frames. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** + * Builds a frame from rows described by a struct schema. Column types come + * from the schema's fields: `String` is Utf8, `Number` is Float64, + * `Boolean` is Bool, `BigInt` is Int64, `DateTimeUtc` and `Duration` + * are microsecond timestamps and durations, arrays are lists, structs are + * structs, and `NullOr`/optional fields are nullable. `options.types` + * overrides the Arrow type of individual columns (for example + * `ArrowSchema.int32`); `BigDecimal` fields need one, a 128-bit decimal + * with the precision and scale (decimals of other widths fail). + * + * **Details** + * + * The rows are decoded values (the schema's `Type`). The schema describes + * them and is not run as a decoder. The frame is built and then checked as + * {@link validate} checks one, so it validates against the same schema and + * options. The schema's refinements, literals and enums are tested on the + * values the frame stores, in every field that has them (inside structs, + * arrays and union members too), and a struct's own checks + * (`Schema.Struct(...).check(...)`) on every stored row. Fields without + * them are not read back as values. + * + * A field that has them accepts the stored values that `Schema.is` accepts + * for it: a check on a nullable field runs on its missing values too, a + * check on a union member runs on that member's values, and a union in + * `oneOf` mode takes a value of exactly one member. + * + * A check is handed what the schema describes. The frame holds `null` for + * every missing value, and a check sees it as the schema writes it: an + * absent key for an optional field, `undefined` for `Schema.UndefinedOr` + * and `null` for `Schema.NullOr`. The rows `{}` and `{ a: undefined }` + * are one row to a frame, so a check sees an absent key for both. + * + * A value is stored exactly or the row is rejected. A `DateTime` or + * `Duration` finer than its column's unit fails, and so do a `DateTime` + * beyond the range of valid dates, a `BigDecimal` with more digits than + * its column's scale or precision, and a number an integer column can't + * hold. A `Float32` override is the exception: it rounds each number to + * the nearest 32-bit float, and the schema's checks test the rounded number. + * + * Missing values read back as `null` (see {@link Normalize}). An override + * must hold its field's values, as {@link validate} checks columns: a + * `DateTimeUtc` override is a timestamp without a time zone or with a UTC + * one, which the frame drops. + * + * Throws a {@link DataFrameError} when a value doesn't fit its column or + * fails a check of the schema, an override can't hold its field's values, + * or the schema has no column types. + * + * **Gotchas** + * + * - A schema that transforms its rows is rejected, as frames hold decoded + * values and run no transformation: `Schema.NumberFromString`, + * `Schema.decodeTo`, decoding defaults, renamed keys. Pass + * `Schema.toType(schema)` to describe the decoded rows of such a schema. + * - Tuples, records, classes, `DateTimeZoned`, unions of different column + * types and recursive schemas have no column type. + * - A schema is rejected when a check would see a missing value that the + * schema writes in several ways, as the frame holds `null` for each of + * them: a struct with checks and a + * `Schema.optional(Schema.NullOr(...))` field, or a check on + * `Schema.UndefinedOr(Schema.NullOr(...))`. Such a field is fine where + * no check is handed its value. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromRows = ( + schema: S, + rows: Iterable, + options?: TypeOptions +): DataFrame> => + unsafe(() => { + const { checks, columns } = Schema_.declare(schema, options?.types as any) + const input: ReadonlyArray = Array.isArray(rows) ? rows : Array.from(rows) + const table = Result.getOrThrowWith(ArrowTable.fromRows(T.toArrowSchema(columns), input), fromDefect) + T.checkExact(columns, input) + return wrap(T.conform(F.fromArrowTable(table), columns, checks)) + }) + +type ColumnInput = + | Float64Array + | Float32Array + | Int32Array + | Int16Array + | Int8Array + | Uint32Array + | Uint16Array + | Uint8Array + | BigInt64Array + | BigUint64Array + | ReadonlyArray + +type ColumnValue = C extends BigInt64Array | BigUint64Array ? bigint + : C extends ArrayBufferView ? number + : C extends ReadonlyArray ? A + : never + +const typedArrayTypes: ReadonlyArray = [ + [Float64Array, ArrowSchema.float64], + [Float32Array, ArrowSchema.float32], + [Int32Array, ArrowSchema.int32], + [Int16Array, ArrowSchema.int16], + [Int8Array, ArrowSchema.int8], + [Uint32Array, ArrowSchema.uint32], + [Uint16Array, ArrowSchema.uint16], + [Uint8Array, ArrowSchema.uint8], + [BigInt64Array, ArrowSchema.int64], + [BigUint64Array, ArrowSchema.uint64] +] + +const inferType = (values: ReadonlyArray): ArrowSchema.DataType => { + for (const v of values) { + switch (typeof v) { + case "number": + return ArrowSchema.float64 + case "string": + return ArrowSchema.utf8 + case "boolean": + return ArrowSchema.bool + case "bigint": + return ArrowSchema.int64 + case "object": + if (DateTime.isDateTime(v)) return ArrowSchema.timestamp("microsecond") + if (Duration.isDuration(v)) return ArrowSchema.duration("microsecond") + if (v !== null) return V.fail(`cannot infer a column type for ${String(v)}: use fromRows with a schema`) + } + } + return ArrowSchema.nullType +} + +/** + * Builds a frame from columns. Typed arrays keep their element type + * (`Int32Array` is Int32, `Float64Array` is Float64, …) and are used without + * copying where possible; arrays infer their type from their values + * (`number` is Float64, `string` Utf8, `boolean` Bool, `bigint` Int64, + * `DateTime` a microsecond timestamp, `Duration` a microsecond duration) and + * use `null` for missing values. + * + * **Details** + * + * Throws a {@link DataFrameError} when columns have different lengths or an + * array mixes value types. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromColumns = ( + columns: C +): DataFrame }>> => + unsafe(() => { + const names = Object.keys(columns) + const types = names.map((name) => { + const c = columns[name] + return ArrayBuffer.isView(c) + ? typedArrayTypes.find(([ctor]) => c instanceof ctor)![1] + : inferType(c as ReadonlyArray) + }) + const lengths = new Set(names.map((n) => columns[n].length)) + if (lengths.size > 1) V.fail(`columns have different lengths: ${[...lengths].join(", ")}`) + const length = names.length === 0 ? 0 : columns[names[0]].length + const vectors = names.map((name, i) => { + const c = columns[name] + if (ArrayBuffer.isView(c)) { + const bytes = new Uint8Array(c.buffer, c.byteOffset, c.byteLength) + return V.fromArrowData(ArrowData.make({ type: types[i], length: c.length, buffers: [bytes] })) + } + const schema = ArrowSchema.make([ArrowSchema.field(name, types[i])]) + const table = Result.getOrThrowWith(ArrowTable.fromColumns(schema, [c]), fromDefect) + return F.fromArrowTable(table).columns[0].vector + }) + return wrap(F.fromVectors(names, vectors, length)) + }) + +/** + * A frame over an Arrow table, with untyped rows. Use {@link validate} to + * check it against a schema and recover the row type. + * + * **Details** + * + * Throws a {@link DataFrameError} for column types frames can't hold, such + * as Float16, intervals, maps, unions, or decimals beyond 18 digits. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromArrowTable = (table: ArrowTable.Table): DataFrame<{ readonly [name: string]: unknown }> => + unsafe(() => wrap(F.fromArrowTable(table))) + +/** + * A frame over one Arrow record batch, with untyped rows. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromRecordBatch = (batch: ArrowTable.RecordBatch): DataFrame<{ readonly [name: string]: unknown }> => + fromArrowTable(ArrowTable.make(batch.schema, [batch])) + +/** + * Decodes Arrow IPC stream bytes (from a file, a socket or Arrow Flight) + * into a stream of frames, one per record batch, ready to be a streamed + * `Catalog` table. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const decodeArrowStream = ( + bytes: Stream.Stream +): Stream.Stream, DataFrameError | E, R> => + bytes.pipe( + Stream.pipeThroughChannel(ArrowIpc.decode()), + Stream.mapError((error) => + ArrowIpc.isArrowIpcError(error) ? new DataFrameError({ message: error.message, cause: error }) : error as E + ), + Stream.mapEffect((batch) => + Effect.try({ + try: () => fromRecordBatch(batch), + catch: (cause) => cause instanceof DataFrameError ? cause : fromDefect(cause) + }) + ) + ) + +/** + * An empty frame with the columns of a struct schema, typed as in + * {@link fromRows}, which also describes `options.types`. + * + * **Gotchas** + * + * Throws a {@link DataFrameError} for the schemas {@link fromRows} rejects: + * one that transforms its rows, has fields without a column type, or has a + * check that would see a missing value the schema writes in several ways. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const empty = ( + schema: S, + options?: TypeOptions +): DataFrame> => fromRows(schema, [], options) + +/** + * Checks that a frame has the columns a struct schema or an Arrow schema + * declares, with compatible types, and returns it with only those columns, + * in the declared order, converted to the declared types. + * + * **Details** + * + * A struct schema declares the columns {@link fromRows} gives its fields + * (`options.types` overrides some, and `BigDecimal` fields need one), and + * the result is typed by it. A column is compatible when its values read + * back as the field's: `Number` takes floats and integers below 64 bits, + * `BigInt` takes 64-bit integers, `BigDecimal` takes decimals of any + * precision and scale, `DateTimeUtc` and `Duration` take timestamps and + * durations in any unit, and structs and arrays need compatible fields and + * items. Every value must convert to the declared type exactly: a decimal + * with more digits than the declared scale or precision, or a timestamp + * finer than the declared unit, fails. + * + * The types are checked and converted on the typed columns, without reading + * values. A field the schema constrains further (literals, enums, + * refinements, including inside structs, arrays and union members) has its + * values read and tested one by one, and a struct with checks of its own + * (`Schema.Struct(...).check(...)`) has every row read and tested. The + * checks see missing values as the schema writes them (an absent key, + * `undefined` or `null`), and a schema that writes one in several ways + * under a check fails, as in {@link fromRows}. The schema is not run as a + * decoder: one that transforms its rows fails, as in {@link fromRows}. + * + * An Arrow schema declares its fields as they are: type, nullability, + * nested fields, decimal precision and scale, and time unit. The result has + * untyped rows, and no values are tested beyond their column type. The + * fields can be `Null` (nullable), `Bool`, integers of any width, `Float32`, + * `Float64`, `Utf8`, `Binary`, `Date` in days, `Time` in microseconds, + * `Timestamp` and `Duration` in any unit, 128-bit `Decimal` (a precision + * from 1 to 38 and a 32-bit scale, or it fails), and `List` and `Struct` of + * these. Other Arrow types fail: frames can't hold some + * (`Float16`, intervals, maps, unions) and hold others as one of the types + * above, which is the type to declare (`Utf8` for `LargeUtf8` and + * `Utf8View`, `List` for `LargeList`, the value type for a dictionary). A + * list's item field is named `item` in the result. Dates in milliseconds + * and times in other units convert to the declared days and microseconds. + * + * **Gotchas** + * + * `DateTimeUtc` fields and Arrow timestamps take timestamps without a time + * zone, or with a UTC one (`UTC`, `Etc/UTC`, `+00:00`), which the result + * drops; other time zones, and values outside the range `DateTime` + * represents, fail. `DateTimeZoned` fields are not supported: read zoned + * timestamps from an untyped frame, where they are `DateTime.Zoned` values. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const validate: { + ( + schema: S, + options?: TypeOptions + ): (self: DataFrame) => Result.Result>, DataFrameError> + ( + schema: ArrowSchema.Schema + ): (self: DataFrame) => Result.Result, DataFrameError> + ( + self: DataFrame, + schema: S, + options?: TypeOptions + ): Result.Result>, DataFrameError> + ( + self: DataFrame, + schema: ArrowSchema.Schema + ): Result.Result, DataFrameError> +} = dual((args) => isDataFrame(args[0]), ( + self: DataFrame, + schema: { readonly ast: any } | ArrowSchema.Schema, + options?: { readonly types?: { readonly [name: string]: ArrowSchema.DataType | undefined } | undefined } +) => + Result.try({ + try: () => { + const { checks, columns } = Schema_.declare(schema, options?.types) + return wrap(T.conform(frame(self), columns, checks)) + }, + catch: fromDefect + })) + +// ----------------------------------------------------------------------------- +// Conversions +// ----------------------------------------------------------------------------- + +/** + * The rows as plain objects. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toRows = (self: DataFrame): Array => F.toRows(frame(self)) + +/** + * The values of one column. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toArray: { + (name: K): (self: DataFrame) => Array + (self: DataFrame, name: K): Array +} = dual(2, (self: DataFrame, name: string) => F.toArray(frame(self), name)) as any + +/** + * One column as an Arrow column, without building rows or values. + * + * **When to use** + * + * Use to read or compute over a column's stored numbers, or to read single + * values, without paying for a row object or a `DateTime`, `Duration` or + * `BigDecimal` per value. + * + * **Details** + * + * The column has one chunk, laid out as {@link toArrowTable} lays it out: + * its type, validity bitmap, buffers and child arrays are those of the + * Arrow table, and share the frame's memory where the layouts agree. + * `ArrowData.values` and `ArrowData.offsets` give its numbers and slot + * boundaries as typed arrays, with timestamps, dates and durations as the + * integers they are stored as, in the unit of the column's type. + * `ArrowColumn.get`, `ArrowColumn.toArray` and iteration build values + * as {@link toArray} does, typed by the frame's rows, one at a time. + * + * **Gotchas** + * + * - The typed array to expect follows from the column's Arrow type + * (`self.schema`), not from the row type: a `number` column can be + * `Int32` or `Float64`. + * - Treat the buffers as read-only: they can be the frame's own. + * - Frames hold dictionary-encoded and chunked input as one plain array per + * column, so the column has no dictionary and a single chunk. + * + * **Example** (Reading timestamps as stored) + * + * ```ts import.meta.vitest + * import { DateTime, Option, Schema } from "effect" + * import { ArrowColumn, ArrowData } from "effect/arrow" + * import { DataFrame } from "effect/dataframe" + * + * const Event = Schema.Struct({ at: Schema.NullOr(Schema.DateTimeUtc) }) + * const events = DataFrame.fromRows(Event, [{ at: DateTime.makeUnsafe(1500) }, { at: null }]) + * + * const at = DataFrame.column(events, "at") + * const micros = Option.getOrThrow(ArrowData.values(at.chunks[0])) + * micros[0] // => 1500000n + * ArrowData.isValid(at.chunks[0], 1) // => false + * ArrowColumn.getUnsafe(at, 0) // => DateTime.makeUnsafe(1500) + * ``` + * + * @see {@link toArray} for the values of a column as an array + * @see {@link toArrowTable} for every column as one record batch + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const column: { + (name: K): (self: DataFrame) => ArrowColumn.ArrowColumn> + (self: DataFrame, name: K): ArrowColumn.ArrowColumn> +} = dual( + 2, + (self: DataFrame, name: string) => unsafe(() => F.toArrowColumn(frame(self), name)) +) as any + +/** + * The frame as an Arrow table with one record batch. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toArrowTable = (self: DataFrame): ArrowTable.Table => F.toArrowTable(frame(self)) + +/** + * The column names, in order. + * + * @stability experimental + * @category getters + * @since 4.1.5 + */ +export const columnNames = (self: DataFrame): Array => + frame(self).columns.map((c) => c.name) as Array + +/** + * A text table of the frame (its first and last rows when it is long). + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const format = (self: DataFrame, options?: { readonly maxRows?: number | undefined }): string => + F.render(frame(self), options?.maxRows ?? 10) + +/** + * Whether two frames have the same columns (names and types) and values. + * `NaN` equals `NaN`. + * + * @stability experimental + * @category equivalence + * @since 4.1.5 + */ +export const equals: { + (that: DataFrame): (self: DataFrame) => boolean + (self: DataFrame, that: DataFrame): boolean +} = dual(2, (self: DataFrame, that: DataFrame) => F.equals(frame(self), frame(that))) diff --git a/packages/effect/src/dataframe/Engine.ts b/packages/effect/src/dataframe/Engine.ts new file mode 100644 index 00000000000..ec5378e6a2f --- /dev/null +++ b/packages/effect/src/dataframe/Engine.ts @@ -0,0 +1,88 @@ +/** + * What runs queries: the in-memory engine by default, or another engine + * (for example `Sql.layer`, which runs plans on a database) provided as a + * layer. + * + * **Mental model** + * + * - A query is a `Plan.Node`. Running it (`yield* query`, `Query.stream`, + * `Query.explain`) hands the plan to the {@link Engine} in the context. + * - The {@link memory} engine runs plans in this process: it reads tables + * from the `Catalog`, rewrites the plan, and runs it. + * - Other engines run the same plans elsewhere. Their results are frames + * like any other. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Context from "../Context.ts" +import type * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Stream from "../Stream.ts" +import { Catalog } from "./Catalog.ts" +import type { DataFrame, DataFrameError } from "./DataFrame.ts" +import * as Q from "./internal/query.ts" +import * as Streaming from "./internal/stream.ts" +import type * as Plan from "./Plan.ts" + +/** + * Types of the {@link Engine} service. + * + * @stability experimental + * @since 4.1.5 + */ +export declare namespace Engine { + /** + * An engine. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ + export interface Service { + /** A short name, for diagnostics. */ + readonly name: string + /** The plan's result as one frame. */ + readonly collect: (plan: Plan.Node) => Effect.Effect, DataFrameError> + /** The plan's result as batches. */ + readonly stream: (plan: Plan.Node) => Stream.Stream, DataFrameError> + /** What running the plan does, as text. */ + readonly explain: (plan: Plan.Node) => Effect.Effect + } +} + +/** + * The in-memory engine: tables from the `Catalog`, the plan rewritten + * (predicate, slice and projection pushdown, top-k), streamed tables batch by + * batch. + * + * @stability experimental + * @category engines + * @since 4.1.5 + */ +export const memory: Engine.Service = { + name: "memory", + collect: (plan) => Catalog.use((catalog) => Q.collect(plan, catalog) as any), + stream: (plan) => Stream.unwrap(Catalog.useSync((catalog) => Streaming.stream(plan, catalog) as any)), + explain: (plan) => Catalog.use((catalog) => Q.explain(plan, catalog)) +} + +/** + * Runs queries. Defaults to the {@link memory} engine. + * + * @stability experimental + * @category services + * @since 4.1.5 + */ +export class Engine extends Context.Reference("effect/dataframe/Engine", { + defaultValue: () => memory +}) {} + +/** + * A layer that runs queries with `engine`. + * + * @stability experimental + * @category layers + * @since 4.1.5 + */ +export const layer = (engine: Engine.Service): Layer.Layer => Layer.succeed(Engine, engine) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts new file mode 100644 index 00000000000..eb8e11100a9 --- /dev/null +++ b/packages/effect/src/dataframe/Expr.ts @@ -0,0 +1,1486 @@ +/** + * Column expressions: typed, inspectable descriptions of a computation over + * the columns of a query. + * + * **Mental model** + * + * - An `Expr` describes one value per row of type `A`. `A` includes + * `null` when the value can be missing. + * - Expressions are data: a typed wrapper around a `Plan.Expression` + * ({@link toPlan}, {@link fromPlan}). `Query` functions take a callback + * that receives one typed reference per column (`c.temp`, `c.city`) and + * returns expressions; the callback runs once, when the query is built. + * The resulting tree runs over whole columns at a time. + * - Every function is dual: `Expr.gt(c.temp, 20)` and + * `c.temp.pipe(Expr.gt(20))` build the same expression. Plain values in + * argument position are literals. + * - Missing values propagate: an operation on `null` gives `null`. + * `and`/`or` use three-valued logic (`false && null` is `false`). + * - An {@link Aggregate} reduces many rows to one (`sum`, `mean`, …). It is + * only accepted by `Query.agg` and `Query.aggregate`. + * + * **Details** + * + * The semantics follow Polars: + * + * - Float comparisons use a total order: `NaN` equals `NaN` and is greater + * than every other number. `-0` equals `0`. + * - Strings compare by Unicode code point (UTF-8 byte order). + * - Integer arithmetic wraps on overflow; `divide` is true division and + * returns a float; `floorDivide` and `mod` round towards negative infinity, + * and an integer division by zero gives `null`. + * - An integer literal takes the integer type of the columns it meets + * (`c.int8Column + 1` stays Int8) when the type holds it, and widens it + * otherwise (`c.int8Column + 300` is Int16), wherever it appears among + * the operands (`coalesce(300, 70000, c.int8Column)` is Int32). + * A `when`, `coalesce` or `fillNull` of literals fits all its literals, + * whichever rows select them (`c.int8Column + when(cond, 300, 1)` is + * Int16), so result types never depend on the rows. Other values computed + * from integer literals (`negate`, arithmetic) widen narrower columns to + * Int32, and a {@link cast} fixes a literal's type. + * - Operands of different types meet in their common type. A value the + * common type can't hold fails the query with a `DataFrameError`; only + * an explicit {@link cast} makes it `null`. + * An integer and a decimal meet in a decimal with a scale of at least + * zero, so the integer keeps its value next to a negative scale. + * - Decimals are stored as 64-bit unscaled integers, so they hold up to 18 + * digits. Unlike integers they never wrap: a decimal result outside that + * storage (from arithmetic, sums, `abs`, `negate`, `round` or a + * literal) fails the query with a `DataFrameError`, and a cast makes it + * `null`. + * + * **Example** (Building expressions) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const Reading = Schema.Struct({ city: Schema.String, temp: Schema.Number, rain: Schema.NullOr(Schema.Number) }) + * const df = DataFrame.fromRows(Reading, [ + * { city: "Vienna", temp: 21.5, rain: 0 }, + * { city: "Berlin", temp: 26.5, rain: null } + * ]) + * + * const query = Query.from(df).pipe( + * Query.select((c) => ({ + * city: c.city, + * fahrenheit: c.temp.pipe(Expr.multiply(1.8), Expr.add(32)), + * wet: Expr.gt(c.rain, 0) + * })) + * ) + * + * const rows = DataFrame.toRows(Effect.runSync(query)) + * rows[0] // => { city: "Vienna", fahrenheit: 70.7, wet: false } + * rows[1] // => { city: "Berlin", fahrenheit: 79.7, wet: null } + * ``` + * + * @stability experimental + * @since 4.1.5 + */ +import type * as BigDecimal from "../BigDecimal.ts" +import type * as DateTime from "../DateTime.ts" +import type * as Duration from "../Duration.ts" +import { dual } from "../Function.ts" +import type { Pipeable } from "../Pipeable.ts" +import type * as Types from "../Types.ts" +import * as internal from "./internal/expr.ts" +import type * as Plan from "./Plan.ts" + +// ----------------------------------------------------------------------------- +// Models +// ----------------------------------------------------------------------------- + +/** + * Identifies values of this module's main type. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const TypeId: "~effect/dataframe/Expr" = internal.ExprTypeId + +/** + * A per-row expression whose value has type `A`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Expr extends Pipeable { + readonly [TypeId]: { + readonly _A: Types.Covariant + } +} + +/** + * Identifies aggregate expressions. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const AggregateTypeId: "~effect/dataframe/Expr/Aggregate" = internal.AggregateTypeId + +/** + * An expression that reduces the rows of a group to one value of type `A`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Aggregate extends Pipeable { + readonly [AggregateTypeId]: { + readonly _A: Types.Covariant + } +} + +/** + * A sort key: an expression and a direction. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface SortOrder { + readonly expr: Expr + readonly descending: boolean +} + +/** + * An expression or a plain value (a literal). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Input = Expr | A + +/** + * The value type of an expression or aggregate. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Type = E extends Expr ? A : E extends Aggregate ? A : never + +/** + * `null` when `A` includes it. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Nullish = null extends A ? null : never + +/** + * The widened primitive of `A`: what `A` can be compared with. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Base = [NonNullable] extends [never] ? never + : NonNullable extends number ? number + : NonNullable extends bigint ? bigint + : NonNullable extends string ? string + : NonNullable extends boolean ? boolean + : NonNullable + +type Numeric = number | bigint | BigDecimal.BigDecimal +/** The numeric family of `A`: bigint, BigDecimal or number. */ +type NumericOf = NonNullable extends bigint ? bigint + : NonNullable extends BigDecimal.BigDecimal ? BigDecimal.BigDecimal + : number +/** The type of a sum of `A`s: durations add up to a duration, the rest to their numeric family. */ +type SumOf = NonNullable extends Numeric | boolean ? NumericOf : Duration.Duration +type InputType = I extends Expr ? A : I + +/** + * Names accepted by {@link cast}, and the value type each produces. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface CastTargets { + readonly Int8: number + readonly Int16: number + readonly Int32: number + readonly Int64: bigint + readonly UInt8: number + readonly UInt16: number + readonly UInt32: number + readonly UInt64: bigint + readonly Float32: number + readonly Float64: number + readonly Utf8: string + readonly Bool: boolean + readonly Date: DateTime.Utc + readonly "Timestamp[s]": DateTime.Utc + readonly "Timestamp[ms]": DateTime.Utc + readonly "Timestamp[us]": DateTime.Utc + readonly "Timestamp[ns]": DateTime.Utc + readonly "Duration[s]": Duration.Duration + readonly "Duration[ms]": Duration.Duration + readonly "Duration[us]": Duration.Duration + readonly "Duration[ns]": Duration.Duration +} + +/** + * Returns `true` for expressions. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isExpr = (u: unknown): u is Expr => internal.isExpr(u) && !internal.isAggregate(u) + +/** + * Returns `true` for aggregate expressions. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isAggregate = (u: unknown): u is Aggregate => internal.isAggregate(u) + +/** + * The expression as plan data. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toPlan = (self: Expr | Aggregate): Plan.Expression => + (self as unknown as internal.Handle).node + +/** + * An expression from plan data, with an unknown value type. Aggregate calls + * become aggregates. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const fromPlan: { + (expression: Plan.AggregateCall): Aggregate + (expression: Plan.Expression): Expr +} = (expression: Plan.Expression): any => + expression._tag === "AggregateCall" ? internal.makeAggregate(expression) : internal.makeExpr(expression) + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** + * A constant. Integers between -2^31 and 2^31 - 1 adopt the integer type of + * the column they meet when it holds them; other numbers are Float64; + * bigints are Int64. A `BigDecimal` is a decimal of its scale. Its precision + * is the larger of its number of digits and its scale, capped at 38. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const lit = < + const A extends number | bigint | string | boolean | DateTime.Utc | Duration.Duration | BigDecimal.BigDecimal | null +>(value: A): Expr | Nullish> => internal.makeExpr(internal.literalNode(value)) + +// ----------------------------------------------------------------------------- +// Arithmetic +// ----------------------------------------------------------------------------- + +type ArithResult = Expr | Nullish | Nullish> + +const unary = (fn: Plan.FunctionName) => (self: Expr) => internal.call(fn, [self]) + +const binary = (fn: Plan.FunctionName) => + dual(2, (self: unknown, that: unknown) => internal.call(fn, [self, that])) as any + +/** + * `self + that`. Integers wrap on overflow. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const add: { + ( + that: Input + ): | null>(self: Expr) => ArithResult + ( + that: Input + ): ( + self: Expr + ) => Expr | Nullish | Nullish> + | null>(self: Expr, that: Input): ArithResult + ( + self: Expr, + that: Input + ): Expr | Nullish | Nullish> +} = binary("add") + +/** + * `self - that`. Integers wrap on overflow. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const subtract: { + ( + that: Input + ): | null>(self: Expr) => ArithResult + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + that: Input + ): ( + self: Expr + ) => Expr | Nullish | Nullish> + | null>(self: Expr, that: Input): ArithResult + ( + self: Expr, + that: Input + ): Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish | Nullish> +} = binary("subtract") + +/** + * `self * that`. Integers wrap on overflow. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const multiply: { + ( + that: Input + ): | null>(self: Expr) => ArithResult + | null>(self: Expr, that: Input): ArithResult +} = binary("multiply") + +/** + * True division: always a float (`7 / 2` is `3.5`). Division by zero gives + * an infinity or NaN. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const divide: { + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish> +} = binary("divide") + +type FloorDivision = { + ( + that: Input + ): | null>(self: Expr) => Expr | null> + | null>(self: Expr, that: Input): Expr | null> +} + +/** + * Division rounded towards negative infinity. Integer division by zero + * gives `null`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const floorDivide: FloorDivision = binary("floorDivide") + +/** + * The remainder of {@link floorDivide}: it has the sign of the divisor + * (`-7 mod 3` is `2`). Integer division by zero gives `null`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const mod: FloorDivision = binary("mod") + +/** + * `-self`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const negate = (self: Expr): Expr => internal.call("negate", [self]) + +/** + * The absolute value. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const abs = (self: Expr): Expr => internal.call("abs", [self]) + +/** + * The square root, as a float. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const sqrt = (self: Expr): Expr> => internal.call("sqrt", [self]) + +/** + * Rounds to `decimals` places (default 0), halves to even. + * + * **Details** + * + * Floats round `x * 10^decimals` in float64 arithmetic, so 2.675 rounds to + * 2.68 and 1.005 to 1 at two places. Integers are returned unchanged. + * Decimals round half to even at `decimals` places and keep their scale; + * when `decimals` is below the scale, the precision grows by one digit (up + * to 38), as in Polars. + * + * **Gotchas** + * + * `decimals` must be a non-negative whole number; other values fail the + * query with a `DataFrameError`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const round: { + (decimals?: number): (self: Expr) => Expr + (self: Expr, decimals?: number): Expr +} = dual( + (args) => isExpr(args[0]), + (self: Expr, decimals = 0) => internal.call("round", [self], { decimals }) +) as any + +// ----------------------------------------------------------------------------- +// Comparisons +// ----------------------------------------------------------------------------- + +type Comparison = { + (that: Input): | null>(self: Expr) => Expr | Nullish> + | null>(self: Expr, that: Input): Expr | Nullish> +} + +/** + * `self === that`. `null` when either side is `null`; `NaN` equals `NaN`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const eq: Comparison = binary("eq") + +/** + * `self !== that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const neq: Comparison = binary("neq") + +/** + * `self < that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const lt: Comparison = binary("lt") + +/** + * `self <= that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const lte: Comparison = binary("lte") + +/** + * `self > that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const gt: Comparison = binary("gt") + +/** + * `self >= that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const gte: Comparison = binary("gte") + +/** + * `lower <= self && self <= upper`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const between: { + (lower: Input, upper: Input): | null>( + self: Expr + ) => Expr | Nullish | Nullish> + | null, C extends Base | null>( + self: Expr, + lower: Input, + upper: Input + ): Expr | Nullish | Nullish> +} = dual(3, (self: unknown, lower: unknown, upper: unknown) => internal.call("between", [self, lower, upper])) as any + +/** + * Whether the value is one of `values`; `null` for `null`. A decimal is one + * of the numbers equal to it, whatever its scale. `BigDecimal`, `DateTime` + * and `Duration` candidates are compared as {@link eq} compares them. + * + * **Details** + * + * The supported candidates are those of the value's kind: strings for + * strings, booleans for booleans, numbers for floats and for integers of + * up to 32 bits, numbers and bigints for 64-bit integers and decimals, and + * `BigDecimal`, `DateTime` and `Duration` for decimals, timestamps and + * durations. A number with a fraction is compared with a decimal as a + * float and is never a 64-bit integer. + * + * Each engine compares them by its own rules, as it does for {@link eq}, + * so two comparisons can differ between engines: + * + * - A decimal with a number that is not an integer (`NaN` and the + * infinities included). Engines round a decimal of more digits than a + * float holds differently, and a database can fail on a decimal outside + * the range of a float. + * - A timestamp that has a time zone with a `DateTime`, which is a + * timestamp without one. A database reads it in its session's time + * zone, so the results agree in a UTC session only. + * + * Other candidates (a string or a bigint for an Int32, a number for a + * timestamp) are not portable: what they match depends on the engine, and + * a database can reject them. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const isIn: { + (values: ReadonlyArray): | null>(self: Expr) => Expr> + (self: Expr, values: ReadonlyArray>): Expr> +} = dual(2, internal.isIn) as any + +// ----------------------------------------------------------------------------- +// Logic +// ----------------------------------------------------------------------------- + +type Logical = { + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish> +} + +/** + * Three-valued AND: `false` if either side is `false`, otherwise `null` if + * either side is `null`. + * + * @stability experimental + * @category logic + * @since 4.1.5 + */ +export const and: Logical = binary("and") + +/** + * Three-valued OR: `true` if either side is `true`, otherwise `null` if + * either side is `null`. + * + * @stability experimental + * @category logic + * @since 4.1.5 + */ +export const or: Logical = binary("or") + +/** + * Negation; `null` stays `null`. + * + * @stability experimental + * @category logic + * @since 4.1.5 + */ +export const not = (self: Expr): Expr => internal.call("not", [self]) + +// ----------------------------------------------------------------------------- +// Missing values +// ----------------------------------------------------------------------------- + +/** + * Whether the value is missing. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const isNull = (self: Expr): Expr => internal.call("isNull", [self]) + +/** + * Whether the value is present. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const isNotNull = (self: Expr): Expr => internal.call("isNotNull", [self]) + +/** + * Whether the value is `NaN` (`false` for integers; `null` stays `null`). + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const isNaN = (self: Expr): Expr> => + internal.call("isNaN", [self]) + +/** + * Replaces missing values with `value`. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const fillNull: { + (value: Input): | null>(self: Expr) => Expr | Base | Nullish> + | null>(self: Expr, value: Input): Expr | Base | Nullish> +} = binary("fillNull") + +type Coalesced> = Expr> | AllNullable> +type AllNullable> = Es extends readonly [infer H, ...infer T] + ? null extends InputType ? AllNullable : never + : null + +/** + * The first argument that is not `null`, per row. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const coalesce = , ...Array>]>(...exprs: Es): Coalesced => + internal.call("coalesce", exprs) + +/** + * `then` where `condition` is `true`, `otherwise` where it is `false` or + * `null`. + * + * @stability experimental + * @category conditionals + * @since 4.1.5 + */ +export const when = | null>( + condition: Expr, + then: Input, + otherwise: Input +): Expr | Nullish | Base | Nullish> => internal.call("when", [condition, then, otherwise]) + +// ----------------------------------------------------------------------------- +// Casts +// ----------------------------------------------------------------------------- + +/** + * Converts to another type. Values that don't fit (out of range, unparsable + * strings, NaN to an integer) become `null`; floats truncate towards zero + * when converted to integers. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const cast: { + (to: To): (self: Expr) => Expr + (self: Expr, to: To): Expr +} = dual(2, (self: unknown, to: string) => internal.call("cast", [self], { to })) as any + +/** + * Converts to a decimal with `precision` digits, `scale` of them after the + * point. Values that don't fit (more than `precision` digits, or outside + * 64-bit decimal storage) become null; extra digits round half to even. A + * negative scale rounds before the point: at scale `-2`, `1250` is `1200`. + * The query fails for a precision outside 1 to 38 or a scale that isn't a + * 32-bit integer. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const castDecimal: { + (precision: number, scale: number): (self: Expr) => Expr + (self: Expr, precision: number, scale: number): Expr +} = dual( + 3, + (self: unknown, precision: number, scale: number) => + internal.call("cast", [self], { to: "Decimal", precision, scale }) +) as any + +// ----------------------------------------------------------------------------- +// Strings +// ----------------------------------------------------------------------------- + +/** + * The number of Unicode code points. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const length = (self: Expr): Expr> => + internal.call("length", [self]) + +/** + * Lower case. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const toLowerCase = (self: Expr): Expr => internal.call("toLowerCase", [self]) + +/** + * Upper case. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const toUpperCase = (self: Expr): Expr => internal.call("toUpperCase", [self]) + +type Pattern = { + (text: string): (self: Expr) => Expr> + (self: Expr, text: string): Expr> +} + +const pattern = (fn: "includes" | "startsWith" | "endsWith"): Pattern => + dual(2, (self: unknown, text: string) => internal.call(fn, [self], { pattern: text })) as any + +/** + * Whether the string contains `text` (literally). + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const includes: Pattern = pattern("includes") + +/** + * Whether the string starts with `text`. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const startsWith: Pattern = pattern("startsWith") + +/** + * Whether the string ends with `text`. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const endsWith: Pattern = pattern("endsWith") + +/** + * Concatenates the arguments as strings; `null` if any argument is `null`. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const concat = , ...Array>]>( + ...parts: Es +): Expr> }[number]> => internal.call("concat", parts) + +// ----------------------------------------------------------------------------- +// Sorting +// ----------------------------------------------------------------------------- + +/** + * Sorts by `self`, smallest first. + * + * @stability experimental + * @category sorting + * @since 4.1.5 + */ +export const asc = (self: Expr): SortOrder => ({ expr: self, descending: false }) + +/** + * Sorts by `self`, largest first. + * + * @stability experimental + * @category sorting + * @since 4.1.5 + */ +export const desc = (self: Expr): SortOrder => ({ expr: self, descending: true }) + +// ----------------------------------------------------------------------------- +// Aggregates +// ----------------------------------------------------------------------------- + +const agg = (fn: Plan.AggregateName, self?: unknown) => + internal.makeAggregate({ + _tag: "AggregateCall", + fn, + arg: self === undefined ? undefined : internal.toNode(self) + }) + +/** + * The sum of the values; 0 for a group without values. Sums of 8- and 16-bit + * integers are Int32. + * + * **Details** + * + * Durations sum to a duration in the column's unit, the zero duration for a + * group without values. Like 64-bit integer sums, they wrap on overflow. + * + * **Gotchas** + * + * Dates, times and timestamps have no sum: a query that sums them fails with + * a `DataFrameError`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const sum = ( + self: Expr +): Aggregate> => agg("sum", self) + +/** + * The mean of the values; `null` for a group without values. + * + * **Gotchas** + * + * Durations, dates, times and timestamps are not numbers: a query that takes + * their {@link mean}, {@link median}, {@link std} or {@link variance} fails + * with a `DataFrameError`. Aggregate a count of units instead, such as + * {@link totalMilliseconds} of a duration. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const mean = (self: Expr): Aggregate => agg("mean", self) + +/** + * The median of the values (the mean of the middle two for an even count). + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const median = (self: Expr): Aggregate => agg("median", self) + +/** + * The sample standard deviation (`n - 1` denominator); `null` for fewer than + * two values. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const std = (self: Expr): Aggregate => agg("std", self) + +/** + * The sample variance (`n - 1` denominator); `null` for fewer than two + * values. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const variance = (self: Expr): Aggregate => agg("variance", self) + +/** + * The smallest value. Float `NaN`s are skipped unless every value is `NaN`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const min = (self: Expr): Aggregate | null> => agg("min", self) + +/** + * The largest value. Float `NaN`s are skipped unless every value is `NaN`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const max = (self: Expr): Aggregate | null> => agg("max", self) + +/** + * Without an argument: the number of rows. With one: the number of rows + * where the expression is not `null`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const count = (self?: Expr): Aggregate => agg("count", self) + +/** + * The number of distinct values, `null` included. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const countDistinct = (self: Expr): Aggregate => agg("countDistinct", self) + +/** + * The value in the group's first row (`null` if the group is empty). + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const first = (self: Expr): Aggregate => agg("first", self) + +/** + * The value in the group's last row (`null` if the group is empty). + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const last = (self: Expr): Aggregate => agg("last", self) + +// ----------------------------------------------------------------------------- +// Windows +// ----------------------------------------------------------------------------- + +/** + * Evaluates `self` separately within each partition of rows with equal + * `partitionBy` values. An aggregate gives its partition's value on every + * row; a window function (`cumSum`, `shift`, `rank`, the rolling functions) + * runs over its partition's rows in row order. + * + * **Example** (Share of each city's total) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const sales = DataFrame.fromColumns({ city: ["Vienna", "Berlin", "Vienna"], units: [2, 5, 6] }) + * + * const rows = Query.from(sales).pipe( + * Query.withColumns((c) => ({ + * share: Expr.divide(c.units, Expr.over(Expr.sum(c.units), [c.city])), + * running: Expr.over(Expr.cumSum(c.units), [c.city]) + * })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows[2] // => { city: "Vienna", units: 6, share: 0.75, running: 8 } + * ``` + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const over: { + (partitionBy: ReadonlyArray>): (self: Expr | Aggregate) => Expr + (self: Expr | Aggregate, partitionBy: ReadonlyArray>): Expr +} = dual( + 2, + (self: unknown, partitionBy: ReadonlyArray) => + internal.makeExpr({ + _tag: "Over", + expression: internal.toNode(self), + partitionBy: partitionBy.map(internal.toNode) + }) +) as any + +/** + * The running sum: on each row, the sum of the values so far (in the + * partition, with {@link over}). Nulls stay null and are skipped; NaN + * propagates. Durations sum to durations, as with {@link sum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumSum = ( + self: Expr +): Expr | Nullish> => internal.call("cumSum", [self]) + +/** + * The running minimum. Nulls stay null; NaN is skipped. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumMin = (self: Expr): Expr => internal.call("cumMin", [self]) + +/** + * The running maximum. Nulls stay null; NaN is skipped. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumMax = (self: Expr): Expr => internal.call("cumMax", [self]) + +/** + * The number of non-null values so far. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumCount = (self: Expr): Expr => internal.call("cumCount", [self]) + +const lag = (fn: "shift" | "diff") => + dual((args) => isExpr(args[0]), (self: unknown, n = 1) => internal.call(fn, [self], { n })) as any + +/** + * The value `n` rows earlier (1 by default); a negative `n` looks ahead. + * Null where there is no such row. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const shift: { + (n?: number): (self: Expr) => Expr + (self: Expr, n?: number): Expr +} = lag("shift") + +/** + * The difference to the value `n` rows earlier (1 by default), of numbers + * or of durations. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const diff: { + (n?: number): (self: Expr) => Expr + (self: Expr, n?: number): Expr +} = lag("diff") + +type RankOptions = { + readonly method?: Plan.RankMethod | undefined + readonly descending?: boolean | undefined +} + +/** + * The rank of each value, 1 for the smallest (largest with `descending`). + * Ties get the mean of their positions by default; `method` picks `min`, + * `max`, `dense` (consecutive) or `ordinal` (by row order). Nulls get no + * rank; NaN ranks above every number. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rank: { + (options?: RankOptions): (self: Expr) => Expr> + (self: Expr, options?: RankOptions): Expr> +} = dual( + (args) => isExpr(args[0]), + (self: unknown, options?: RankOptions) => + internal.call("rank", [self], { + method: options?.method ?? "average", + descending: options?.descending ?? false + }) +) as any + +type RollingOptions = { readonly minPeriods?: number | undefined } + +type Rolling = { + (window: number, options?: RollingOptions): ( + self: Expr + ) => Expr | null> + ( + self: Expr, + window: number, + options?: RollingOptions + ): Expr | null> +} + +const rolling = (fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax"): Rolling => + dual( + (args) => isExpr(args[0]), + (self: unknown, window: number, options?: RollingOptions) => + internal.call(fn, [self], { window, minPeriods: options?.minPeriods ?? window }) + ) as any + +/** + * The sum of the last `window` values (the current one included). Null + * while fewer than `minPeriods` (default: `window`) of them are non-null. + * Durations sum to durations, as with {@link sum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingSum: Rolling = rolling("rollingSum") + +/** + * The mean of the last `window` values; see {@link rollingSum}. Durations + * have no mean, as with {@link mean}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingMean: { + (window: number, options?: RollingOptions): ( + self: Expr + ) => Expr + ( + self: Expr, + window: number, + options?: RollingOptions + ): Expr +} = rolling("rollingMean") as any + +/** + * The minimum of the last `window` values; see {@link rollingSum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingMin: Rolling = rolling("rollingMin") + +/** + * The maximum of the last `window` values; see {@link rollingSum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingMax: Rolling = rolling("rollingMax") + +// ----------------------------------------------------------------------------- +// Dates and times +// ----------------------------------------------------------------------------- + +type Field = (self: Expr) => Expr> + +/** + * The calendar year of a date or timestamp (UTC). + * + * **Example** (Calendar fields) + * + * ```ts import.meta.vitest + * import { DateTime, Effect } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const events = DataFrame.fromColumns({ at: [DateTime.makeUnsafe("2024-02-29T13:45:07Z")] }) + * + * const rows = Query.from(events).pipe( + * Query.select((c) => ({ year: Expr.year(c.at), weekday: Expr.weekday(c.at), hour: Expr.hour(c.at) })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ year: 2024, weekday: 4, hour: 13 }] + * ``` + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const year: Field = unary("year") + +/** + * The month (1–12) of a date or timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const month: Field = unary("month") + +/** + * The day of the month (1–31) of a date or timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const day: Field = unary("day") + +/** + * The hour (0–23) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const hour: Field = unary("hour") + +/** + * The minute (0–59) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const minute: Field = unary("minute") + +/** + * The second (0–59) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const second: Field = unary("second") + +/** + * The millisecond within the second (0–999) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const millisecond: Field = unary("millisecond") + +/** + * The ISO weekday of a date or timestamp: 1 for Monday to 7 for Sunday. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const weekday: Field = unary("weekday") + +/** + * The day of the year (1–366) of a date or timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const ordinalDay: Field = unary("ordinalDay") + +/** + * Rounds dates and timestamps down to a multiple of `every`: `"30s"`, + * `"15m"`, `"1h"`, `"1d"` (from the epoch), `"1w"` (weeks start on Monday), + * `"1mo"`, `"3mo"` or `"1y"` (calendar months and years). + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const truncate: { + (every: string): (self: Expr) => Expr + (self: Expr, every: string): Expr +} = dual(2, (self: unknown, every: string) => internal.call("truncate", [self], { every })) as any + +type Total = (self: Expr) => Expr> + +/** + * A duration in whole days, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalDays: Total = unary("totalDays") + +/** + * A duration in whole hours, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalHours: Total = unary("totalHours") + +/** + * A duration in whole minutes, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalMinutes: Total = unary("totalMinutes") + +/** + * A duration in whole seconds, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalSeconds: Total = unary("totalSeconds") + +/** + * A duration in whole milliseconds, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalMilliseconds: Total = unary("totalMilliseconds") + +// ----------------------------------------------------------------------------- +// Lists and structs +// ----------------------------------------------------------------------------- + +type Element = NonNullable extends ReadonlyArray ? E : never + +/** + * The number of elements of a list; null for a null list. + * + * **Example** (Working with list columns) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const Order = Schema.Struct({ id: Schema.String, items: Schema.Array(Schema.Number) }) + * const orders = DataFrame.fromRows(Order, [{ id: "a", items: [3, 5] }, { id: "b", items: [] }]) + * + * const rows = Query.from(orders).pipe( + * Query.select((c) => ({ id: c.id, n: Expr.listLength(c.items), total: Expr.listSum(c.items) })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ id: "a", n: 2, total: 8 }, { id: "b", n: 0, total: 0 }] + * ``` + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listLength = | null>(self: Expr): Expr> => + internal.call("listLength", [self]) + +/** + * The element at `index` (negative counts from the end); null when the + * list is null or too short. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listGet: { + (index: number): | null>(self: Expr) => Expr | null> + | null>(self: Expr, index: number): Expr | null> +} = dual(2, (self: unknown, index: number) => internal.call("listGet", [self], { index })) as any + +/** + * The sum of each list's elements: 0 for an empty list, null for a null + * list. Lists of durations sum to durations, as with {@link sum}. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listSum = | null>( + self: Expr +): Expr> | Nullish> => internal.call("listSum", [self]) + +/** + * The mean of each list's elements; null for an empty or null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listMean = | null>( + self: Expr +): Expr => internal.call("listMean", [self]) + +/** + * The smallest element of each list; null for an empty or null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listMin = | null>(self: Expr): Expr> | null> => + internal.call("listMin", [self]) + +/** + * The largest element of each list; null for an empty or null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listMax = | null>(self: Expr): Expr> | null> => + internal.call("listMax", [self]) + +/** + * Whether each list holds `item`; null for a null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listContains: { + (item: Input): | null>(self: Expr) => Expr> + | null>(self: Expr, item: Input>>): Expr> +} = dual(2, (self: unknown, item: unknown) => internal.call("listContains", [self, item])) as any + +/** + * A field of a struct; null where the struct is null. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const field: { + & string>( + name: K + ): (self: Expr) => Expr[K] | Nullish> + & string>( + self: Expr, + name: K + ): Expr[K] | Nullish> +} = dual(2, (self: unknown, name: string) => internal.call("field", [self], { name })) as any + +/** + * A struct of the given expressions; the record's keys name its fields. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const struct = }>( + fields: O +): Expr<{ readonly [K in keyof O]: Type }> => + internal.call("struct", Object.values(fields), { names: Object.keys(fields) }) + +/** + * Each group's values as a list, in row order. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const list = (self: Expr): Aggregate> => agg("list", self) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts new file mode 100644 index 00000000000..4c4572cf91c --- /dev/null +++ b/packages/effect/src/dataframe/Plan.ts @@ -0,0 +1,1267 @@ +/** + * Queries as plain data: the nodes a `Query` is made of, and schemas + * that validate them and encode them to JSON. + * + * **Mental model** + * + * - A plan is a tree of {@link Node}s. Leaves are inputs ({@link Values} + * holds a frame, {@link Table} names a table the `Catalog` provides); + * every other node transforms the rows of its input(s). + * - Column values are computed by {@link Expression}s: column references, + * literals, function calls and aggregate calls. Function and aggregate + * names are the names of the `Expr` constructors (`add`, `gt`, `mean`, …). + * - Plans are what `Query` builds, what `Query.explain` prints and what + * `Query.fromPlan` runs. Building them from data (a request body, a stored + * report definition, a tool call) needs no TypeScript types: decode with + * {@link Node} or {@link NodeFromJson}, then run. + * + * **Example** (A plan from JSON) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { Catalog, DataFrame, Plan, Query } from "effect/dataframe" + * + * const json = { + * _tag: "Filter", + * input: { _tag: "Table", name: "readings" }, + * predicate: { + * _tag: "Call", + * fn: "gt", + * args: [{ _tag: "Column", name: "temp" }, { _tag: "Literal", value: 20 }] + * } + * } + * + * const plan = Schema.decodeUnknownSync(Plan.NodeFromJson)(json) + * const readings = DataFrame.fromColumns({ temp: [18, 21.5, 24] }) + * + * const result = Query.fromPlan(plan).pipe( + * Effect.provide(Catalog.layer({ readings })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * result // => [{ temp: 21.5 }, { temp: 24 }] + * ``` + * + * **Details** + * + * In JSON, literal values are JSON values where that is unambiguous: strings, + * booleans, `null` and finite numbers. A `bigint` is written as + * `{ "bigint": "" }` and `NaN`, `Infinity`, `-Infinity` and `-0` as + * `{ "float": "NaN" }` and so on. A {@link Values} frame is written as a + * base64 Arrow IPC file. + * + * @stability experimental + * @since 4.1.5 + */ +import * as ArrowIpc from "../arrow/ArrowIpc.ts" +import * as Effect from "../Effect.ts" +import * as Result from "../Result.ts" +import * as Schema from "../Schema.ts" +import * as SchemaAST from "../SchemaAST.ts" +import * as SchemaIssue from "../SchemaIssue.ts" +import * as SchemaParser from "../SchemaParser.ts" +import * as SchemaTransformation from "../SchemaTransformation.ts" +import { type DataFrame, isDataFrame } from "./DataFrame.ts" +import * as F from "./internal/frame.ts" +import * as Q from "./internal/query.ts" + +// ----------------------------------------------------------------------------- +// Values +// ----------------------------------------------------------------------------- + +/** + * A literal value. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Value = number | bigint | string | boolean | null + +const isValue = (u: unknown): u is Value => + u === null || typeof u === "number" || typeof u === "bigint" || typeof u === "string" || typeof u === "boolean" + +const FloatName = Schema.Literals(["NaN", "Infinity", "-Infinity", "-0"]) + +const ValueJson = Schema.Union([ + Schema.String, + Schema.Boolean, + Schema.Null, + Schema.Finite, + Schema.Struct({ bigint: Schema.String }), + Schema.Struct({ float: FloatName }) +]) + +const floatValues: Record = { + NaN: NaN, + Infinity: Infinity, + "-Infinity": -Infinity, + "-0": -0 +} + +const valueFromJson: SchemaTransformation.Transformation = SchemaTransformation + .transformEffect({ + decode: (json, options): Effect.Effect => { + if (json === null || typeof json !== "object") return Effect.succeed(json) + if ("float" in json) return Effect.succeed(floatValues[json.float]) + try { + return Effect.succeed(BigInt(json.bigint)) + } catch { + return Effect.fail(new SchemaIssue.InvalidValue({ expected: "an integer" }, json, options)) + } + }, + encode: (value) => { + if (typeof value === "bigint") return Effect.succeed({ bigint: value.toString() }) + if (typeof value === "number" && (!Number.isFinite(value) || Object.is(value, -0))) { + return Effect.succeed({ float: (Object.is(value, -0) ? "-0" : String(value)) as typeof FloatName.Type }) + } + return Effect.succeed(value as string | number | boolean | null) + } + }) + +/** + * Schema of {@link Value}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Value: Schema.declare = Schema.declare(isValue, { + expected: "number | bigint | string | boolean | null", + toCodecJson: () => Schema.link()(ValueJson, valueFromJson) +}) + +const frameFromBytes: SchemaTransformation.Transformation, Uint8Array> = SchemaTransformation + .transformEffect({ + decode: (bytes, options) => { + const table = ArrowIpc.decodeFile(bytes) + if (Result.isFailure(table)) { + return Effect.fail(new SchemaIssue.InvalidValue({ expected: "an Arrow IPC file" }, bytes, options)) + } + try { + return Effect.succeed(F.fromArrowTable(table.success) as unknown as DataFrame) + } catch (cause) { + return Effect.fail(new SchemaIssue.InvalidValue({ expected: `a supported table: ${String(cause)}` }, bytes)) + } + }, + encode: (frame, options) => { + const bytes = ArrowIpc.encodeFile(F.toArrowTable(frame as unknown as F.Frame)) + return Result.isSuccess(bytes) + ? Effect.succeed(bytes.success) + : Effect.fail(new SchemaIssue.InvalidValue({ expected: "a frame Arrow can encode" }, frame, options)) + } + }) + +/** + * Schema of a {@link DataFrame} inside a plan. Its JSON form is a base64 + * Arrow IPC file. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Frame: Schema.declare> = Schema.declare(isDataFrame, { + expected: "DataFrame", + toCodecJson: () => Schema.link>()(Schema.Uint8ArrayFromBase64, frameFromBytes) +}) + +// ----------------------------------------------------------------------------- +// Expressions +// ----------------------------------------------------------------------------- + +/** + * Names of the functions a {@link Call} can apply; each is the `Expr` + * constructor of the same name. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type FunctionName = typeof FunctionName.Type + +/** + * Schema of {@link FunctionName}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const FunctionName = Schema.Literals([ + "add", + "subtract", + "multiply", + "divide", + "floorDivide", + "mod", + "negate", + "abs", + "sqrt", + "round", + "eq", + "neq", + "lt", + "lte", + "gt", + "gte", + "between", + "isIn", + "and", + "or", + "not", + "isNull", + "isNotNull", + "isNaN", + "fillNull", + "coalesce", + "when", + "cast", + "length", + "toLowerCase", + "toUpperCase", + "includes", + "startsWith", + "endsWith", + "concat", + "cumSum", + "cumMin", + "cumMax", + "cumCount", + "shift", + "diff", + "rank", + "rollingSum", + "rollingMean", + "rollingMin", + "rollingMax", + "year", + "month", + "day", + "hour", + "minute", + "second", + "millisecond", + "weekday", + "ordinalDay", + "truncate", + "totalDays", + "totalHours", + "totalMinutes", + "totalSeconds", + "totalMilliseconds", + "listLength", + "listGet", + "listSum", + "listMean", + "listMin", + "listMax", + "listContains", + "field", + "struct" +]) + +/** + * Names of the aggregates an {@link AggregateCall} can compute; each is the + * `Expr` constructor of the same name. `count` without an argument counts + * rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type AggregateName = typeof AggregateName.Type + +/** + * Schema of {@link AggregateName}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const AggregateName = Schema.Literals([ + "sum", + "mean", + "median", + "std", + "variance", + "min", + "max", + "count", + "countDistinct", + "first", + "last", + "list" +]) + +/** + * Names of the types `cast` converts to. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type CastTarget = typeof CastTarget.Type + +/** + * Schema of {@link CastTarget}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const CastTarget = Schema.Literals([ + "Int8", + "Int16", + "Int32", + "Int64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "Float32", + "Float64", + "Utf8", + "Bool", + "Date", + "Timestamp[s]", + "Timestamp[ms]", + "Timestamp[us]", + "Timestamp[ns]", + "Duration[s]", + "Duration[ms]", + "Duration[us]", + "Duration[ns]", + "Decimal" +]) + +/** + * The type of a {@link Literal} whose value is the integer storage of a + * richer type: days since the epoch for `Date`, units since the epoch for + * `Timestamp`, units for `Duration`, and the unscaled integer for `Decimal`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type LiteralType = + | { readonly _tag: "Date" } + | { readonly _tag: "Timestamp"; readonly unit: TimeUnit; readonly timezone?: string | undefined } + | { readonly _tag: "Duration"; readonly unit: TimeUnit } + | { readonly _tag: "Decimal"; readonly precision: number; readonly scale: number } + +/** + * Units of timestamps and durations. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TimeUnit = typeof TimeUnit.Type + +/** + * Schema of {@link TimeUnit}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const TimeUnit = Schema.Literals(["second", "millisecond", "microsecond", "nanosecond"]) + +const LiteralTypeSchema: Schema.Codec = Schema.Union([ + Schema.TaggedStruct("Date", {}), + Schema.TaggedStruct("Timestamp", { unit: TimeUnit, timezone: Schema.optional(Schema.String) }), + Schema.TaggedStruct("Duration", { unit: TimeUnit }), + Schema.TaggedStruct("Decimal", { precision: Schema.Int, scale: Schema.Int }) +]) as any + +/** + * A value computed per row (or per group, for aggregate calls). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Expression = Column | Literal | Call | AggregateCall | Over + +/** + * The value of a column. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Column { + readonly _tag: "Column" + readonly name: string +} + +/** + * A constant. Integral numbers take the integer type of the column they meet, + * as Polars integer literals do. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Literal { + readonly _tag: "Literal" + readonly value: Value + readonly type?: LiteralType | undefined +} + +/** + * Options of the functions that take more than expressions: `round` + * (`decimals`), `cast` (`to`), `isIn` (`values`), `includes`, `startsWith` + * and `endsWith` (`pattern`), `shift` and `diff` (`n`), `rank` (`method`, + * `descending`), the rolling functions (`window`, `minPeriods`) and + * `truncate` (`every`: `"1h"`, `"1d"`, `"1w"`, `"1mo"`, `"1y"`, …), `listGet` + * (`index`), `field` (`name`), `struct` (`names`, one per argument) and + * `cast` to `Decimal` (`precision`, `scale`). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface CallOptions { + readonly decimals?: number | undefined + readonly to?: CastTarget | undefined + readonly values?: ReadonlyArray | undefined + readonly pattern?: string | undefined + readonly n?: number | undefined + readonly method?: RankMethod | undefined + readonly descending?: boolean | undefined + readonly window?: number | undefined + readonly minPeriods?: number | undefined + readonly every?: string | undefined + readonly index?: number | undefined + readonly name?: string | undefined + readonly names?: ReadonlyArray | undefined + readonly precision?: number | undefined + readonly scale?: number | undefined +} + +/** + * How `rank` numbers ties: `average` (the mean of their positions, a float), + * `min`, `max`, `dense` (consecutive ranks) or `ordinal` (by row order). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type RankMethod = typeof RankMethod.Type + +/** + * Schema of {@link RankMethod}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const RankMethod = Schema.Literals(["average", "min", "max", "dense", "ordinal"]) + +/** + * A function applied to argument expressions, row by row. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Call { + readonly _tag: "Call" + readonly fn: FunctionName + readonly args: ReadonlyArray + readonly options?: CallOptions | undefined +} + +/** + * An aggregate over the rows of a group. Only valid as an output of + * {@link GroupBy} or {@link Aggregate}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface AggregateCall { + readonly _tag: "AggregateCall" + readonly fn: AggregateName + readonly arg?: Expression | undefined +} + +/** + * An expression evaluated separately within each partition of rows with + * equal `partitionBy` values, its results in row order: aggregates give + * their partition's value on every row, window functions (`cumSum`, + * `shift`, `rank`, …) run over their partition's rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Over { + readonly _tag: "Over" + readonly expression: Expression + readonly partitionBy: ReadonlyArray +} + +const ExpressionSchema = Schema.suspend((): Schema.Codec => ExpressionUnion) + +const ColumnSchema = Schema.TaggedStruct("Column", { name: Schema.String }) +const LiteralSchema = Schema.TaggedStruct("Literal", { value: Value, type: Schema.optional(LiteralTypeSchema) }) +const PositiveInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) +const CallOptionsSchema = Schema.Struct({ + decimals: Schema.optional(Schema.Int), + to: Schema.optional(CastTarget), + values: Schema.optional(Schema.Array(Value)), + pattern: Schema.optional(Schema.String), + n: Schema.optional(Schema.Int), + method: Schema.optional(RankMethod), + descending: Schema.optional(Schema.Boolean), + window: Schema.optional(PositiveInt), + minPeriods: Schema.optional(PositiveInt), + every: Schema.optional(Schema.String), + index: Schema.optional(Schema.Int), + name: Schema.optional(Schema.String), + names: Schema.optional(Schema.Array(Schema.String)), + precision: Schema.optional(PositiveInt), + scale: Schema.optional(NonNegativeInt) +}) +const CallSchema = Schema.TaggedStruct("Call", { + fn: FunctionName, + args: Schema.Array(ExpressionSchema), + options: Schema.optional(CallOptionsSchema) +}) +const AggregateCallSchema = Schema.TaggedStruct("AggregateCall", { + fn: AggregateName, + arg: Schema.optional(ExpressionSchema) +}) +const OverSchema = Schema.TaggedStruct("Over", { + expression: ExpressionSchema, + partitionBy: Schema.Array(ExpressionSchema) +}) +const ExpressionUnion = Schema.Union([ColumnSchema, LiteralSchema, CallSchema, AggregateCallSchema, OverSchema]) + +/** + * Schema of {@link Expression}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Expression: Schema.Codec = ExpressionSchema + +/** + * An expression and the name of the column it produces. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Named { + readonly name: string + readonly expression: Expression +} + +const NamedSchema = Schema.Struct({ name: Schema.String, expression: ExpressionSchema }) + +/** + * A sort key. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface SortKey { + readonly expression: Expression + readonly descending: boolean +} + +const SortKeySchema = Schema.Struct({ expression: ExpressionSchema, descending: Schema.Boolean }) + +// ----------------------------------------------------------------------------- +// Nodes +// ----------------------------------------------------------------------------- + +/** + * A query plan. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Node = + | Values + | Table + | Filter + | Select + | WithColumns + | Sort + | Slice + | Unique + | DropNulls + | Pick + | Omit + | Rename + | GroupBy + | Aggregate + | Join + | Concat + | Pivot + | Unpivot + | Explode + | Unnest + +/** + * The rows of a frame. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Values { + readonly _tag: "Values" + readonly frame: DataFrame +} + +/** + * The type names a {@link Table} can declare for its columns. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TypeName = typeof TypeName.Type + +/** + * Schema of {@link TypeName}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const TypeName = Schema.Literals([ + "Int8", + "Int16", + "Int32", + "Int64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "Float16", + "Float32", + "Float64", + "Utf8", + "Binary", + "Bool", + "Null", + "Date", + "Time", + "Timestamp", + "Duration", + "Decimal", + "List", + "Struct" +]) + +/** + * A column a {@link Table} is declared to have, and the type it is read as. + * + * **Details** + * + * `fields` describes the children of a nested column: the single `item` of + * a `List`, or the fields of a `Struct`. `precision` and `scale` give the + * type of a `Decimal` column, and `unit` the unit of a `Timestamp` or + * `Duration` column (microseconds when absent). + * + * **Gotchas** + * + * A `Decimal` column fails when its table is read unless it has a + * `precision` that is an integer from 1 to 38 and a `scale` that is a + * 32-bit integer. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface TableColumn { + readonly name: string + readonly type: TypeName + readonly nullable: boolean + readonly fields?: ReadonlyArray | undefined + readonly precision?: number | undefined + readonly scale?: number | undefined + readonly unit?: TimeUnit | undefined +} + +/** + * The tests of a {@link Table}'s values that its column types don't cover, + * compiled once from a struct schema's literals, enums and refinements when + * the table is declared. + * + * **Details** + * + * `columns` pairs the name of a declared column with a test of one value + * of it. Every value of that column is read (as `DataFrame.toArray` gives + * it) and tested, `null` included: a nullable column's test decides + * whether its missing values pass, as a schema's check on a nullable field + * does. `rows` tests one row object, as `DataFrame.toRows` gives it, and + * is `undefined` when there is nothing to test across columns. A table + * fails with a `DataFrameError` when a test returns `false`. + * + * **Gotchas** + * + * The tests are functions and have no JSON form: {@link NodeFromJson} fails + * for a table that has them (see {@link withoutChecks}). A SQL engine runs + * `columns` tests on the distinct values of each column and fails for a + * table with a `rows` test. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface TableChecks { + readonly columns: ReadonlyArray boolean]> + readonly rows: ((row: unknown) => boolean) | undefined +} + +/** + * The rows of a table the `Catalog` provides by name. + * + * **Details** + * + * When `columns` is given, the table must have these columns and no nulls + * in non-nullable ones; the input (every batch of a streamed table) is + * narrowed to these columns, in this order, and converted to their declared + * types, so results don't depend on the types the input happens to have. A + * column converts when its type holds the declared values (integers below + * 64 bits for floats and narrower integers, any 64-bit integer for `Int64` + * and `UInt64`, any decimal for `Decimal`, the same temporal type in any + * unit, lists and structs with such children) and every value converts + * exactly. `Timestamp` columns read as `DateTime.Utc` values: their + * timestamps must have no time zone or a UTC one (dropped), and lie in the + * range `DateTime` represents. `Date` columns are in days and `Time` + * columns in microseconds: dates in milliseconds and times in other units + * convert to them. + * + * Every batch of a streamed table, after this conversion (as given, without + * `columns`), must have the column names and types of its first batch. + * + * Without `columns`, a streamed table's columns come from its batches. When + * no batch is read (the table has none, or a slice without rows skips it), + * they are unknown: a result that needs them (collecting one, concatenating, + * joining, aggregating, sorting) fails with a `DataFrameError` asking for + * them to be declared, and `Query.stream` emits no batch for one that has no + * rows. + * + * `checks`, set by `Query.table` for a struct schema that constrains + * values beyond their column type, holds the tests the rows must also pass + * (see {@link TableChecks}). They run after the conversion, on every batch + * of a streamed table. A column without a test is checked by its type alone + * and its values are never read. + * + * **Gotchas** + * + * `checks` has no JSON form. See {@link NodeFromJson} and + * {@link withoutChecks}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Table { + readonly _tag: "Table" + readonly name: string + readonly columns?: ReadonlyArray | undefined + readonly checks?: TableChecks | undefined +} + +/** + * The rows where `predicate` is `true`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Filter { + readonly _tag: "Filter" + readonly input: Node + readonly predicate: Expression +} + +/** + * Only the given computed columns. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Select { + readonly _tag: "Select" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * The input's columns plus the given computed columns (replacing columns of + * the same name in place). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface WithColumns { + readonly _tag: "WithColumns" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * The rows in key order, stable, nulls first unless `nullsLast`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Sort { + readonly _tag: "Sort" + readonly input: Node + readonly keys: ReadonlyArray + readonly nullsLast: boolean +} + +/** + * `length` rows (all remaining when absent) from `offset`; a negative offset + * counts from the end. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Slice { + readonly _tag: "Slice" + readonly input: Node + readonly offset: number + readonly length?: number | undefined +} + +/** + * The first row of every distinct combination of `columns` (all when absent). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Unique { + readonly _tag: "Unique" + readonly input: Node + readonly columns?: ReadonlyArray | undefined +} + +/** + * The rows without nulls in `columns` (all when absent). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface DropNulls { + readonly _tag: "DropNulls" + readonly input: Node + readonly columns?: ReadonlyArray | undefined +} + +/** + * Only the given columns, in the given order. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Pick { + readonly _tag: "Pick" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * All columns but the given ones. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Omit { + readonly _tag: "Omit" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * The input with columns renamed: `{ old: "new" }`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Rename { + readonly _tag: "Rename" + readonly input: Node + readonly mapping: { readonly [name: string]: string } +} + +/** + * One row per distinct key, in order of first appearance: the keys, then the + * aggregates. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface GroupBy { + readonly _tag: "GroupBy" + readonly input: Node + readonly keys: ReadonlyArray + readonly aggregates: ReadonlyArray +} + +/** + * One row of aggregates over all rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Aggregate { + readonly _tag: "Aggregate" + readonly input: Node + readonly aggregates: ReadonlyArray +} + +/** + * How a {@link Join} combines rows. + * + * **Details** + * + * - `inner`: rows whose key exists on both sides + * - `left`: every left row; right columns are `null` where nothing matched + * - `full`: every row of both sides; keys are taken from whichever side has them + * - `semi`: left rows that have a match (left columns only) + * - `anti`: left rows without a match (left columns only) + * + * Null keys never match. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type JoinHow = typeof JoinHow.Type + +/** + * Schema of {@link JoinHow}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const JoinHow = Schema.Literals(["inner", "left", "full", "semi", "anti"]) + +/** + * Rows of `left` and `right` with equal `on` columns, in left row order. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Join { + readonly _tag: "Join" + readonly left: Node + readonly right: Node + readonly on: ReadonlyArray + readonly how: JoinHow +} + +/** + * The rows of every input, one after the other. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Concat { + readonly _tag: "Concat" + readonly inputs: ReadonlyArray +} + +/** + * Aggregates a {@link Pivot} can put in its cells: the {@link AggregateName}s + * and `len` (the number of rows). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type PivotAggregate = AggregateName | "len" + +/** + * One row per distinct `index`, one column per distinct value of `on`, and + * in each cell the aggregate of `values` over the rows with that index and + * `on` value. + * + * **Details** + * + * Columns are named by the `on` values as strings (`null` for a null), in + * order of first appearance; with `columns`, exactly those columns, in that + * order. A cell without rows holds the aggregate of no values: 0 for `sum`, + * `count` and `len`, otherwise null. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Pivot { + readonly _tag: "Pivot" + readonly input: Node + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: PivotAggregate + readonly columns?: ReadonlyArray | undefined +} + +/** + * The `on` columns stacked into one `value` column, one block of rows per + * column, with the column's name in `variable` and the `index` columns + * repeated. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Unpivot { + readonly _tag: "Unpivot" + readonly input: Node + readonly index: ReadonlyArray + readonly on: ReadonlyArray + readonly variable: string + readonly value: string +} + +/** + * One row per element of the list column `column`, the other columns + * repeated. A null list gives one row with a null element; an empty list + * gives no rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Explode { + readonly _tag: "Explode" + readonly input: Node + readonly column: string +} + +/** + * The fields of the struct column `column` as columns, in its place. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Unnest { + readonly _tag: "Unnest" + readonly input: Node + readonly column: string +} + +const TableColumnSchema: Schema.Codec = Schema.Struct({ + name: Schema.String, + type: TypeName, + nullable: Schema.Boolean, + fields: Schema.optional(Schema.Array(Schema.suspend(() => TableColumnSchema))), + precision: Schema.optional(Schema.Int), + scale: Schema.optional(Schema.Int), + unit: Schema.optional(TimeUnit) +}) + +const isTableChecks = (u: unknown): u is TableChecks => + typeof u === "object" && u !== null && "columns" in u && Array.isArray(u.columns) && + u.columns.every((c) => + Array.isArray(c) && c.length === 2 && typeof c[0] === "string" && typeof c[1] === "function" + ) && + (!("rows" in u) || u.rows === undefined || typeof u.rows === "function") + +const tableFields = { name: Schema.String, columns: Schema.optional(Schema.Array(TableColumnSchema)) } + +const TableStruct = Schema.TaggedStruct("Table", { + ...tableFields, + checks: Schema.optional(Schema.declare(isTableChecks, { expected: "TableChecks" })) +}) + +/** A table in JSON; `checks` is only there to be refused. */ +const TableJson = Schema.TaggedStruct("Table", { ...tableFields, checks: Schema.optional(Schema.Unknown) }) + +/** Fails for a table with checks: dropping them would drop the validation they do. */ +const layoutOnly = ( + table: A +): Effect.Effect => + table.checks === undefined + ? Effect.succeed(table) + : Effect.fail( + new SchemaIssue.Forbidden({ + message: `table ${JSON.stringify(table.name)} has checks, which are functions without a JSON form: ` + + "encode Plan.withoutChecks(plan), and declare the table again with Query.table to check the decoded plan's rows" + }, table) + ) + +const tableFromJson: SchemaTransformation.Transformation = SchemaTransformation + .transformEffect({ + decode: (json) => layoutOnly(json as Table), + encode: (table) => layoutOnly(table as typeof TableJson.Type) + }) + +/** + * A table as it is, checks included, so that validating a plan keeps them; + * the JSON form refuses a table with checks instead of dropping them. + */ +const TableSchema = Schema.declareConstructor()( + [], + () => { + const decode = SchemaParser.decodeUnknownEffect(TableStruct) + return (u, _, options) => Effect.as(decode(u, options), u as Table) + }, + { + expected: "Table", + // A union picks this member by its tag, as it does a struct. + "~sentinels": SchemaAST.collectSentinels(TableStruct.ast), + toCodecJson: () => Schema.link
()(TableJson, tableFromJson) + } +) + +const NodeSchema = Schema.suspend((): Schema.Codec => NodeUnion) +const Columns = Schema.Array(Schema.String) + +const NodeUnion = Schema.Union([ + Schema.TaggedStruct("Values", { frame: Frame }), + TableSchema, + Schema.TaggedStruct("Filter", { input: NodeSchema, predicate: ExpressionSchema }), + Schema.TaggedStruct("Select", { input: NodeSchema, columns: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("WithColumns", { input: NodeSchema, columns: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("Sort", { input: NodeSchema, keys: Schema.Array(SortKeySchema), nullsLast: Schema.Boolean }), + Schema.TaggedStruct("Slice", { input: NodeSchema, offset: Schema.Int, length: Schema.optional(NonNegativeInt) }), + Schema.TaggedStruct("Unique", { input: NodeSchema, columns: Schema.optional(Columns) }), + Schema.TaggedStruct("DropNulls", { input: NodeSchema, columns: Schema.optional(Columns) }), + Schema.TaggedStruct("Pick", { input: NodeSchema, columns: Columns }), + Schema.TaggedStruct("Omit", { input: NodeSchema, columns: Columns }), + Schema.TaggedStruct("Rename", { input: NodeSchema, mapping: Schema.Record(Schema.String, Schema.String) }), + Schema.TaggedStruct("GroupBy", { input: NodeSchema, keys: Columns, aggregates: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("Aggregate", { input: NodeSchema, aggregates: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("Join", { left: NodeSchema, right: NodeSchema, on: Columns, how: JoinHow }), + Schema.TaggedStruct("Concat", { inputs: Schema.Array(NodeSchema) }), + Schema.TaggedStruct("Pivot", { + input: NodeSchema, + on: Schema.String, + index: Columns, + values: Schema.String, + aggregate: Schema.Union([AggregateName, Schema.Literal("len")]), + columns: Schema.optional(Columns) + }), + Schema.TaggedStruct("Explode", { input: NodeSchema, column: Schema.String }), + Schema.TaggedStruct("Unnest", { input: NodeSchema, column: Schema.String }), + Schema.TaggedStruct("Unpivot", { + input: NodeSchema, + index: Columns, + on: Columns, + variable: Schema.String, + value: Schema.String + }) +]) + +/** + * Schema of {@link Node}. + * + * **Details** + * + * A valid plan decodes and encodes to itself: tables keep their `checks`. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Node: Schema.Codec = NodeSchema + +/** + * A {@link Node} from its JSON form. + * + * **Gotchas** + * + * A table's `checks` are functions and have no JSON form. Encoding a plan + * fails for a table that has them, naming the table, instead of leaving + * them out, and so does decoding JSON with a `checks` key (the plan as + * `JSON.stringify` writes it). Encode {@link withoutChecks} of the plan to + * send it without them. A decoded plan never has checks: to test its + * tables' rows, build the query from `Query.table` with the schema where + * the plan is decoded. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const NodeFromJson: Schema.Codec = Schema.toCodecJson(Node) as any + +/** + * The plan with the {@link TableChecks} of its tables removed, so that + * {@link NodeFromJson} encodes it. + * + * **When to use** + * + * Use to send or store a plan whose tables were declared by a struct schema + * with refinements, literals or enums, when the reader of the plan doesn't + * need those values tested. + * + * **Details** + * + * Tables keep their names and declared columns: the result still fails for a + * table without those columns, with nulls in non-nullable ones or with values + * a column's type can't hold. + * + * **Gotchas** + * + * The result accepts values the schema rejects, here and after decoding. + * {@link NodeFromJson} says how to test the rows of a decoded plan. + * + * **Example** (Encoding a plan over a table with refinements) + * + * ```ts import.meta.vitest + * import { Result, Schema } from "effect" + * import { Plan, Query } from "effect/dataframe" + * + * const Reading = Schema.Struct({ temp: Schema.Number.check(Schema.isGreaterThan(-273.15)) }) + * const query = Query.table("readings", Reading) + * + * const encode = Schema.encodeResult(Plan.NodeFromJson) + * Result.isFailure(encode(query.plan)) // => true + * + * const columns = [{ name: "temp", type: "Float64", nullable: false }] + * Result.getOrThrow(encode(Plan.withoutChecks(query.plan))) // => { _tag: "Table", name: "readings", columns } + * ``` + * + * @stability experimental + * @category combinators + * @since 4.1.5 + */ +export const withoutChecks = (plan: Node): Node => + Q.mapTables(plan as Q.Node, (table) => { + if (table.checks === undefined) return table + const { checks: _, ...layout } = table + return layout + }) as Node + +/** + * An {@link Expression} from its JSON form. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const ExpressionFromJson: Schema.Codec = Schema.toCodecJson(Expression) as any diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts new file mode 100644 index 00000000000..040437a2eb6 --- /dev/null +++ b/packages/effect/src/dataframe/Query.ts @@ -0,0 +1,995 @@ +/** + * Lazy, typed queries over data frames: describe filters, derived columns, + * sorts, groups and joins, then run them as an `Effect`. + * + * **Mental model** + * + * - A `Query` describes how to compute a table whose rows have type + * `Row`. Building one does no work; every function returns a new query + * with one more operation, and the row type follows along. + * - A query is a {@link Plan.Node} underneath, plain data that can be + * inspected, stored, encoded to JSON and run again ({@link fromPlan}). + * - A query is an `Effect, DataFrameError>`: `yield*` it, + * or pass it to any `Effect` function. Running it reads the tables it + * names from the `Catalog`, rewrites the plan to do less work, and + * executes it. + * - Expressions come from {@link Expr}. Functions that take expressions take + * a callback that receives one typed reference per column; it runs once, + * when the query is built. + * + * **Example** (Summarising readings per city) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const Reading = Schema.Struct({ city: Schema.String, temp: Schema.Number }) + * const readings = DataFrame.fromRows(Reading, [ + * { city: "Vienna", temp: 21.5 }, + * { city: "Berlin", temp: 26.5 }, + * { city: "Vienna", temp: 19 }, + * { city: "Zurich", temp: 15 } + * ]) + * + * const summary = Query.from(readings).pipe( + * Query.filter((c) => Expr.gt(c.temp, 16)), + * Query.groupBy("city"), + * Query.agg((c) => ({ readings: Expr.count(), mean: Expr.mean(c.temp) })), + * Query.sort((c) => [Expr.desc(c.mean)]) + * ) + * + * const rows = Effect.runSync(Effect.map(summary, DataFrame.toRows)) + * rows[0] // => { city: "Berlin", readings: 1, mean: 26.5 } + * rows[1] // => { city: "Vienna", readings: 2, mean: 20.25 } + * ``` + * + * **Details** + * + * Results match Polars for the same operations: groups and `unique` keep + * first-appearance order, sorting is stable with nulls first unless + * `nullsLast` is set, and joins keep the left row order (with right rows in + * their order within a key). See {@link Expr} for value semantics. + * + * Before running, the plan is rewritten: predicates move below the + * operations they commute with, filters merge, slices move below row-wise + * operations (and a slice of a sort keeps only the rows it needs), and + * columns and aggregates nothing reads are not computed. {@link explain} + * shows the plan that runs. + * + * **Gotchas** + * + * - The column references passed to callbacks are created on access, so + * spreading or enumerating them yields nothing. + * - Joins require the non-key columns of both sides to have different + * names; rename first. + * - A long query yields to other fibers between operations, so it can be + * interrupted or timed out, but not in the middle of one operation. + * + * @stability experimental + * @since 4.1.5 + */ +import type * as ArrowSchema from "../arrow/ArrowSchema.ts" +import type * as BigDecimal from "../BigDecimal.ts" +import type * as Duration from "../Duration.ts" +import type * as Effect from "../Effect.ts" +import * as Effectable from "../Effectable.ts" +import { dual } from "../Function.ts" +import { NodeInspectSymbol } from "../Inspectable.ts" +import * as Stream from "../Stream.ts" +import type * as Types from "../Types.ts" +import type { DataFrame, DataFrameError, Normalize } from "./DataFrame.ts" +import { Engine } from "./Engine.ts" +import type * as Expr from "./Expr.ts" +import * as E from "./internal/expr.ts" +import * as Q from "./internal/query.ts" +import * as Schema_ from "./internal/schema.ts" +import type * as Plan from "./Plan.ts" + +// ----------------------------------------------------------------------------- +// Models +// ----------------------------------------------------------------------------- + +/** + * Identifies queries. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const TypeId = "~effect/dataframe/Query" + +/** + * A lazy computation of a table whose rows have type `Row`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Query extends Effect.Effect, DataFrameError> { + readonly [TypeId]: { + readonly _Row: Types.Covariant + } + /** The query as data. */ + readonly plan: Plan.Node +} + +/** + * One typed column reference per column, as passed to expression callbacks. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Columns = { readonly [K in keyof Row & string]: Expr.Expr } + +/** + * A query split into groups by key columns, waiting for {@link agg}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface GroupBy { + readonly _tag: "GroupBy" + readonly query: Query + readonly keys: ReadonlyArray +} + +/** + * The row type of a query. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Row = Q extends Query ? R : never + +/** + * How {@link join} combines rows; see {@link Plan.JoinHow}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type JoinHow = Plan.JoinHow + +/** + * The row type of a join. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Joined = Simplify< + How extends "semi" | "anti" ? R + : How extends "inner" ? R & Omit + : How extends "left" ? R & Nullable> + : Pick & Nullable> & Nullable> +> + +type Simplify = { readonly [K in keyof A]: A[K] } extends infer B ? B : never +type Nullable = { readonly [K in keyof R]: R[K] | null } + +type JoinCheck = [Exclude] extends [never] ? unknown + : { readonly "~error": `rename the columns that exist on both sides: ${Exclude & string}` } + +interface QueryImpl extends Query {} + +const Proto = { + ...Effectable.Prototype({ + label: "Query", + evaluate(fiber) { + return fiber.getRef(Engine).collect(this.plan) as any + } + }), + [TypeId]: { _Row: (_: never) => _ }, + toJSON(this: QueryImpl) { + return { _id: "Query", plan: this.plan } + }, + toString(this: QueryImpl) { + return Q.show(this.plan) + }, + [NodeInspectSymbol](this: QueryImpl) { + return this.toJSON() + } +} + +const make = (plan: Plan.Node): Query => { + const self = Object.create(Proto) + self.plan = plan + return self +} + +/** + * Returns `true` for queries. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isQuery = (u: unknown): u is Query => typeof u === "object" && u !== null && TypeId in u + +const columns: any = new Proxy({}, { + get: (_, name) => typeof name === "string" ? E.makeExpr({ _tag: "Column", name }) : undefined +}) + +const named = (exprs: { readonly [name: string]: unknown }): Array => + Object.keys(exprs).map((name) => ({ name, expression: E.toNode(exprs[name]) })) + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** + * A query over the rows of a frame. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const from = (frame: DataFrame): Query => make({ _tag: "Values", frame: frame as DataFrame }) + +/** + * A query over the table called `name`, provided by the `Catalog` when + * the query runs, with the columns a struct schema or an Arrow schema + * declares. The table must have those columns, as `DataFrame.validate` + * checks a frame; the query sees only them, in the declared order and with + * the declared types. + * + * **Details** + * + * A struct schema declares the columns `DataFrame.fromRows` gives its + * fields (`options.types` overrides some, and `BigDecimal` fields need + * one) and types the rows. The schema is read once, here, and compiled to + * the tests the table's rows must pass (`Plan.TableChecks`). A field with + * refinements, literals or enums has its values read and tested one by one, + * and a struct with checks of its own (`Schema.Struct(...).check(...)`) has + * every row read and tested. Other fields are checked by column type only, + * on the typed columns. The tests see missing values as the schema writes + * them (an absent key, `undefined` or `null`), as in `DataFrame.fromRows`. + * + * An Arrow schema declares its fields as they are (type, nullability, + * nested fields, decimal precision and scale, time unit), with untyped rows + * and no tests of values. `DataFrame.validate` lists the Arrow types a + * declaration takes. + * + * Every batch of a streamed table is converted to the declared types, so + * results have the same types whatever the batches hold, and when there are + * none. A table without the precision and scale of a `BigDecimal` field + * fails when it is read. + * + * **Gotchas** + * + * - Throws a `DataFrameError` for a schema that declares no columns: a + * struct schema that transforms its rows, has fields without a column + * type or has a check that would see a missing value the schema writes in + * several ways (see `DataFrame.fromRows`), or an Arrow schema with types + * a declaration can't take. + * - A SQL engine can't check a struct's own checks on whole rows: a query + * over such a table fails. + * - The tests are functions and have no JSON form: `Plan.NodeFromJson` + * fails to encode a plan over a table that has them. Encode + * `Plan.withoutChecks` of the plan to send it without them, and declare + * the table again where the plan is decoded to test its rows there. + * + * **Example** (Declaring a table by an Arrow schema) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { ArrowSchema } from "effect/arrow" + * import { Catalog, DataFrame, Query } from "effect/dataframe" + * + * const readings = Query.table( + * "readings", + * ArrowSchema.make([ + * ArrowSchema.field("city", ArrowSchema.utf8, { nullable: false }), + * ArrowSchema.field("temp", ArrowSchema.float32) + * ]) + * ) + * + * const frame = DataFrame.fromColumns({ temp: Int32Array.of(21, 19), city: ["Vienna", "Berlin"] }) + * const result = Effect.runSync(readings.pipe(Effect.provide(Catalog.layer({ readings: frame })))) + * result.schema.fields.map((f) => f.type) // => [ArrowSchema.utf8, ArrowSchema.float32] + * DataFrame.toRows(result) // => [{ city: "Vienna", temp: 21 }, { city: "Berlin", temp: 19 }] + * ``` + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const table: { + ( + name: string, + schema: S, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } + ): Query> + (name: string, schema: ArrowSchema.Schema): Query<{ readonly [name: string]: unknown }> +} = ( + name: string, + schema: { readonly ast: any } | ArrowSchema.Schema, + options?: { readonly types?: { readonly [name: string]: ArrowSchema.DataType | undefined } | undefined } +): Query => + Q.unsafe(() => { + const { checks, columns } = Schema_.declare(schema, options?.types) + return make(checks === undefined ? { _tag: "Table", name, columns } : { _tag: "Table", name, columns, checks }) + }) + +/** + * A query from its plan, with untyped rows. Decode untrusted plans with + * {@link Plan.Node} or {@link Plan.NodeFromJson} first. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromPlan = (plan: Plan.Node): Query<{ readonly [name: string]: unknown }> => make(plan) + +/** + * The query's result as a stream of batches. + * + * **Details** + * + * Over streamed tables (a `Stream` in the `Catalog`), row-wise operations + * run on each batch as it arrives and a `head` stops reading once it has its + * rows; grouping and aggregation fold the batches into one result. Over + * frames, the result is one batch. + * + * **Example** (Filtering a stream of batches) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" + * + * const batch = (from: number) => DataFrame.fromColumns({ n: Int32Array.from({ length: 3 }, (_, i) => from + i) }) + * const numbers = Stream.make(batch(0), batch(3), batch(6)) + * + * const sizes = Query.fromPlan({ _tag: "Table", name: "numbers" }).pipe( + * Query.filter((c) => Expr.gt(c.n as Expr.Expr, 2)), + * Query.stream, + * Stream.map((frame) => frame.length), + * Stream.runCollect, + * Effect.provide(Catalog.layer({ numbers })), + * Effect.runSync + * ) + * sizes // => [3, 3] + * ``` + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const stream = (self: Query): Stream.Stream, DataFrameError> => + Stream.unwrap(Engine.useSync((engine) => engine.stream(self.plan) as any)) + +/** + * The plan that runs for a query, after tables are resolved and the plan is + * rewritten, as an indented tree with one operation per line. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const explain = (self: Query): Effect.Effect => + Engine.use((engine) => engine.explain(self.plan)) + +// ----------------------------------------------------------------------------- +// Rows +// ----------------------------------------------------------------------------- + +/** + * Keeps the rows where the predicate is `true` (`null` counts as `false`). + * + * @stability experimental + * @category filtering + * @since 4.1.5 + */ +export const filter: { + (predicate: (columns: Columns) => Expr.Expr): (self: Query) => Query + (self: Query, predicate: (columns: Columns) => Expr.Expr): Query +} = dual( + 2, + (self: Query, predicate: (columns: Columns) => Expr.Expr): Query => + make({ _tag: "Filter", input: self.plan, predicate: E.toNode(predicate(columns)) }) +) + +/** + * Sorts by the given keys, first key first; ties keep their order. Nulls + * come first unless `nullsLast` is set. + * + * @stability experimental + * @category sorting + * @since 4.1.5 + */ +export const sort: { + ( + keys: (columns: Columns) => ReadonlyArray | Expr.SortOrder>, + options?: { readonly nullsLast?: boolean | undefined } + ): (self: Query) => Query + ( + self: Query, + keys: (columns: Columns) => ReadonlyArray | Expr.SortOrder>, + options?: { readonly nullsLast?: boolean | undefined } + ): Query +} = dual((args) => isQuery(args[0]), ( + self: Query, + keys: (columns: Columns) => ReadonlyArray | Expr.SortOrder>, + options?: { readonly nullsLast?: boolean | undefined } +): Query => + make({ + _tag: "Sort", + input: self.plan, + keys: keys(columns).map((k) => { + const { expr, descending } = E.isExpr(k) ? { expr: k, descending: false } : k as Expr.SortOrder + return { expression: E.toNode(expr), descending } + }), + nullsLast: options?.nullsLast ?? false + })) + +/** + * `length` rows (all remaining when omitted) starting at `offset`; a negative + * offset counts from the end. + * + * **Details** + * + * As in Polars, the slice covers `offset` up to `offset + length` and is then + * clipped to the rows there are: over 3 rows, `slice(-5, 3)` keeps the first + * row and `slice(-5, 1)` none. + * + * @stability experimental + * @category slicing + * @since 4.1.5 + */ +export const slice: { + (offset: number, length?: number): (self: Query) => Query + (self: Query, offset: number, length?: number): Query +} = dual( + (args) => isQuery(args[0]), + (self: Query, offset: number, length?: number): Query => + make({ _tag: "Slice", input: self.plan, offset, length }) +) + +/** + * The first `n` rows. + * + * @stability experimental + * @category slicing + * @since 4.1.5 + */ +export const head: { + (n: number): (self: Query) => Query + (self: Query, n: number): Query +} = dual(2, (self: Query, n: number): Query => slice(self, 0, Math.max(0, n))) + +/** + * The last `n` rows. + * + * @stability experimental + * @category slicing + * @since 4.1.5 + */ +export const tail: { + (n: number): (self: Query) => Query + (self: Query, n: number): Query +} = dual(2, (self: Query, n: number): Query => n <= 0 ? slice(self, 0, 0) : slice(self, -n)) + +/** + * Keeps the first row of every distinct combination of the given columns + * (all columns by default). + * + * @stability experimental + * @category filtering + * @since 4.1.5 + */ +export const unique: { + (columns?: ReadonlyArray): (self: Query) => Query + (self: Query, columns?: ReadonlyArray): Query +} = dual( + (args) => isQuery(args[0]), + (self: Query, columns?: ReadonlyArray): Query => + make({ _tag: "Unique", input: self.plan, columns }) +) + +type NonNull = Simplify & { [P in K]: NonNullable }> + +/** + * Removes the rows that are `null` in any of the given columns (all + * columns by default). + * + * @stability experimental + * @category filtering + * @since 4.1.5 + */ +export const dropNulls: { + ( + columns?: ReadonlyArray + ): (self: Query) => Query> + ( + self: Query, + columns?: ReadonlyArray + ): Query> +} = dual( + (args) => isQuery(args[0]), + (self: Query, columns?: ReadonlyArray) => make({ _tag: "DropNulls", input: self.plan, columns }) +) + +/** + * The rows of every query, one after the other. The queries must have the + * same columns. + * + * **Gotchas** + * + * Columns of different Arrow types (decimals of different scales, integers + * of different widths) take their common type, as in Polars. A value the + * common type can't hold, such as a decimal that leaves 64-bit storage at + * the larger scale, fails with a `DataFrameError` instead of becoming null. + * + * Over streamed tables, the concatenation reads the first batch of every + * query, which gives its types, before it returns its first row. + * + * @stability experimental + * @category combining + * @since 4.1.5 + */ +export const concat = (queries: readonly [Query, ...Array>]): Query => + make({ _tag: "Concat", inputs: queries.map((q) => q.plan) }) + +/** + * The rows of `self`, then the rows of `that`. + * + * @stability experimental + * @category combining + * @since 4.1.5 + */ +export const append: { + (that: Query): (self: Query) => Query + (self: Query, that: Query): Query +} = dual(2, (self: Query, that: Query) => concat([self, that])) + +// ----------------------------------------------------------------------------- +// Columns +// ----------------------------------------------------------------------------- + +/** + * Keeps the given columns, in the given order. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const pick: { + >( + ...names: K + ): (self: Query) => Query>> +} = (...names: ReadonlyArray) => (self: Query) => + make({ _tag: "Pick", input: self.plan, columns: names }) as any + +/** + * Removes the given columns. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const omit: { + >( + ...names: K + ): (self: Query) => Query>> +} = (...names: ReadonlyArray) => (self: Query) => + make({ _tag: "Omit", input: self.plan, columns: names }) as any + +type Renamed = Simplify< + Omit & { [K in keyof M & keyof R as M[K] extends string ? M[K] : never]: R[K] } +> + +/** + * Renames columns: `{ old: "new" }`. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const rename: { + ( + mapping: M + ): (self: Query) => Query> + (self: Query, mapping: M): Query> +} = dual( + 2, + (self: Query, mapping: { readonly [name: string]: string | undefined }) => { + const defined: { [name: string]: string } = {} + for (const [from, to] of Object.entries(mapping)) if (to !== undefined) defined[from] = to + return make({ _tag: "Rename", input: self.plan, mapping: defined }) + } +) + +type Computed = Simplify<{ [K in keyof O]: Expr.Type }> + +/** + * Replaces the columns with the given expressions; the record's keys name + * the new columns. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const select: { + }>( + f: (columns: Columns) => O + ): (self: Query) => Query> + }>( + self: Query, + f: (columns: Columns) => O + ): Query> +} = dual( + 2, + (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ _tag: "Select", input: self.plan, columns: named(f(columns)) }) +) + +/** + * Adds columns, or replaces columns of the same name in place. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const withColumns: { + }>( + f: (columns: Columns) => O + ): (self: Query) => Query & Computed>> + }>( + self: Query, + f: (columns: Columns) => O + ): Query & Computed>> +} = dual( + 2, + (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ _tag: "WithColumns", input: self.plan, columns: named(f(columns)) }) +) + +// ----------------------------------------------------------------------------- +// Grouping +// ----------------------------------------------------------------------------- + +/** + * Groups rows by the given key columns; finish with {@link agg}. + * + * @stability experimental + * @category grouping + * @since 4.1.5 + */ +export const groupBy: { + >(...keys: K): (self: Query) => GroupBy +} = (...keys: ReadonlyArray) => (self: Query) => ({ _tag: "GroupBy", query: self, keys }) as any + +/** + * Computes aggregates per group: one row per distinct key, in order of first + * appearance, with the key columns first. + * + * **Details** + * + * An aggregate's argument is evaluated per group: window functions in it + * (`Expr.cumSum`, `Expr.over`, …) see only the group's rows, in row order. + * + * @stability experimental + * @category grouping + * @since 4.1.5 + */ +export const agg: { + }>( + f: (columns: Columns) => O + ): (self: GroupBy) => Query & Computed>> + }>( + self: GroupBy, + f: (columns: Columns) => O + ): Query & Computed>> +} = dual( + 2, + (self: GroupBy, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ + _tag: "GroupBy", + input: self.query.plan, + keys: self.keys as ReadonlyArray, + aggregates: named(f(columns)) + }) +) + +/** + * Computes aggregates over all rows: a table of one row. + * + * @stability experimental + * @category grouping + * @since 4.1.5 + */ +export const aggregate: { + }>( + f: (columns: Columns) => O + ): (self: Query) => Query> + }>( + self: Query, + f: (columns: Columns) => O + ): Query> +} = dual( + 2, + (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ _tag: "Aggregate", input: self.plan, aggregates: named(f(columns)) }) +) + +// ----------------------------------------------------------------------------- +// Joins +// ----------------------------------------------------------------------------- + +/** + * Joins `that` onto `self` by equal key columns. Output rows keep the left + * row order; several right matches follow in their order. The non-key + * columns of the two sides must have different names. + * + * @stability experimental + * @category combining + * @since 4.1.5 + */ +export const join: { + ( + that: Query, + options: { readonly on: On | ReadonlyArray; readonly how?: How | undefined } & JoinCheck + ): (self: Query) => Query> + ( + self: Query, + that: Query, + options: { readonly on: On | ReadonlyArray; readonly how?: How | undefined } & JoinCheck + ): Query> +} = dual( + 3, + ( + self: Query, + that: Query, + options: { readonly on: string | ReadonlyArray; readonly how?: JoinHow | undefined } + ) => + make({ + _tag: "Join", + left: self.plan, + right: that.plan, + on: typeof options.on === "string" ? [options.on] : options.on, + how: options.how ?? "inner" + }) +) as any + +// ----------------------------------------------------------------------------- +// Reshaping +// ----------------------------------------------------------------------------- + +/** + * The value type of a pivot cell for an aggregate of values of type `A`. + * + * **Details** + * + * Sums have the type `Expr.sum` gives: durations sum to a duration. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type PivotCell = F extends "len" | "count" | "countDistinct" ? number + : F extends "sum" ? (NonNullable extends bigint ? bigint + : NonNullable extends BigDecimal.BigDecimal ? BigDecimal.BigDecimal + : NonNullable extends Duration.Duration ? Duration.Duration + : number) + : F extends "mean" | "median" | "std" | "variance" ? number | null + : NonNullable | null + +interface PivotOptions { + /** The column whose distinct values become columns. */ + readonly on: On + /** The columns whose distinct combinations become rows. */ + readonly index: Index + /** The column aggregated into the cells. */ + readonly values: Values + /** The aggregate of each cell. */ + readonly aggregate: F + readonly "~row"?: R +} + +/** + * One row per distinct `index`, one column per distinct value of `on`, and + * in each cell the `aggregate` of `values` over the rows with that index and + * value (`Plan.Pivot` has the details). + * + * **Details** + * + * Without `columns`, the output columns depend on the data, so the rows are + * untyped. With `columns`, the output has exactly those value columns, in + * that order, and is typed. + * + * **Example** (Units per region and quarter) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { DataFrame, Query } from "effect/dataframe" + * + * const sales = DataFrame.fromColumns({ + * region: ["north", "north", "south"], + * quarter: ["q1", "q2", "q1"], + * units: [10, 20, 5] + * }) + * + * const rows = Query.from(sales).pipe( + * Query.pivot({ on: "quarter", index: ["region"], values: "units", aggregate: "sum", columns: ["q1", "q2"] }), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ region: "north", q1: 10, q2: 20 }, { region: "south", q1: 5, q2: 0 }] + * ``` + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const pivot: { + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate, + const Columns extends ReadonlyArray + >( + options: PivotOptions & { readonly columns: Columns } + ): ( + self: Query + ) => Query & { readonly [C in Columns[number]]: PivotCell }>> + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate + >( + options: PivotOptions + ): (self: Query) => Query<{ readonly [name: string]: unknown }> + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate, + const Columns extends ReadonlyArray + >( + self: Query, + options: PivotOptions & { readonly columns: Columns } + ): Query & { readonly [C in Columns[number]]: PivotCell }>> + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate + >( + self: Query, + options: PivotOptions + ): Query<{ readonly [name: string]: unknown }> +} = dual(2, (self: Query, options: { + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: Plan.PivotAggregate + readonly columns?: ReadonlyArray +}) => + make({ + _tag: "Pivot", + input: self.plan, + on: options.on, + index: options.index, + values: options.values, + aggregate: options.aggregate, + columns: options.columns + })) as any + +interface UnpivotOptions { + readonly on: On + readonly index?: Index | undefined + readonly variable?: Variable | undefined + readonly value?: Value | undefined +} + +type Unpivoted< + R, + On extends ReadonlyArray, + Index extends ReadonlyArray, + Variable extends string, + Value extends string +> = Simplify< + & Pick + & { readonly [K in Variable]: On[number] } + & { readonly [K in Value]: R[On[number]] } +> + +/** + * Stacks the `on` columns into one `value` column (`"value"` by default), + * one block of rows per column, with the column's name in `variable` + * (`"variable"` by default) and the `index` columns repeated. The values + * take the common type of the stacked columns; a value it can't hold fails. + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const unpivot: { + < + R, + const On extends ReadonlyArray, + const Index extends ReadonlyArray = readonly [], + const Variable extends string = "variable", + const Value extends string = "value" + >( + options: UnpivotOptions + ): (self: Query) => Query> + < + R, + const On extends ReadonlyArray, + const Index extends ReadonlyArray = readonly [], + const Variable extends string = "variable", + const Value extends string = "value" + >( + self: Query, + options: UnpivotOptions + ): Query> +} = dual( + 2, + (self: Query, options: UnpivotOptions, ReadonlyArray, string, string>) => + make({ + _tag: "Unpivot", + input: self.plan, + index: options.index ?? [], + on: options.on, + variable: options.variable ?? "variable", + value: options.value ?? "value" + }) +) as any + +/** + * One row per element of the list column `column`, the other columns + * repeated. A null list gives one row with a null element; an empty list + * gives no rows. + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const explode: { + >( + column: K + ): (self: Query) => Query> + >( + self: Query, + column: K + ): Query> +} = dual(2, (self: Query, column: string) => make({ _tag: "Explode", input: self.plan, column })) as any + +/** + * The fields of the struct column `column` as columns, in its place; they + * are null where the struct is null. + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const unnest: { + >( + column: K + ): (self: Query) => Query> + >(self: Query, column: K): Query> +} = dual(2, (self: Query, column: string) => make({ _tag: "Unnest", input: self.plan, column })) as any + +type ListKeys = { + [K in keyof R & string]: NonNullable extends ReadonlyArray ? K : never +}[keyof R & string] +type ElementOf = NonNullable extends ReadonlyArray ? E : never +type Exploded = Simplify<{ readonly [P in keyof R]: P extends K ? ElementOf | null : R[P] }> +type StructKeys = { + [K in keyof R & string]: NonNullable extends ReadonlyArray ? never + : NonNullable extends object ? K + : never +}[keyof R & string] +type Fields = null extends A ? { readonly [K in keyof NonNullable]: NonNullable[K] | null } + : { readonly [K in keyof NonNullable]: NonNullable[K] } +type Unnested = Simplify & Fields> diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts new file mode 100644 index 00000000000..82639421174 --- /dev/null +++ b/packages/effect/src/dataframe/Sql.ts @@ -0,0 +1,211 @@ +/** + * Running queries on PostgreSQL: plans compiled to SQL and run through the + * `effect/sql` client in the context. + * + * **Mental model** + * + * - {@link compile} turns a plan into one SQL query whose rows come back in + * the plan's row order: stable sorts, groups in order of first appearance, + * joins in left row order, as the in-memory engine. + * - {@link layer} provides an `Engine` that compiles every query and runs it + * with the `SqlClient` in the context, so `yield* query` reads from the + * database. `Query.table` names become table names, each one quoted + * identifier: `"a.b"` reads the table called `a.b`, not table `b` of + * schema `a`. + * - The SQL is PostgreSQL's (PGlite included); other databases are future + * work. + * + * **Example** (Compiling a query) + * + * ```ts import.meta.vitest + * import { Result, Schema } from "effect" + * import { Expr, Query, Sql } from "effect/dataframe" + * + * const Trip = Schema.Struct({ city: Schema.String, km: Schema.Number }) + * const query = Query.table("trips", Trip).pipe( + * Query.filter((c) => Expr.gt(c.km, 10)), + * Query.groupBy("city"), + * Query.agg((c) => ({ total: Expr.sum(c.km) })) + * ) + * + * const compiled = Result.getOrThrow(Sql.compile(query.plan)) + * compiled.schema.fields.map((f) => f.name) // => ["city", "total"] + * ``` + * + * **Details** + * + * Results have the column types the in-memory engine gives for the same + * plan (see {@link Compiled}). Values follow the database's semantics where + * they differ from the in-memory engine's: integer overflow raises an error + * instead of wrapping, division by zero follows the database, string order + * follows its collation, and timestamps and durations keep PostgreSQL's + * microseconds (finer timestamps round down, finer durations toward zero). + * Decimals need a scale PostgreSQL's `numeric` takes, from -1000 to 1000. + * Comparisons and `Expr.isIn` follow the database too. A decimal next to + * a number that is not an integer is cast to `DOUBLE PRECISION`, with + * PostgreSQL's rounding and its error for a decimal outside that type's + * range. A timestamp without a time zone (a `DateTime` literal is one) + * next to one that has a time zone is read in the session's time zone, so + * it gives the in-memory result in a UTC session only. `Expr.isIn` + * candidates of another kind than the value are written as they are and + * are not portable: PostgreSQL converts some (the string `"1"` is one of + * the integers equal to 1) and rejects others with its own error. + * Pivots, unpivots, `explode` and `unnest` are not supported yet; tables + * need declared columns (`Query.table` with a schema), and {@link engine} + * checks that tables satisfy them, as the in-memory engine does. + * + * @stability experimental + * @since 4.1.5 + */ +import type * as ArrowSchema from "../arrow/ArrowSchema.ts" +import * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Result from "../Result.ts" +import { SqlClient } from "../sql/SqlClient.ts" +import * as Stream from "../Stream.ts" +import { type DataFrame, DataFrameError } from "./DataFrame.ts" +import { Engine } from "./Engine.ts" +import * as internal from "./internal/sql.ts" +import * as V from "./internal/vector.ts" +import type * as Plan from "./Plan.ts" + +/** + * A compiled plan: the query, and the schema of its result. + * + * **Details** + * + * The schema has the column types the in-memory engine gives for the same + * plan, whatever the database's own types are; every field is nullable. The + * query renders each column in a form {@link engine} reads back as that + * type, so its raw rows are meant for that decoding rather than for direct + * use. + * + * **Gotchas** + * + * The query doesn't check that the tables satisfy their declared columns and + * schemas: {@link engine} runs a query that checks the rows it reads + * instead. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Compiled { + readonly sql: string + readonly schema: ArrowSchema.Schema +} + +/** The messages of an error and its causes, outermost first, without repeats. */ +const toError = (cause: unknown) => { + const messages: Array = [] + for (let c: unknown = cause; c instanceof Error && messages.length < 8; c = c.cause) { + if (c.message !== "" && !messages.includes(c.message)) messages.push(c.message) + } + return new DataFrameError({ message: messages.length === 0 ? String(cause) : messages.join(": "), cause }) +} + +/** + * Compiles a plan to one PostgreSQL query. Fails for operations the SQL + * engine doesn't support, and for columns of one relation whose names + * share their first 63 bytes (in UTF-8): PostgreSQL keeps only those of a + * name, so it can't tell such columns apart. Declarations of tables, or of + * their columns, whose names share those bytes read one table or column, + * each with its own checks. + * + * @stability experimental + * @category compiling + * @since 4.1.5 + */ +export const compile = (plan: Plan.Node): Result.Result => + Result.map(attempt(() => internal.compile(plan)), ({ schema, sql }) => ({ sql, schema })) + +const attempt = (f: () => A): Result.Result => { + try { + return Result.succeed(f()) + } catch (cause) { + if (cause instanceof V.DataFrameDefect) return Result.fail(toError(cause)) + throw cause + } +} + +/** + * An engine that compiles queries to SQL and runs them with the + * `SqlClient` in the context. + * + * **Details** + * + * Tables must satisfy their declared columns and schemas, as with the + * in-memory engine, or the query fails. First the engine queries the + * PostgreSQL types of the declared columns: `smallint` or `integer` for + * integers up to 32 bits, `bigint` for 64-bit ones, `smallint`, + * `integer`, `real` or `double precision` for floats, string or enum + * types for strings, `bytea`, `boolean`, `date`, `timestamp` or + * `timestamptz` (read as UTC), `interval`, `numeric` for decimals, and + * any type for `Null` columns. Then one query reads each table once, as a + * `MATERIALIZED` common table expression (computed once per statement), + * checks those rows and computes the result from exactly them, whatever + * the relation: views that sample rows or call functions included, and a + * table read twice gives the same rows twice. The checks: only nulls in + * `Null` columns, no nulls in other non-nullable columns, values their + * declared types hold exactly (integers in range, numbers that `Float32` + * holds, decimals within their precision, scale and 64-bit storage, finite + * dates, timestamps and durations in whole units), timestamps in the range + * of `DateTime`, intervals without months, and the distinct values of the + * columns the schema constrains beyond their type (literals, enums, + * refinements) accepted by the schema. Columns the query doesn't read are + * checked too. The query reads every column as its declared type: strings, + * enums and `char(n)` as text (`char(n)` without trailing spaces, as + * PostgreSQL converts it), numbers as the PostgreSQL type of their + * declared type. `explain` shows both queries. Tables whose schemas have + * checks on whole rows are rejected when the query compiles. + * + * **Gotchas** + * + * Every query reads all rows of its tables, which the checks need, and + * copies their declared columns once: filters can't use the tables' + * indexes. The column types are checked in a separate + * statement first: run both inside a `REPEATABLE READ` transaction when + * the tables may be altered concurrently. + * + * @stability experimental + * @category engines + * @since 4.1.5 + */ +export const engine: Effect.Effect = Effect.gen(function*() { + const sql = yield* SqlClient + const compiled = (plan: Plan.Node) => Effect.fromResult(attempt(() => internal.compile(plan))) + const run = (query: string) => sql.unsafe(query).values.pipe(Effect.mapError(toError)) + const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => + Effect.gen(function*() { + const query = yield* compiled(plan) + if (query.types !== undefined) { + const types = query.types + const rows = yield* run(types.sql) + yield* Effect.fromResult(attempt(() => internal.verify(types, rows))) + } + const rows = yield* run(query.query.sql) + return yield* Effect.try({ + try: () => internal.read(query, rows) as unknown as DataFrame, + catch: toError + }) + }) + return { + name: "sql (postgres)", + collect, + stream: (plan) => Stream.fromEffect(collect(plan)), + explain: (plan) => + Effect.map(compiled(plan), (c) => [...(c.types === undefined ? [] : [c.types.sql]), c.query.sql].join(";\n")) + } satisfies Engine.Service +}) + +/** + * A layer that runs queries on the database of the `SqlClient` in the + * context, with {@link engine}: each query reads its tables once, checks + * those rows against their declared columns and schemas, and computes its + * result from exactly those rows. + * + * @stability experimental + * @category layers + * @since 4.1.5 + */ +export const layer: Layer.Layer = Layer.effect(Engine, engine) diff --git a/packages/effect/src/dataframe/index.ts b/packages/effect/src/dataframe/index.ts new file mode 100644 index 00000000000..2594495cb34 --- /dev/null +++ b/packages/effect/src/dataframe/index.ts @@ -0,0 +1,48 @@ +/** + * @stability experimental + * @since 4.1.5 + */ + +// @barrel: Auto-generated exports. Do not edit manually. + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Catalog from "./Catalog.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as DataFrame from "./DataFrame.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Engine from "./Engine.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Expr from "./Expr.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Plan from "./Plan.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Query from "./Query.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Sql from "./Sql.ts" diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts new file mode 100644 index 00000000000..e7d74692d70 --- /dev/null +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -0,0 +1,420 @@ +/** + * Grouped aggregation kernels. Every kernel takes the input column (one value + * per row), the group id of every row and the group count, and returns one + * value per group. + * + * Null inputs are skipped. Over a group without values `sum` is 0, `count` + * is 0 and the other aggregates are null. `min`/`max` skip NaN unless every + * value is NaN; `sum` and `mean` propagate it. This matches Polars. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as Hash from "./hash.ts" +import * as Sort from "./sort.ts" +import * as T from "./temporal.ts" +import * as V from "./vector.ts" + +const I0 = BigInt(0) + +/** @internal */ +export type AggFn = + | "sum" + | "mean" + | "min" + | "max" + | "count" + | "len" + | "first" + | "last" + | "countDistinct" + | "std" + | "variance" + | "median" + | "list" + +/** + * The kernel of an aggregate call with argument `arg`: `count` without an + * argument counts rows; every other function needs one. + * + * @internal + */ +export const kernel = (fn: AggFn, arg: unknown): AggFn => + arg !== undefined ? fn : fn === "count" ? "len" : V.fail(`${fn} needs an argument`) + +/** @internal */ +export interface Grouping { + readonly ids: Int32Array + readonly count: number +} + +const validCounts = (v: V.Vector, g: Grouping): Float64Array => { + const out = new Float64Array(g.count) + const ids = g.ids + const n = ids.length + if (v.kind === "null") return out + const valid = v.valid + if (valid === undefined) { for (let i = 0; i < n; i++) out[ids[i]]++ } + else for (let i = 0; i < n; i++) out[ids[i]] += valid[i] + return out +} + +const invalidWhereZero = (counts: Float64Array): Uint8Array | undefined => { + let valid: Uint8Array | undefined + for (let g = 0; g < counts.length; g++) { + if (counts[g] === 0) { + if (valid === undefined) valid = new Uint8Array(counts.length).fill(1) + valid[g] = 0 + } + } + return valid +} + +/** @internal */ +export const aggregate = (fn: AggFn, v: V.Vector | undefined, g: Grouping): V.Vector => { + const ids = g.ids + const n = ids.length + const groups = g.count + if (fn === "len") { + const out = new Float64Array(groups) + for (let i = 0; i < n; i++) out[ids[i]]++ + return V.f64(S.uint32, out) + } + if (v === undefined) return V.fail(`${fn} needs an argument`) + switch (fn) { + case "list": { + const { rows, starts } = Hash.rowsByGroup(ids, groups) + return V.list(S.list(S.field("item", v.type)), starts, V.take(v, rows)) + } + case "count": + return V.f64(S.uint32, validCounts(v, g)) + case "first": + case "last": { + const rows = new Int32Array(groups).fill(-1) + if (fn === "first") { for (let i = n - 1; i >= 0; i--) rows[ids[i]] = i } + else for (let i = 0; i < n; i++) rows[ids[i]] = i + return V.take(v, rows) + } + case "countDistinct": { + const values = Hash.ids(v) + const distinct = Hash.groups([V.i32(S.int32, ids), V.i32(S.int32, values.ids)], n) + const out = new Float64Array(groups) + for (let d = 0; d < distinct.count; d++) out[ids[distinct.firstRow[d]]]++ + return V.f64(S.uint32, out) + } + case "sum": + return sum(v, g) + case "mean": + return mean(v, g) + case "min": + case "max": + return minMax(fn, v, g) + case "std": + case "variance": + return variance(fn, v, g) + case "median": + return median(v, g) + } +} + +const sum = (v: V.Vector, g: Grouping): V.Vector => sumColumn(v.type, sums(v, g)) + +/** + * Per-group sums of the valid values: Float64 for float columns, exact + * integers (unscaled for decimals) for every other column `sum` takes. + * + * @internal + */ +export type Sums = Float64Array | ReadonlyArray + +/** Per-group {@link Sums} of a column. @internal */ +export const sums = (v: V.Vector, g: Grouping): Sums => { + const ids = g.ids + const n = ids.length + if (D.isPointInTime(v.type)) return V.fail(`sum is not defined for ${D.describe(v.type)}`) + switch (v.kind) { + case "null": + return new Array(g.count).fill(I0) + case "i64": { + const x = V.isUnsigned64(v.type) ? new BigUint64Array(v.values.buffer, v.values.byteOffset, v.length) : v.values + const acc = new Array(g.count).fill(I0) + for (let i = 0; i < n; i++) if (V.isValid(v, i)) acc[ids[i]] += x[i] + return acc + } + case "f64": + case "i32": + case "bool": { + const x = v.values + const acc = new Float64Array(g.count) + const valid = v.valid + if (valid === undefined) { for (let i = 0; i < n; i++) acc[ids[i]] += x[i] } + else for (let i = 0; i < n; i++) if (valid[i] === 1) acc[ids[i]] += x[i] + // Integers accumulate exactly in Float64 up to 2^53. + return v.type._tag === "Float" ? acc : Array.from(acc, BigInt) + } + default: + return V.fail(`sum is not defined for ${D.describe(v.type)}`) + } +} + +/** + * The `sum` column of an input of type `inputType` from its {@link Sums}: + * integer sums wrap to the sum type, decimal sums fail outside 64 bits. + * + * @internal + */ +export const sumColumn = (inputType: S.DataType, sums: Sums): V.Vector => { + const type = D.sumType(inputType) + if (sums instanceof Float64Array || type._tag === "Float") { + const out = Float64Array.from(sums as ArrayLike, Number) + V.roundFloat32InPlace(type, out) + return V.f64(type, out) + } + switch (V.kindOf(type)) { + case "null": + return V.i32(S.int32, new Int32Array(sums.length)) + case "i32": + return V.i32(type, Int32Array.from(sums, (x) => Number(BigInt.asIntN(32, x)))) + case "f64": + return V.f64(type, Float64Array.from(sums, (x) => Number(BigInt.asUintN(32, x)))) + default: + return V.i64(type, BigInt64Array.from(sums, Dec.fit(type))) + } +} + +/** + * Every value as a float, with validity. Fails for durations and points in + * time: their integers count units, which a mean would lose. + */ +const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { + if (T.isTemporal(v.type)) { + return V.fail(`expected a numeric column, got ${D.describe(v.type)}`) + } + switch (v.kind) { + case "f64": + return { x: v.values, valid: v.valid } + case "i32": + return { x: Float64Array.from(v.values), valid: v.valid } + case "i64": + if (v.type._tag === "Decimal") return { x: Dec.toFloat(v.values, v.type.scale), valid: v.valid } + return { + x: V.isUnsigned64(v.type) + ? Float64Array.from(new BigUint64Array(v.values.buffer, v.values.byteOffset, v.length), Number) + : Float64Array.from(v.values, Number), + valid: v.valid + } + case "bool": + return { x: Float64Array.from(v.values), valid: v.valid } + case "null": + return { x: new Float64Array(v.length), valid: new Uint8Array(v.length) } + default: + return V.fail(`expected a numeric column, got ${D.describe(v.type)}`) + } +} + +/** + * Per-group count and sum of the valid values, as floats: the parts of a + * mean that combine across row ranges. + * + * @internal + */ +export const sumCount = ( + v: V.Vector, + g: Grouping +): { readonly sum: Float64Array; readonly count: Float64Array } => { + const { valid, x } = asFloat(v) + return sumCountOf(x, valid, g) +} + +const sumCountOf = ( + x: Float64Array, + valid: Uint8Array | undefined, + g: Grouping +): { readonly sum: Float64Array; readonly count: Float64Array } => { + const ids = g.ids + const n = ids.length + const sum = new Float64Array(g.count) + const count = new Float64Array(g.count) + if (valid === undefined) { + for (let i = 0; i < n; i++) { + const k = ids[i] + sum[k] += x[i] + count[k]++ + } + } else { + for (let i = 0; i < n; i++) { + if (valid[i] === 0) continue + const k = ids[i] + sum[k] += x[i] + count[k]++ + } + } + return { sum, count } +} + +/** The mean column from per-group sums and counts (consumes `sum`). @internal */ +export const meanFrom = (inputType: S.DataType, sum: Float64Array, count: Float64Array): V.Vector => { + for (let k = 0; k < sum.length; k++) sum[k] /= count[k] + const type = D.meanType(inputType) + V.roundFloat32InPlace(type, sum) + return V.f64(type, sum, invalidWhereZero(count)) +} + +const mean = (v: V.Vector, g: Grouping): V.Vector => { + const { count, sum } = sumCount(v, g) + return meanFrom(v.type, sum, count) +} + +/** + * Per-group count, mean and sum of squared deviations from the mean (M2): + * the parts of a variance that combine across row ranges. + * + * @internal + */ +export const moments = ( + v: V.Vector, + g: Grouping +): { readonly count: Float64Array; readonly mean: Float64Array; readonly m2: Float64Array } => { + const ids = g.ids + const n = ids.length + const { valid, x } = asFloat(v) + const groups = g.count + const { count, sum } = sumCountOf(x, valid, g) + for (let k = 0; k < groups; k++) sum[k] /= count[k] + const m2 = new Float64Array(groups) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + const d = x[i] - sum[ids[i]] + m2[ids[i]] += d * d + } + return { count, mean: sum, m2 } +} + +/** The std or variance column from per-group counts and M2. @internal */ +export const varianceFrom = ( + fn: "std" | "variance", + inputType: S.DataType, + count: Float64Array, + m2: Float64Array +): V.Vector => { + const groups = count.length + const out = new Float64Array(groups) + let outValid: Uint8Array | undefined + for (let k = 0; k < groups; k++) { + if (count[k] < 2) { + if (outValid === undefined) outValid = new Uint8Array(groups).fill(1) + outValid[k] = 0 + continue + } + const vr = m2[k] / (count[k] - 1) + out[k] = fn === "std" ? Math.sqrt(vr) : vr + } + const type = D.meanType(inputType) + V.roundFloat32InPlace(type, out) + return V.f64(type, out, outValid) +} + +const variance = (fn: "std" | "variance", v: V.Vector, g: Grouping): V.Vector => { + const { count, m2 } = moments(v, g) + return varianceFrom(fn, v.type, count, m2) +} + +const minMax = (fn: "min" | "max", v: V.Vector, g: Grouping): V.Vector => { + if (v.kind === "list" || v.kind === "struct") return V.fail(`${fn} is not defined for ${v.type._tag} columns`) + const ids = g.ids + const n = ids.length + const groups = g.count + const isMin = fn === "min" + switch (v.kind) { + case "null": + return V.nulls(groups) + case "f64": { + const x = v.values + const valid = v.valid + const acc = new Float64Array(groups).fill(NaN) + const seen = new Uint8Array(groups) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + const k = ids[i] + const a = x[i] + seen[k] = 1 + if (a !== a) continue + const c = acc[k] + if (c !== c || (isMin ? a < c : a > c)) acc[k] = a + } + return V.f64(v.type, acc, V.compactValid(seen)) + } + case "i32": { + const x = v.values + const valid = v.valid + const acc = new Int32Array(groups) + const seen = new Uint8Array(groups) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + const k = ids[i] + const a = x[i] + if (seen[k] === 0) { + seen[k] = 1 + acc[k] = a + } else if (isMin ? a < acc[k] : a > acc[k]) acc[k] = a + } + return V.i32(v.type, acc, V.compactValid(seen)) + } + case "bool": { + const x = v.values + const acc = new Uint8Array(groups).fill(isMin ? 1 : 0) + const seen = new Uint8Array(groups) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const k = ids[i] + seen[k] = 1 + acc[k] = isMin ? acc[k] & x[i] : acc[k] | x[i] + } + return V.bool(acc, V.compactValid(seen)) + } + case "i64": + case "str": { + // Track the best row per group, then gather it. + const best = new Int32Array(groups).fill(-1) + const cmp = Sort.rowOrder(v) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const k = ids[i] + const b = best[k] + if (b < 0 || (isMin ? cmp(i, b) < 0 : cmp(i, b) > 0)) best[k] = i + } + return V.take(v, best) + } + } +} + +const median = (v: V.Vector, g: Grouping): V.Vector => { + const ids = g.ids + const n = ids.length + const groups = g.count + const { valid, x } = asFloat(v) + // Counting sort of the present values by group, then a numeric sort per group. + const start = new Int32Array(groups + 1) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) start[ids[i] + 1]++ + for (let k = 0; k < groups; k++) start[k + 1] += start[k] + const values = new Float64Array(start[groups]) + const fill = start.slice(0, groups) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) values[fill[ids[i]]++] = x[i] + const out = new Float64Array(groups) + const outValid = new Uint8Array(groups) + for (let k = 0; k < groups; k++) { + const s = start[k] + const len = start[k + 1] - s + if (len === 0) continue + const group = values.subarray(s, s + len) + group.sort() + const mid = len >> 1 + out[k] = len % 2 === 1 ? group[mid] : (group[mid - 1] + group[mid]) / 2 + outValid[k] = 1 + } + const type = D.meanType(v.type) + V.roundFloat32InPlace(type, out) + return V.f64(type, out, V.compactValid(outValid)) +} diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts new file mode 100644 index 00000000000..cd153cd20b4 --- /dev/null +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -0,0 +1,423 @@ +/** + * Decimals: unscaled 64-bit integers with a scale (`value / 10^scale`), so + * up to 18 significant digits. Arithmetic follows Polars' result types + * (precision 38, scales added for products, the larger scale for sums). + * Every computed decimal is stored through `checked` (or `fit`), so a + * value outside 64 bits fails instead of wrapping; casts make such values + * null instead. Rescaling down rounds half to even. Scales are any 32-bit + * integers, negative ones included; see {@link FAR} for scales far apart. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as V from "./vector.ts" + +/** + * From this many digits on, the 64 bits of storage decide a rescale, so no + * larger power of ten is built for one: `10^20` is more than twice any + * 64-bit value (a quotient rounds to zero, see {@link divRound}), and a + * nonzero value times `10^20` is outside 64 bits and stays outside with a + * 64-bit value added (see {@link factor}). + */ +const FAR = 20 + +const powers: Array = [] +/** `10^k` as a bigint, for `k >= 0`. Only the powers up to {@link FAR} are kept. */ +const pow10 = (k: number): bigint => k <= FAR ? (powers[k] ??= BigInt(10) ** BigInt(k)) : BigInt(10) ** BigInt(k) + +/** + * What to multiply by to add `k >= 0` digits: `10^k`, or `10^FAR` from + * {@link FAR} digits on. The product is then zero for zero and outside 64 + * bits otherwise, as it is with `10^k`, and every product is range-checked. + */ +const factor = (k: number): bigint => pow10(Math.min(k, FAR)) + +const I0 = BigInt(0) +const I1 = BigInt(1) +const I2 = BigInt(2) +const MAX = BigInt("9223372036854775807") +const MIN = -BigInt("9223372036854775808") +const UMAX = BigInt("18446744073709551615") + +/** @internal */ +export const isDecimal = (t: S.DataType): t is S.Decimal => t._tag === "Decimal" + +/** Precision of results, as in Polars. */ +const PRECISION = 38 + +/** @internal */ +export const decimal = (scale: number): S.Decimal => S.decimal(PRECISION, scale) + +/** + * The common type of a decimal and another numeric type. It holds every + * value of both exactly or not at all: the larger scale of two decimals, + * and with an integer a scale of at least zero, as a negative scale would + * round the integer. + * + * @internal + */ +export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined => { + if (isDecimal(a) && isDecimal(b)) { + return a.scale === b.scale && a.precision === b.precision + ? a + : decimal(Math.max(a.scale, b.scale)) + } + const [d, other] = isDecimal(a) ? [a, b] : [b as S.Decimal, a] + if (other._tag === "Int") return decimal(Math.max(d.scale, 0)) + if (other._tag === "Float") return S.float64 + return undefined +} + +const inRange = (x: bigint): boolean => x <= MAX && x >= MIN + +/** `x`, or a failure when it overflows 64-bit decimal storage. @internal */ +export const checked = (x: bigint): bigint => (inRange(x) ? x : V.fail("decimal value overflows 64 bits")) + +/** + * `x` as the unscaled value of a literal of `type`, or a failure when it + * overflows 64-bit storage or has more digits than the type's precision. + * + * @internal + */ +export const literal = (x: bigint, type: S.Decimal): bigint => { + const limit = factor(type.precision) + return checked(x) < limit && -x < limit + ? x + : V.fail(`decimal literal ${x} has more digits than its precision ${type.precision}`) +} + +/** + * The largest power of ten a float holds is `10^308`. A larger one is + * applied in two steps, so that a finite result isn't lost to an infinite + * factor. + */ +const FLOAT_DIGITS = 308 + +/** `x * 10^scale` as a float. */ +const scaleFloat = (x: number, scale: number): number => + // Zero is zero at any scale, also where `10^scale` is infinite. + x === 0 + ? 0 + : scale > FLOAT_DIGITS + ? x * 10 ** FLOAT_DIGITS * 10 ** (scale - FLOAT_DIGITS) + : x * 10 ** scale + +/** + * `unscaled / 10^scale` as a float. Past {@link FLOAT_DIGITS} digits it + * reads the value's decimal notation, which rounds once. + * + * @internal + */ +export const toNumber = (unscaled: bigint, scale: number): number => + // Zero is zero at any scale, also where `10^scale` is zero. + unscaled === I0 + ? 0 + : scale > FLOAT_DIGITS + ? Number(`${unscaled}e-${scale}`) + : Number(unscaled) / 10 ** scale + +const wrap = (x: bigint): bigint => BigInt.asIntN(64, x) + +/** + * How an exact result of `type` is stored in 64 bits: decimals are + * `checked`, 64-bit integers wrap. + * + * @internal + */ +export const fit = (type: S.DataType): (x: bigint) => bigint => isDecimal(type) ? checked : wrap + +/** Rounds `x / 10^k` half to even. */ +const divRound = (x: bigint, k: number): bigint => { + if (k === 0) return x + // `10^k` is more than twice `x`, which has fewer than `k` digits. + if (k >= FAR && (inRange(x) || k > x.toString().length)) return I0 + const divisor = pow10(k) + // Truncated towards zero; the remainder has the sign of `x`. + const quotient = x / divisor + const remainder = x % divisor + const twice = (remainder < I0 ? -remainder : remainder) * I2 + if (twice < divisor || (twice === divisor && quotient % I2 === I0)) return quotient + return x < I0 ? quotient - I1 : quotient + I1 +} + +/** + * `unscaled / 10^scale` as a decimal string, all `scale` digits shown. A + * negative scale writes its zeros out (`7` at scale `-2` is `700`). + * + * @internal + */ +export const format = (unscaled: bigint, scale: number): string => { + const negative = unscaled < I0 + const digits = (negative ? -unscaled : unscaled).toString() + if (scale <= 0) return unscaled === I0 ? digits : `${negative ? "-" : ""}${digits}${"0".repeat(-scale)}` + const padded = digits.padStart(scale + 1, "0") + const text = `${padded.slice(0, padded.length - scale)}.${padded.slice(padded.length - scale)}` + return negative ? `-${text}` : text +} + +/** + * {@link format} for every value. The text has as many digits as the scale, + * so it fails where the scale is more than a string can hold. + */ +const formatAll = (v: V.I64, type: S.Decimal): Array => { + try { + return Array.from({ length: v.length }, (_, i) => (V.isValid(v, i) ? format(v.values[i], type.scale) : null)) + } catch (error) { + if (!(error instanceof RangeError)) throw error + return V.fail(`Decimal[${type.precision}, ${type.scale}] values are too long to write as text`) + } +} + +/** + * The integer part of `unscaled / 10^scale`, truncated towards zero. A + * negative scale can take it outside 64 bits (see {@link factor}). + */ +const integerPart = (unscaled: bigint, scale: number): bigint => + scale < 0 ? unscaled * factor(-scale) : scale >= FAR ? I0 : unscaled / pow10(scale) + +/** + * The integer `x` in unscaled units of `scale`: rounded half to even for a + * negative scale, and possibly outside 64 bits otherwise (see {@link factor}). + */ +const toScale = (x: bigint, scale: number): bigint => scale < 0 ? divRound(x, -scale) : x * factor(scale) + +const DECIMAL_SYNTAX = /^\s*([+-]?)(\d*)(?:\.(\d*))?\s*$/ +/** {@link DECIMAL_SYNTAX} without the spaces around it. */ +const PLAIN_SYNTAX = /^([+-]?)(\d*)(?:\.(\d*))?$/ + +/** + * Parses a decimal string to unscaled units of `scale`; `undefined` when + * invalid. `drop` removes the `k` digits a fraction has beyond the scale. + */ +const parse = ( + s: string, + scale: number, + syntax: RegExp = DECIMAL_SYNTAX, + drop: (x: bigint, k: number) => bigint | undefined = divRound +): bigint | undefined => { + const m = syntax.exec(s) + if (m === null || (m[2] === "" && (m[3] ?? "") === "")) return undefined + const fraction = m[3] ?? "" + const all = BigInt(`${m[1]}${m[2] || "0"}${fraction}`) + return fraction.length >= scale ? drop(all, fraction.length - scale) : all * factor(scale - fraction.length) +} + +/** For {@link parse}: a fraction longer than the scale is invalid. */ +const dropNone = (x: bigint, k: number): bigint | undefined => (k === 0 ? x : undefined) + +/** + * A decimal vector of `type` from decimal strings; `null` and `undefined` + * are nulls. Reads plain decimal notation (`-12.34`) only: `undefined` when + * a string has another form, more fraction digits than the scale, or a value + * outside the type's bit width or the 64 bits of a vector. + * + * @internal + */ +export const fromStrings = (type: S.Decimal, values: ReadonlyArray): V.I64 | undefined => { + const n = values.length + const out = new BigInt64Array(n) + const valid = new Uint8Array(n) + const limit = I1 << BigInt(Math.min(type.bitWidth, 64) - 1) + let nulls = false + for (let i = 0; i < n; i++) { + const v = values[i] + if (v === null || v === undefined) { + nulls = true + continue + } + const x = parse(String(v), type.scale, PLAIN_SYNTAX, dropNone) + if (x === undefined || x >= limit || x < -limit) return undefined + out[i] = x + valid[i] = 1 + } + return V.i64(type, out, nulls ? valid : undefined) +} + +/** + * Casts to and from decimals; `undefined` when neither side is a decimal. + * Values that don't fit the target become null. + * + * @internal + */ +export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { + const from = v.type + if (!isDecimal(from) && !isDecimal(target)) return undefined + const n = v.length + const valid = new Uint8Array(n) + if (isDecimal(target)) { + const out = new BigInt64Array(n) + // At most `precision` digits: |x| < 10^precision. + // From `FAR` digits on, the 64 bits of storage are the tighter limit. + const limit = target.precision >= 1 ? factor(target.precision) : I0 + const put = (i: number, x: bigint | undefined) => { + if (x !== undefined && inRange(x) && x < limit && -x < limit) { + out[i] = x + valid[i] = 1 + } + } + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + switch (v.kind) { + case "i64": { + const x = V.isUnsigned64(from) ? BigInt.asUintN(64, v.values[i]) : v.values[i] + put( + i, + isDecimal(from) + ? from.scale >= target.scale + ? divRound(x, from.scale - target.scale) + : x * factor(target.scale - from.scale) + : toScale(x, target.scale) + ) + break + } + case "i32": + case "bool": + put(i, toScale(BigInt(v.values[i]), target.scale)) + break + case "f64": { + const scaled = scaleFloat(v.values[i], target.scale) + if (Number.isFinite(scaled)) { + const floor = Math.floor(scaled) + const rest = scaled - floor + put(i, BigInt(rest > 0.5 || (rest === 0.5 && floor % 2 !== 0) ? floor + 1 : floor)) + } + break + } + case "str": + put(i, parse(V.decodeString(v, i), target.scale)) + break + } + } + return V.i64(target, out, valid) + } + // from a decimal + if (v.kind !== "i64" || !isDecimal(from)) return V.fail("expected decimal storage") + switch (target._tag) { + case "Float": + return V.f64(target, toFloat(v.values, from.scale), v.valid) + case "Int": { + if (target.bitWidth === 64) { + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) { + const x = integerPart(v.values[i], from.scale) + if (V.isValid(v, i) && (target.signed ? inRange(x) : x >= I0 && x <= UMAX)) { + out[i] = x + valid[i] = 1 + } + } + return V.i64(target, out, valid) + } + const [lo, hi] = V.intRange(target) + const out = new Float64Array(n) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const x = integerPart(v.values[i], from.scale) + if (x >= lo && x <= hi) { + out[i] = Number(x) + valid[i] = 1 + } + } + return V.kindOf(target) === "f64" ? V.f64(target, out, valid) : V.i32(target, Int32Array.from(out), valid) + } + case "Utf8": + return V.stringsFrom(S.utf8, formatAll(v, from)) + case "Bool": + return V.bool(Uint8Array.from(v.values, (x) => (x === I0 ? 0 : 1)), v.valid) + default: + return V.fail(`cannot cast a decimal to ${target._tag}`) + } +} + +/** Decimal addition, subtraction and multiplication of unscaled values. @internal */ +export const arith = ( + op: "add" | "sub" | "mul", + a: BigInt64Array, + aScale: number, + b: BigInt64Array, + bScale: number, + scalar: boolean +): { readonly values: BigInt64Array; readonly type: S.Decimal } => { + const n = a.length + const out = new BigInt64Array(n) + if (op === "mul") { + const scale = aScale + bScale + if ((scale | 0) !== scale) V.fail(`the scale ${scale} of a decimal product is not a 32-bit integer`) + for (let i = 0; i < n; i++) out[i] = checked(a[i] * (scalar ? b[0] : b[i])) + return { values: out, type: decimal(scale) } + } + const scale = Math.max(aScale, bScale) + const fa = factor(scale - aScale) + const fb = factor(scale - bScale) + for (let i = 0; i < n; i++) { + const x = a[i] * fa + const y = (scalar ? b[0] : b[i]) * fb + out[i] = checked(op === "add" ? x + y : x - y) + } + return { values: out, type: decimal(scale) } +} + +/** + * Rounds to `places` decimal places, halves to even, keeping the scale. As + * in Polars, the precision grows by one digit when values can round up. + * + * @internal + */ +export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { + if (places >= type.scale) return v + const k = type.scale - places + const up = factor(k) + return V.i64( + S.decimal(Math.min(type.precision + 1, PRECISION), type.scale), + BigInt64Array.from(v.values, (x) => checked(divRound(x, k) * up)), + v.valid + ) +} + +/** + * The integer `x` in unscaled units of `scale`, or `undefined` when the + * scale or the 64 bits of storage can't hold it exactly. + */ +const exactly = (x: bigint, scale: number): bigint | undefined => { + if (scale >= 0) { + const unscaled = x * factor(scale) + return inRange(unscaled) ? unscaled : undefined + } + if (x === I0) return I0 + // A nonzero multiple of `10^-scale` has more digits than that. + if (-scale >= (x < I0 ? -x : x).toString().length) return undefined + const divisor = pow10(-scale) + return x % divisor === I0 && inRange(x / divisor) ? x / divisor : undefined +} + +/** + * Whether each decimal is one of `candidates`. A candidate is a value and + * not a number of unscaled units: an integer is the decimal equal to it, + * and another number is the decimals whose float it is, as in a comparison + * with a float. Candidates that aren't numbers match nothing. + * + * @internal + */ +export const isIn = (values: BigInt64Array, scale: number, candidates: ReadonlyArray): Uint8Array => { + const exact = new Set() + const floats = new Set() + for (const c of candidates) { + if (typeof c === "bigint" || (typeof c === "number" && Number.isInteger(c))) { + const unscaled = exactly(BigInt(c), scale) + if (unscaled !== undefined) exact.add(unscaled) + } else if (typeof c === "number" && Number.isFinite(c)) { + floats.add(c) + } + } + const out = new Uint8Array(values.length) + for (let i = 0; i < values.length; i++) { + out[i] = +(exact.has(values[i]) || (floats.size > 0 && floats.has(toNumber(values[i], scale)))) + } + return out +} + +/** Unscaled values as floats. @internal */ +export const toFloat = (values: BigInt64Array, scale: number): Float64Array => { + const out = new Float64Array(values.length) + for (let i = 0; i < values.length; i++) out[i] = toNumber(values[i], scale) + return out +} diff --git a/packages/effect/src/dataframe/internal/dtype.ts b/packages/effect/src/dataframe/internal/dtype.ts new file mode 100644 index 00000000000..b6f1f51cf58 --- /dev/null +++ b/packages/effect/src/dataframe/internal/dtype.ts @@ -0,0 +1,191 @@ +/** + * Result types of expressions: numeric supertypes and literal adoption, + * following Polars where the TypeScript types allow it. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Dec from "./decimal.ts" +import * as T from "./temporal.ts" + +/** + * A literal that has not committed to a width yet. Like Polars' dynamic + * literals, an integer literal adopts the integer type of the column it + * meets, and a float literal adopts Float32 next to a Float32 column. + * + * @internal + */ +export type Dynamic = "int" | "float" | undefined + +/** @internal */ +export const isNumeric = (t: S.DataType): t is S.Int | S.Float => t._tag === "Int" || t._tag === "Float" +/** @internal */ +export const isInteger = (t: S.DataType): t is S.Int => t._tag === "Int" +/** @internal */ +export const isFloat = (t: S.DataType): t is S.Float => t._tag === "Float" +/** @internal */ +export const isBinaryLike = (t: S.DataType): boolean => t._tag === "Utf8" || t._tag === "Binary" + +/** @internal */ +export const sameType = (a: S.DataType, b: S.DataType): boolean => { + if (a._tag !== b._tag) return false + switch (a._tag) { + case "Int": + return a.bitWidth === (b as S.Int).bitWidth && a.signed === (b as S.Int).signed + case "Float": + return a.bitWidth === (b as S.Float).bitWidth + case "Date": + return a.unit === (b as S.Date).unit + case "Time": + return a.unit === (b as S.Time).unit + case "Timestamp": + return a.unit === (b as S.Timestamp).unit && (a.timezone ?? null) === ((b as S.Timestamp).timezone ?? null) + case "Duration": + return a.unit === (b as S.Duration).unit + case "Decimal": + return a.precision === (b as S.Decimal).precision && a.scale === (b as S.Decimal).scale + case "List": + case "LargeList": + return sameType(a.child.type, (b as S.List).child.type) + case "Struct": { + const fields = (b as S.Struct).fields + return a.fields.length === fields.length && + a.fields.every((f, i) => f.name === fields[i].name && sameType(f.type, fields[i].type)) + } + default: + return true + } +} + +/** The default type of a literal value. @internal */ +export const literalType = (value: unknown): { readonly type: S.DataType; readonly dynamic: Dynamic } => { + switch (typeof value) { + case "number": + return Number.isInteger(value) && value >= -2147483648 && value <= 2147483647 && !Object.is(value, -0) + ? { type: S.int32, dynamic: "int" } + : { type: S.float64, dynamic: "float" } + case "bigint": + return { type: S.int64, dynamic: undefined } + case "string": + return { type: S.utf8, dynamic: undefined } + case "boolean": + return { type: S.bool, dynamic: undefined } + default: + return { type: S.nullType, dynamic: undefined } + } +} + +const intSupertype = (a: S.Int, b: S.Int): S.DataType => { + if (a.signed === b.signed) return a.bitWidth >= b.bitWidth ? a : b + const [s, u] = a.signed ? [a, b] : [b, a] + if (s.bitWidth > u.bitWidth) return s + // Polars widens to the next signed width; beyond 32 bits that is a + // bigint, which the number-typed API can't express, so use Float64. + return u.bitWidth === 8 ? S.int16 : u.bitWidth === 16 ? S.int32 : u.bitWidth === 32 ? S.float64 : S.int64 +} + +/** + * The common type of two operands. Returns `undefined` when they have none. + * + * @internal + */ +export const supertype = ( + a: S.DataType, + ad: Dynamic, + b: S.DataType, + bd: Dynamic +): { readonly type: S.DataType; readonly dynamic: Dynamic } | undefined => { + if (a._tag === "Null") return { type: b, dynamic: bd } + if (b._tag === "Null") return { type: a, dynamic: ad } + if (Dec.isDecimal(a) || Dec.isDecimal(b)) { + const t = Dec.supertype(a, b) + return t === undefined ? undefined : { type: t, dynamic: undefined } + } + if (isNumeric(a) && isNumeric(b)) { + if (ad !== undefined && bd !== undefined) { + return ad === "float" || bd === "float" + ? { type: S.float64, dynamic: "float" } + : { type: S.int32, dynamic: "int" } + } + if (ad !== undefined || bd !== undefined) { + const [lit, col] = ad !== undefined ? [ad, b] : [bd, a] + if (lit === "int") return { type: col, dynamic: undefined } + return { type: col._tag === "Float" ? col : S.float64, dynamic: undefined } + } + if (a._tag === "Int" && b._tag === "Int") return { type: intSupertype(a, b), dynamic: undefined } + const f32 = (t: S.DataType) => t._tag === "Float" ? t.bitWidth === 32 : (t as S.Int).bitWidth <= 16 + return { type: f32(a) && f32(b) ? S.float32 : S.float64, dynamic: undefined } + } + if (sameType(a, b)) return { type: a, dynamic: undefined } + const t = T.supertype(a, b) + return t === undefined ? undefined : { type: t, dynamic: undefined } +} + +/** Type of a true division. @internal */ +export const divisionType = (t: S.DataType): S.DataType => + t._tag === "Float" && t.bitWidth === 32 ? S.float32 : S.float64 + +/** Type of `sum` over a column of type `t`. @internal */ +export const sumType = (t: S.DataType): S.DataType => { + if (t._tag === "Bool") return S.uint32 + if (t._tag === "Decimal") return Dec.decimal(t.scale) + if (t._tag === "Int" && t.bitWidth < 32) return S.int32 + return t +} + +/** + * Whether values of type `t` are points in time, which have neither a sum + * nor a mean: dates, times of day and timestamps. Durations add up. + * + * @internal + */ +export const isPointInTime = (t: S.DataType): boolean => + t._tag === "Date" || t._tag === "Time" || t._tag === "Timestamp" + +/** Type of `mean`, `median`, `std` and `var` over a column of type `t`. @internal */ +export const meanType = divisionType + +/** @internal */ +export const describe = (t: S.DataType): string => { + switch (t._tag) { + case "Int": + return `${t.signed ? "Int" : "UInt"}${t.bitWidth}` + case "Float": + return `Float${t.bitWidth}` + case "Timestamp": + return `Timestamp[${unitName[t.unit]}${t.timezone === undefined ? "" : `, ${t.timezone}`}]` + case "Duration": + return `Duration[${unitName[t.unit]}]` + case "Decimal": + return `Decimal[${t.precision}, ${t.scale}]` + default: + return t._tag + } +} + +const unitName: Record = { second: "s", millisecond: "ms", microsecond: "us", nanosecond: "ns" } + +/** Names accepted by `Expr.cast`. @internal */ +export const castTargets: Record = { + Int8: S.int8, + Int16: S.int16, + Int32: S.int32, + Int64: S.int64, + UInt8: S.uint8, + UInt16: S.uint16, + UInt32: S.uint32, + UInt64: S.uint64, + Float32: S.float32, + Float64: S.float64, + Utf8: S.utf8, + Bool: S.bool, + Date: S.date("day"), + "Timestamp[s]": S.timestamp("second"), + "Timestamp[ms]": S.timestamp("millisecond"), + "Timestamp[us]": S.timestamp("microsecond"), + "Timestamp[ns]": S.timestamp("nanosecond"), + "Duration[s]": S.duration("second"), + "Duration[ms]": S.duration("millisecond"), + "Duration[us]": S.duration("microsecond"), + "Duration[ns]": S.duration("nanosecond") +} diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts new file mode 100644 index 00000000000..041adf5219a --- /dev/null +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -0,0 +1,569 @@ +/** + * Expression trees and their evaluation over a frame. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Layout from "../../arrow/internal/layout.ts" +import * as BigDecimal from "../../BigDecimal.ts" +import * as DateTime from "../../DateTime.ts" +import * as Duration from "../../Duration.ts" +import { pipeArguments } from "../../Pipeable.ts" +import type * as Plan from "../Plan.ts" +import * as Agg from "./aggregate.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as Hash from "./hash.ts" +import * as K from "./kernels.ts" +import * as Nested from "./nested.ts" +import * as T from "./temporal.ts" +import * as V from "./vector.ts" +import * as W from "./window.ts" + +/** @internal */ +export const ExprTypeId = "~effect/dataframe/Expr" +/** @internal */ +export const AggregateTypeId = "~effect/dataframe/Expr/Aggregate" + +/** @internal */ +export type Node = Plan.Expression + +/** @internal */ +export interface Handle { + readonly node: Node +} + +const ExprProto = { + [ExprTypeId]: { _A: (_: never) => _ }, + pipe() { + return pipeArguments(this, arguments) + }, + toJSON(this: Handle) { + return { _id: "Expr", node: this.node } + }, + toString(this: Handle) { + return show(this.node) + } +} + +const AggregateProto = { + ...ExprProto, + [AggregateTypeId]: { _A: (_: never) => _ }, + toJSON(this: Handle) { + return { _id: "Aggregate", node: this.node } + } +} + +/** @internal */ +export const makeExpr = (node: Node): E => { + const self = Object.create(ExprProto) + self.node = node + return self +} + +/** @internal */ +export const makeAggregate = (node: Node): E => { + const self = Object.create(AggregateProto) + self.node = node + return self +} + +/** @internal */ +export const isExpr = (u: unknown): u is Handle => typeof u === "object" && u !== null && ExprTypeId in u + +/** @internal */ +export const isAggregate = (u: unknown): u is Handle => typeof u === "object" && u !== null && AggregateTypeId in u + +/** @internal */ +export const toNode = (u: unknown): Node => isExpr(u) ? u.node : literalNode(u) + +/** + * A plan literal for a JS value: `DateTime`s become microsecond timestamps, + * `Duration`s microsecond durations and `BigDecimal`s decimals. + * + * @internal + */ +export const literalNode = (u: unknown): Plan.Literal => { + if (DateTime.isDateTime(u)) { + return { + _tag: "Literal", + value: BigInt(DateTime.toEpochMillis(u)) * BigInt(1000), + type: { _tag: "Timestamp", unit: "microsecond" } + } + } + if (Duration.isDuration(u)) { + const nanos = Duration.toNanos(u) + if (nanos._tag === "None") return V.fail("infinite durations cannot be literals") + const micros = nanos.value / BigInt(1000) + return { _tag: "Literal", value: micros, type: { _tag: "Duration", unit: "microsecond" } } + } + if (BigDecimal.isBigDecimal(u)) { + const digits = (u.value < BigInt(0) ? -u.value : u.value).toString().length + return { + _tag: "Literal", + value: u.value, + // The larger of the digits and the scale, as SQL has it, capped at the 38 digits of a 128-bit decimal. + type: { _tag: "Decimal", precision: Math.min(Math.max(digits, u.scale, 1), 38), scale: u.scale } + } + } + return { _tag: "Literal", value: u as Plan.Value } +} + +/** @internal */ +export const call = ( + fn: Plan.FunctionName, + args: ReadonlyArray, + options?: { readonly [key: string]: unknown } +): E => makeExpr({ _tag: "Call", fn, args: args.map(toNode), options }) + +/** + * `isIn`: plain candidates are the `values` of the call. A `BigDecimal`, + * `DateTime` or `Duration` has a literal type, which a plan value can't + * carry, so it is compared with `eq`. The tests are joined in halves, so + * the expression is as deep as the logarithm of their number. + * + * @internal + */ +export const isIn = (self: unknown, values: ReadonlyArray): E => { + const literals = values.map(literalNode) + const typed = literals.filter((literal) => literal.type !== undefined) + if (typed.length === 0) return call("isIn", [self], { values }) + const node = toNode(self) + const test = (fn: Plan.FunctionName, args: ReadonlyArray, options?: Plan.CallOptions): Node => ({ + _tag: "Call", + fn, + args, + options + }) + const plain = literals.filter((literal) => literal.type === undefined).map((literal) => literal.value) + const tests = typed.map((literal) => test("eq", [node, literal])) + if (plain.length > 0) tests.unshift(test("isIn", [node], { values: plain })) + const anyOf = (tests: ReadonlyArray): Node => { + if (tests.length === 1) return tests[0] + const half = tests.length >> 1 + return test("or", [anyOf(tests.slice(0, half)), anyOf(tests.slice(half))]) + } + return makeExpr(anyOf(tests)) +} + +/** Readable form of an expression, used by `toString`. @internal */ +export const show = (node: Node): string => { + switch (node._tag) { + case "Column": + return `col(${JSON.stringify(node.name)})` + case "Literal": { + const value = typeof node.value === "bigint" ? `${node.value}n` : JSON.stringify(node.value) ?? "undefined" + return node.type === undefined ? value : `${node.type._tag}(${value})` + } + case "Call": + return `${node.fn}(${node.args.map(show).join(", ")}${ + node.options === undefined + ? "" + : `, ${JSON.stringify(node.options, (_, v) => typeof v === "bigint" ? `${v}n` : v)}` + })` + case "AggregateCall": + return `${node.fn}(${node.arg === undefined ? "" : show(node.arg)})` + case "Over": + return `over(${show(node.expression)}, [${node.partitionBy.map(show).join(", ")}])` + } +} + +// ----------------------------------------------------------------------------- +// Evaluation +// ----------------------------------------------------------------------------- + +/** What an expression is evaluated against. @internal */ +export interface Context { + readonly length: number + readonly column: (name: string) => V.Vector + /** + * The partitions of an enclosing `Over` or grouped aggregate: window + * functions run per partition, and aggregates broadcast over it. + */ + readonly partition?: W.Partition | undefined +} + +const literalOfType = (value: unknown, type: Plan.LiteralType): K.Value => { + const big = typeof value === "bigint" ? value : BigInt(value as number) + switch (type._tag) { + case "Date": + return K.scalar(V.i32(S.date("day"), Int32Array.of(Number(big)))) + case "Timestamp": + return K.scalar(V.i64(S.timestamp(type.unit, type.timezone), BigInt64Array.of(big))) + case "Duration": + return K.scalar(V.i64(S.duration(type.unit), BigInt64Array.of(big))) + case "Decimal": { + const decimal = S.decimal(type.precision, type.scale) + Layout.checkDecimal(decimal, (message) => V.fail(`decimal literal: ${message}`)) + return K.scalar(V.i64(decimal, BigInt64Array.of(Dec.literal(big, decimal)))) + } + } +} + +const literal = (value: unknown): K.Value => { + const { dynamic, type } = D.literalType(value) + switch (typeof value) { + case "number": + return K.scalar( + type._tag === "Int" ? V.i32(type, Int32Array.of(value)) : V.f64(type, Float64Array.of(value)), + dynamic + ) + case "bigint": + return K.scalar(V.i64(type, BigInt64Array.of(BigInt.asIntN(64, value)))) + case "string": + return K.scalar(V.stringsFrom(S.utf8, [value])) + case "boolean": + return K.scalar(V.bool(Uint8Array.of(value ? 1 : 0))) + default: + return K.scalar(V.nulls(1)) + } +} + +/** @internal */ +export const evaluate = (node: Node, ctx: Context): K.Value => { + switch (node._tag) { + case "Column": + return K.column(ctx.column(node.name)) + case "Literal": + return node.type === undefined || node.value === null ? literal(node.value) : literalOfType(node.value, node.type) + case "AggregateCall": { + // Inside `over` or an aggregate: the partition's aggregate on each of its rows. + if (ctx.partition === undefined) return V.fail(`aggregate ${show(node)} can only be used in agg or over`) + // `count()` without an argument counts rows; `Agg.aggregate` fails for other aggregates without one. + const fn = node.fn === "count" && node.arg === undefined ? "len" : node.fn + const values = Agg.aggregate(fn, node.arg && evaluateColumn(node.arg, ctx), ctx.partition) + return K.column(V.take(values, ctx.partition.ids)) + } + case "Call": + return evaluateCall(node, ctx) + case "Over": { + const keys = node.partitionBy.map((k) => evaluateColumn(k, ctx)) + if (ctx.partition !== undefined) keys.unshift(V.i32(S.int32, ctx.partition.ids)) + const g = Hash.groups(keys, ctx.length) + return evaluate(node.expression, { ...ctx, partition: { ids: g.ids, count: g.count } }) + } + } +} + +const arity: { readonly [F in Plan.FunctionName]: readonly [number, number] } = { + add: [2, 2], + subtract: [2, 2], + multiply: [2, 2], + divide: [2, 2], + floorDivide: [2, 2], + mod: [2, 2], + negate: [1, 1], + abs: [1, 1], + sqrt: [1, 1], + round: [1, 1], + eq: [2, 2], + neq: [2, 2], + lt: [2, 2], + lte: [2, 2], + gt: [2, 2], + gte: [2, 2], + between: [3, 3], + isIn: [1, 1], + and: [2, 2], + or: [2, 2], + not: [1, 1], + isNull: [1, 1], + isNotNull: [1, 1], + isNaN: [1, 1], + fillNull: [2, 2], + coalesce: [1, Infinity], + when: [3, 3], + cast: [1, 1], + length: [1, 1], + toLowerCase: [1, 1], + toUpperCase: [1, 1], + includes: [1, 1], + startsWith: [1, 1], + endsWith: [1, 1], + concat: [1, Infinity], + cumSum: [1, 1], + cumMin: [1, 1], + cumMax: [1, 1], + cumCount: [1, 1], + shift: [1, 1], + diff: [1, 1], + rank: [1, 1], + rollingSum: [1, 1], + rollingMean: [1, 1], + rollingMin: [1, 1], + rollingMax: [1, 1], + year: [1, 1], + month: [1, 1], + day: [1, 1], + hour: [1, 1], + minute: [1, 1], + second: [1, 1], + millisecond: [1, 1], + weekday: [1, 1], + ordinalDay: [1, 1], + truncate: [1, 1], + totalDays: [1, 1], + totalHours: [1, 1], + totalMinutes: [1, 1], + totalSeconds: [1, 1], + totalMilliseconds: [1, 1], + listLength: [1, 1], + listGet: [1, 1], + listSum: [1, 1], + listMean: [1, 1], + listMin: [1, 1], + listMax: [1, 1], + listContains: [2, 2], + field: [1, 1], + struct: [1, Infinity] +} + +const windowFunctions: ReadonlySet = new Set([ + "cumSum", + "cumMin", + "cumMax", + "cumCount", + "shift", + "diff", + "rank", + "rollingSum", + "rollingMean", + "rollingMin", + "rollingMax" +]) + +/** + * Whether an expression's value on a row depends only on that row: no + * window functions, `over` or aggregates. Only such expressions commute + * with filters and slices. + * + * @internal + */ +export const isRowWise = (node: Node): boolean => { + switch (node._tag) { + case "Column": + case "Literal": + return true + case "Call": + return !windowFunctions.has(node.fn) && node.args.every(isRowWise) + case "AggregateCall": + case "Over": + return false + } +} + +const arithmetic = { + add: "add", + subtract: "sub", + multiply: "mul", + divide: "div", + floorDivide: "floorDiv", + mod: "mod" +} as const + +const listAggregates = { listSum: "sum", listMean: "mean", listMin: "min", listMax: "max" } as const + +const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { + const n = ctx.length + const bounds = arity[node.fn] + if (bounds === undefined) return V.fail(`unknown function ${String(node.fn)}`) + if (node.args.length < bounds[0] || node.args.length > bounds[1]) { + return V.fail( + `${node.fn} takes ${ + bounds[0] === bounds[1] + ? bounds[0] + : bounds[1] === Infinity + ? `at least ${bounds[0]}` + : `${bounds[0]}–${bounds[1]}` + } argument(s), got ${node.args.length}` + ) + } + const arg = (i: number) => evaluate(node.args[i], ctx) + const col = (i: number) => K.materialize(arg(i), n) + const args = () => node.args.map((a) => evaluate(a, ctx)) + const options = node.options ?? {} + const option = (name: keyof Plan.CallOptions): A => { + const value = options[name] + return value === undefined ? V.fail(`${node.fn} needs the ${name} option`) : value as A + } + switch (node.fn) { + case "add": + case "subtract": + case "multiply": + case "divide": + case "floorDivide": + case "mod": + return K.arith(arithmetic[node.fn], arg(0), arg(1), n) + case "negate": + return K.unaryNumeric("neg", arg(0)) + case "abs": + case "sqrt": + return K.unaryNumeric(node.fn, arg(0)) + case "round": + return K.unaryNumeric("round", arg(0), options.decimals ?? 0) + case "eq": + case "neq": + case "lt": + case "lte": + case "gt": + case "gte": + return K.compare(node.fn, arg(0), arg(1), n) + case "and": + case "or": + return K.logical(node.fn, arg(0), arg(1), n) + case "not": + return K.not(arg(0)) + case "isNull": + return K.isNull(arg(0), false) + case "isNotNull": + return K.isNull(arg(0), true) + case "isNaN": + return K.isNaN_(arg(0)) + case "fillNull": + case "coalesce": + return K.coalesce(args(), n) + case "when": + return K.when(arg(0), arg(1), arg(2), n) + case "cast": { + const to = option("to") + if (to === "Decimal") { + const type = S.decimal(option("precision"), option("scale")) + Layout.checkDecimal(type, (message) => V.fail(`cast: ${message}`)) + return K.castValue(arg(0), type) + } + const target = D.castTargets[to] + if (target === undefined) return V.fail(`unknown cast target ${to}`) + return K.castValue(arg(0), target) + } + case "isIn": + return K.isIn(arg(0), option>("values")) + case "between": { + const x = arg(0) + return K.logical("and", K.compare("gte", x, arg(1), n), K.compare("lte", x, arg(2), n), n) + } + case "length": + return K.strLength(arg(0)) + case "toLowerCase": + return K.strMap(arg(0), (s) => s.toLowerCase()) + case "toUpperCase": + return K.strMap(arg(0), (s) => s.toUpperCase()) + case "includes": + case "startsWith": + case "endsWith": + return K.strMatch(arg(0), node.fn, option("pattern")) + case "concat": + return K.strConcat(args(), n) + case "year": + case "month": + case "day": + case "hour": + case "minute": + case "second": + case "millisecond": + case "weekday": + case "ordinalDay": + return K.column(T.field(node.fn, col(0))) + case "truncate": + return K.column(T.truncate(col(0), option("every"))) + case "listLength": + return K.column(Nested.length(col(0))) + case "listGet": + return K.column(Nested.get(col(0), option("index"))) + case "listSum": + case "listMean": + case "listMin": + case "listMax": + return K.column(Nested.aggregate(listAggregates[node.fn], col(0))) + case "listContains": + return K.column(Nested.contains(col(0), arg(1))) + case "field": + return K.column(Nested.field(col(0), option("name"))) + case "struct": { + const names = option>("names") + if (names.length !== node.args.length) return V.fail("struct needs one name per argument") + return K.column(Nested.struct(names, node.args.map((_, i) => col(i)), n)) + } + case "totalDays": + case "totalHours": + case "totalMinutes": + case "totalSeconds": + case "totalMilliseconds": + return K.column(T.total(node.fn, col(0))) + case "cumSum": + case "cumMin": + case "cumMax": + case "cumCount": + case "shift": + case "diff": + case "rank": + case "rollingSum": + case "rollingMean": + case "rollingMin": + case "rollingMax": { + const x = col(0) + const p = ctx.partition ?? W.whole(n) + switch (node.fn) { + case "cumSum": + return K.column(W.cumSum(x, p)) + case "cumMin": + case "cumMax": + return K.column(W.cumMinMax(node.fn, x, p)) + case "cumCount": + return K.column(W.cumCount(x, p)) + case "shift": + return K.column(W.shift(x, p, options.n ?? 1)) + case "diff": + return K.arith("sub", K.column(x), K.column(W.shift(x, p, options.n ?? 1)), n) + case "rank": + return K.column(W.rank(x, p, options.method ?? "average", options.descending ?? false)) + default: { + const window = option("window") + return K.column(W.rolling(node.fn, x, p, window, options.minPeriods ?? window)) + } + } + } + } +} + +/** Column names an expression reads. @internal */ +export const columns = (node: Node, out: Set = new Set()): Set => { + switch (node._tag) { + case "Column": + out.add(node.name) + break + case "Call": + for (const a of node.args) columns(a, out) + break + case "AggregateCall": + if (node.arg !== undefined) columns(node.arg, out) + break + case "Over": + columns(node.expression, out) + for (const k of node.partitionBy) columns(k, out) + break + } + return out +} + +/** Replaces column references. @internal */ +export const mapColumns = (node: Node, f: (name: string) => Node): Node => { + switch (node._tag) { + case "Column": + return f(node.name) + case "Literal": + return node + case "Call": + return { ...node, args: node.args.map((a) => mapColumns(a, f)) } + case "AggregateCall": + return node.arg === undefined ? node : { ...node, arg: mapColumns(node.arg, f) } + case "Over": + return { + ...node, + expression: mapColumns(node.expression, f), + partitionBy: node.partitionBy.map((k) => mapColumns(k, f)) + } + } +} + +/** Evaluates to a column of the context's length. @internal */ +export const evaluateColumn = (node: Node, ctx: Context): V.Vector => K.materialize(evaluate(node, ctx), ctx.length) diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts new file mode 100644 index 00000000000..8c8bd72ed45 --- /dev/null +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -0,0 +1,758 @@ +/** + * The DataFrame representation and its operations. + * + * A frame is a list of named columns of equal length. Each column is a + * physical vector plus an optional *selection*: the row indices of the + * vector that make up the column. `filter`, `sort`, `take` and joins only + * compose selections; the values are gathered the first time a column is + * read, and the gathered vector is cached on the column. Columns sharing a + * selection keep sharing it, so composing it happens once per operation. + * + * @internal + */ +import * as ArrowColumn from "../../arrow/ArrowColumn.ts" +import * as S from "../../arrow/ArrowSchema.ts" +import * as ArrowTable from "../../arrow/ArrowTable.ts" +import * as Equal from "../../Equal.ts" +import { format } from "../../Formatter.ts" +import * as Hash_ from "../../Hash.ts" +import { NodeInspectSymbol, toJson } from "../../Inspectable.ts" +import { pipeArguments } from "../../Pipeable.ts" +import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" +import * as E from "./expr.ts" +import * as Hash from "./hash.ts" +import * as K from "./kernels.ts" +import * as Sort from "./sort.ts" +import * as V from "./vector.ts" + +/** @internal */ +export const TypeId = "~effect/dataframe/DataFrame" + +/** @internal */ +export class Column { + readonly name: string + readonly vector: V.Vector + readonly selection: Int32Array | undefined + /** Whether the column may hold nulls (the Arrow field's `nullable`). */ + readonly nullable: boolean + private gathered: V.Vector | undefined + constructor(name: string, vector: V.Vector, selection: Int32Array | undefined, nullable?: boolean) { + this.name = name + this.vector = vector + this.selection = selection + this.nullable = nullable ?? (vector.kind === "null" || vector.valid !== undefined) + } + get type(): S.DataType { + return this.vector.type + } + /** The column's values, gathered through its selection (cached). */ + values(): V.Vector { + if (this.selection === undefined) return this.vector + if (this.gathered === undefined) this.gathered = V.take(this.vector, this.selection) + return this.gathered + } + rename(name: string): Column { + const c = new Column(name, this.vector, this.selection, this.nullable) + c.gathered = this.gathered + return c + } +} + +/** @internal */ +export interface Frame { + readonly [TypeId]: unknown + readonly columns: ReadonlyArray + readonly length: number + readonly schema: S.Schema +} + +const Proto = { + [TypeId]: { _Row: (_: never) => _ }, + pipe() { + return pipeArguments(this, arguments) + }, + [Symbol.iterator](this: Frame) { + return rows(this)[Symbol.iterator]() + }, + toJSON(this: Frame) { + return { + _id: "DataFrame", + schema: this.columns.map((c) => ({ name: c.name, type: D.describe(c.type) })), + length: this.length, + rows: toJson(toRows(head(this, 10))) + } + }, + toString(this: Frame) { + return render(this) + }, + [NodeInspectSymbol](this: Frame) { + return render(this) + }, + [Equal.symbol](this: Frame, that: unknown): boolean { + return isFrame(that) && equals(this, that) + }, + [Hash_.symbol](this: Frame): number { + let h = Hash_.number(this.length) + for (const c of this.columns) h = Hash_.combine(h)(Hash_.string(c.name)) + return h + } +} + +/** @internal */ +export const isFrame = (u: unknown): u is Frame => typeof u === "object" && u !== null && TypeId in u + +/** @internal */ +export const make = (columns: ReadonlyArray, length: number): Frame => { + const self = Object.create(Proto) + self.columns = columns + self.length = length + let schema: S.Schema | undefined + Object.defineProperty(self, "schema", { + enumerable: true, + get: () => schema ??= S.make(columns.map((c) => S.field(c.name, c.type, { nullable: c.nullable }))) + }) + return self +} + +/** + * A frame without rows with the given columns. Its vectors have no validity + * bitmap, which would make a column nullable where it meets others. + * + * @internal + */ +export const empty = ( + fields: ReadonlyArray<{ readonly name: string; readonly type: S.DataType; readonly nullable: boolean }> +): Frame => + make( + fields.map((f) => { + const vector = V.withValid(K.nullsOf(f.type, 0), undefined) + return new Column(f.name, vector, undefined, f.nullable || vector.kind === "null") + }), + 0 + ) + +/** @internal */ +export const fromVectors = (names: ReadonlyArray, vectors: ReadonlyArray, length?: number): Frame => { + const n = length ?? vectors[0]?.length ?? 0 + return make(names.map((name, i) => new Column(name, vectors[i], undefined)), n) +} + +/** @internal */ +export const column = (self: Frame, name: string): Column => + self.columns.find((c) => c.name === name) ?? V.fail(`unknown column ${JSON.stringify(name)}`) + +/** @internal */ +export const context = (self: Frame): E.Context => ({ + length: self.length, + column: (name) => column(self, name).values() +}) + +// ----------------------------------------------------------------------------- +// Arrow interop and rows +// ----------------------------------------------------------------------------- + +/** @internal */ +export const fromArrowTable = (table: ArrowTable.Table): Frame => { + if (table.batches.length === 0) { + return empty(table.schema.fields.map((f) => ({ name: f.name, type: V.storageType(f.type), nullable: f.nullable }))) + } + const length = table.batches.reduce((s, b) => s + b.length, 0) + const vectors = table.schema.fields.map((_, c) => V.concat(table.batches.map((b) => V.fromArrowData(b.columns[c])))) + return make( + table.schema.fields.map((f, c) => + new Column( + f.name, + vectors[c], + undefined, + f.nullable || vectors[c].kind === "null" || vectors[c].valid !== undefined + ) + ), + length + ) +} + +/** @internal */ +export const toArrowTable = (self: Frame): ArrowTable.Table => { + const schema = self.schema + const data = self.columns.map((c) => V.toArrowData(c.values())) + return ArrowTable.make(schema, [ArrowTable.recordBatch(schema, self.length, data)]) +} + +const rows = (self: Frame): Iterable => ArrowTable.rows(toArrowTable(self)) + +/** @internal */ +export const toRows = (self: Frame): Array => ArrowTable.toRows(toArrowTable(self)) + +/** One column as an Arrow column of one chunk, over the frame's buffers where the layouts agree. @internal */ +export const toArrowColumn = (self: Frame, name: string): ArrowColumn.ArrowColumn => { + const c = column(self, name) + const vector = c.values() + return ArrowColumn.make(S.field(name, vector.type, { nullable: c.nullable }), [V.toArrowData(vector)]) +} + +/** @internal */ +export const toArray = (self: Frame, name: string): Array => ArrowColumn.toArray(toArrowColumn(self, name)) + +// ----------------------------------------------------------------------------- +// Row selection +// ----------------------------------------------------------------------------- + +/** Re-selects every column through `indices` (positions in the current frame; -1 = null). */ +const take = (self: Frame, indices: Int32Array): Frame => { + let missing = false + for (let i = 0; i < indices.length; i++) { + if (indices[i] < 0) { + missing = true + break + } + } + const composed = new Map() + const columns = self.columns.map((c) => { + let sel = composed.get(c.selection) + if (sel === undefined) { + if (c.selection === undefined) sel = indices + else { + const base = c.selection + sel = new Int32Array(indices.length) + for (let i = 0; i < indices.length; i++) { + const j = indices[i] + sel[i] = j < 0 ? -1 : base[j] + } + } + composed.set(c.selection, sel) + } + return new Column(c.name, c.vector, sel, c.nullable || missing) + }) + return make(columns, indices.length) +} + +const maskIndices = (mask: V.Bool): Int32Array => { + const n = mask.length + const m = mask.values + const v = mask.valid + const out = new Int32Array(n) + let k = 0 + if (v === undefined) { + for (let i = 0; i < n; i++) { + out[k] = i + k += m[i] + } + } else { + for (let i = 0; i < n; i++) { + out[k] = i + k += m[i] & v[i] + } + } + return out.subarray(0, k) +} + +/** @internal */ +export const filter = (self: Frame, predicate: E.Node): Frame => { + const value = E.evaluate(predicate, context(self)) + const mask = K.materialize(value, self.length) + if (mask.kind === "null") return take(self, new Int32Array(0)) + if (mask.kind !== "bool") return V.fail(`filter predicate must be boolean, got ${D.describe(mask.type)}`) + const indices = maskIndices(mask) + return indices.length === self.length ? self : take(self, indices) +} + +/** @internal */ +export const slice = (self: Frame, start: number, end: number): Frame => { + const n = self.length + const s = Math.max(0, Math.min(n, start)) + const e = Math.max(s, Math.min(n, end)) + if (s === 0 && e === n) return self + return make( + self.columns.map((c) => + c.selection === undefined + ? new Column(c.name, V.slice(c.vector, s, e), undefined, c.nullable) + : new Column(c.name, c.vector, c.selection.subarray(s, e), c.nullable) + ), + e - s + ) +} + +const head = (self: Frame, n: number): Frame => slice(self, 0, n) + +const tail = (self: Frame, n: number): Frame => slice(self, Math.max(0, self.length - n), self.length) + +/** @internal */ +export const sort = ( + self: Frame, + keys: ReadonlyArray<{ readonly node: E.Node; readonly descending: boolean }>, + nullsLast: boolean, + limit?: number +): Frame => { + const ctx = context(self) + const sortKeys = keys.map((k) => ({ vector: E.evaluateColumn(k.node, ctx), descending: k.descending })) + return take( + self, + limit === undefined + ? Sort.permutation(sortKeys, nullsLast, self.length) + : Sort.topK(sortKeys, nullsLast, self.length, limit) + ) +} + +// ----------------------------------------------------------------------------- +// Projection +// ----------------------------------------------------------------------------- + +/** @internal */ +export const pick = (self: Frame, names: ReadonlyArray): Frame => + make(names.map((n) => column(self, n)), self.length) + +/** @internal */ +export const omit = (self: Frame, names: ReadonlyArray): Frame => { + const drop = new Set(names) + return make(self.columns.filter((c) => !drop.has(c.name)), self.length) +} + +/** @internal */ +export const rename = (self: Frame, mapping: { readonly [name: string]: string | undefined }): Frame => + make( + self.columns.map((c) => { + const to = mapping[c.name] + return to === undefined ? c : c.rename(to) + }), + self.length + ) + +const evaluateAll = (self: Frame, exprs: { readonly [name: string]: E.Node }): Array => { + const ctx = context(self) + return Object.keys(exprs).map((name) => new Column(name, E.evaluateColumn(exprs[name], ctx), undefined)) +} + +/** @internal */ +export const select = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => + // A frame of only scalars (e.g. `select(() => ({ one: Expr.lit(1) }))`) keeps the frame's length. + make(evaluateAll(self, exprs), self.length) + +/** @internal */ +export const withColumns = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => { + const added = evaluateAll(self, exprs) + const byName = new Map(added.map((c) => [c.name, c])) + const columns: Array = self.columns.map((c) => { + const replacement = byName.get(c.name) + if (replacement !== undefined) byName.delete(c.name) + return replacement ?? c + }) + for (const c of added) if (byName.has(c.name)) columns.push(c) + return make(columns, self.length) +} + +// ----------------------------------------------------------------------------- +// Aggregation +// ----------------------------------------------------------------------------- + +const aggregateColumns = ( + self: Frame, + aggs: { readonly [name: string]: E.Node }, + grouping: Agg.Grouping +): Array => { + const ctx: E.Context = { ...context(self), partition: grouping } + return Object.keys(aggs).map((name) => { + const node = aggs[name] + if (node._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) + const fn = Agg.kernel(node.fn, node.arg) + const input = node.arg === undefined ? undefined : E.evaluateColumn(node.arg, ctx) + return new Column(name, Agg.aggregate(fn, input, grouping), undefined) + }) +} + +/** @internal */ +export const groupBy = (self: Frame, keys: ReadonlyArray, aggs: { readonly [name: string]: E.Node }): Frame => { + const keyVectors = keys.map((k) => column(self, k).values()) + const groups = Hash.groups(keyVectors, self.length) + const keyColumns = keys.map((k, i) => + new Column(k, V.take(keyVectors[i], groups.firstRow), undefined, column(self, k).nullable) + ) + return make([...keyColumns, ...aggregateColumns(self, aggs, groups)], groups.count) +} + +/** @internal */ +export const aggregate = (self: Frame, aggs: { readonly [name: string]: E.Node }): Frame => + make(aggregateColumns(self, aggs, { ids: new Int32Array(self.length), count: 1 }), 1) + +// ----------------------------------------------------------------------------- +// Joins +// ----------------------------------------------------------------------------- + +/** @internal */ +export type JoinHow = "inner" | "left" | "full" | "semi" | "anti" + +const unifyKeys = (l: V.Vector, r: V.Vector): [V.Vector, V.Vector] => { + if (D.sameType(l.type, r.type)) return [l, r] + const st = D.supertype(l.type, undefined, r.type, undefined) + if (st === undefined) return V.fail(`cannot join ${D.describe(l.type)} with ${D.describe(r.type)}`) + return [K.coerce(l, st.type), K.coerce(r, st.type)] +} + +/** + * Key ids for both sides of a join; equal keys get equal ids and rows with a + * null key (or, on the left, a key the right side lacks) get -1. + */ +const joinIds = ( + pairs: ReadonlyArray<[V.Vector, V.Vector]>, + nl: number, + nr: number +): { leftIds: Int32Array; rightIds: Int32Array; count: number } => { + if (pairs.length === 1 && pairs[0][0].kind === "i32" && pairs[0][1].kind === "i32") { + // One Int32 key: index the right keys directly by value when their range is small. + const l = pairs[0][0] as V.I32 + const r = pairs[0][1] as V.I32 + const { max, min } = Hash.int32Range(r.values, r.valid, nr) + if (max < min || max - min < Hash.directLimit(nl + nr)) { + const range = max < min ? 0 : max - min + 1 + const table = new Int32Array(range).fill(-1) + const rightIds = new Int32Array(nr) + let count = 0 + for (let j = 0; j < nr; j++) { + if (r.valid !== undefined && r.valid[j] === 0) { + rightIds[j] = -1 + continue + } + const k = r.values[j] - min + let id = table[k] + if (id < 0) id = table[k] = count++ + rightIds[j] = id + } + const leftIds = new Int32Array(nl) + const lv = l.values + const la = l.valid + for (let i = 0; i < nl; i++) { + const k = lv[i] - min + leftIds[i] = (la !== undefined && la[i] === 0) || k < 0 || k >= range ? -1 : table[k] + } + return { leftIds, rightIds, count } + } + } + // General case: ids over both sides at once. + const joint = pairs.map(([l, r]) => V.concat([l, r])) + const g = Hash.groups(joint, nl + nr) + const nulls = Hash.anyNull(joint, nl + nr) + const leftIds = g.ids.slice(0, nl) + const rightIds = g.ids.slice(nl) + if (nulls !== undefined) { + for (let i = 0; i < nl; i++) if (nulls[i] === 1) leftIds[i] = -1 + for (let j = 0; j < nr; j++) if (nulls[nl + j] === 1) rightIds[j] = -1 + } + return { leftIds, rightIds, count: g.count } +} + +/** @internal */ +export const join = (self: Frame, that: Frame, on: ReadonlyArray, how: JoinHow): Frame => { + const nl = self.length + const nr = that.length + const pairs = on.map((k) => unifyKeys(column(self, k).values(), column(that, k).values())) + const { count, leftIds: ids, rightIds } = joinIds(pairs, nl, nr) + const { rows, starts: start } = Hash.rowsByGroup(rightIds, count) + + if (how === "semi" || how === "anti") { + const want = how === "semi" ? 1 : 0 + const keep = new Int32Array(nl) + let kept = 0 + for (let i = 0; i < nl; i++) { + const id = ids[i] + const has = id >= 0 && start[id + 1] > start[id] ? 1 : 0 + keep[kept] = i + kept += has === want ? 1 : 0 + } + return kept === nl ? self : take(self, keep.subarray(0, kept)) + } + + let total = 0 + const keepUnmatched = how === "inner" ? 0 : 1 + for (let i = 0; i < nl; i++) { + const id = ids[i] + const m = id < 0 ? 0 : start[id + 1] - start[id] + total += m === 0 ? keepUnmatched : m + } + let matchedRight: Uint8Array | undefined + if (how === "full") matchedRight = new Uint8Array(nr) + let li = new Int32Array(total) + let ri = new Int32Array(total) + let k = 0 + for (let i = 0; i < nl; i++) { + const id = ids[i] + const s = id < 0 ? 0 : start[id] + const e = id < 0 ? 0 : start[id + 1] + if (s === e) { + if (keepUnmatched === 1) { + li[k] = i + ri[k++] = -1 + } + continue + } + for (let t = s; t < e; t++) { + li[k] = i + ri[k++] = rows[t] + } + if (matchedRight !== undefined) { for (let t = s; t < e; t++) matchedRight[rows[t]] = 1 } + } + if (matchedRight !== undefined) { + let extra = 0 + for (let j = 0; j < nr; j++) extra += matchedRight[j] ^ 1 + if (extra > 0) { + const l2 = new Int32Array(total + extra) + const r2 = new Int32Array(total + extra) + l2.set(li) + r2.set(ri) + for (let j = 0; j < nr; j++) { + if (matchedRight[j] === 0) { + l2[k] = -1 + r2[k++] = j + } + } + li = l2 + ri = r2 + } + } + const left = take(self, li) + const keySet = new Set(on) + const right = take(make(that.columns.filter((c) => !keySet.has(c.name)), nr), ri) + let leftColumns: ReadonlyArray = left.columns + if (how === "full") { + // Coalesce the key columns: rows that only exist on the right take their keys from there. + const rightKeys = take(make(on.map((k) => column(that, k)), nr), ri) + leftColumns = left.columns.map((c) => { + if (!keySet.has(c.name)) return c + const [l, r] = unifyKeys(c.values(), column(rightKeys, c.name).values()) + return new Column(c.name, K.coalesce([K.column(l), K.column(r)], li.length).vector, undefined) + }) + } + return make([...leftColumns, ...right.columns], li.length) +} + +// ----------------------------------------------------------------------------- +// Other row operations +// ----------------------------------------------------------------------------- + +/** @internal */ +export const unique = (self: Frame, names: ReadonlyArray | undefined): Frame => { + const keys = (names ?? self.columns.map((c) => c.name)).map((n) => column(self, n).values()) + const g = Hash.groups(keys, self.length) + return g.count === self.length ? self : take(self, g.firstRow) +} + +/** @internal */ +export const dropNulls = (self: Frame, names: ReadonlyArray | undefined): Frame => { + const keys = (names ?? self.columns.map((c) => c.name)).map((n) => column(self, n).values()) + const nulls = Hash.anyNull(keys, self.length) + if (nulls === undefined) return self + const keep: Array = [] + for (let i = 0; i < self.length; i++) if (nulls[i] === 0) keep.push(i) + return take(self, Int32Array.from(keep)) +} + +/** + * `self` with the columns of `like`, each converted to its type in `like` + * (`K.coerce`: a value that type can't hold fails). + * + * @internal + */ +export const coerce = (self: Frame, like: Frame): Frame => + make( + like.columns.map((c) => { + const own = column(self, c.name) + return D.sameType(own.type, c.type) + ? own + : new Column(c.name, K.coerce(own.values(), c.type), undefined, own.nullable) + }), + self.length + ) + +/** @internal */ +export const concat = (frames: ReadonlyArray): Frame => { + const first = frames[0] + const columns = first.columns.map((c) => + new Column(c.name, K.concat(frames.map((f) => column(f, c.name).values())), undefined) + ) + return make(columns, frames.reduce((s, f) => s + f.length, 0)) +} + +// ----------------------------------------------------------------------------- +// Equality and rendering +// ----------------------------------------------------------------------------- + +/** @internal */ +export const equals = (a: Frame, b: Frame): boolean => { + if (a === b) return true + if (a.length !== b.length || a.columns.length !== b.columns.length) return false + for (let c = 0; c < a.columns.length; c++) { + const x = a.columns[c] + const y = b.columns[c] + if (x.name !== y.name || !D.sameType(x.type, y.type)) return false + const vx = x.values() + const vy = y.values() + const eq = K.compare("eq", K.column(vx), K.column(vy), a.length).vector as V.Bool + for (let i = 0; i < a.length; i++) { + const px = V.isValid(vx, i) + const py = V.isValid(vy, i) + if (px !== py || (px && eq.values[i] === 0)) return false + } + } + return true +} + +const cell = (u: unknown): string => { + if (u === null || u === undefined) return "null" + if (typeof u === "string") return JSON.stringify(u) + if (typeof u === "bigint") return `${u}` + if (typeof u === "number") return Object.is(u, -0) ? "-0" : String(u) + return format(u) +} + +/** A text table of the first and last rows. @internal */ +export const render = (self: Frame, maxRows = 10): string => { + const n = self.length + const shown = n <= maxRows ? toRows(self) : [...toRows(head(self, maxRows / 2)), ...toRows(tail(self, maxRows / 2))] + const header = self.columns.map((c) => c.name) + const types = self.columns.map((c) => D.describe(c.type)) + const body = shown.map((r) => header.map((h) => cell(r[h]))) + if (n > maxRows) body.splice(maxRows / 2, 0, header.map(() => "…")) + const widths = header.map((h, i) => Math.max(h.length, types[i].length, ...body.map((r) => r[i].length))) + const line = (cells: ReadonlyArray) => `│ ${cells.map((c, i) => c.padEnd(widths[i])).join(" │ ")} │` + const rule = (l: string, m: string, r: string) => `${l}${widths.map((w) => "─".repeat(w + 2)).join(m)}${r}` + return [ + `DataFrame (${n} ${n === 1 ? "row" : "rows"} × ${header.length} ${header.length === 1 ? "column" : "columns"})`, + rule("┌", "┬", "┐"), + line(header), + line(types), + rule("├", "┼", "┤"), + ...body.map(line), + rule("└", "┴", "┘") + ].join("\n") +} + +// ----------------------------------------------------------------------------- +// Reshaping +// ----------------------------------------------------------------------------- + +/** @internal */ +export const pivot = ( + self: Frame, + options: { + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: Agg.AggFn + readonly columns?: ReadonlyArray | undefined + } +): Frame => { + const n = self.length + const indexVectors = options.index.map((k) => column(self, k).values()) + const rows = Hash.groups(indexVectors, n) + const on = column(self, options.on).values() + const cols = Hash.groups([on], n) + const cells = Hash.groups([V.i32(S.int32, rows.ids), V.i32(S.int32, cols.ids)], n) + const input = column(self, options.values).values() + const fn = options.aggregate + const values = Agg.aggregate(fn, fn === "len" ? undefined : input, { ids: cells.ids, count: cells.count }) + // The aggregate of no rows, for cells without rows, at index `cells.count`. + const empty = Agg.aggregate(fn, fn === "len" ? undefined : V.slice(input, 0, 0), { + ids: new Int32Array(0), + count: 1 + }) + const all = V.concat([values, K.cast(empty, values.type)]) + const cellOf = new Int32Array(rows.count * cols.count).fill(cells.count) + for (let c = 0; c < cells.count; c++) { + const r = cells.firstRow[c] + cellOf[rows.ids[r] * cols.count + cols.ids[r]] = c + } + const names = K.cast(V.take(on, cols.firstRow), S.utf8) + const nameOf = (j: number): string => V.isValid(names, j) ? V.decodeString(names as V.Str, j) : "null" + const byName = new Map() + for (let j = 0; j < cols.count; j++) if (!byName.has(nameOf(j))) byName.set(nameOf(j), j) + const outputs = options.columns ?? Array.from({ length: cols.count }, (_, j) => nameOf(j)) + const columns = outputs.map((name) => { + const j = byName.get(name) + const indices = new Int32Array(rows.count) + for (let r = 0; r < rows.count; r++) indices[r] = j === undefined ? cells.count : cellOf[r * cols.count + j] + return new Column(name, V.take(all, indices), undefined) + }) + const keys = options.index.map((k, i) => + new Column(k, V.take(indexVectors[i], rows.firstRow), undefined, column(self, k).nullable) + ) + return make([...keys, ...columns], rows.count) +} + +/** + * One row per list element, the other columns repeated. A null list gives one + * row with a null element; an empty list gives no rows (Polars). + * + * @internal + */ +export const explode = (self: Frame, name: string): Frame => { + const v = column(self, name).values() + if (v.kind !== "list") return V.fail(`explode is defined for list columns, not ${D.describe(v.type)}`) + const rows: Array = [] + const elements: Array = [] + for (let i = 0; i < v.length; i++) { + if (!V.isValid(v, i)) { + rows.push(i) + elements.push(-1) + continue + } + for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) { + rows.push(i) + elements.push(e) + } + } + const repeated = take(self, Int32Array.from(rows)) + const exploded = new Column(name, V.take(v.child, Int32Array.from(elements)), undefined) + return make(repeated.columns.map((c) => (c.name === name ? exploded : c)), rows.length) +} + +/** The fields of a struct column as columns, in its place. @internal */ +export const unnest = (self: Frame, name: string): Frame => { + const v = column(self, name).values() + if (v.kind !== "struct") return V.fail(`unnest is defined for struct columns, not ${D.describe(v.type)}`) + const fields = (v.type as S.Struct).fields + const existing = new Set(self.columns.map((c) => c.name).filter((n) => n !== name)) + const columns: Array = [] + for (const c of self.columns) { + if (c.name !== name) { + columns.push(c) + continue + } + fields.forEach((f, k) => { + if (existing.has(f.name)) V.fail(`unnest would duplicate the column ${JSON.stringify(f.name)}`) + const child = v.children[k] + const valid = V.andValid(child.kind === "null" ? undefined : child.valid, v.valid, v.length) + columns.push(new Column(f.name, V.withValid(child, valid), undefined)) + }) + } + return make(columns, self.length) +} + +/** @internal */ +export const unpivot = ( + self: Frame, + options: { + readonly index: ReadonlyArray + readonly on: ReadonlyArray + readonly variable: string + readonly value: string + } +): Frame => { + const n = self.length + const k = options.on.length + const repeat = new Int32Array(n * k) + for (let b = 0; b < k; b++) for (let i = 0; i < n; i++) repeat[b * n + i] = i + const keys = options.index.map((name) => { + const c = column(self, name) + return new Column(name, V.take(c.values(), repeat), undefined, c.nullable) + }) + const parts = options.on.map((name) => column(self, name).values()) + const type = parts.reduce((t, p) => { + const st = D.supertype(t, undefined, p.type, undefined) + return st === undefined ? V.fail(`cannot unpivot ${D.describe(t)} with ${D.describe(p.type)}`) : st.type + }, parts.length === 0 ? S.nullType : parts[0].type) + const variable = V.stringsFrom(S.utf8, options.on.flatMap((name) => Array.from({ length: n }, () => name))) + const value = k === 0 ? V.nulls(0) : V.concat(parts.map((p) => K.coerce(p, type))) + return make( + [...keys, new Column(options.variable, variable, undefined, false), new Column(options.value, value, undefined)], + n * k + ) +} diff --git a/packages/effect/src/dataframe/internal/hash.ts b/packages/effect/src/dataframe/internal/hash.ts new file mode 100644 index 00000000000..302ce2b4e1a --- /dev/null +++ b/packages/effect/src/dataframe/internal/hash.ts @@ -0,0 +1,471 @@ +/** + * Dense group ids: every distinct key gets an id in order of first + * appearance. Grouping, joins, `unique` and `countDistinct` all build on it. + * + * Keys hash with open addressing over typed arrays. Multi-column keys are + * grouped column by column and the per-column ids combined, so every column + * type needs only its own single-column table. + * + * Float keys follow Polars: `-0` and `0` are one key, and so are all NaNs. + * Null is a key of its own. + * + * @internal + */ +import * as V from "./vector.ts" + +/** @internal */ +export interface Groups { + /** Group id of every row. */ + readonly ids: Int32Array + /** Number of groups. */ + readonly count: number + /** The first row of every group, by group id. */ + readonly firstRow: Int32Array +} + +const mix = (h: number): number => { + h = Math.imul(h ^ (h >>> 16), 0x85ebca6b) + h = Math.imul(h ^ (h >>> 13), 0xc2b2ae35) + return (h ^ (h >>> 16)) >>> 0 +} + +/** Largest table the direct (array-indexed) strategy may allocate for `n` rows. @internal */ +export const directLimit = (n: number) => Math.max(1 << 16, n * 2) + +/** + * Ids for keys in `[0, range)`, by direct indexing: no hashing, one array + * read per row. + */ +const directIds = ( + x: Int32Array, + offset: number, + range: number, + valid: Uint8Array | undefined, + n: number +): Groups => { + const table = new Int32Array(range + 1).fill(-1) + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(Math.min(range + 1, Math.max(64, n))) + let count = 0 + for (let i = 0; i < n; i++) { + const k = valid !== undefined && valid[i] === 0 ? range : x[i] - offset + let id = table[k] + if (id < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + id = table[k] = count + firstRow[count++] = i + } + ids[i] = id + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** The smallest and largest valid value; `max < min` when there is none. @internal */ +export const int32Range = ( + x: Int32Array, + valid: Uint8Array | undefined, + n: number +): { readonly min: number; readonly max: number } => { + let min = 2147483647 + let max = -2147483648 + if (valid === undefined) { + for (let i = 0; i < n; i++) { + const v = x[i] + if (v < min) min = v + if (v > max) max = v + } + } else { + for (let i = 0; i < n; i++) { + if (valid[i] === 0) continue + const v = x[i] + if (v < min) min = v + if (v > max) max = v + } + } + return { min, max } +} + +/** Ids for one Int32 column. */ +const int32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + const { max, min } = int32Range(x, valid, n) + if (max >= min && max - min < directLimit(n)) return directIds(x, min, max - min + 1, valid, n) + return hashInt32Ids(x, valid, n) +} + +const hashInt32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let nullId = -1 + let cap = 1024 + let mask = cap - 1 + let keys = new Int32Array(cap) + let slots = new Int32Array(cap).fill(-1) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) { + if (nullId < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + nullId = count + firstRow[count++] = i + } + ids[i] = nullId + continue + } + const k = x[i] + let s = mix(k) & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + keys[s] = k + ids[i] = count++ + if (count * 2 > cap) { + const ok = keys + const os = slots + cap <<= 1 + mask = cap - 1 + keys = new Int32Array(cap) + slots = new Int32Array(cap).fill(-1) + for (let t = 0; t < os.length; t++) { + const id2 = os[t] + if (id2 === -1) continue + let u = mix(ok[t]) & mask + while (slots[u] !== -1) u = (u + 1) & mask + slots[u] = id2 + keys[u] = ok[t] + } + } + break + } + if (keys[s] === k) { + ids[i] = id + break + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** Ids for 64-bit keys given as (lo, hi) pairs. */ +const pairIds = (pairs: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let nullId = -1 + let cap = 1024 + let mask = cap - 1 + let keys = new Int32Array(cap * 2) + let slots = new Int32Array(cap).fill(-1) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) { + if (nullId < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + nullId = count + firstRow[count++] = i + } + ids[i] = nullId + continue + } + const lo = pairs[2 * i] + const hi = pairs[2 * i + 1] + let s = mix(lo ^ Math.imul(hi, 0x9e3779b1)) & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + keys[2 * s] = lo + keys[2 * s + 1] = hi + ids[i] = count++ + if (count * 2 > cap) { + const ok = keys + const os = slots + cap <<= 1 + mask = cap - 1 + keys = new Int32Array(cap * 2) + slots = new Int32Array(cap).fill(-1) + for (let t = 0; t < os.length; t++) { + const id2 = os[t] + if (id2 === -1) continue + const l = ok[2 * t] + const h = ok[2 * t + 1] + let u = mix(l ^ Math.imul(h, 0x9e3779b1)) & mask + while (slots[u] !== -1) u = (u + 1) & mask + slots[u] = id2 + keys[2 * u] = l + keys[2 * u + 1] = h + } + } + break + } + if (keys[2 * s] === lo && keys[2 * s + 1] === hi) { + ids[i] = id + break + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** `x` with `-0` as `0` and every NaN as one NaN, so equal keys have equal bits. @internal */ +export const canonicalBits = (x: Float64Array, n: number): Float64Array => { + const out = new Float64Array(n) + for (let i = 0; i < n; i++) { + const v = x[i] + out[i] = v === 0 ? 0 : v !== v ? NaN : v + } + return out +} + +/** Moves a table of group ids and their stored hashes into one of `cap` slots. */ +const rehash = ( + slots: Int32Array, + hashes: Int32Array, + cap: number +): [Int32Array, Int32Array] => { + const mask = cap - 1 + const s = new Int32Array(cap).fill(-1) + const h = new Int32Array(cap) + for (let t = 0; t < slots.length; t++) { + if (slots[t] === -1) continue + let u = hashes[t] & mask + while (s[u] !== -1) u = (u + 1) & mask + s[u] = slots[t] + h[u] = hashes[t] + } + return [s, h] +} + +const stringIds = (v: V.Str, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let nullId = -1 + const valid = v.valid + const d = v.data + const off = v.offsets + let cap = 1024 + let mask = cap - 1 + let slots = new Int32Array(cap).fill(-1) + let hashes = new Int32Array(cap) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) { + if (nullId < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + nullId = count + firstRow[count++] = i + } + ids[i] = nullId + continue + } + const s0 = off[i] + const e0 = off[i + 1] + let h = 0x811c9dc5 | 0 + for (let k = s0; k < e0; k++) h = Math.imul(h ^ d[k], 0x01000193) + h = mix(h ^ (e0 - s0)) + let s = h & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + hashes[s] = h + ids[i] = count++ + if (count * 2 > cap) { + cap <<= 1 + mask = cap - 1 + ;[slots, hashes] = rehash(slots, hashes, cap) + } + break + } + if (hashes[s] === (h | 0)) { + const r = firstRow[id] + const s1 = off[r] + const e1 = off[r + 1] + if (e1 - s1 === e0 - s0) { + let same = true + for (let k = 0; k < e0 - s0; k++) { + if (d[s0 + k] !== d[s1 + k]) { + same = false + break + } + } + if (same) { + ids[i] = id + break + } + } + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +const smallIds = (values: Uint8Array, valid: Uint8Array | undefined, n: number): Groups => { + const ids = new Int32Array(n) + const map = new Int32Array(3).fill(-1) + const firstRow = new Int32Array(3) + let count = 0 + for (let i = 0; i < n; i++) { + const k = valid !== undefined && valid[i] === 0 ? 2 : values[i] + let id = map[k] + if (id < 0) { + id = map[k] = count + firstRow[count++] = i + } + ids[i] = id + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +const growInt = (a: Int32Array): Int32Array => { + const out = new Int32Array(a.length * 2) + out.set(a) + return out +} + +/** Dense ids for one column. @internal */ +export const ids = (v: V.Vector): Groups => { + const n = v.length + switch (v.kind) { + case "i32": + return int32Ids(v.values, v.valid, n) + case "f64": + return pairIds(new Int32Array(canonicalBits(v.values, n).buffer), v.valid, n) + case "i64": + return pairIds(new Int32Array(v.values.buffer, v.values.byteOffset, n * 2), v.valid, n) + case "bool": + return smallIds(v.values, v.valid, n) + case "str": + return stringIds(v, n) + case "null": + return single(n) + case "list": + case "struct": + return V.fail(`cannot group, join or deduplicate by ${v.type._tag} columns yet`) + } +} + +/** Every row in one group (none without rows). */ +const single = (n: number): Groups => ({ + ids: new Int32Array(n), + count: n > 0 ? 1 : 0, + firstRow: new Int32Array(n > 0 ? 1 : 0) +}) + +/** Ids of the pairs of two id columns whose product of counts is at most `directLimit(n)`. */ +const combine = (a: Groups, b: Groups, n: number): Groups => { + if (a.count === 1) return b + if (b.count === 1) return a + const bc = b.count + const ai = a.ids + const bi = b.ids + const keys = new Int32Array(n) + for (let i = 0; i < n; i++) keys[i] = ai[i] * bc + bi[i] + return directIds(keys, 0, a.count * bc, undefined, n) +} + +/** + * Ids of tuples of per-column ids, in one hashing pass: the tuple hashes as a + * whole and compares column by column against its group's first row. + */ +const tupleIds = (cols: ReadonlyArray, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let cap = 1024 + let mask = cap - 1 + let slots = new Int32Array(cap).fill(-1) + let hashes = new Int32Array(cap) + const k = cols.length + for (let i = 0; i < n; i++) { + let h = 0x2545f491 + for (let c = 0; c < k; c++) h = Math.imul(h ^ cols[c][i], 0x9e3779b1) ^ (h >>> 15) + h = mix(h) + let s = h & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + hashes[s] = h + ids[i] = count++ + if (count * 2 > cap) { + cap <<= 1 + mask = cap - 1 + ;[slots, hashes] = rehash(slots, hashes, cap) + } + break + } + if (hashes[s] === (h | 0)) { + const r = firstRow[id] + let same = true + for (let c = 0; c < k; c++) { + if (cols[c][i] !== cols[c][r]) { + same = false + break + } + } + if (same) { + ids[i] = id + break + } + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** Dense ids for the rows of several key columns together. @internal */ +export const groups = (keys: ReadonlyArray, n: number): Groups => { + if (keys.length === 0) return single(n) + const parts = keys.map(ids) + // Combine by direct indexing while the key space stays small, then hash + // the remaining columns as one tuple. + let acc = parts[0] + let k = 1 + while (k < parts.length && acc.count * parts[k].count <= directLimit(n)) acc = combine(acc, parts[k++], n) + if (k === parts.length) return acc + return tupleIds([acc.ids, ...parts.slice(k).map((p) => p.ids)], n) +} + +/** + * The rows of every group, in row order: `rows[starts[g]..starts[g + 1])`. + * Rows with a negative id are left out. + * + * @internal + */ +export const rowsByGroup = ( + ids: Int32Array, + count: number +): { readonly starts: Int32Array; readonly rows: Int32Array } => { + const n = ids.length + const starts = new Int32Array(count + 1) + for (let i = 0; i < n; i++) if (ids[i] >= 0) starts[ids[i] + 1]++ + for (let g = 0; g < count; g++) starts[g + 1] += starts[g] + const at = starts.slice(0, count) + const rows = new Int32Array(starts[count]) + for (let i = 0; i < n; i++) if (ids[i] >= 0) rows[at[ids[i]]++] = i + return { starts, rows } +} + +/** Rows where any key column is null. @internal */ +export const anyNull = (keys: ReadonlyArray, n: number): Uint8Array | undefined => { + let out: Uint8Array | undefined + for (const k of keys) { + if (k.kind === "null") return new Uint8Array(n).fill(1) + if (k.valid === undefined) continue + if (out === undefined) out = new Uint8Array(n) + for (let i = 0; i < n; i++) out[i] |= k.valid[i] ^ 1 + } + return out +} diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts new file mode 100644 index 00000000000..663ddce5edc --- /dev/null +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -0,0 +1,1278 @@ +/** + * Element-wise kernels. Loops are specialised per physical layout so that + * typed-array access stays monomorphic. A scalar right operand is read with a + * `step` of 0 (`y[i * step]`), except in Int32 arithmetic and Int32 and + * Float64 comparisons, which keep a separate, faster scalar loop. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as T from "./temporal.ts" +import * as V from "./vector.ts" + +const I0 = BigInt(0) +const I2 = BigInt(2) +const I63 = BigInt(63) +const I64 = BigInt(64) + +/** + * An evaluated expression: a vector of the frame's length, or a scalar (a + * vector of length 1) that broadcasts. + * + * @internal + */ +export interface Value { + readonly vector: V.Vector + readonly scalar: boolean + readonly dynamic: D.Dynamic + /** + * For a dynamic integer column (a `when` or `coalesce` of literals), the + * bounds of its literals, known from the plan; Int32's when unknown. See + * {@link literalRange}. + */ + readonly range?: readonly [number, number] | undefined +} + +/** @internal */ +export const column = (vector: V.Vector): Value => ({ vector, scalar: false, dynamic: undefined }) + +/** @internal */ +export const scalar = (vector: V.Vector, dynamic: D.Dynamic = undefined): Value => ({ vector, scalar: true, dynamic }) + +const result = (vector: V.Vector, scalar: boolean, dynamic?: D.Dynamic): Value => ({ vector, scalar, dynamic }) + +/** Expands a scalar to `length` slots. @internal */ +export const materialize = (value: Value, length: number): V.Vector => + value.scalar ? V.broadcast(value.vector, length) : value.vector + +const scalarNull = (value: Value): boolean => value.scalar && !V.isValid(value.vector, 0) + +/** Validity of a binary result. */ +const resultValid = (a: Value, b: Value, length: number): Uint8Array | undefined | "all-null" => { + if (scalarNull(a) || scalarNull(b)) return "all-null" + const av = a.scalar ? undefined : a.vector.valid + const bv = b.scalar ? undefined : b.vector.valid + return V.andValid(av, bv, length) +} + +// ----------------------------------------------------------------------------- +// Null vectors of any type +// ----------------------------------------------------------------------------- + +/** A vector of `length` nulls with the layout of `type`. @internal */ +export const nullsOf = (type: S.DataType, length: number): V.Vector => { + const valid = new Uint8Array(length) + switch (V.kindOf(type)) { + case "f64": + return V.f64(type, new Float64Array(length), valid) + case "i32": + return V.i32(type, new Int32Array(length), valid) + case "i64": + return V.i64(type, new BigInt64Array(length), valid) + case "bool": + return V.bool(new Uint8Array(length), valid) + case "str": + return V.str(type, new Int32Array(length + 1), new Uint8Array(0), valid) + case "list": + return V.list(type, new Int32Array(length + 1), nullsOf(V.elementType(type), 0), valid) + case "struct": { + const fields = (type as S.Struct).fields + return V.struct(type, fields.map((f) => nullsOf(f.type, length)), length, valid) + } + default: + return V.nulls(length) + } +} + +// ----------------------------------------------------------------------------- +// Casts +// ----------------------------------------------------------------------------- + +const FLOAT_SYNTAX = /^[+-]?(?:\d+\.?\d*(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?|inf|infinity|nan)$/i +const INT_SYNTAX = /^[+-]?\d+$/ + +/** + * Converts `v` to `target`, as `Expr.cast` does. Values that don't fit + * become null (Polars' non-strict casts). + * + * @internal + */ +export const cast = (v: V.Vector, target: S.DataType): V.Vector => { + if (D.sameType(v.type, target)) return v + const n = v.length + if (v.kind === "null") return nullsOf(target, n) + const temporal = T.cast(v, target) + if (temporal !== undefined) return temporal + const decimal = Dec.cast(v, target) + if (decimal !== undefined) return decimal + const targetKind = V.kindOf(target) + if (targetKind === undefined) return V.fail(`cannot cast to ${target._tag}`) + let valid = v.valid === undefined ? undefined : v.valid.slice() + const invalidate = (i: number) => { + if (valid === undefined) valid = new Uint8Array(n).fill(1) + valid[i] = 0 + } + // Same physical layout and both temporal/integer: reinterpret. + if ( + targetKind === v.kind && (v.kind === "i32" || v.kind === "i64") && + (!D.isInteger(target) || !D.isInteger(v.type)) && target._tag !== "Float" + ) { + return V.withType(v, target) + } + if (targetKind === "str") { + if (target._tag === "Binary" && v.kind === "str") return V.withType(v, target) + const out: Array = new Array(n) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) out[i] = null + else { + switch (v.kind) { + case "f64": + case "i32": + out[i] = String(v.values[i]) + break + case "i64": + out[i] = (V.isUnsigned64(v.type) ? BigInt.asUintN(64, v.values[i]) : v.values[i]).toString() + break + case "bool": + out[i] = v.values[i] === 1 ? "true" : "false" + break + case "str": + out[i] = V.decodeString(v, i) + break + } + } + } + return V.stringsFrom(target, out) + } + if (targetKind === "bool") { + const out = new Uint8Array(n) + switch (v.kind) { + case "f64": + case "i32": { + const a = v.values + for (let i = 0; i < n; i++) out[i] = a[i] !== 0 ? 1 : 0 + break + } + case "i64": { + const a = v.values + for (let i = 0; i < n; i++) out[i] = a[i] !== I0 ? 1 : 0 + break + } + case "str": + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const s = V.decodeString(v, i) + if (s === "true") out[i] = 1 + else if (s !== "false") invalidate(i) + } + break + case "bool": + return v + } + return V.bool(out, V.compactValid(valid)) + } + // numeric targets: read every source slot as a number (or bigint for i64 targets) + const [lo, hi] = V.intRange(target) + const isIntTarget = D.isInteger(target) || (target._tag !== "Float" && targetKind !== "f64") + if (targetKind === "i64") { + const out = new BigInt64Array(n) + const unsigned = V.isUnsigned64(target) + const min = unsigned ? I0 : -(I2 ** I63) + const max = unsigned ? I2 ** I64 : I2 ** I63 + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + switch (v.kind) { + case "i64": { + const x = v.values[i] + if (unsigned !== V.isUnsigned64(v.type) && x < I0) invalidate(i) + else out[i] = x + break + } + case "f64": + case "i32": { + const x = Math.trunc(v.values[i]) + if (!Number.isFinite(x) || x < (unsigned ? 0 : -(2 ** 63)) || x >= (unsigned ? 2 ** 64 : 2 ** 63)) { + invalidate(i) + } else out[i] = BigInt.asIntN(64, BigInt(x)) + break + } + case "bool": + out[i] = BigInt(v.values[i]) + break + case "str": { + const s = V.decodeString(v, i) + if (!INT_SYNTAX.test(s)) invalidate(i) + else { + const x = BigInt(s) + if (x < min || x >= max) invalidate(i) + else out[i] = BigInt.asIntN(64, x) + } + break + } + } + } + return V.i64(target, out, V.compactValid(valid)) + } + const out = targetKind === "i32" ? new Int32Array(n) : new Float64Array(n) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + let x: number + switch (v.kind) { + case "f64": + case "i32": + x = v.values[i] + break + case "i64": + x = Number(V.isUnsigned64(v.type) ? BigInt.asUintN(64, v.values[i]) : v.values[i]) + break + case "bool": + x = v.values[i] + break + case "str": { + const s = V.decodeString(v, i) + if (isIntTarget ? !INT_SYNTAX.test(s) : !FLOAT_SYNTAX.test(s)) { + invalidate(i) + continue + } + const lower = s.toLowerCase() + x = lower.endsWith("nan") ? NaN : lower.endsWith("inf") || lower.endsWith("infinity") + ? (s.startsWith("-") ? -Infinity : Infinity) + : Number(s) + break + } + case "list": + case "struct": + return V.fail(`cannot cast ${D.describe(v.type)} to ${D.describe(target)}`) + } + if (isIntTarget) { + const t = Math.trunc(x) + if (!Number.isFinite(t) || t < lo || t > hi) { + invalidate(i) + continue + } + out[i] = t + } else { + out[i] = x + } + } + if (target._tag === "Float" && target.bitWidth === 32) V.roundFloat32InPlace(target, out as Float64Array) + return targetKind === "i32" + ? V.i32(target, out as Int32Array, V.compactValid(valid)) + : V.f64(target, out as Float64Array, V.compactValid(valid)) +} + +/** + * The common type of values of `types` (the larger decimal scale, the finer + * time unit, …); fails when they have none. + * + * @internal + */ +export const commonType = (types: ReadonlyArray): S.DataType => + types.reduce((t, u) => { + const common = D.supertype(t, undefined, u, undefined) + return common === undefined ? V.fail(`cannot concatenate ${D.describe(t)} with ${D.describe(u)}`) : common.type + }) + +/** + * Converts `v` to `target`, a type it shares with other values (their + * {@link commonType}, an operation's supertype) without being asked to: a + * value `target` can't represent fails, where {@link cast} makes it null. + * Conversions to the common type may still round, as Polars' do (integers + * to floats, timestamps to a coarser unit). + * + * @internal + */ +export const coerce = (v: V.Vector, target: S.DataType): V.Vector => { + const out = cast(v, target) + if (out.valid !== undefined || out.kind === "null") { + for (let i = 0; i < v.length; i++) { + if (V.isValid(v, i) && !V.isValid(out, i)) { + return V.fail(`${D.describe(v.type)} values don't fit their common type ${D.describe(target)}`) + } + } + } + return out +} + +/** Concatenates vectors, coerced to their {@link commonType}. @internal */ +export const concat = (parts: ReadonlyArray): V.Vector => { + const type = commonType(parts.map((p) => p.type)) + return V.concat(parts.map((p) => coerce(p, type))) +} + +/** Casts a value ({@link cast}), keeping its shape; a cast literal has a fixed type. @internal */ +export const castValue = (value: Value, target: S.DataType): Value => result(cast(value.vector, target), value.scalar) + +/** Coerces a value ({@link coerce}), keeping its shape. */ +const coerceValue = (value: Value, target: S.DataType): Value => + D.sameType(value.vector.type, target) + ? value + : result(coerce(value.vector, target), value.scalar) + +// ----------------------------------------------------------------------------- +// Arithmetic +// ----------------------------------------------------------------------------- + +/** @internal */ +export type ArithOp = "add" | "sub" | "mul" | "div" | "floorDiv" | "mod" + +/** + * The bounds of a dynamic integer's values, from the plan alone, so that its + * type never depends on the rows (a batch, a slice of them): a scalar's + * value (built from literals only), or the {@link Value.range} of a column. + */ +const literalRange = (v: Value): readonly [number, number] => { + if (!v.scalar) return v.range ?? V.intRange(S.int32) + const x = (v.vector as V.I32).values[0] + return V.isValid(v.vector, 0) ? [x, x] : [0, 0] +} + +/** The range of `when` or `coalesce` of `values` whose common type is `dynamic`. */ +const rangeOf = (values: ReadonlyArray, dynamic: D.Dynamic): readonly [number, number] | undefined => { + if (dynamic !== "int") return undefined + const ranges = values.flatMap((v) => v.dynamic === "int" ? [literalRange(v)] : []) + return [Math.min(...ranges.map((r) => r[0])), Math.max(...ranges.map((r) => r[1]))] +} + +/** + * The type and dynamism of an operand next to one of type `other`. Integer + * literals that `other` can't hold take the smallest signed integer type + * that holds them instead of adopting `other`, as in Polars (`Int8 + 300` + * is Int16). An operand built only from literals is fitted as a whole. + */ +const operand = (v: Value, other: S.DataType): readonly [S.DataType, D.Dynamic] => { + if (v.dynamic !== "int" || other._tag !== "Int" || v.vector.kind !== "i32") return [v.vector.type, v.dynamic] + const [lo, hi] = literalRange(v) + const [min, max] = V.intRange(other) + if (lo >= min && hi <= max) return [v.vector.type, v.dynamic] + return [lo >= -128 && hi <= 127 ? S.int8 : lo >= -32768 && hi <= 32767 ? S.int16 : S.int32, undefined] +} + +/** + * The common type of operands (`D.supertype`): integer literals are fitted + * by {@link operand} to the common type of the other operands, wherever they + * appear among them, so the result doesn't depend on their order. + */ +const supertype = (...values: ReadonlyArray) => { + const fold = (types: ReadonlyArray) => + types.reduce>( + (acc, [type, dynamic]) => acc && D.supertype(acc.type, acc.dynamic, type, dynamic), + { type: S.nullType, dynamic: undefined } + ) + const fixed = fold(values.flatMap((v) => v.dynamic === undefined ? [[v.vector.type, undefined] as const] : [])) + return fixed && fold([ + [fixed.type, undefined], + ...values.flatMap((v) => v.dynamic === undefined ? [] : [operand(v, fixed.type)]) + ]) +} + +const notDefined = (op: string, a: Value, b: Value): never => + V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) + +const unify = (op: string, a: Value, b: Value) => { + const st = supertype(a, b) + return st === undefined || !(D.isNumeric(st.type) || st.type._tag === "Null") ? notDefined(op, a, b) : st +} + +/** Relabels a value's vector (same storage). */ +const relabel = (value: Value, type: S.DataType): Value => ({ ...value, vector: V.withType(value.vector, type) }) + +/** + * Arithmetic with dates, timestamps and durations, on their integers: + * differences of dates or timestamps are durations, durations shift + * timestamps, durations add up and scale by integers. Mixed units use the + * coarser one. + */ +const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { + const at = a.vector.type + const bt = b.vector.type + const asInt = (v: Value, type: S.DataType) => relabel(coerceValue(v, type), S.int64) + const point = (t: S.DataType) => t._tag === "Date" || t._tag === "Timestamp" + if (op === "sub" && point(at) && point(bt)) { + if (at._tag === "Date" && bt._tag === "Date") { + const ts = S.timestamp("microsecond") + return relabel(arith("sub", asInt(a, ts), asInt(b, ts), length), S.duration("microsecond")) + } + const st = T.supertype(at, bt) as S.Timestamp | undefined + if (st === undefined) return notDefined(op, a, b) + return relabel(arith("sub", asInt(a, st), asInt(b, st), length), S.duration(st.unit)) + } + if ((op === "add" || op === "sub") && at._tag === "Timestamp" && bt._tag === "Duration") { + const unit = T.coarser(at.unit, bt.unit) + const ts = S.timestamp(unit, at.timezone) + return relabel(arith(op, asInt(a, ts), asInt(b, S.duration(unit)), length), ts) + } + if (op === "add" && at._tag === "Duration" && bt._tag === "Timestamp") return temporalArith(op, b, a, length) + if ((op === "add" || op === "sub") && at._tag === "Duration" && bt._tag === "Duration") { + const d = S.duration(T.coarser(at.unit, bt.unit)) + return relabel(arith(op, asInt(a, d), asInt(b, d), length), d) + } + if (op === "mul" && at._tag === "Duration" && D.isInteger(bt)) { + return relabel(arith("mul", relabel(a, S.int64), coerceValue(b, S.int64), length), at) + } + if (op === "mul" && bt._tag === "Duration" && D.isInteger(at)) return temporalArith(op, b, a, length) + return notDefined(op, a, b) +} + +/** + * Arithmetic with decimals: exact for `add`, `sub` and `mul` (Polars result + * types); other operations and mixes with floats run on floats. + */ +const decimalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { + const st = supertype(a, b) + if (st === undefined) return notDefined(op, a, b) + if (!Dec.isDecimal(st.type) || (op !== "add" && op !== "sub" && op !== "mul")) { + return arith(op, coerceValue(a, S.float64), coerceValue(b, S.float64), length) + } + const asDecimal = (v: Value): Value => (Dec.isDecimal(v.vector.type) ? v : coerceValue(v, Dec.decimal(0))) + let left = asDecimal(a) + const right = asDecimal(b) + const n = left.scalar && right.scalar ? 1 : length + if (left.scalar && !right.scalar) left = column(materialize(left, n)) + const valid = resultValid(left, right, n) + const ls = (left.vector.type as S.Decimal).scale + const rs = (right.vector.type as S.Decimal).scale + const { type, values } = Dec.arith( + op, + (left.vector as V.I64).values, + ls, + (right.vector as V.I64).values, + rs, + right.scalar && !left.scalar + ) + return result(valid === "all-null" ? nullsOf(type, n) : V.i64(type, values, valid), a.scalar && b.scalar) +} + +/** @internal */ +export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => { + if (T.isTemporal(a.vector.type) || T.isTemporal(b.vector.type)) return temporalArith(op, a, b, length) + if (Dec.isDecimal(a.vector.type) || Dec.isDecimal(b.vector.type)) return decimalArith(op, a, b, length) + let { dynamic, type } = unify(op, a, b) + if (op === "div") { + type = D.divisionType(type) + dynamic = dynamic === undefined ? undefined : "float" + } + if (type._tag === "Null") type = S.int32 + const n = a.scalar && b.scalar ? 1 : length + const valid = resultValid(a, b, n) + if (valid === "all-null") return result(nullsOf(type, n), a.scalar && b.scalar, dynamic) + // A scalar on the left: materialise it (rare) so kernels only see col op col / col op scalar. + let left = coerceValue(a, type) + const right = coerceValue(b, type) + if (left.scalar && !right.scalar) left = column(materialize(left, n)) + const lv = left.vector + const rv = right.vector + const step = right.scalar ? 0 : 1 + let out: V.Vector + let nullOut: Uint8Array | undefined + switch (lv.kind) { + case "f64": { + const x = lv.values + const o = new Float64Array(n) + const y = (rv as V.F64).values + const unsigned32 = type._tag === "Int" + arithF64(op, x, y, step, o, n, unsigned32) + if (unsigned32 && (op === "floorDiv" || op === "mod")) { + nullOut = Uint8Array.from({ length: n }, (_, i) => y[i * step] === 0 ? 0 : 1) + } + V.roundFloat32InPlace(type, o) + out = V.f64(type, o, valid) + break + } + case "i32": { + const x = lv.values + const o = new Int32Array(n) + if (op === "floorDiv" || op === "mod") { + nullOut = new Uint8Array(n).fill(1) + intDivMod(op, x, (rv as V.I32).values, step, o, n, nullOut) + } else if (step === 0) arithI32Scalar(op, x, (rv as V.I32).values[0], o, n) + else arithI32(op, x, (rv as V.I32).values, o, n) + V.wrapInPlace(type, o) + out = V.i32(type, o, valid) + break + } + case "i64": { + const x = lv.values + const y = (rv as V.I64).values + const o = new BigInt64Array(n) + const unsigned = V.isUnsigned64(type) + for (let i = 0; i < n; i++) { + let p = x[i] + let q = y[i * step] + if (unsigned) { + p = BigInt.asUintN(64, p) + q = BigInt.asUintN(64, q) + } + let r: bigint + switch (op) { + case "add": + r = p + q + break + case "sub": + r = p - q + break + case "mul": + r = p * q + break + default: { + if (q === I0) { + if (nullOut === undefined) nullOut = new Uint8Array(n).fill(1) + nullOut[i] = 0 + r = I0 + break + } + const d = T.floorDiv(p, q) + r = op === "floorDiv" ? d : p - d * q + } + } + o[i] = BigInt.asIntN(64, r) + } + out = V.i64(type, o, valid) + break + } + default: + return V.fail(`${op} is not defined for ${D.describe(type)}`) + } + if (nullOut !== undefined) out = V.withValid(out, V.compactValid(V.andValid(out.valid, nullOut, n))) + return result(out, a.scalar && b.scalar, dynamic) +} + +/** Float modulo as Polars computes it: `a - b * floor(a / b)`. */ +const floatMod = (a: number, b: number): number => a - b * Math.floor(a / b) + +const floorMod = (a: number, b: number): number => { + const r = a % b + return r !== 0 && (r < 0) !== (b < 0) ? r + b : r +} + +/** Float and UInt32 kernels; `step` 0 reads `y[0]` for every row (a scalar). */ +const arithF64 = ( + op: ArithOp, + x: Float64Array, + y: Float64Array, + step: 0 | 1, + o: Float64Array, + n: number, + u32: boolean +) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = x[i] + y[i * step] + break + case "sub": + for (let i = 0; i < n; i++) o[i] = x[i] - y[i * step] + break + case "mul": + if (u32) { for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y[i * step]) >>> 0 } + else for (let i = 0; i < n; i++) o[i] = x[i] * y[i * step] + break + case "div": + for (let i = 0; i < n; i++) o[i] = x[i] / y[i * step] + break + case "floorDiv": + for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y[i * step]) + break + case "mod": + if (u32) { for (let i = 0; i < n; i++) o[i] = floorMod(x[i], y[i * step]) } + else for (let i = 0; i < n; i++) o[i] = floatMod(x[i], y[i * step]) + break + } + if (u32) { for (let i = 0; i < n; i++) o[i] = o[i] >>> 0 } +} + +const arithI32 = (op: ArithOp, x: Int32Array, y: Int32Array, o: Int32Array, n: number) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = (x[i] + y[i]) | 0 + break + case "sub": + for (let i = 0; i < n; i++) o[i] = (x[i] - y[i]) | 0 + break + case "mul": + for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y[i]) + break + } +} + +const arithI32Scalar = (op: ArithOp, x: Int32Array, y: number, o: Int32Array, n: number) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = (x[i] + y) | 0 + break + case "sub": + for (let i = 0; i < n; i++) o[i] = (x[i] - y) | 0 + break + case "mul": + for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y) + break + } +} + +const intDivMod = ( + op: "floorDiv" | "mod", + x: Int32Array, + y: Int32Array, + step: 0 | 1, + o: Int32Array, + n: number, + valid: Uint8Array +) => { + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i * step] + if (b === 0) { + valid[i] = 0 + continue + } + if (op === "floorDiv") o[i] = Math.floor(a / b) | 0 + else o[i] = floorMod(a, b) | 0 + } +} + +/** Unary numeric operations. @internal */ +export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, decimals = 0): Value => { + const v = a.vector + const n = v.length + if (v.kind === "null") return a + if (op === "sqrt" && v.kind !== "f64") return unaryNumeric(op, coerceValue(a, S.float64)) + if (op === "round" && !(Number.isInteger(decimals) && decimals >= 0)) { + return V.fail(`round needs a non-negative whole number of decimals, got ${decimals}`) + } + if (op === "round" && v.kind === "i64" && Dec.isDecimal(v.type)) { + return result(Dec.round(v, v.type, decimals), a.scalar, a.dynamic) + } + // Rounding leaves integers (and dates) as they are. + if (op === "round" && (v.kind === "i32" || v.kind === "i64" || (v.kind === "f64" && !D.isFloat(v.type)))) { + return result(v, a.scalar, a.dynamic) + } + switch (v.kind) { + case "f64": { + const x = v.values + const o = new Float64Array(n) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = -x[i] } + else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) } + else if (op === "sqrt") { for (let i = 0; i < n; i++) o[i] = Math.sqrt(x[i]) } + else roundHalfEven(x, o, decimals) + V.wrapInPlace(v.type, o) + V.roundFloat32InPlace(v.type, o) + return result(V.f64(v.type, o, v.valid), a.scalar, a.dynamic) + } + case "i32": { + const x = v.values + const o = new Int32Array(n) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = -x[i] | 0 } + else for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) | 0 + V.wrapInPlace(v.type, o) + return result(V.i32(v.type, o, v.valid), a.scalar, a.dynamic) + } + case "i64": { + const x = v.values + const o = new BigInt64Array(n) + const fit = Dec.fit(v.type) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = fit(-x[i]) } + else for (let i = 0; i < n; i++) o[i] = fit(x[i] < I0 ? -x[i] : x[i]) + return result(V.i64(v.type, o, v.valid), a.scalar, a.dynamic) + } + default: + return V.fail(`${op} is not defined for ${D.describe(v.type)}`) + } +} + +const roundHalfEven = (x: Float64Array, o: Float64Array, decimals: number) => { + const scale = 10 ** decimals + for (let i = 0; i < x.length; i++) { + const v = x[i] + if (!Number.isFinite(v)) { + o[i] = v + continue + } + const y = decimals === 0 ? v : v * scale + let r = Math.round(y) + // Math.round rounds halves up; move exact halves to the even neighbour. + if (r - y === 0.5 && r % 2 !== 0) r -= 1 + if (r === 0 && (y < 0 || Object.is(y, -0))) r = -0 + o[i] = decimals === 0 ? r : r / scale + } +} + +// ----------------------------------------------------------------------------- +// Comparisons +// ----------------------------------------------------------------------------- + +/** @internal */ +export type CompareOp = "eq" | "neq" | "lt" | "lte" | "gt" | "gte" + +const mirror: Record = { eq: "eq", neq: "neq", lt: "gt", lte: "gte", gt: "lt", gte: "lte" } + +/** Lexicographic comparison of two byte ranges. @internal */ +export const compareBytes = ( + a: Uint8Array, + as: number, + ae: number, + b: Uint8Array, + bs: number, + be: number +): number => { + const la = ae - as + const lb = be - bs + const l = la < lb ? la : lb + for (let k = 0; k < l; k++) { + const d = a[as + k] - b[bs + k] + if (d !== 0) return d + } + return la - lb +} + +const fromOrdering = (op: CompareOp, c: number): number => { + switch (op) { + case "eq": + return c === 0 ? 1 : 0 + case "neq": + return c !== 0 ? 1 : 0 + case "lt": + return c < 0 ? 1 : 0 + case "lte": + return c <= 0 ? 1 : 0 + case "gt": + return c > 0 ? 1 : 0 + case "gte": + return c >= 0 ? 1 : 0 + } +} + +/** @internal */ +export const compare = (op: CompareOp, a: Value, b: Value, length: number): Value => { + if (a.scalar && !b.scalar) return compare(mirror[op], b, a, length) + const at = a.vector.type + const bt = b.vector.type + const n = a.scalar ? 1 : length + let left = a + let right = b + if (D.isNumeric(at) && D.isNumeric(bt) && !D.sameType(at, bt)) { + const st = supertype(a, b)!.type + // Mixed widths compare as Float64 unless the common type has Int32 storage + // or both sides are 64-bit integers. + const k = V.kindOf(st) + const target = k === "i32" || (k === "i64" && V.kindOf(at) === "i64" && V.kindOf(bt) === "i64") ? st : S.float64 + left = coerceValue(a, target) + right = coerceValue(b, target) + } else if (at._tag === "Null" || bt._tag === "Null") { + return result(nullsOf(S.bool, n), a.scalar) + } else if (!D.sameType(at, bt) && !(D.isBinaryLike(at) && D.isBinaryLike(bt))) { + // Decimals and temporal values compare in their common type. + const st = supertype(a, b) + if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) + left = coerceValue(a, st.type) + right = coerceValue(b, st.type) + } + const valid = resultValid(left, right, n) + if (valid === "all-null") return result(nullsOf(S.bool, n), a.scalar) + const o = new Uint8Array(n) + const lv = left.vector + const rv = right.vector + const rs = right.scalar + const step = rs ? 0 : 1 + switch (lv.kind) { + case "f64": + if (rs) compareF64Scalar(op, lv.values, (rv as V.F64).values[0], o, n) + else compareF64(op, lv.values, (rv as V.F64).values, o, n) + break + case "i32": + if (rs) compareI32Scalar(op, lv.values, (rv as V.I32).values[0], o, n) + else compareI32(op, lv.values, (rv as V.I32).values, o, n) + break + case "bool": { + const x = lv.values + const y = (rv as V.Bool).values + for (let i = 0; i < n; i++) o[i] = fromOrdering(op, x[i] - y[i * step]) + break + } + case "i64": { + const unsigned = V.isUnsigned64(lv.type) + const x = new Int32Array(lv.values.buffer, lv.values.byteOffset, n * 2) + const yv = (rv as V.I64).values + const y = new Int32Array(yv.buffer, yv.byteOffset, yv.length * 2) + for (let i = 0; i < n; i++) { + const j = i * step + const xh = x[2 * i + 1] + const yh = y[2 * j + 1] + let c: number + if (xh !== yh) c = unsigned ? (xh >>> 0) - (yh >>> 0) : xh - yh + else c = (x[2 * i] >>> 0) - (y[2 * j] >>> 0) + o[i] = fromOrdering(op, c) + } + break + } + case "str": { + const r = rv as V.Str + const ao = lv.offsets + const ad = lv.data + const bo = r.offsets + const bd = r.data + if (rs && (op === "eq" || op === "neq")) { + const bs = bo[0] + const blen = bo[1] - bs + const yes = op === "eq" ? 1 : 0 + for (let i = 0; i < n; i++) { + const s = ao[i] + if (ao[i + 1] - s !== blen) { + o[i] = 1 - yes + continue + } + let same = 1 + for (let k = 0; k < blen; k++) { + if (ad[s + k] !== bd[bs + k]) { + same = 0 + break + } + } + o[i] = same === 1 ? yes : 1 - yes + } + } else { + for (let i = 0; i < n; i++) { + const j = i * step + o[i] = fromOrdering(op, compareBytes(ad, ao[i], ao[i + 1], bd, bo[j], bo[j + 1])) + } + } + break + } + default: + return V.fail(`cannot compare ${D.describe(at)}`) + } + return result(V.bool(o, valid), a.scalar) +} + +// Float64 comparisons use a total order: NaN equals NaN and sorts above every +// other value, as in Polars. +const compareF64 = (op: CompareOp, x: Float64Array, y: Float64Array, o: Uint8Array, n: number) => { + switch (op) { + case "eq": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +(a === b || (a !== a && b !== b)) + } + break + case "neq": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +!(a === b || (a !== a && b !== b)) + } + break + case "lt": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +(a < b || (b !== b && a === a)) + } + break + case "gte": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +!(a < b || (b !== b && a === a)) + } + break + case "gt": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +(a > b || (a !== a && b === b)) + } + break + case "lte": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +!(a > b || (a !== a && b === b)) + } + break + } +} + +const compareF64Scalar = (op: CompareOp, x: Float64Array, b: number, o: Uint8Array, n: number) => { + if (b !== b) { + // Against NaN: only NaN is equal; everything else is smaller. + for (let i = 0; i < n; i++) { + const isNaN = x[i] !== x[i] + o[i] = op === "eq" || op === "gte" || op === "lte" + ? +(op === "lte" ? true : isNaN) + : op === "neq" || op === "lt" + ? +!isNaN + : 0 + } + return + } + switch (op) { + case "eq": + for (let i = 0; i < n; i++) o[i] = +(x[i] === b) + break + case "neq": + for (let i = 0; i < n; i++) o[i] = +(x[i] !== b) + break + case "lt": + for (let i = 0; i < n; i++) o[i] = +(x[i] < b) + break + case "lte": + for (let i = 0; i < n; i++) o[i] = +(x[i] <= b) + break + case "gt": + for (let i = 0; i < n; i++) { + const a = x[i] + o[i] = +(a > b || a !== a) + } + break + case "gte": + for (let i = 0; i < n; i++) { + const a = x[i] + o[i] = +(a >= b || a !== a) + } + break + } +} + +const compareI32 = (op: CompareOp, x: Int32Array, y: Int32Array, o: Uint8Array, n: number) => { + switch (op) { + case "eq": + for (let i = 0; i < n; i++) o[i] = +(x[i] === y[i]) + break + case "neq": + for (let i = 0; i < n; i++) o[i] = +(x[i] !== y[i]) + break + case "lt": + for (let i = 0; i < n; i++) o[i] = +(x[i] < y[i]) + break + case "lte": + for (let i = 0; i < n; i++) o[i] = +(x[i] <= y[i]) + break + case "gt": + for (let i = 0; i < n; i++) o[i] = +(x[i] > y[i]) + break + case "gte": + for (let i = 0; i < n; i++) o[i] = +(x[i] >= y[i]) + break + } +} + +const compareI32Scalar = (op: CompareOp, x: Int32Array, b: number, o: Uint8Array, n: number) => { + switch (op) { + case "eq": + for (let i = 0; i < n; i++) o[i] = +(x[i] === b) + break + case "neq": + for (let i = 0; i < n; i++) o[i] = +(x[i] !== b) + break + case "lt": + for (let i = 0; i < n; i++) o[i] = +(x[i] < b) + break + case "lte": + for (let i = 0; i < n; i++) o[i] = +(x[i] <= b) + break + case "gt": + for (let i = 0; i < n; i++) o[i] = +(x[i] > b) + break + case "gte": + for (let i = 0; i < n; i++) o[i] = +(x[i] >= b) + break + } +} + +// ----------------------------------------------------------------------------- +// Boolean logic (Kleene) +// ----------------------------------------------------------------------------- + +const asBool = (value: Value, length: number): V.Bool => { + const v = materialize(value, length) + if (v.kind === "bool") return v + if (v.kind === "null") return V.bool(new Uint8Array(length), new Uint8Array(length)) + return V.fail(`expected a boolean expression, got ${D.describe(v.type)}`) +} + +/** @internal */ +export const logical = (op: "and" | "or", a: Value, b: Value, length: number): Value => { + const n = a.scalar && b.scalar ? 1 : length + const x = asBool(a, n) + const y = asBool(b, n) + const o = new Uint8Array(n) + const xv = x.values + const yv = y.values + if (x.valid === undefined && y.valid === undefined) { + if (op === "and") { for (let i = 0; i < n; i++) o[i] = xv[i] & yv[i] } + else for (let i = 0; i < n; i++) o[i] = xv[i] | yv[i] + return result(V.bool(o), a.scalar && b.scalar) + } + const valid = new Uint8Array(n) + const xa = x.valid + const ya = y.valid + for (let i = 0; i < n; i++) { + const p = xa === undefined ? 1 : xa[i] + const q = ya === undefined ? 1 : ya[i] + const pv = xv[i] & p + const qv = yv[i] & q + if (op === "and") { + // false wins over null + const definitelyFalse = (p === 1 && pv === 0) || (q === 1 && qv === 0) + o[i] = pv & qv + valid[i] = definitelyFalse || (p === 1 && q === 1) ? 1 : 0 + } else { + const definitelyTrue = pv === 1 || qv === 1 + o[i] = pv | qv + valid[i] = definitelyTrue || (p === 1 && q === 1) ? 1 : 0 + } + } + return result(V.bool(o, V.compactValid(valid)), a.scalar && b.scalar) +} + +/** @internal */ +export const not = (a: Value): Value => { + const v = a.vector + if (v.kind === "null") return a + if (v.kind !== "bool") return V.fail(`not is not defined for ${D.describe(v.type)}`) + const o = new Uint8Array(v.length) + const x = v.values + for (let i = 0; i < o.length; i++) o[i] = x[i] ^ 1 + return result(V.bool(o, v.valid), a.scalar) +} + +/** @internal */ +export const isNull = (a: Value, negate: boolean): Value => { + const v = a.vector + const o = new Uint8Array(v.length) + if (v.kind === "null") o.fill(negate ? 0 : 1) + else if (v.valid === undefined) o.fill(negate ? 1 : 0) + else if (negate) o.set(v.valid) + else for (let i = 0; i < o.length; i++) o[i] = v.valid[i] ^ 1 + return result(V.bool(o), a.scalar) +} + +/** @internal */ +export const isNaN_ = (a: Value): Value => { + const v = a.vector + const o = new Uint8Array(v.length) + if (v.kind === "f64" && v.type._tag === "Float") { + const x = v.values + for (let i = 0; i < o.length; i++) o[i] = +(x[i] !== x[i]) + } else if (v.kind !== "f64" && v.kind !== "i32" && v.kind !== "i64" && v.kind !== "null") { + return V.fail(`isNaN is not defined for ${D.describe(v.type)}`) + } + return result(V.bool(o, v.kind === "null" ? new Uint8Array(v.length) : v.valid), a.scalar) +} + +// ----------------------------------------------------------------------------- +// Conditionals +// ----------------------------------------------------------------------------- + +/** + * `cond ? then : otherwise` per slot; a null condition takes `otherwise`. + * + * @internal + */ +export const when = (cond: Value, then: Value, otherwise: Value, length: number): Value => { + const st = supertype(then, otherwise) + if (st === undefined) { + return V.fail( + `when branches have incompatible types ${D.describe(then.vector.type)} and ${D.describe(otherwise.vector.type)}` + ) + } + const allScalar = cond.scalar && then.scalar && otherwise.scalar + const n = allScalar ? 1 : length + const c = asBool(cond, n) + const mask = new Int32Array(n) + const cv = c.values + const ca = c.valid + for (let i = 0; i < n; i++) mask[i] = (cv[i] & (ca === undefined ? 1 : ca[i])) === 1 ? 0 : 1 + const branches = [materialize(coerceValue(then, st.type), n), materialize(coerceValue(otherwise, st.type), n)] + return { + vector: select(branches, mask), + scalar: allScalar, + dynamic: st.dynamic, + range: rangeOf([then, otherwise], st.dynamic) + } +} + +/** Picks slot `i` from `branches[choice[i]]`; all branches share a type. */ +const select = (branches: ReadonlyArray, choice: Int32Array): V.Vector => { + const n = choice.length + const indices = new Int32Array(n) + for (let i = 0; i < n; i++) indices[i] = choice[i] * n + i + const out = V.take(V.concat(branches), indices) + return V.withValid(out, V.compactValid(out.valid)) +} + +/** First non-null value per slot. @internal */ +export const coalesce = (values: ReadonlyArray, length: number): Value => { + const st = supertype(...values) + if (st === undefined) return V.fail(`coalesce arguments have incompatible types`) + const allScalar = values.every((v) => v.scalar) + const n = allScalar ? 1 : length + const vectors = values.map((v) => materialize(coerceValue(v, st.type), n)) + const choice = new Int32Array(n) + for (let i = 0; i < n; i++) { + let k = 0 + while (k < vectors.length - 1 && !V.isValid(vectors[k], i)) k++ + choice[i] = k + } + return { vector: select(vectors, choice), scalar: allScalar, dynamic: st.dynamic, range: rangeOf(values, st.dynamic) } +} + +// ----------------------------------------------------------------------------- +// Membership +// ----------------------------------------------------------------------------- + +/** @internal */ +export const isIn = (a: Value, candidates: ReadonlyArray): Value => { + const v = a.vector + const n = v.length + const o = new Uint8Array(n) + switch (v.kind) { + case "str": { + const encoded = candidates.filter((c): c is string => typeof c === "string").map(V.encodeString) + for (let i = 0; i < n; i++) { + const s = v.offsets[i] + const e = v.offsets[i + 1] + for (const c of encoded) { + if (compareBytes(v.data, s, e, c, 0, c.length) === 0) { + o[i] = 1 + break + } + } + } + break + } + case "f64": + case "i32": { + const set = new Set(candidates.filter((c) => typeof c === "number").map((c) => (c as number) === 0 ? 0 : c)) + const hasNaN = candidates.some((c) => typeof c === "number" && Number.isNaN(c)) + const x = v.values + for (let i = 0; i < n; i++) o[i] = +(set.has(x[i] === 0 ? 0 : x[i]) || (hasNaN && x[i] !== x[i])) + break + } + case "i64": { + // A decimal's candidates are values, not the unscaled units it stores. + if (Dec.isDecimal(v.type)) return result(V.bool(Dec.isIn(v.values, v.type.scale, candidates), v.valid), a.scalar) + // Only an integer can be one of these values. + const set = new Set( + candidates.filter((c): c is bigint | number => typeof c === "bigint" || Number.isInteger(c)).map(BigInt) + ) + // A UInt64 is stored as the signed integer with the same bits. + const unsigned = V.isUnsigned64(v.type) + for (let i = 0; i < n; i++) o[i] = +set.has(unsigned ? BigInt.asUintN(64, v.values[i]) : v.values[i]) + break + } + case "bool": { + const t = candidates.includes(true) + const f = candidates.includes(false) + for (let i = 0; i < n; i++) o[i] = +(v.values[i] === 1 ? t : f) + break + } + case "null": + return result(nullsOf(S.bool, n), a.scalar) + } + return result(V.bool(o, v.valid), a.scalar) +} + +// ----------------------------------------------------------------------------- +// Strings +// ----------------------------------------------------------------------------- + +const asStr = (value: Value, name: string): V.Str => { + const v = value.vector + if (v.kind === "str") return v + return V.fail(`${name} is not defined for ${D.describe(v.type)}`) +} + +/** Number of code points. @internal */ +export const strLength = (a: Value): Value => { + if (a.vector.kind === "null") return result(nullsOf(S.uint32, a.vector.length), a.scalar) + const v = asStr(a, "length") + const n = v.length + const o = new Float64Array(n) + const d = v.data + const off = v.offsets + for (let i = 0; i < n; i++) { + let c = 0 + for (let k = off[i]; k < off[i + 1]; k++) c += (d[k] & 0xc0) !== 0x80 ? 1 : 0 + o[i] = c + } + return result(V.f64(S.uint32, o, v.valid), a.scalar) +} + +/** @internal */ +export const strMap = (a: Value, f: (s: string) => string): Value => { + if (a.vector.kind === "null") return a + const v = asStr(a, "string function") + const out: Array = new Array(v.length) + for (let i = 0; i < v.length; i++) out[i] = V.isValid(v, i) ? f(V.decodeString(v, i)) : null + return result(V.stringsFrom(v.type, out), a.scalar) +} + +/** @internal */ +export const strMatch = (a: Value, mode: "includes" | "startsWith" | "endsWith", pattern: string): Value => { + if (a.vector.kind === "null") return result(nullsOf(S.bool, a.vector.length), a.scalar) + const v = asStr(a, mode) + const p = V.encodeString(pattern) + const m = p.length + const n = v.length + const o = new Uint8Array(n) + const d = v.data + const off = v.offsets + for (let i = 0; i < n; i++) { + const s = off[i] + const e = off[i + 1] + if (e - s < m) continue + if (mode === "startsWith" || mode === "endsWith") { + const at = mode === "startsWith" ? s : e - m + let ok = 1 + for (let k = 0; k < m; k++) { + if (d[at + k] !== p[k]) { + ok = 0 + break + } + } + o[i] = ok + } else { + outer: for (let at = s; at <= e - m; at++) { + for (let k = 0; k < m; k++) if (d[at + k] !== p[k]) continue outer + o[i] = 1 + break + } + } + } + return result(V.bool(o, v.valid), a.scalar) +} + +/** Concatenation; a null argument makes the slot null. @internal */ +export const strConcat = (values: ReadonlyArray, length: number): Value => { + const allScalar = values.every((v) => v.scalar) + const n = allScalar ? 1 : length + const parts = values.map((v) => { + const vector = materialize(v, n) + return vector.kind === "str" ? vector : (cast(vector, S.utf8) as V.Str | V.Null) + }) + let valid: Uint8Array | undefined + for (const p of parts) { + if (p.kind === "null") valid = new Uint8Array(n) + else valid = V.andValid(valid, p.valid, n) + } + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + for (const p of parts) if (p.kind === "str") total += p.offsets[i + 1] - p.offsets[i] + offsets[i + 1] = total + } + const data = new Uint8Array(total) + for (let i = 0; i < n; i++) { + let at = offsets[i] + for (const p of parts) { + if (p.kind !== "str") continue + const s = p.offsets[i] + const e = p.offsets[i + 1] + data.set(p.data.subarray(s, e), at) + at += e - s + } + } + return result(V.str(S.utf8, offsets, data, valid), allScalar) +} diff --git a/packages/effect/src/dataframe/internal/nested.ts b/packages/effect/src/dataframe/internal/nested.ts new file mode 100644 index 00000000000..d9be6608401 --- /dev/null +++ b/packages/effect/src/dataframe/internal/nested.ts @@ -0,0 +1,83 @@ +/** + * Lists and structs: element access and aggregates of list columns, struct + * fields and construction (Polars semantics). + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" +import * as K from "./kernels.ts" +import * as V from "./vector.ts" + +const asList = (v: V.Vector, fn: string): V.List => + v.kind === "list" ? v : V.fail(`${fn} is defined for lists, not ${D.describe(v.type)}`) + +/** The row of every element of a list column. */ +const elementRows = (v: V.List): Int32Array => { + const ids = new Int32Array(v.child.length) + for (let i = 0; i < v.length; i++) for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) ids[e] = i + return ids +} + +/** Number of elements; null for a null list. @internal */ +export const length = (x: V.Vector): V.Vector => { + const v = asList(x, "listLength") + const out = new Float64Array(v.length) + for (let i = 0; i < v.length; i++) out[i] = v.offsets[i + 1] - v.offsets[i] + return V.f64(S.uint32, out, v.valid) +} + +/** The element at `index` (negative counts from the end); null when out of range. @internal */ +export const get = (x: V.Vector, index: number): V.Vector => { + const v = asList(x, "listGet") + const idx = new Int32Array(v.length) + for (let i = 0; i < v.length; i++) { + const start = v.offsets[i] + const end = v.offsets[i + 1] + const at = index < 0 ? end + index : start + index + idx[i] = V.isValid(v, i) && at >= start && at < end ? at : -1 + } + return V.take(v.child, idx) +} + +/** An aggregate of every list's elements; null for a null list. @internal */ +export const aggregate = (fn: "sum" | "mean" | "min" | "max", x: V.Vector): V.Vector => { + const v = asList(x, `list${fn[0].toUpperCase()}${fn.slice(1)}`) + const result = Agg.aggregate(fn, v.child, { ids: elementRows(v), count: v.length }) + return v.valid === undefined ? result : V.withValid(result, V.andValid(result.valid, v.valid, v.length)) +} + +/** Whether a list holds `item`; null for a null list. @internal */ +export const contains = (x: V.Vector, item: K.Value): V.Vector => { + const v = asList(x, "listContains") + const n = v.length + const items = V.take(K.materialize(item, n), elementRows(v)) + const eq = K.compare("eq", K.column(v.child), K.column(items), v.child.length).vector as V.Bool + const out = new Uint8Array(n) + for (let i = 0; i < n; i++) { + for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) { + if (V.isValid(eq, e) && eq.values[e] === 1) { + out[i] = 1 + break + } + } + } + return V.bool(out, v.valid) +} + +/** A struct field; null where the struct is null. @internal */ +export const field = (x: V.Vector, name: string): V.Vector => { + if (x.kind !== "struct") return V.fail(`field is defined for structs, not ${D.describe(x.type)}`) + const fields = (x.type as S.Struct).fields + const k = fields.findIndex((f) => f.name === name) + if (k < 0) return V.fail(`struct has no field ${JSON.stringify(name)}`) + const child = x.children[k] + return x.valid === undefined + ? child + : V.withValid(child, V.andValid(child.kind === "null" ? undefined : child.valid, x.valid, x.length)) +} + +/** A struct of the given columns. @internal */ +export const struct = (names: ReadonlyArray, values: ReadonlyArray, n: number): V.Vector => + V.struct(S.struct(values.map((v, k) => S.field(names[k], v.type))), values, n) diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts new file mode 100644 index 00000000000..2b7fca9f6b1 --- /dev/null +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -0,0 +1,185 @@ +/** + * Partial aggregation: grouping a part of the rows (a row range, a stream + * batch) into partial aggregates that combine with other parts' into the + * aggregates of all rows. + * + * Counts, minima, maxima, firsts and lasts combine by aggregating the + * partials again; sums combine from per-part sums (exact for integers and + * decimals); means from sums and counts; variances from counts, means and + * M2 (Chan et al.). Medians and distinct counts don't combine. + * + * Parts have the types of the whole input (every batch of a streamed node + * has the types of its whole result), so their partials combine as they are + * and finish in that input type. Float results may differ in rounding, as + * float sums of the same values in another order do. + * + * @internal + */ +import type * as S from "../../arrow/ArrowSchema.ts" +import * as Agg from "./aggregate.ts" +import * as Hash from "./hash.ts" +import * as K from "./kernels.ts" +import * as V from "./vector.ts" + +type Part = + | { readonly _tag: "vector"; readonly vector: V.Vector } + | { readonly _tag: "sums"; readonly sums: Agg.Sums } + | { readonly _tag: "mean"; readonly sum: Float64Array; readonly count: Float64Array } + | { + readonly _tag: "moments" + readonly count: Float64Array + readonly mean: Float64Array + readonly m2: Float64Array + } + +const reaggregated = new Set(["count", "len", "min", "max", "first", "last"]) + +/** Whether partial aggregates of `fn` combine. @internal */ +export const combinable = (fn: Agg.AggFn): boolean => + reaggregated.has(fn) || fn === "sum" || fn === "mean" || fn === "std" || fn === "variance" + +/** The partial aggregate of `fn` over `v` grouped by `g`. */ +const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping): Part => { + if (reaggregated.has(fn)) return { _tag: "vector", vector: Agg.aggregate(fn, v, g) } + if (v === undefined) return V.fail(`${fn} needs an argument`) + switch (fn) { + case "sum": + return { _tag: "sums", sums: Agg.sums(v, g) } + case "mean": { + const { count, sum } = Agg.sumCount(v, g) + return { _tag: "mean", sum, count } + } + case "std": + case "variance": + return { _tag: "moments", ...Agg.moments(v, g) } + default: + return V.fail(`${fn} cannot be computed from partial aggregates`) + } +} + +/** + * Combines the partials of several parts: `parts[c]` holds part `c`'s + * partials for its groups; `ids` maps the concatenated groups of all parts + * to `count` combined groups. + */ +const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Array, count: number): Part => { + const grouping: Agg.Grouping = { ids, count } + if (reaggregated.has(fn)) { + const vector = K.concat(parts.map((p) => (p as Extract).vector)) + return { _tag: "vector", vector: Agg.aggregate(fn === "count" || fn === "len" ? "sum" : fn, vector, grouping) } + } + if (fn === "sum") { + const float = (parts[0] as Extract).sums instanceof Float64Array + const sums = float ? new Float64Array(count) : new Array(count).fill(BigInt(0)) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.sums.length; j++) { + if (float) (sums as Float64Array)[ids[at + j]] += p.sums[j] as number + else (sums as Array)[ids[at + j]] += p.sums[j] as bigint + } + at += p.sums.length + } + return { _tag: "sums", sums } + } + if (fn === "mean") { + const sum = new Float64Array(count) + const total = new Float64Array(count) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.sum.length; j++) { + sum[ids[at + j]] += p.sum[j] + total[ids[at + j]] += p.count[j] + } + at += p.sum.length + } + return { _tag: "mean", sum, count: total } + } + // std and variance + const n = new Float64Array(count) + const mean = new Float64Array(count) + const m2 = new Float64Array(count) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.count.length; j++) { + const nb = p.count[j] + if (nb === 0) continue + const g = ids[at + j] + const na = n[g] + const nab = na + nb + const delta = p.mean[j] - mean[g] + mean[g] += delta * nb / nab + m2[g] += p.m2[j] + delta * delta * na * nb / nab + n[g] = nab + } + at += p.count.length + } + return { _tag: "moments", count: n, mean, m2 } +} + +/** The aggregate column of a combined partial over inputs of `type`. @internal */ +export const finish = (fn: Agg.AggFn, part: Part, type: S.DataType | undefined): V.Vector => { + switch (part._tag) { + case "vector": + return part.vector + case "sums": + return Agg.sumColumn(type!, part.sums) + case "mean": + return Agg.meanFrom(type!, part.sum, part.count) + case "moments": + return Agg.varianceFrom(fn as "std" | "variance", type!, part.count, part.m2) + } +} + +// ----------------------------------------------------------------------------- +// Grouped states +// ----------------------------------------------------------------------------- + +/** + * The groups of some rows: their keys (one row per group), partial + * aggregates and the type of each aggregate's input. + * + * @internal + */ +export interface State { + readonly keys: ReadonlyArray + readonly types: ReadonlyArray + readonly parts: ReadonlyArray + readonly count: number + readonly rows: number +} + +/** Groups some rows; without keys, they form one group even when there are none. @internal */ +export const group = ( + keys: ReadonlyArray, + inputs: ReadonlyArray<{ readonly fn: Agg.AggFn; readonly vector: V.Vector | undefined }>, + n: number +): State => { + const g = keys.length === 0 ? { ids: new Int32Array(n), count: 1, firstRow: new Int32Array(1) } : Hash.groups(keys, n) + return { + keys: keys.map((k) => V.take(k, g.firstRow)), + types: inputs.map(({ vector }) => vector?.type), + parts: inputs.map(({ fn, vector }) => partial(fn, vector, g)), + count: g.count, + rows: n + } +} + +/** + * Combines states of rows of the same types, keeping groups in order of + * first appearance. States without rows add nothing. + * + * @internal + */ +export const merge = (fns: ReadonlyArray, states: ReadonlyArray): State => { + const filled = states.filter((s) => s.rows > 0) + if (filled.length <= 1) return filled[0] ?? states[0] + const keys = filled[0].keys.map((_, k) => K.concat(filled.map((s) => s.keys[k]))) + const g = Hash.groups(keys, filled.reduce((n, s) => n + s.count, 0)) + return { + keys: keys.map((k) => V.take(k, g.firstRow)), + types: filled[0].types, + parts: fns.map((fn, a) => combine(fn, filled.map((s) => s.parts[a]), g.ids, g.count)), + count: g.count, + rows: filled.reduce((n, s) => n + s.rows, 0) + } +} diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts new file mode 100644 index 00000000000..a8afc4b8426 --- /dev/null +++ b/packages/effect/src/dataframe/internal/query.ts @@ -0,0 +1,668 @@ +/** + * Running plans: resolving tables, rewriting the plan, and executing it. + * + * Execution is a post-order walk that calls the frame operations. Before it, + * `optimize` rewrites the plan so less work happens: + * + * - row-wise predicates move below the row-wise operations they commute + * with (sorts, projections, derived columns they don't read, group keys, + * either side of a join), and filters that meet merge. A predicate only + * moves where it sees the values it sees in place: it stays above a concat + * (whose inputs may differ in type from their concatenation), and one on + * join keys moves to the left side only (the side whose keys the join + * keeps); + * - slices move below row-wise operations that keep every row in place, and + * a slice of a sort becomes a top-k selection; + * - columns nothing reads are dropped at the inputs, and derived columns and + * aggregates nothing reads are not computed. + * + * An operation is row-wise when its expressions are (see `isRowWise`): a + * window in a predicate, a sort key or a derived column sees every row of its + * input, so moving a filter or slice below it changes its result. + * + * @internal + */ +import * as Effect from "../../Effect.ts" +import type { Catalog } from "../Catalog.ts" +import { DataFrameError } from "../DataFrame.ts" +import type * as Plan from "../Plan.ts" +import * as E from "./expr.ts" +import * as F from "./frame.ts" +import * as Streaming from "./stream.ts" +import { conform } from "./table.ts" +import * as V from "./vector.ts" + +/** A sort that only keeps its first `limit` rows. @internal */ +export interface TopK { + readonly _tag: "TopK" + readonly input: Node + readonly keys: ReadonlyArray + readonly nullsLast: boolean + readonly limit: number +} + +/** Plans after optimisation. @internal */ +export type Node = + | Exclude + | Rebuild> + | Omit & { readonly left: Node; readonly right: Node } + | Omit & { readonly inputs: ReadonlyArray } + | TopK + +type Rebuild = N extends { readonly input: Plan.Node } ? Omit & { readonly input: Node } : never + +/** Runs a kernel computation, turning its failures into `DataFrameError`s. @internal */ +export const attempt = (f: () => A): Effect.Effect => { + try { + return Effect.succeed(f()) + } catch (cause) { + if (cause instanceof V.DataFrameDefect) return Effect.fail(new DataFrameError({ message: cause.message, cause })) + if (cause instanceof DataFrameError) return Effect.fail(cause) + throw cause + } +} + +/** Runs a kernel computation, throwing its failures as `DataFrameError`s. @internal */ +export const unsafe = (f: () => A): A => { + try { + return f() + } catch (cause) { + throw cause instanceof V.DataFrameDefect ? new DataFrameError({ message: cause.message, cause }) : cause + } +} + +/** @internal */ +export const children = (node: Node): ReadonlyArray => { + switch (node._tag) { + case "Values": + case "Table": + return [] + case "Join": + return [node.left, node.right] + case "Concat": + return node.inputs + default: + return [node.input] + } +} + +const mapChildren = (node: Node, f: (child: Node) => Node): Node => { + switch (node._tag) { + case "Values": + case "Table": + return node + case "Join": + return { ...node, left: f(node.left), right: f(node.right) } + case "Concat": + return { ...node, inputs: node.inputs.map(f) } + default: + return { ...node, input: f(node.input) } as Node + } +} + +// ----------------------------------------------------------------------------- +// Resolving tables +// ----------------------------------------------------------------------------- + +/** A frame as a plan node. @internal */ +export const values = (frame: F.Frame): Node => ({ _tag: "Values", frame: frame as any }) +const frameOf = (node: { readonly frame: unknown }): F.Frame => node.frame as F.Frame + +/** How many times a plan names each table. @internal */ +export const tables = (plan: Plan.Node): Map => { + const out = new Map() + const walk = (node: Node): void => { + if (node._tag === "Table") out.set(node.name, (out.get(node.name) ?? 0) + 1) + for (const c of children(node)) walk(c) + } + walk(plan as Node) + return out +} + +/** A plan with each `Table` replaced by `f` of it. @internal */ +export const mapTables = (plan: Node, f: (table: Plan.Table) => Node): Node => { + const replace = (node: Node): Node => node._tag === "Table" ? f(node) : mapChildren(node, replace) + return replace(plan) +} + +/** A table's frame, checked against and narrowed to its declared columns. */ +const tableValues = (table: Plan.Table, frame: F.Frame): Node => + values(table.columns === undefined ? frame : conform(frame, table.columns, table.checks)) + +/** Replaces every `Table` with the catalog's frame, reading each table once. */ +const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { + const names = [...tables(plan).keys()] + if (names.length === 0) return Effect.succeed(plan as Node) + return Effect.forEach( + names, + (name) => Effect.map(catalog.table(name), (df) => [name, df as unknown as F.Frame] as const) + ) + .pipe( + Effect.flatMap((entries) => { + const frames = new Map(entries) + return attempt(() => mapTables(plan as Node, (table) => tableValues(table, frames.get(table.name)!))) + }) + ) +} + +// ----------------------------------------------------------------------------- +// Column names +// ----------------------------------------------------------------------------- + +const namesCache = new WeakMap | undefined }>() + +/** + * The output column names of a resolved plan, or `undefined` when they + * depend on the data (a pivot without declared columns). + */ +const names = (node: Node): ReadonlyArray | undefined => { + const cached = namesCache.get(node) + if (cached !== undefined) return cached.names + const result = computeNames(node) + namesCache.set(node, { names: result }) + return result +} + +const computeNames = (node: Node): ReadonlyArray | undefined => { + switch (node._tag) { + case "Values": + return frameOf(node).columns.map((c) => c.name) + case "Table": + return node.columns?.map((c) => c.name) + case "Filter": + case "Sort": + case "TopK": + case "Slice": + case "Unique": + case "DropNulls": + case "Explode": + return names(node.input) + case "Select": + return node.columns.map((c) => c.name) + case "WithColumns": { + const input = names(node.input) + if (input === undefined) return undefined + const existing = new Set(input) + return [...input, ...node.columns.map((c) => c.name).filter((n) => !existing.has(n))] + } + case "Pick": + return node.columns + case "Omit": { + const drop = new Set(node.columns) + return names(node.input)?.filter((n) => !drop.has(n)) + } + case "Rename": + return names(node.input)?.map((n) => node.mapping[n] ?? n) + case "GroupBy": + return [...node.keys, ...node.aggregates.map((a) => a.name)] + case "Aggregate": + return node.aggregates.map((a) => a.name) + case "Join": { + const left = names(node.left) + if (node.how === "semi" || node.how === "anti") return left + const right = names(node.right) + if (left === undefined || right === undefined) return undefined + const on = new Set(node.on) + return [...left, ...right.filter((n) => !on.has(n))] + } + case "Concat": + return node.inputs.length === 0 ? [] : names(node.inputs[0]) + case "Pivot": + return node.columns === undefined ? undefined : [...node.index, ...node.columns] + case "Unnest": + return undefined + case "Unpivot": + return [...node.index, node.variable, node.value] + } +} + +// ----------------------------------------------------------------------------- +// Optimisation +// ----------------------------------------------------------------------------- + +/** The expressions an operation evaluates over its input rows. */ +const expressions = (node: Node): ReadonlyArray => { + switch (node._tag) { + case "Filter": + return [node.predicate] + case "Select": + case "WithColumns": + return node.columns.map((c) => c.expression) + case "Sort": + case "TopK": + return node.keys.map((k) => k.expression) + case "GroupBy": + case "Aggregate": + return node.aggregates.map((a) => a.expression._tag === "AggregateCall" ? a.expression.arg : a.expression) + .filter((e) => e !== undefined) + default: + return [] + } +} + +/** + * Whether an operation's expressions are row-wise (for aggregations, their + * arguments). Only such operations let filters and slices move below them, + * and run batch by batch. + * + * @internal + */ +export const isRowWise = (node: Node): boolean => expressions(node).every(E.isRowWise) + +/** @internal */ +export const optimize = (plan: Node): Node => prune(pushSlices(pushFilters(plan)), undefined) + +const conjuncts = (e: Plan.Expression): Array => + e._tag === "Call" && e.fn === "and" && e.args.length === 2 + ? [...conjuncts(e.args[0]), ...conjuncts(e.args[1])] + : [e] + +const and = (es: ReadonlyArray): Plan.Expression => + es.reduce((a, b) => ({ _tag: "Call", fn: "and", args: [a, b] })) + +const subset = (a: Set, b: ReadonlySet): boolean => { + for (const x of a) if (!b.has(x)) return false + return true +} + +const inverse = (mapping: { readonly [from: string]: string }): Map => + new Map(Object.entries(mapping).map(([from, to]) => [to, from])) + +const pushFilters = (node: Node): Node => { + const optimized = mapChildren(node, pushFilters) + return optimized._tag === "Filter" ? push(optimized.input, conjuncts(optimized.predicate)) : optimized +} + +const filtered = (input: Node, predicates: ReadonlyArray): Node => + predicates.length === 0 ? input : { _tag: "Filter", input, predicate: and(predicates) } + +/** + * Places each predicate as deep as it can go below `node`. Predicates that + * are not row-wise (windows) depend on exactly the rows they see, so a set + * containing one stays where it is. + */ +const push = (node: Node, predicates: ReadonlyArray): Node => { + if (predicates.length === 0) return node + if (!predicates.every(E.isRowWise) || !isRowWise(node)) return filtered(node, predicates) + /** Pushes the predicates reading only columns that pass `ok` into the input, rewritten. */ + const below = (ok: (name: string) => boolean, rewrite = (p: Plan.Expression) => p): Node => { + const down: Array = [] + const here: Array = [] + for (const p of predicates) ([...E.columns(p)].every(ok) ? down : here).push(p) + const input = push((node as { readonly input: Node }).input, down.map(rewrite)) + return filtered({ ...node, input } as Node, here) + } + switch (node._tag) { + case "Filter": + return push(node.input, [...conjuncts(node.predicate), ...predicates]) + case "Sort": + case "DropNulls": + case "Pick": + case "Omit": + return below(() => true) + case "Rename": { + const from = inverse(node.mapping) + return below(() => true, (p) => E.mapColumns(p, (name) => ({ _tag: "Column", name: from.get(name) ?? name }))) + } + case "WithColumns": { + const added = new Set(node.columns.map((c) => c.name)) + return below((r) => !added.has(r)) + } + case "Select": { + const byName = new Map(node.columns.map((c) => [c.name, c.expression])) + return below((r) => byName.get(r)?._tag === "Column", (p) => E.mapColumns(p, (name) => byName.get(name)!)) + } + case "Unique": { + const columns = node.columns + return below((r) => columns === undefined || columns.includes(r)) + } + case "GroupBy": + return below((r) => node.keys.includes(r)) + case "Join": { + const leftNames = names(node.left) + const rightNames = names(node.right) + if (node.how === "full" || leftNames === undefined || rightNames === undefined) return filtered(node, predicates) + const left = new Set(leftNames) + // The join keeps the left side's keys, which the right side's may differ from in type. + const right = new Set(rightNames.filter((n) => !node.on.includes(n))) + const toLeft: Array = [] + const toRight: Array = [] + const here: Array = [] + for (const p of predicates) { + const refs = E.columns(p) + if (subset(refs, left)) toLeft.push(p) + else if (node.how === "inner" && subset(refs, right)) toRight.push(p) + else here.push(p) + } + return filtered({ ...node, left: push(node.left, toLeft), right: push(node.right, toRight) }, here) + } + default: + return filtered(node, predicates) + } +} + +/** One output row per input row, in order, so a slice can move below it. */ +const keepsRows = (node: Node): boolean => + (node._tag === "Select" || node._tag === "WithColumns" || node._tag === "Rename" || node._tag === "Pick" || + node._tag === "Omit") && isRowWise(node) + +const pushSlices = (node: Node): Node => { + const optimized = mapChildren(node, pushSlices) + return optimized._tag === "Slice" ? slice(optimized.input, optimized.offset, optimized.length) : optimized +} + +const slice = (input: Node, offset: number, length: number | undefined): Node => { + if (keepsRows(input)) { + return { ...input, input: slice((input as { readonly input: Node }).input, offset, length) } as Node + } + if (input._tag === "Slice" && input.offset >= 0 && offset >= 0) { + const inner = input.length === undefined ? Infinity : input.length + const remaining = Math.max(0, inner - offset) + const combined = Math.min(remaining, length === undefined ? Infinity : length) + return slice(input.input, input.offset + offset, combined === Infinity ? undefined : combined) + } + if (input._tag === "Sort" && offset >= 0 && length !== undefined) { + const topK: Node = { + _tag: "TopK", + input: input.input, + keys: input.keys, + nullsLast: input.nullsLast, + limit: offset + length + } + return offset === 0 ? topK : { _tag: "Slice", input: topK, offset, length } + } + return { _tag: "Slice", input, offset, length } +} + +type Required = ReadonlySet | undefined + +const union = (required: Required, more: Iterable): Required => { + if (required === undefined) return undefined + const out = new Set(required) + for (const x of more) out.add(x) + return out +} + +const exprColumns = (es: ReadonlyArray): Set => { + const out = new Set() + for (const e of es) E.columns(e, out) + return out +} + +/** Keeps at least one item, so a frame never loses its row count. */ +const keep = (all: ReadonlyArray, wanted: (a: A) => boolean): ReadonlyArray => { + const kept = all.filter(wanted) + return kept.length === 0 && all.length > 0 ? [all[0]] : kept +} + +/** Drops columns nothing reads. `required` undefined means all. */ +const prune = (node: Node, required: Required): Node => { + switch (node._tag) { + case "Values": { + if (required === undefined) return node + const all = names(node)! + const kept = keep(all, (n) => required.has(n)) + return kept.length === all.length ? node : { _tag: "Pick", input: node, columns: kept } + } + case "Table": + return node + case "Filter": + case "Sort": + case "TopK": + case "Slice": + return { ...node, input: prune(node.input, union(required, exprColumns(expressions(node)))) } + case "Unique": + case "DropNulls": + return { + ...node, + input: prune(node.input, node.columns === undefined ? undefined : union(required, node.columns)) + } + case "Select": { + const columns = required === undefined ? node.columns : keep(node.columns, (c) => required.has(c.name)) + return { ...node, columns, input: prune(node.input, exprColumns(columns.map((c) => c.expression))) } + } + case "WithColumns": { + if (required === undefined) return { ...node, input: prune(node.input, undefined) } + const columns = node.columns.filter((c) => required.has(c.name)) + const produced = new Set(columns.map((c) => c.name)) + const below = new Set([...required].filter((n) => !produced.has(n))) + const input = prune(node.input, union(below, exprColumns(columns.map((c) => c.expression)))) + return columns.length === 0 ? input : { ...node, columns, input } + } + case "Pick": { + const columns = required === undefined ? node.columns : keep(node.columns, (c) => required.has(c)) + return { ...node, columns, input: prune(node.input, new Set(columns)) } + } + case "Omit": { + const all = names(node.input) + if (all === undefined) return { ...node, input: prune(node.input, undefined) } + const drop = new Set(node.columns) + const below = keep(all, (n) => !drop.has(n) && (required === undefined || required.has(n))) + return { ...node, input: prune(node.input, new Set(below)) } + } + case "Rename": { + if (required === undefined) return { ...node, input: prune(node.input, undefined) } + const from = inverse(node.mapping) + return { ...node, input: prune(node.input, new Set([...required].map((n) => from.get(n) ?? n))) } + } + case "GroupBy": { + const aggregates = required === undefined + ? node.aggregates + : node.aggregates.filter((a) => required.has(a.name)) + return { + ...node, + aggregates, + input: prune(node.input, new Set([...node.keys, ...exprColumns(aggregates.map((a) => a.expression))])) + } + } + case "Aggregate": { + const aggregates = required === undefined + ? node.aggregates + : keep(node.aggregates, (a) => required.has(a.name)) + return { ...node, aggregates, input: prune(node.input, exprColumns(aggregates.map((a) => a.expression))) } + } + case "Join": { + const side = (input: Node): Required => { + const all = names(input) + if (required === undefined || all === undefined) return undefined + const available = new Set(all) + return new Set([...node.on, ...[...required].filter((n) => available.has(n))]) + } + const right = node.how === "semi" || node.how === "anti" ? new Set(node.on) : side(node.right) + return { ...node, left: prune(node.left, side(node.left)), right: prune(node.right, right) } + } + case "Concat": + return { ...node, inputs: node.inputs.map((input) => prune(input, required)) } + case "Pivot": + return { ...node, input: prune(node.input, new Set([...node.index, node.on, node.values])) } + case "Explode": + return { ...node, input: prune(node.input, union(required, [node.column])) } + case "Unnest": + return { ...node, input: prune(node.input, undefined) } + case "Unpivot": + return { ...node, input: prune(node.input, new Set([...node.index, ...node.on])) } + } +} + +// ----------------------------------------------------------------------------- +// Explaining +// ----------------------------------------------------------------------------- + +const list = (items: ReadonlyArray) => `[${items.join(", ")}]` +const named = (items: ReadonlyArray) => list(items.map((c) => `${c.name} = ${E.show(c.expression)}`)) +const keys = (items: ReadonlyArray) => + list(items.map((k) => `${k.descending ? "desc" : "asc"}(${E.show(k.expression)})`)) + +const line = (node: Node): string => { + switch (node._tag) { + case "Values": { + const f = frameOf(node) + return `Values ${f.length} rows ${list(f.columns.map((c) => c.name))}` + } + case "Table": + return `Table ${node.name}${node.columns === undefined ? "" : ` ${list(node.columns.map((c) => c.name))}`}` + case "Filter": + return `Filter ${E.show(node.predicate)}` + case "Select": + case "WithColumns": + return `${node._tag} ${named(node.columns)}` + case "Sort": + return `Sort ${keys(node.keys)}${node.nullsLast ? " nulls last" : ""}` + case "TopK": + return `TopK ${node.limit} ${keys(node.keys)}${node.nullsLast ? " nulls last" : ""}` + case "Slice": + return `Slice offset ${node.offset}${node.length === undefined ? "" : ` length ${node.length}`}` + case "Unique": + case "DropNulls": + return `${node._tag}${node.columns === undefined ? "" : ` ${list(node.columns)}`}` + case "Pick": + case "Omit": + return `${node._tag} ${list(node.columns)}` + case "Rename": + return `Rename ${list(Object.entries(node.mapping).map(([a, b]) => `${a} -> ${b}`))}` + case "GroupBy": + return `GroupBy ${list(node.keys)} ${named(node.aggregates)}` + case "Aggregate": + return `Aggregate ${named(node.aggregates)}` + case "Join": + return `Join ${node.how} on ${list(node.on)}` + case "Concat": + return `Concat` + case "Pivot": + return `Pivot on ${node.on} index ${list(node.index)} ${node.aggregate}(${node.values})${ + node.columns === undefined ? "" : ` columns ${list(node.columns)}` + }` + case "Unpivot": + return `Unpivot ${list(node.on)} index ${list(node.index)} as ${node.variable}, ${node.value}` + case "Explode": + case "Unnest": + return `${node._tag} ${node.column}` + } +} + +/** A plan as an indented tree, one operation per line. @internal */ +export const show = (node: Node | Plan.Node, indent = ""): string => + [indent + line(node as Node), ...children(node as Node).map((c) => show(c, indent + " "))].join("\n") + +// ----------------------------------------------------------------------------- +// Execution +// ----------------------------------------------------------------------------- + +const exprs = (items: ReadonlyArray): { [name: string]: Plan.Expression } => { + const out: { [name: string]: Plan.Expression } = {} + for (const c of items) out[c.name] = c.expression + return out +} + +const sortKeys = (items: ReadonlyArray) => + items.map((k) => ({ node: k.expression, descending: k.descending })) + +/** Runs one operation over its inputs. @internal */ +export const step = (node: Node, inputs: ReadonlyArray): F.Frame => { + const input = inputs[0] + switch (node._tag) { + case "Values": + return frameOf(node) + case "Table": + return V.fail(`table ${node.name} is not resolved`) + case "Filter": + return F.filter(input, node.predicate) + case "Select": + return F.select(input, exprs(node.columns)) + case "WithColumns": + return F.withColumns(input, exprs(node.columns)) + case "Sort": + return F.sort(input, sortKeys(node.keys), node.nullsLast) + case "TopK": + return F.sort(input, sortKeys(node.keys), node.nullsLast, node.limit) + case "Slice": { + const n = input.length + const from = node.offset < 0 ? n + node.offset : node.offset + const start = Math.min(Math.max(from, 0), n) + const end = node.length === undefined ? n : Math.min(Math.max(from + node.length, start), n) + return F.slice(input, start, end) + } + case "Unique": + return F.unique(input, node.columns) + case "DropNulls": + return F.dropNulls(input, node.columns) + case "Pick": + return F.pick(input, node.columns) + case "Omit": + return F.omit(input, node.columns) + case "Rename": + return F.rename(input, node.mapping) + case "GroupBy": + return F.groupBy(input, node.keys, exprs(node.aggregates)) + case "Aggregate": + return F.aggregate(input, exprs(node.aggregates)) + case "Join": + return F.join(inputs[0], inputs[1], node.on, node.how) + case "Concat": + return inputs.length === 0 ? V.fail("concat needs at least one input") : F.concat(inputs) + case "Pivot": + return F.pivot(input, node) + case "Unpivot": + return F.unpivot(input, node) + case "Explode": + return F.explode(input, node.column) + case "Unnest": + return F.unnest(input, node.column) + } +} + +/** Yield to other fibers when an operation ran for longer than this (ms). */ +const SLICE = 10 + +/** Runs a resolved plan, yielding between long-running operations. @internal */ +export const execute = (plan: Node): Effect.Effect => { + let last = performance.now() + const run = (node: Node): Effect.Effect => + Effect.flatMap( + Effect.forEach(children(node), run), + (inputs) => { + const result = attempt(() => step(node, inputs)) + const now = performance.now() + if (now - last < SLICE) return result + last = now + return Effect.tap(result, () => Effect.yieldNow) + } + ) + return run(plan) +} + +/** + * Resolves, optimises and runs a plan. Plans that read streamed tables run + * batch by batch (see `stream.ts`). + * + * @internal + */ +export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { + const streaming = isStreaming(plan, catalog) + const result = streaming + ? Streaming.collect(plan, catalog) + : resolve(plan, catalog).pipe( + Effect.flatMap((resolved) => attempt(() => optimize(resolved))), + Effect.flatMap(execute) + ) + return result.pipe(Effect.withSpan("dataframe.Query", { + attributes: { "dataframe.operations": count(plan), ...(streaming ? { "dataframe.streaming": true } : {}) } + })) +} + +/** + * The optimised plan that runs, as text. Plans over streamed tables show + * them as `Table` nodes, without reading them. + * + * @internal + */ +export const explain = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => + Effect.flatMap( + isStreaming(plan, catalog) ? Effect.succeed(plan as Node) : resolve(plan, catalog), + (resolved) => attempt(() => show(optimize(resolved))) + ) + +const isStreaming = (plan: Plan.Node, catalog: Catalog.Service): boolean => + [...tables(plan).keys()].some(catalog.streaming) + +const count = (node: Plan.Node): number => 1 + children(node as Node).reduce((n, c) => n + count(c as Plan.Node), 0) diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts new file mode 100644 index 00000000000..c88321a1747 --- /dev/null +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -0,0 +1,378 @@ +/** + * Effect Schema at the boundary of frames: the declared columns of a struct + * schema's fields, and its checks compiled to plain tests. + * + * A schema is read once, when a frame is built from rows or validated, or a + * table is declared (`declare`). Everything after that works on the declared + * columns and the compiled tests (see `./table.ts`), without the schema. + * + * Frames hold decoded values, so a schema describes the columns of its type + * side and must not transform: a node with an encoding fails. Numbers are + * Float64, timestamps and durations microseconds, unless an override says + * otherwise, and decimals need one. `DateTimeZoned` fields have no column + * type. + * + * @internal + */ +import type * as S from "../../arrow/ArrowSchema.ts" +import * as Values from "../../arrow/internal/values.ts" +import { identity } from "../../Function.ts" +import * as Result from "../../Result.ts" +import * as AST from "../../SchemaAST.ts" +import * as Parser from "../../SchemaParser.ts" +import type * as Plan from "../Plan.ts" +import * as T from "./table.ts" +import * as V from "./vector.ts" + +type Column = Omit + +const leaf = (type: Plan.TypeName, nullable = false): Column => ({ type, nullable }) + +const literalType = (literal: unknown): Plan.TypeName | undefined => { + switch (typeof literal) { + case "string": + return "Utf8" + case "number": + return "Float64" + case "boolean": + return "Bool" + case "bigint": + return "Int64" + default: + return undefined + } +} + +const sameShape = (a: Column, b: Column): boolean => + a.type === b.type && JSON.stringify(a.fields ?? []) === JSON.stringify(b.fields ?? []) + +const suspended = new Set() + +/** `f` of a suspended schema, failing (after `prefix`) when it recurses. */ +const unsuspend = (ast: AST.Suspend, prefix: string, f: (ast: AST.AST) => A): A => { + if (suspended.has(ast)) return V.fail(`${prefix}recursive schemas have no DataFrame column type`) + suspended.add(ast) + try { + return f(ast.thunk()) + } finally { + suspended.delete(ast) + } +} + +const columnOf = (ast: AST.AST, path: string): Column => { + switch (ast._tag) { + case "String": + case "TemplateLiteral": + return leaf("Utf8") + case "Number": + return leaf("Float64") + case "Boolean": + return leaf("Bool") + case "BigInt": + return leaf("Int64") + case "Null": + case "Undefined": + return leaf("Null", true) + case "Literal": { + const type = literalType(ast.literal) + return type === undefined ? unsupported(ast, path) : leaf(type) + } + case "Enum": { + const types = new Set(ast.enums.map(([, v]) => literalType(v))) + return types.size === 1 ? leaf([...types][0]!) : unsupported(ast, path) + } + case "Union": { + let column: Column | undefined + let nullable = false + for (const member of ast.types) { + const m = columnOf(member, path) + nullable ||= m.nullable + if (m.type === "Null") continue + if (column === undefined) column = m + else if (!sameShape(column, m)) return unsupported(ast, path) + } + return { ...(column ?? leaf("Null")), nullable } + } + case "Suspend": + return unsuspend(ast, `field ${JSON.stringify(path)}: `, (ast) => columnOf(ast, path)) + case "Arrays": { + const element = ast.rest[0] + if (ast.elements.length > 0 || element === undefined || ast.rest.length > 1) return unsupported(ast, path) + return { type: "List", nullable: false, fields: [{ name: "item", ...columnOf(element, `${path}[]`) }] } + } + case "Objects": + return ast.indexSignatures.length > 0 + ? unsupported(ast, path) + : { type: "Struct", nullable: false, fields: fieldsOf(ast, path) } + case "Declaration": { + const id = (ast.annotations as { readonly representation?: { readonly id?: unknown } } | undefined) + ?.representation?.id + if (id === "effect/schema/DateTimeUtc") return leaf("Timestamp") + if (id === "effect/schema/Duration") return leaf("Duration") + if (id === "effect/schema/BigDecimal") return leaf("Decimal") + return unsupported(ast, path) + } + default: + return unsupported(ast, path) + } +} + +const unsupported = (ast: AST.AST, path: string): never => + V.fail(`field ${JSON.stringify(path)}: schemas of kind ${ast._tag} have no DataFrame column type yet`) + +const fieldsOf = (ast: AST.Objects, path: string): Array => + ast.propertySignatures.map((ps) => { + const name = String(ps.name) + const column = columnOf(ps.type, T.field(path, name)) + return { name, ...column, nullable: column.nullable || AST.isOptional(ps.type) } + }) + +/** The struct a schema describes its rows by. */ +const structOf = (ast: AST.AST): AST.Objects => { + const type = AST.toType(ast) + return type._tag === "Objects" && type.indexSignatures.length === 0 + ? type + : V.fail("DataFrame rows must be described by a struct schema") +} + +const transformed = (path: string): never => + V.fail( + `${path === "" ? "the schema" : `field ${JSON.stringify(path)}`} has a transformation: ` + + "frames hold decoded values and transformations are not run; Schema.toType(schema) describes the decoded rows" + ) + +/** + * Fails when a schema on the way to a column has an encoding: its decoded + * values differ from the encoded ones, and frames run no transformation. + */ +const rejectEncodings = (ast: AST.AST, path: string): void => { + if (ast.encoding !== undefined) return transformed(path) + switch (ast._tag) { + case "Objects": + return ast.propertySignatures.forEach((ps) => rejectEncodings(ps.type, T.field(path, String(ps.name)))) + case "Arrays": + return [...ast.elements, ...ast.rest].forEach((item) => rejectEncodings(item, `${path}[]`)) + case "Union": + return ast.types.forEach((member) => rejectEncodings(member, path)) + case "Suspend": + return unsuspend(ast, `field ${JSON.stringify(path)}: `, (ast) => rejectEncodings(ast, path)) + } +} + +// ----------------------------------------------------------------------------- +// Missing values +// ----------------------------------------------------------------------------- + +type Missing = "null" | "undefined" | "absent" + +/** + * How a schema writes the values a frame stores. A frame holds `null` for + * every missing value, and a schema writes one as `null`, as `undefined` + * or as an absent key: `missing` is the one it writes where values can be + * missing, for a list's items and a struct's fields too. + */ +interface Shape { + readonly missing?: Missing | undefined + readonly item?: Shape | undefined + readonly fields?: ReadonlyArray | undefined +} + +const written: { readonly [M in Missing]: string } = { null: "null", undefined: "undefined", absent: "an absent key" } + +/** + * The shape of the values of `ast`. `optional` says that `ast` is the type + * of an optional key. `seen` says that a check of a struct, array or union + * around the values is handed them. + * + * A check must be handed what the schema describes, so a missing value it + * sees needs one way of being written. For an optional field it is an + * absent key, whether or not its type has `undefined` too. Fails when a + * check sees a missing value the schema writes as `null` and another way. + */ +const shapeOf = (ast: AST.AST, path: string, seen: boolean, optional = false): Shape => { + const ways = new Set(optional ? ["absent"] : []) + const members: Array = [] + // Whether a check inside this field is handed a missing value. + let own = false + const visit = (ast: AST.AST, checked: boolean): void => { + checked ||= ast.checks !== undefined + switch (ast._tag) { + case "Null": + case "Undefined": + ways.add(ast._tag === "Null" ? "null" : "undefined") + own ||= checked + return + case "Suspend": + return unsuspend(ast, "", (inner) => visit(inner, checked)) + case "Union": + // Matching exactly one member tells the members apart, as a check does. + return ast.types.forEach((member) => visit(member, checked || ast.options?.mode === "oneOf")) + case "Arrays": + members.push({ item: shapeOf(ast.rest[0], `${path}[]`, seen || checked) }) + return + case "Objects": + members.push({ + fields: ast.propertySignatures.map((ps) => { + const name = String(ps.name) + return [name, shapeOf(ps.type, T.field(path, name), seen || checked, AST.isOptional(ps.type))] as const + }) + }) + } + } + visit(ast, false) + if (ways.size > 1 && ways.has("null") && (own || seen)) { + V.fail( + `field ${JSON.stringify(path)}: a check sees a missing value that the schema writes in several ways (${ + [...ways].map((way) => written[way]).join(", ") + }), and a frame holds null for each of them` + ) + } + if (new Set(members.map((member) => JSON.stringify(member))).size > 1) { + V.fail( + `field ${JSON.stringify(path)}: the members of a union write missing values in different ways, ` + + "and a frame holds null for each of them" + ) + } + const missing = ways.has("null") + ? "null" + : ways.has("absent") + ? "absent" + : ways.has("undefined") + ? "undefined" + : undefined + return { missing, ...members[0] } +} + +/** + * The function that turns a value a frame stores for `shape` into the form + * the schema writes. `undefined` when the frame stores that form already. + */ +const restorer = (shape: Shape): ((value: unknown) => unknown) | undefined => { + const present = shape.item !== undefined + ? items(shape.item) + : shape.fields !== undefined + ? struct(shape.fields) + : undefined + const missing = shape.missing === "undefined" ? undefined : null + if (present === undefined && missing === null) return undefined + return (value) => value === null ? missing : present === undefined ? value : present(value) +} + +const items = (shape: Shape): ((value: unknown) => unknown) | undefined => { + const item = restorer(shape) + return item && ((value) => (value as ReadonlyArray).map(item)) +} + +const struct = (shapes: NonNullable): ((value: unknown) => unknown) | undefined => { + const fields = shapes.map(([name, shape]) => [name, shape.missing === "absent", restorer(shape)] as const) + if (fields.every(([, absent, restore]) => !absent && restore === undefined)) return undefined + return (value) => { + const out: Record = {} + for (const [name, absent, restore] of fields) { + const v = Values.getProperty(value as object, name) + if (v === null && absent) continue + Values.setProperty(out, name, restore === undefined ? v : restore(v)) + } + return out + } +} + +// ----------------------------------------------------------------------------- +// Checks +// ----------------------------------------------------------------------------- + +/** + * Whether a schema constrains its values beyond their column type and + * nullability, anywhere inside it: by a check, a literal, an enum, a + * template literal, or a union that takes the values of exactly one member. + */ +const constrains = (ast: AST.AST): boolean => { + if (ast.checks !== undefined) return true + switch (ast._tag) { + case "Literal": + case "Enum": + case "TemplateLiteral": + return true + case "Suspend": + return unsuspend(ast, "", constrains) + case "Union": + return ast.options?.mode === "oneOf" || ast.types.some(constrains) + case "Arrays": + return constrains(ast.rest[0]) + case "Objects": + return ast.propertySignatures.some((ps) => constrains(ps.type)) + default: + return false + } +} + +/** + * The tests of a struct schema: one for each field whose values it + * constrains beyond their column type, and one of whole rows when the struct + * has checks of its own. `undefined` when the column types say it all. + * + * The tests take values and rows as a frame stores them, and hand them to + * the schema as it writes them (see {@link shapeOf}). A field's test is the + * schema's own: a value passes when the field's type has it, so checks run + * on every value that reaches them (`null` too) and unions match as they + * do in Schema. An absent key has no value to test. + */ +const checksOf = (ast: AST.Objects): Plan.TableChecks | undefined => { + const checks = ast.checks + const shapes: Array = [] + const columns = ast.propertySignatures.flatMap((ps) => { + const name = String(ps.name) + const constrained = constrains(ps.type) + if (!constrained && checks === undefined) return [] + const shape = shapeOf(ps.type, name, checks !== undefined, AST.isOptional(ps.type)) + shapes.push([name, shape]) + if (!constrained) return [] + const is = Parser._is(ps.type) + const restore = restorer(shape) ?? identity + const absent = shape.missing === "absent" + return [[name, (value: unknown) => (absent && value === null) || is(restore(value))] as const] + }) + // When the struct has checks, `shapes` holds every field. + const restore = checks && struct(shapes) + const rows = checks && + ((row: unknown) => Result.isSuccess(AST.runChecks(checks, restore === undefined ? row : restore(row)))) + return columns.length === 0 && rows === undefined ? undefined : { columns, rows } +} + +// ----------------------------------------------------------------------------- +// Declarations +// ----------------------------------------------------------------------------- + +/** + * A table as declared by a schema: its columns, and the tests of the values + * and rows the columns' types don't rule out. + * + * @internal + */ +export interface Declared { + readonly columns: Array + readonly checks: Plan.TableChecks | undefined +} + +/** + * The table a struct schema declares: the columns of its fields, with + * `overrides` for the Arrow types of some of them, checked against the + * schema, and its checks compiled. Decimals have no default precision and + * scale. A schema that transforms its rows fails. + * + * An Arrow schema declares its fields as columns, without checks: no Effect + * Schema is involved. + * + * @internal + */ +export const declare = ( + schema: { readonly ast: AST.AST } | S.Schema, + overrides?: { readonly [name: string]: S.DataType | undefined } | undefined +): Declared => { + if (!("ast" in schema)) return { columns: T.fromArrowSchema(schema), checks: undefined } + const struct = structOf(schema.ast) + const columns = fieldsOf(struct, "").map((c) => T.declareColumn(c.name, c, overrides?.[c.name])) + // After the columns: a decoded side without column types fails for that first. + rejectEncodings(schema.ast, "") + return { columns, checks: checksOf(struct) } +} diff --git a/packages/effect/src/dataframe/internal/sort.ts b/packages/effect/src/dataframe/internal/sort.ts new file mode 100644 index 00000000000..816f54a905f --- /dev/null +++ b/packages/effect/src/dataframe/internal/sort.ts @@ -0,0 +1,276 @@ +/** + * Stable multi-key sorting. + * + * Keys are sorted least significant first, each with a stable pass, so the + * result is ordered by the first key, then the second, and so on. Numeric + * keys turn into unsigned words whose order matches the key order (Float64 + * keys: NaN last, `-0` equal to `0`) and go through an LSD radix sort with + * 16-bit digits. String keys are ranked: their distinct values are sorted + * once by UTF-8 bytes and the rows radix-sorted by rank. + * + * @internal + */ +import * as Hash from "./hash.ts" +import { compareBytes } from "./kernels.ts" +import * as V from "./vector.ts" + +/** @internal */ +export interface SortKey { + readonly vector: V.Vector + readonly descending: boolean +} + +const SMALL = 512 + +/** The row order that sorts `keys`; nulls first unless `nullsLast`. @internal */ +export const permutation = (keys: ReadonlyArray, nullsLast: boolean, n: number): Int32Array => { + let perm: Int32Array = new Int32Array(n) + for (let i = 0; i < n; i++) perm[i] = i + if (n < 2) return perm + if (n <= SMALL) return smallSort(keys, nullsLast, perm) + for (let k = keys.length - 1; k >= 0; k--) perm = sortByKey(keys[k], nullsLast, perm) + return perm +} + +// ----------------------------------------------------------------------------- +// Keys as unsigned words +// ----------------------------------------------------------------------------- + +/** One or two 32-bit words per row (most significant first), in row order. @internal */ +export const words = (key: SortKey): { readonly hi: Uint32Array | undefined; readonly lo: Uint32Array } => { + const v = key.vector + const n = v.length + if (v.kind === "list" || v.kind === "struct") return V.fail(`cannot sort by ${v.type._tag} columns`) + const flip = key.descending ? 0xffffffff : 0 + switch (v.kind) { + case "i32": { + const lo = new Uint32Array(n) + const x = v.values + for (let i = 0; i < n; i++) lo[i] = ((x[i] ^ 0x80000000) ^ flip) >>> 0 + return { hi: undefined, lo } + } + case "bool": { + const lo = new Uint32Array(n) + const x = v.values + for (let i = 0; i < n; i++) lo[i] = (x[i] ^ flip) >>> 0 + return { hi: undefined, lo } + } + case "f64": { + const bits = new Uint32Array(Hash.canonicalBits(v.values, n).buffer) + const hi = new Uint32Array(n) + const lo = new Uint32Array(n) + for (let i = 0; i < n; i++) { + let l = bits[2 * i] + let h = bits[2 * i + 1] + if ((h & 0x80000000) !== 0) { + h = ~h + l = ~l + } else h ^= 0x80000000 + hi[i] = (h ^ flip) >>> 0 + lo[i] = (l ^ flip) >>> 0 + } + return { hi, lo } + } + case "i64": { + const bits = new Uint32Array(v.values.buffer, v.values.byteOffset, n * 2) + const signFlip = V.isUnsigned64(v.type) ? 0 : 0x80000000 + const hi = new Uint32Array(n) + const lo = new Uint32Array(n) + for (let i = 0; i < n; i++) { + hi[i] = ((bits[2 * i + 1] ^ signFlip) ^ flip) >>> 0 + lo[i] = (bits[2 * i] ^ flip) >>> 0 + } + return { hi, lo } + } + case "str": { + const g = Hash.ids(v) + const off = v.offsets + const d = v.data + const sorted = Int32Array.from({ length: g.count }, (_, i) => i).sort((a, b) => { + const ra = g.firstRow[a] + const rb = g.firstRow[b] + return compareBytes(d, off[ra], off[ra + 1], d, off[rb], off[rb + 1]) + }) + const rank = new Uint32Array(g.count) + for (let r = 0; r < sorted.length; r++) rank[sorted[r]] = r + const lo = new Uint32Array(n) + for (let i = 0; i < n; i++) lo[i] = (rank[g.ids[i]] ^ flip) >>> 0 + return { hi: undefined, lo } + } + case "null": + return { hi: undefined, lo: new Uint32Array(n) } + } +} + +// ----------------------------------------------------------------------------- +// Radix passes +// ----------------------------------------------------------------------------- + +const counts = new Uint32Array(65537) + +/** + * Stable LSD radix sort of `perm` by `keys` (in perm order), 16 bits per + * pass; skips passes whose digit is constant. + */ +const radix = (perm: Int32Array, hi: Uint32Array | undefined, lo: Uint32Array): Int32Array => { + const n = perm.length + let p: Int32Array = perm + let h: Uint32Array | undefined = hi + let l: Uint32Array = lo + let p2: Int32Array = new Int32Array(n) + let h2: Uint32Array | undefined = hi === undefined ? undefined : new Uint32Array(n) + let l2: Uint32Array = new Uint32Array(n) + const passes: Array<[boolean, number]> = [[false, 0], [false, 16]] + if (hi !== undefined) passes.push([true, 0], [true, 16]) + for (const [useHi, shift] of passes) { + const src = useHi ? h! : l + counts.fill(0) + for (let i = 0; i < n; i++) counts[((src[i] >>> shift) & 0xffff) + 1]++ + let constant = false + for (let b = 1; b <= 65536; b++) { + if (counts[b] === n) { + constant = true + break + } + if (counts[b] !== 0) break + } + if (constant) continue + for (let b = 1; b <= 65536; b++) counts[b] += counts[b - 1] + for (let i = 0; i < n; i++) { + const at = counts[(src[i] >>> shift) & 0xffff]++ + p2[at] = p[i] + l2[at] = l[i] + if (h !== undefined) h2![at] = h[i] + } + ;[p, p2] = [p2, p] + ;[l, l2] = [l2, l] + if (h !== undefined) [h, h2] = [h2, h] + } + return p +} + +const sortByKey = (key: SortKey, nullsLast: boolean, perm: Int32Array): Int32Array => { + const n = perm.length + const w = words(key) + // The key's words in the current perm order. + const lo = new Uint32Array(n) + const hi = w.hi === undefined ? undefined : new Uint32Array(n) + for (let i = 0; i < n; i++) { + const r = perm[i] + lo[i] = w.lo[r] + if (hi !== undefined) hi[i] = w.hi![r] + } + const sorted = radix(perm, hi, lo) + return partitionNulls(key.vector, sorted, nullsLast) +} + +/** Moves null keys to the front (or back), keeping order on both sides. */ +const partitionNulls = (v: V.Vector, perm: Int32Array, nullsLast: boolean): Int32Array => { + if (v.kind !== "null" && v.valid === undefined) return perm + const n = perm.length + const out = new Int32Array(n) + let nulls = 0 + for (let i = 0; i < n; i++) if (!V.isValid(v, perm[i])) nulls++ + let a = nullsLast ? 0 : nulls + let b = nullsLast ? n - nulls : 0 + for (let i = 0; i < n; i++) { + const r = perm[i] + if (V.isValid(v, r)) out[a++] = r + else out[b++] = r + } + return out +} + +// ----------------------------------------------------------------------------- +// Small inputs: one comparator sort +// ----------------------------------------------------------------------------- + +/** Compares two rows by the keys; ties go to the earlier row. */ +const comparator = (keys: ReadonlyArray, nullsLast: boolean): (a: number, b: number) => number => { + const prepared = keys.map((k) => ({ k, w: words(k) })) + return (a, b) => { + for (const { k, w } of prepared) { + const av = V.isValid(k.vector, a) + const bv = V.isValid(k.vector, b) + if (!av || !bv) { + if (av === bv) continue + return (av ? 1 : -1) * (nullsLast ? -1 : 1) + } + if (w.hi !== undefined && w.hi[a] !== w.hi[b]) return w.hi[a] < w.hi[b] ? -1 : 1 + if (w.lo[a] !== w.lo[b]) return w.lo[a] < w.lo[b] ? -1 : 1 + } + return a - b + } +} + +const smallSort = (keys: ReadonlyArray, nullsLast: boolean, perm: Int32Array): Int32Array => + perm.sort(comparator(keys, nullsLast)) + +/** + * The first `k` rows of the order that sorts `keys`, without sorting the + * rest: one pass that keeps the best `k` rows in a heap. + * + * @internal + */ +export const topK = (keys: ReadonlyArray, nullsLast: boolean, n: number, k: number): Int32Array => { + if (k <= 0) return new Int32Array(0) + if (n <= SMALL || k * 8 > n) return permutation(keys, nullsLast, n).subarray(0, Math.min(k, n)) + const cmp = comparator(keys, nullsLast) + // A max-heap: the root is the worst of the best `k` rows so far. + const heap = new Int32Array(k) + const down = (i: number) => { + while (true) { + const l = 2 * i + 1 + if (l >= k) return + const r = l + 1 + const c = r < k && cmp(heap[r], heap[l]) > 0 ? r : l + if (cmp(heap[c], heap[i]) <= 0) return + const t = heap[c] + heap[c] = heap[i] + heap[i] = t + i = c + } + } + for (let i = 0; i < k; i++) heap[i] = i + for (let i = (k >> 1) - 1; i >= 0; i--) down(i) + for (let i = k; i < n; i++) { + if (cmp(i, heap[0]) < 0) { + heap[0] = i + down(0) + } + } + return heap.sort(cmp) +} + +// ----------------------------------------------------------------------------- +// Row comparison +// ----------------------------------------------------------------------------- + +/** `order(i, j) < 0` when row i sorts before row j (valid, non-NaN rows). @internal */ +export const rowOrder = (v: V.Vector): (i: number, j: number) => number => { + switch (v.kind) { + case "list": + case "struct": + return V.fail(`cannot order ${v.type._tag} values`) + case "f64": { + const x = v.values + return (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) + } + case "i32": + case "bool": { + const x = v.values + return (i, j) => x[i] - x[j] + } + case "i64": { + const x = V.isUnsigned64(v.type) ? new BigUint64Array(v.values.buffer, v.values.byteOffset, v.length) : v.values + return (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) + } + case "str": { + const o = v.offsets + const d = v.data + return (i, j) => compareBytes(d, o[i], o[i + 1], d, o[j], o[j + 1]) + } + case "null": + return () => 0 + } +} diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts new file mode 100644 index 00000000000..1714b9356e6 --- /dev/null +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -0,0 +1,1372 @@ +/** + * Compiling plans to SQL. + * + * Every relation the compiler builds carries a hidden row column: a key + * that increases in row order. Operations keep it (filters, projections), + * redefine it (sorts, joins, concatenations: `ROW_NUMBER()` over the new + * order) or derive it (groups: the first row's key), and the final query + * orders by it. This reproduces the row order of the in-memory engine: + * stable sorts, groups in order of first appearance, joins in left row + * order, `first`/`last` by row order, windows over rows in order. + * + * Column types come from the in-memory engine: every relation carries the + * frame the engine gives for its plan on zero-row inputs, so the compiler + * knows the engine's type of every column and expression. The final query + * renders every column as text (temporal values as integers in their unit), + * and `decode` reads the rows back as columns of those types. + * + * Invented names (the row key, table sources) share a per-plan prefix that + * no name of the plan starts with (see {@link inventor}). PostgreSQL keeps + * the first 63 bytes of a name (see {@link identifier}). + * + * @internal + */ +import * as ArrowColumn from "../../arrow/ArrowColumn.ts" +import * as S from "../../arrow/ArrowSchema.ts" +import * as BigDecimal from "../../BigDecimal.ts" +import * as Result from "../../Result.ts" +import type * as Plan from "../Plan.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as E from "./expr.ts" +import * as F from "./frame.ts" +import * as K from "./kernels.ts" +import * as Q from "./query.ts" +import { arrowType } from "./table.ts" +import { civil, coarser } from "./temporal.ts" +import * as V from "./vector.ts" + +interface Compiler { + /** The engine's frame for every node of the plan, on zero-row inputs. */ + readonly frames: ReadonlyMap + /** A quoted invented name, disjoint from the plan's table and column names (see {@link inventor}). */ + readonly hidden: (name: string) => string + /** The quoted row key. */ + readonly row: string + /** The quoted name of the source of each table name, as PostgreSQL keeps it (see {@link source}). */ + readonly sources: ReadonlyMap +} + +const quote = (name: string): string => `"${name.replaceAll(`"`, `""`)}"` + +const list = (names: ReadonlyArray) => names.map(quote).join(", ") + +const unsupported = (what: string): never => V.fail(`${what} is not supported by the SQL engine`) + +/** + * A decimal scale PostgreSQL's `numeric` can declare (-1000 to 1000). The + * SQL for a decimal writes out as many digits as its scale. + */ +const numericScale = (scale: number): number => + scale >= -1000 && scale <= 1000 ? scale : unsupported(`a decimal scale of ${scale}`) + +// ----------------------------------------------------------------------------- +// Types and literals +// ----------------------------------------------------------------------------- + +const sqlType = (t: S.DataType): string => { + switch (t._tag) { + case "Int": + if (t.bitWidth <= 16) return t.signed ? "SMALLINT" : "INTEGER" + if (t.bitWidth === 32) return t.signed ? "INTEGER" : "BIGINT" + return t.signed ? "BIGINT" : "NUMERIC(20, 0)" + case "Float": + return t.bitWidth === 32 ? "REAL" : "DOUBLE PRECISION" + case "Utf8": + case "LargeUtf8": + case "Utf8View": + return "TEXT" + case "Binary": + case "LargeBinary": + case "BinaryView": + return "BYTEA" + case "Bool": + return "BOOLEAN" + case "Date": + return "DATE" + case "Timestamp": + return t.timezone === undefined ? "TIMESTAMP" : "TIMESTAMPTZ" + case "Duration": + return "INTERVAL" + case "Decimal": + return `DECIMAL(${t.precision}, ${numericScale(t.scale)})` + case "Null": + return "INTEGER" + default: + return unsupported(`columns of type ${t._tag}`) + } +} + +const string = (s: string): string => `'${s.replaceAll("'", "''")}'` + +const pad = (n: number, width = 2) => String(n).padStart(width, "0") + +const perSecond: Record = { + second: BigInt(1), + millisecond: BigInt(1000), + microsecond: BigInt(1_000_000), + nanosecond: BigInt(1_000_000_000) +} + +const floorDiv = (a: bigint, b: bigint) => (a < BigInt(0) ? -((-a + b - BigInt(1)) / b) : a / b) + +const timestampText = (value: bigint, unit: Plan.TimeUnit): string => { + const per = perSecond[unit] + const perDay = per * BigInt(86400) + const days = floorDiv(value, perDay) + const rem = value - days * perDay + const [y, m, d] = civil(Number(days)) + const seconds = Number(rem / per) + const micros = Number((rem % per) * BigInt(1_000_000) / per) + return `${pad(y, 4)}-${pad(m)}-${pad(d)} ${pad(Math.floor(seconds / 3600))}:${pad(Math.floor(seconds / 60) % 60)}:${ + pad(seconds % 60) + }.${pad(micros, 6)}` +} + +const NAN = "CAST('NaN' AS DOUBLE PRECISION)" + +const literal = (value: unknown, type: Plan.LiteralType | undefined): string => { + if (value === null || value === undefined) return "NULL" + if (type !== undefined) { + const big = BigInt(value as number | bigint) + switch (type._tag) { + case "Date": { + const [y, m, d] = civil(Number(big)) + return `DATE '${pad(y, 4)}-${pad(m)}-${pad(d)}'` + } + case "Timestamp": + return type.timezone === undefined + ? `TIMESTAMP '${timestampText(big, type.unit)}'` + : `TIMESTAMPTZ '${timestampText(big, type.unit)}+00'` + case "Duration": + return fromInteger(`CAST(${big} AS BIGINT)`, S.duration(type.unit)) + case "Decimal": { + const scale = numericScale(type.scale) + return `CAST('${Dec.format(big, scale)}' AS DECIMAL(${type.precision}, ${scale}))` + } + } + } + switch (typeof value) { + case "number": + if (Number.isNaN(value)) return NAN + if (!Number.isFinite(value)) return `CAST('${value > 0 ? "" : "-"}Infinity' AS DOUBLE PRECISION)` + if (Object.is(value, -0)) return `CAST('-0' AS DOUBLE PRECISION)` + return Number.isInteger(value) ? String(value) : `CAST(${value} AS DOUBLE PRECISION)` + case "bigint": + return `CAST(${value} AS BIGINT)` + case "boolean": + return value ? "TRUE" : "FALSE" + case "string": + return string(value) + default: + return unsupported(`literals like ${String(value)}`) + } +} + +// ----------------------------------------------------------------------------- +// Temporal values and casts +// ----------------------------------------------------------------------------- + +type Temporal = S.Date | S.Timestamp | S.Duration + +const isTemporal = (t: S.DataType): t is Temporal => + t._tag === "Date" || t._tag === "Timestamp" || t._tag === "Duration" + +/** The integer type of a temporal type's values. */ +const integerOf = (t: Temporal): S.Int => (t._tag === "Date" && t.unit === "day" ? S.int32 : S.int64) + +/** + * A temporal value as its integer: its `unit`s (its own by default) since + * the epoch, rounded down, or its length in them, rounded toward zero. + */ +const toInteger = (x: string, type: Temporal, unit?: Plan.TimeUnit): string => { + if (type._tag !== "Date") { + const rounding = type._tag === "Duration" ? "TRUNC" : "FLOOR" + return `CAST(${rounding}(EXTRACT(EPOCH FROM ${x}) * ${perSecond[unit ?? type.unit]}) AS BIGINT)` + } + const days = `(CAST(${x} AS DATE) - DATE '1970-01-01')` + return type.unit === "day" ? days : `(CAST(${days} AS BIGINT) * 86400000)` +} + +/** + * An interval of `micros` (a BIGINT) microseconds. Intervals multiply in + * double precision, so whole hours and the remaining microseconds are + * multiplied separately, each exactly. + */ +const interval = (micros: string): string => + `((${micros}) / 3600000000 * INTERVAL '1 hour' + (${micros}) % 3600000000 * INTERVAL '1 microsecond')` + +/** The temporal value of `type` whose integer (see {@link toInteger}) is `x`, of type {@link integerOf}. */ +const fromInteger = (x: string, type: Temporal): string => { + if (type._tag === "Date") { + return `(DATE '1970-01-01' + CAST(${type.unit === "day" ? x : `FLOOR(${x} / 86400000.0)`} AS INTEGER))` + } + const micros = type.unit === "nanosecond" + ? `CAST(${type._tag === "Duration" ? "TRUNC" : "FLOOR"}(${x} / 1000.0) AS BIGINT)` + : `(${x} * ${perSecond.microsecond / perSecond[type.unit]})` + if (type._tag === "Duration") return interval(micros) + const epoch = type.timezone === undefined ? "TIMESTAMP '1970-01-01 00:00:00'" : "TIMESTAMPTZ '1970-01-01 00:00:00+00'" + return `(${epoch} + ${interval(micros)})` +} + +/** + * `x` converted from `from` to `to` as the engine converts: timestamps + * and durations rescale to the target unit (rounding as {@link toInteger}), + * timestamps and dates convert at midnight, and other temporal values + * convert as their integers. + */ +const cast = (x: string, from: S.DataType, to: S.DataType): string => { + if (isTemporal(from) && isTemporal(to)) { + if (from._tag !== "Date" && from._tag === to._tag) return fromInteger(toInteger(x, from, to.unit), to) + if (from._tag === "Timestamp" && to._tag === "Date") { + return fromInteger(`CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) / 86400) AS BIGINT)`, S.date("day")) + } + if (from._tag === "Date" && to._tag === "Timestamp") { + return fromInteger(`CAST(${toInteger(x, S.date("day"))} AS BIGINT) * ${BigInt(86400) * perSecond[to.unit]}`, to) + } + } + if (isTemporal(from)) return cast(toInteger(x, from), integerOf(from), to) + if (isTemporal(to)) return fromInteger(cast(x, from, integerOf(to)), to) + // Floats and decimals truncate towards zero. + const truncated = to._tag === "Int" && (from._tag === "Decimal" || from._tag === "Float") ? `TRUNC(${x})` : x + return `CAST(${truncated} AS ${sqlType(to)})` +} + +/** + * Operands as the engine combines them: timestamps and durations of mixed + * units converted to the coarsest of their units first. + */ +const sameUnit = (xs: ReadonlyArray, types: ReadonlyArray): Array => { + const units = types.flatMap((t) => (t._tag === "Timestamp" || t._tag === "Duration" ? [t.unit] : [])) + if (units.every((u) => u === units[0])) return [...xs] + const unit = units.reduce(coarser) + return xs.map((x, i) => { + const t = types[i] + if (t._tag === "Timestamp" && t.unit !== unit) return cast(x, t, S.timestamp(unit, t.timezone)) + if (t._tag === "Duration" && t.unit !== unit) return cast(x, t, S.duration(unit)) + return x + }) +} + +// ----------------------------------------------------------------------------- +// Expressions +// ----------------------------------------------------------------------------- + +interface Scope { + readonly compiler: Compiler + /** The input relation's frame (zero rows). */ + readonly frame: F.Frame + /** The PARTITION BY expressions of an enclosing `over`. */ + readonly partition: ReadonlyArray | undefined +} + +/** The engine's type of an expression's result. */ +const typeOf = (node: Plan.Expression, scope: Scope): S.DataType => + E.evaluate( + scope.partition === undefined ? node : { _tag: "Over", expression: node, partitionBy: [] }, + F.context(scope.frame) + ).vector.type + +const partitionBy = (scope: Scope, separator = ""): string => + scope.partition === undefined || scope.partition.length === 0 + ? "" + : `PARTITION BY ${scope.partition.join(", ")}${separator}` + +const isNaNSql = (x: string) => `(${x} = ${NAN})` + +const window = (scope: Scope, frame?: string): string => + `OVER (${partitionBy(scope, " ")}ORDER BY ${scope.compiler.row}${frame === undefined ? "" : ` ${frame}`})` + +const running = "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" + +const float = (x: string) => `CAST(${x} AS DOUBLE PRECISION)` + +/** A power of ten as an exact SQL literal. */ +const power = (k: number) => k >= 0 ? `1${"0".repeat(k)}` : `0.${"0".repeat(-k - 1)}1` + +/** + * `round` as the engine does it: floats as `rint(x * 10^d) / 10^d` in double + * precision, decimals half to even at `d` places into the engine's result + * type, integers unchanged. + */ +const round = (x: string, type: S.DataType, result: S.DataType, decimals: number): string => { + switch (type._tag) { + case "Int": + case "Null": + return x + case "Decimal": { + if (decimals >= type.scale) return x + const scaled = decimals === 0 ? x : `(${x} * ${power(decimals)})` + const floor = `FLOOR(${scaled})` + const up = `(${scaled} - ${floor} > 0.5 OR (${scaled} - ${floor} = 0.5 AND MOD(${floor}, 2) <> 0))` + const rounded = `(CASE WHEN ${up} THEN ${floor} + 1 ELSE ${floor} END)` + return `CAST(${decimals === 0 ? rounded : `(${rounded} / ${power(decimals)})`} AS ${sqlType(result)})` + } + default: { + if (decimals === 0) return `ROUND(${float(x)})` + const scale = float(literal(10 ** decimals, undefined)) + return `(ROUND(${float(x)} * ${scale}) / ${scale})` + } + } +} + +/** The SQL aggregate for the minimum or maximum of a column of `type` (booleans have their own). */ +const extreme = (fn: "min" | "max", type: S.DataType["_tag"] | undefined): string => + type === "Bool" ? (fn === "min" ? "BOOL_AND" : "BOOL_OR") : fn.toUpperCase() + +/** + * The aggregate `node` of `x`, the SQL of its argument (of type `type`); a + * window function when `over` (its window) is given. + */ +const aggregate = ( + node: Plan.AggregateCall, + x: string, + type: S.DataType["_tag"] | undefined, + scope: Scope, + over: string | undefined +): string => { + const row = scope.compiler.row + const o = over === undefined ? "" : ` ${over}` + switch (node.fn) { + case "count": + return node.arg === undefined ? `COUNT(*)${o}` : `COUNT(${x})${o}` + case "sum": + if (type === "Duration") return `COALESCE(SUM(${x})${o}, INTERVAL '0 seconds')` + return `COALESCE(SUM(${type === "Bool" ? `CAST(${x} AS INTEGER)` : x})${o}, 0)` + case "mean": + return `AVG(${x})${o}` + case "min": + case "max": { + const fn = extreme(node.fn, type) + // Floats: NaN is skipped unless every value is NaN. + return type === "Float" + ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(x)} THEN NULL ELSE ${x} END)${o}, ${fn}(${x})${o})` + : `${fn}(${x})${o}` + } + case "std": + return `STDDEV_SAMP(${x})${o}` + case "variance": + return `VAR_SAMP(${x})${o}` + case "median": + if (over !== undefined) return unsupported("median over partitions") + return `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${x})` + case "countDistinct": + if (over !== undefined) return unsupported("countDistinct over partitions") + // Nulls count as one value. + return `(COUNT(DISTINCT ${x}) + CASE WHEN COUNT(*) > COUNT(${x}) THEN 1 ELSE 0 END)` + case "first": + case "last": { + if (over !== undefined) { + return `${node.fn === "first" ? "FIRST_VALUE" : "LAST_VALUE"}(${x}) ${ + window(scope, "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING") + }` + } + return `(ARRAY_AGG(${x} ORDER BY ${row} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` + } + case "list": + if (over !== undefined) return unsupported("list over partitions") + return `ARRAY_AGG(${x} ORDER BY ${row})` + } +} + +const truncateUnits: Record = { "1h": "hour", "1d": "day", "1w": "week", "1mo": "month", "1y": "year" } +const totalSeconds: Record = { + totalDays: 86400, + totalHours: 3600, + totalMinutes: 60, + totalSeconds: 1, + totalMilliseconds: 0.001 +} + +const infix = { + add: "+", + multiply: "*", + eq: "=", + neq: "<>", + lt: "<", + lte: "<=", + gt: ">", + gte: ">=", + and: "AND", + or: "OR" +} as const +const unary = { abs: "ABS", length: "LENGTH", toLowerCase: "LOWER", toUpperCase: "UPPER" } as const + +const call = (node: Plan.Call, scope: Scope): string => { + const row = scope.compiler.row + const types = node.args.map((x) => typeOf(x, scope)) + const raw = node.args.map((x) => expression(x, scope)) + // between compares its operand with each bound separately. + const a = node.fn === "between" ? raw : sameUnit(raw, types) + const k = types.map((t) => t._tag) + const ints = k[0] === "Int" && k[1] === "Int" + const o = node.options ?? {} + // Floored modulo of integers, exact, null for a zero divisor. + const intMod = () => `((${a[0]} % NULLIF(${a[1]}, 0)) + ${a[1]}) % NULLIF(${a[1]}, 0)` + switch (node.fn) { + case "add": + case "multiply": + case "eq": + case "neq": + case "lt": + case "lte": + case "gt": + case "gte": + case "and": + case "or": + return `(${a[0]} ${infix[node.fn]} ${a[1]})` + case "subtract": + if (k[0] === "Date" && k[1] === "Date") return `((${a[0]} - ${a[1]}) * INTERVAL '1 day')` + return `(${a[0]} - ${a[1]})` + case "divide": + return `(${float(a[0])} / ${a[1]})` + case "floorDivide": + return ints + ? `((${a[0]} - (${intMod()})) / NULLIF(${a[1]}, 0))` + : `FLOOR(${float(a[0])} / ${a[1]})` + case "mod": + return ints ? `(${intMod()})` : `(${a[0]} - ${a[1]} * FLOOR(${float(a[0])} / ${a[1]}))` + case "negate": + return `(-(${a[0]}))` + case "abs": + case "length": + case "toLowerCase": + case "toUpperCase": + return `${unary[node.fn]}(${a[0]})` + case "sqrt": + return `(CASE WHEN ${float(a[0])} < 0 THEN ${NAN} ELSE SQRT(${float(a[0])}) END)` + case "round": + return round(a[0], types[0], typeOf(node, scope), o.decimals ?? 0) + case "between": { + const [x, lo] = sameUnit([a[0], a[1]], [types[0], types[1]]) + const [y, hi] = sameUnit([a[0], a[2]], [types[0], types[2]]) + return `(${x} >= ${lo} AND ${y} <= ${hi})` + } + case "isIn": { + const values = (o.values ?? []).filter((v) => v !== null) + const isIn = (x: string, list: ReadonlyArray) => `(${x} IN (${list.join(", ")}))` + // One float in a list makes PostgreSQL compare the whole list as floats, which + // rounds decimals and 64-bit integers. They take their floats in a list of its own. + const t = types[0] + if (t._tag !== "Decimal" && !(t._tag === "Int" && t.bitWidth === 64)) { + return values.length === 0 ? + `(${a[0]} IS NULL AND NULL)` : + isIn(a[0], values.map((v) => literal(v, undefined))) + } + const fraction = (v: Plan.Value) => typeof v === "number" && !Number.isInteger(v) + // An integer as its digits: exact, whatever its size. + const exact = values.filter((v) => !fraction(v)).map((v) => + typeof v === "string" || typeof v === "boolean" ? literal(v, undefined) : String(BigInt(v as number | bigint)) + ) + const floats = values.filter(fraction).map((v) => literal(v, undefined)) + const tests = [ + ...(exact.length > 0 ? [isIn(a[0], exact)] : []), + ...(floats.length > 0 ? [isIn(`CAST(${a[0]} AS DOUBLE PRECISION)`, floats)] : []) + ] + return tests.length === 0 ? `(${a[0]} IS NULL AND NULL)` : `(${tests.join(" OR ")})` + } + case "not": + return `(NOT ${a[0]})` + case "isNull": + return `(${a[0]} IS NULL)` + case "isNotNull": + return `(${a[0]} IS NOT NULL)` + case "isNaN": + return isNaNSql(a[0]) + case "fillNull": + case "coalesce": + return `COALESCE(${a.join(", ")})` + case "when": + return `(CASE WHEN ${a[0]} THEN ${a[1]} ELSE ${a[2]} END)` + case "cast": + return cast(a[0], types[0], typeOf(node, scope)) + case "includes": + return `(STRPOS(${a[0]}, ${string(o.pattern ?? "")}) > 0)` + case "startsWith": + return `(LEFT(${a[0]}, ${[...(o.pattern ?? "")].length}) = ${string(o.pattern ?? "")})` + case "endsWith": + return `(RIGHT(${a[0]}, ${[...(o.pattern ?? "")].length}) = ${string(o.pattern ?? "")})` + case "concat": + return `(${a.join(" || ")})` + case "cumSum": + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE SUM(${k[0] === "Bool" ? `CAST(${a[0]} AS INTEGER)` : a[0]}) ${ + window(scope, running) + } END)` + case "cumMin": + case "cumMax": { + const fn = extreme(node.fn === "cumMin" ? "min" : "max", k[0]) + // Floats: NaN is skipped (it stays only while no other value was seen). + const value = k[0] === "Float" + ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(a[0])} THEN NULL ELSE ${a[0]} END) ${window(scope, running)}, ${a[0]})` + : `${fn}(${a[0]}) ${window(scope, running)}` + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${value} END)` + } + case "cumCount": + return `COUNT(${a[0]}) ${window(scope, running)}` + case "shift": { + const n = o.n ?? 1 + return n >= 0 ? `LAG(${a[0]}, ${n}) ${window(scope)}` : `LEAD(${a[0]}, ${-n}) ${window(scope)}` + } + case "diff": + return `(${a[0]} - LAG(${a[0]}, ${o.n ?? 1}) ${window(scope)})` + case "rank": { + const order = `${a[0]} ${o.descending ? "DESC" : "ASC"} NULLS LAST` + const over = `OVER (${partitionBy(scope, " ")}ORDER BY ${order})` + const ties = `COUNT(*) OVER (PARTITION BY ${[...(scope.partition ?? []), a[0]].join(", ")})` + const ranked = (() => { + switch (o.method ?? "average") { + case "min": + return `RANK() ${over}` + case "max": + return `(RANK() ${over} + ${ties} - 1)` + case "dense": + return `DENSE_RANK() ${over}` + case "ordinal": + return `ROW_NUMBER() OVER (${partitionBy(scope, " ")}ORDER BY ${order}, ${row})` + case "average": + return `(RANK() ${over} + (${ties} - 1) / 2.0)` + } + })() + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${ranked} END)` + } + case "rollingSum": + case "rollingMean": + case "rollingMin": + case "rollingMax": { + const w = window(scope, `ROWS BETWEEN ${(o.window ?? 1) - 1} PRECEDING AND CURRENT ROW`) + const fn = { rollingSum: "SUM", rollingMean: "AVG", rollingMin: "MIN", rollingMax: "MAX" }[node.fn] + const enough = `COUNT(${a[0]}) ${w} >= ${o.minPeriods ?? o.window ?? 1}` + // Floats: a NaN in the window makes the minimum or maximum NaN. + const nan = `MAX(CASE WHEN ${isNaNSql(a[0])} THEN 1 ELSE 0 END) ${w} = 1` + const value = k[0] === "Float" && (fn === "MIN" || fn === "MAX") + ? `CASE WHEN ${nan} THEN ${NAN} ELSE ${fn}(${a[0]}) ${w} END` + : `${fn}(${a[0]}) ${w}` + return `(CASE WHEN ${enough} THEN ${value} END)` + } + case "year": + case "month": + case "day": + case "hour": + case "minute": + return `CAST(EXTRACT(${node.fn.toUpperCase()} FROM ${a[0]}) AS INTEGER)` + case "second": + return `CAST(FLOOR(EXTRACT(SECOND FROM ${a[0]})) AS INTEGER)` + case "millisecond": + return `CAST(MOD(FLOOR(EXTRACT(MILLISECONDS FROM ${a[0]})), 1000) AS INTEGER)` + case "weekday": + return `CAST(EXTRACT(ISODOW FROM ${a[0]}) AS INTEGER)` + case "ordinalDay": + return `CAST(EXTRACT(DOY FROM ${a[0]}) AS INTEGER)` + case "truncate": { + const unit = truncateUnits[o.every ?? ""] ?? unsupported(`truncate to ${o.every}`) + return k[0] === "Date" ? `CAST(DATE_TRUNC('${unit}', ${a[0]}) AS DATE)` : `DATE_TRUNC('${unit}', ${a[0]})` + } + case "totalDays": + case "totalHours": + case "totalMinutes": + case "totalSeconds": + case "totalMilliseconds": + return `CAST(TRUNC(EXTRACT(EPOCH FROM ${a[0]}) / ${totalSeconds[node.fn]}) AS BIGINT)` + case "listLength": + return `COALESCE(CARDINALITY(${a[0]}), CASE WHEN ${a[0]} IS NULL THEN NULL ELSE 0 END)` + case "listGet": { + const i = o.index ?? 0 + return `${a[0]}[${i >= 0 ? `${i + 1}` : `CARDINALITY(${a[0]}) + ${i + 1}`}]` + } + case "listSum": + case "listMean": + case "listMin": + case "listMax": + case "struct": + return unsupported(node.fn) + case "listContains": + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${a[1]} = ANY(${a[0]}) END)` + case "field": + return `(${a[0]}).${quote(o.name ?? "")}` + } +} + +const expression = (node: Plan.Expression, scope: Scope): string => { + switch (node._tag) { + case "Column": + return quote(node.name) + case "Literal": + return literal(node.value, node.type) + case "Call": + return call(node, scope) + case "AggregateCall": { + if (scope.partition === undefined) return V.fail(`aggregate ${node.fn} can only be used in agg or over`) + const inner = { ...scope, partition: undefined } + const x = node.arg === undefined ? "" : expression(node.arg, inner) + const type = node.arg === undefined ? undefined : typeOf(node.arg, inner)._tag + return aggregate(node, x, type, scope, `OVER (${partitionBy(scope)})`) + } + case "Over": { + // Within an enclosing partition (another `over`, or a group), partitions refine it. + const keys = node.partitionBy.map((k) => expression(k, { ...scope, partition: undefined })) + return expression(node.expression, { ...scope, partition: [...(scope.partition ?? []), ...keys] }) + } + } +} + +// ----------------------------------------------------------------------------- +// Types of relations +// ----------------------------------------------------------------------------- + +const unsupportedNodes: ReadonlySet = new Set(["Pivot", "Unpivot", "Explode", "Unnest"]) + +/** + * Runs the plan through the in-memory engine on zero-row inputs (tables as + * their declared columns), recording every node's frame. + */ +const typed = (node: Plan.Node, frames: Map): F.Frame => { + const known = frames.get(node) + if (known !== undefined) return known + let frame: F.Frame + if (unsupportedNodes.has(node._tag)) return unsupported(node._tag) + if (node._tag === "Table") { + const columns = node.columns ?? V.fail( + `the SQL engine needs the columns of table ${JSON.stringify(node.name)}: use Query.table with a schema` + ) + frame = F.fromVectors( + columns.map((c) => c.name), + columns.map((c) => K.nullsOf(tableType(node.name, c), 0)), + 0 + ) + } else if (node._tag === "Values") { + frame = F.slice(node.frame as unknown as F.Frame, 0, 0) + } else { + const inputs = Q.children(node as Q.Node).map((n) => typed(n as Plan.Node, frames)) + frame = Q.step(node as Q.Node, inputs) + } + frames.set(node, frame) + return frame +} + +// ----------------------------------------------------------------------------- +// Relations +// ----------------------------------------------------------------------------- + +const columnsOf = (node: Plan.Node, c: Compiler): ReadonlyArray => + c.frames.get(node)!.columns.map((col) => col.name) + +const typeIn = (node: Plan.Node, name: string, c: Compiler): S.DataType => + c.frames.get(node)!.columns.find((col) => col.name === name)!.type + +/** + * Column `name` of `input`, as `x`, in the type the column has in `node`: + * joins and concatenations bring timestamps and durations of mixed units to + * the coarser one. + */ +const conformed = (x: string, input: Plan.Node, node: Plan.Node, name: string, c: Compiler): string => { + const from = typeIn(input, name, c) + const to = typeIn(node, name, c) + return isTemporal(from) && isTemporal(to) && !D.sameType(from, to) ? cast(x, from, to) : x +} + +const values = (frame: F.Frame, c: Compiler): string => { + if (frame.length === 0) { + const nulls = frame.columns.map((col) => `CAST(NULL AS ${sqlType(col.type)}) AS ${quote(col.name)}`) + return `SELECT ${[...nulls, `CAST(0 AS BIGINT) AS ${c.row}`].join(", ")} WHERE FALSE` + } + const columns = frame.columns.map((col) => { + const type = col.type + const vector = col.values() + const raw = (vector as V.I32 | V.I64).values + const integers = isTemporal(type) || type._tag === "Decimal" + const values = integers ? undefined : F.toArray(frame, col.name) + // Temporal and decimal values are written from their integers, at full precision. + const value = (i: number): string => { + if (type._tag === "Date" && type.unit !== "day") { + return literal(floorDiv(raw[i] as bigint, BigInt(86_400_000)), type) + } + if (integers) return literal(raw[i], type as Plan.LiteralType) + const v = values![i] + return typeof v === "bigint" ? String(v) : literal(v, undefined) + } + const sql = sqlType(type) + return { name: quote(col.name), cell: (i: number) => `CAST(${V.isValid(vector, i) ? value(i) : "NULL"} AS ${sql})` } + }) + const tuples = Array.from( + { length: frame.length }, + (_, r) => `(${[...columns.map((col) => col.cell(r)), String(r + 1)].join(", ")})` + ) + return `SELECT * FROM (VALUES ${tuples.join(", ")}) AS v(${[...columns.map((col) => col.name), c.row].join(", ")})` +} + +const named = (items: ReadonlyArray, scope: Scope) => + items.map((c) => `${expression(c.expression, scope)} AS ${quote(c.name)}`) + +/** A SELECT of the node's columns plus the row key. */ +const relation = (node: Plan.Node, c: Compiler): string => { + const row = c.row + const columns = columnsOf(node, c) + const keep = [...columns.map(quote), row].join(", ") + const input = (node as { readonly input: Plan.Node }).input + const from = (alias = "t") => `(${relation(input, c)}) AS ${alias}` + const scope = (): Scope => ({ compiler: c, frame: c.frames.get(input)!, partition: undefined }) + // The rows of `query` renumbered in `order`. + const renumbered = (query: string, order: string) => + `SELECT ${ + [...columns.map(quote), `ROW_NUMBER() OVER (ORDER BY ${order}) AS ${row}`].join(", ") + } FROM (${query}) AS t` + switch (node._tag) { + case "Table": + return `SELECT ${ + [ + ...node.columns!.map((column) => `${declared(column)} AS ${quote(column.name)}`), + `ROW_NUMBER() OVER () AS ${row}` + ] + .join(", ") + } FROM ${c.sources.get(identifier(node.name))}` + case "Values": + return values(node.frame as unknown as F.Frame, c) + case "Filter": { + // The predicate is computed over the whole input first: it may hold window functions. + const flag = c.hidden("keep") + return `SELECT ${keep} FROM (SELECT *, ${ + expression(node.predicate, scope()) + } AS ${flag} FROM ${from()}) AS t WHERE ${flag}` + } + case "Select": + return `SELECT ${[...named(node.columns, scope()), row].join(", ")} FROM ${from()}` + case "WithColumns": { + const s = scope() + const added = new Map(node.columns.map((col) => [col.name, col.expression])) + const inputs = columnsOf(node.input, c) + const kept = inputs.map((name) => { + const e = added.get(name) + return e === undefined ? quote(name) : `${expression(e, s)} AS ${quote(name)}` + }) + const extra = node.columns.filter((col) => !inputs.includes(col.name)) + return `SELECT ${[...kept, ...named(extra, s), row].join(", ")} FROM ${from()}` + } + case "Pick": + case "Omit": + return `SELECT ${keep} FROM ${from()}` + case "Rename": + return `SELECT ${ + [...columnsOf(node.input, c).map((name) => `${quote(name)} AS ${quote(node.mapping[name] ?? name)}`), row].join( + ", " + ) + } FROM ${from()}` + case "Sort": { + const s = scope() + const keys = node.keys.map((k) => + `${expression(k.expression, s)} ${k.descending ? "DESC" : "ASC"} NULLS ${node.nullsLast ? "LAST" : "FIRST"}` + ) + return renumbered(`SELECT * FROM ${from("s")}`, [...keys, row].join(", ")) + } + case "Slice": { + if (node.offset >= 0) { + return `SELECT * FROM ${from()} ORDER BY ${row}${ + node.length === undefined ? "" : ` LIMIT ${node.length}` + } OFFSET ${node.offset}` + } + const n = c.hidden("n") + const i = c.hidden("i") + const upper = node.length === undefined ? "" : ` AND ${i} <= ${n} + ${node.offset} + ${node.length}` + return `SELECT ${keep} FROM (SELECT *, COUNT(*) OVER () AS ${n}, ROW_NUMBER() OVER (ORDER BY ${row}) AS ${i} FROM ${ + from("s") + }) AS t WHERE ${i} > ${n} + ${node.offset}${upper}` + } + case "Unique": { + const k = c.hidden("k") + const keys = node.columns ?? columns + const partition = keys.length === 0 ? "" : `PARTITION BY ${list(keys)} ` + return `SELECT ${keep} FROM (SELECT *, ROW_NUMBER() OVER (${partition}ORDER BY ${row}) AS ${k} FROM ${ + from("s") + }) AS t WHERE ${k} = 1` + } + case "DropNulls": { + const keys = node.columns ?? columns + return `SELECT * FROM ${from()}${ + keys.length === 0 ? "" : ` WHERE ${keys.map((k) => `${quote(k)} IS NOT NULL`).join(" AND ")}` + }` + } + case "GroupBy": + case "Aggregate": { + const keys = node._tag === "GroupBy" ? node.keys : [] + // Arguments are computed first, partitioned by the group: windows in them see only its rows. + const group: Scope = { ...scope(), partition: keys.map(quote) } + const args = node.aggregates.map((a, i) => { + const arg = (a.expression as Plan.AggregateCall).arg + return arg === undefined + ? undefined + : { name: c.hidden(`arg${i}`), sql: expression(arg, group), type: typeOf(arg, group)._tag } + }) + const computed = args.flatMap((x) => x === undefined ? [] : [`, ${x.sql} AS ${x.name}`]) + const aggs = node.aggregates.map((a, i) => + `${aggregate(a.expression as Plan.AggregateCall, args[i]?.name ?? "", args[i]?.type, group, undefined)} AS ${ + quote(a.name) + }` + ) + const key = keys.length === 0 ? `1 AS ${row}` : `MIN(${row}) AS ${row}` + return `SELECT ${[...keys.map(quote), ...aggs, key].join(", ")} FROM (SELECT *${ + computed.join("") + } FROM ${from()}) AS t${keys.length === 0 ? "" : ` GROUP BY ${list(keys)}`}` + } + case "Join": { + const l = relation(node.left, c) + const r = relation(node.right, c) + const on = node.on.map((k) => { + const [x, y] = sameUnit([`l.${quote(k)}`, `r.${quote(k)}`], [typeIn(node.left, k, c), typeIn(node.right, k, c)]) + return `${x} = ${y}` + }).join(" AND ") + if (node.how === "semi" || node.how === "anti") { + return `SELECT * FROM (${l}) AS l WHERE ${ + node.how === "anti" ? "NOT " : "" + }EXISTS (SELECT 1 FROM (${r}) AS r WHERE ${on})` + } + const rightOnly = columnsOf(node.right, c).filter((name) => !node.on.includes(name)) + const fromLeft = (name: string) => conformed(`l.${quote(name)}`, node.left, node, name, c) + const fromRight = (name: string) => conformed(`r.${quote(name)}`, node.right, node, name, c) + const left = columnsOf(node.left, c).map((name) => + `${ + node.how === "full" && node.on.includes(name) + ? `COALESCE(${fromLeft(name)}, ${fromRight(name)})` + : fromLeft(name) + } AS ${quote(name)}` + ) + const join = node.how === "inner" ? "INNER JOIN" : node.how === "left" ? "LEFT JOIN" : "FULL JOIN" + return `SELECT ${ + [...left, ...rightOnly.map((name) => `${fromRight(name)} AS ${quote(name)}`)].join(", ") + }, ROW_NUMBER() OVER (ORDER BY l.${row}${ + node.how === "full" ? " NULLS LAST" : "" + }, r.${row}) AS ${row} FROM (${l}) AS l ${join} (${r}) AS r ON ${on}` + } + case "Concat": { + const source = c.hidden("source") + const union = node.inputs + .map((n, i) => + `SELECT ${ + columns.map((name) => `${conformed(quote(name), n, node, name, c)} AS ${quote(name)}`).join(", ") + }, ${row}, ${i} AS ${source} FROM (${relation(n, c)}) AS t${i}` + ) + .join(" UNION ALL ") + return renumbered(union, `${source}, ${row}`) + } + case "Pivot": + case "Unpivot": + case "Explode": + case "Unnest": + return unsupported(node._tag) + } +} + +// ----------------------------------------------------------------------------- +// Results +// ----------------------------------------------------------------------------- + +const scalars: ReadonlySet = new Set(["Null", "Bool", "Int", "Float", "Decimal", "Utf8"]) + +/** SQL that renders a result column of `type` in the form `reader` reads. */ +const output = (x: string, type: S.DataType): string => { + const text = (y: string) => `CAST(${y} AS TEXT)` + switch (type._tag) { + case "Null": + return "NULL" + case "Bool": + case "Int": + case "Float": + case "Decimal": + case "Utf8": + return text(`CAST(${x} AS ${sqlType(type)})`) + case "Date": + case "Timestamp": + case "Duration": + return text(toInteger(x, type)) + case "Binary": + return x + case "List": + if (scalars.has(type.child.type._tag)) return `CAST(${x} AS TEXT[])` + } + return unsupported(`results of type ${D.describe(type)}`) +} + +/** Reads one non-null value of a result column of `type`. */ +const reader = (type: S.DataType): (value: unknown) => unknown => { + switch (type._tag) { + case "Int": + return type.bitWidth === 64 ? (v) => BigInt(String(v)) : Number + case "Float": + return Number + case "Bool": + return (v) => v === true || v === "true" || v === "t" + case "Decimal": + return (v) => BigDecimal.fromStringUnsafe(String(v)) + case "Date": + case "Timestamp": + case "Duration": + return (v) => BigInt(String(v)) + case "List": { + const item = reader(type.child.type) + return (v) => (v as ReadonlyArray).map((x) => (x === null || x === undefined ? null : item(x))) + } + default: + return (v) => v + } +} + +// ----------------------------------------------------------------------------- +// Tables +// ----------------------------------------------------------------------------- + +const smallIntegers = ["smallint", "integer"] + +/** A condition, as SQL on a value `x`, and the failure of a column with values that don't satisfy it. */ +interface Condition { + readonly holds: string + readonly failure: string +} + +const exactly = (type: S.DataType): string => `has values that ${D.describe(type)} can't hold exactly` + +/** Integers in the range of an integer type. */ +const inRange = (x: string, type: S.DataType): Array => { + const { bitWidth, signed } = type as S.Int + const lo = signed ? -(BigInt(1) << BigInt(bitWidth - 1)) : BigInt(0) + const hi = (BigInt(1) << BigInt(signed ? bitWidth - 1 : bitWidth)) - BigInt(1) + return [{ holds: `${x} BETWEEN ${lo} AND ${hi}`, failure: exactly(type) }] +} + +/** + * Numbers that are 32-bit floats: zero, infinities, NaN, or numbers between + * the least and the greatest 32-bit float in magnitude that convert to + * `REAL` and back unchanged (outside that range the conversion fails). + */ +const float32 = (x: string, type: S.DataType): Array => { + const d = float(x) + return [{ + holds: `(CASE WHEN ABS(${d}) BETWEEN ${float(`'${2 ** -149}'`)} AND ${ + float(`'${(2 - 2 ** -23) * 2 ** 127}'`) + } THEN CAST(CAST(${d} AS REAL) AS DOUBLE PRECISION) = ${d} ELSE ${d} = 0 OR ABS(${d}) = ${ + float("'Infinity'") + } OR ${d} = ${NAN} END)`, + failure: exactly(type) + }] +} + +const int64 = BigInt(1) << BigInt(63) + +/** Timestamps and durations in whole units of the type that fit 64 bits (PostgreSQL keeps microseconds). */ +const wholeUnits = (x: string, type: S.DataType): Condition => { + const v = `(EXTRACT(EPOCH FROM ${x}) * ${perSecond[(type as S.Timestamp | S.Duration).unit]})` + return { holds: `${v} = TRUNC(${v}) AND ${v} BETWEEN ${-int64} AND ${int64 - BigInt(1)}`, failure: exactly(type) } +} + +/** + * How the SQL engine reads declared table columns of each type, as the + * in-memory engine conforms table columns to their declaration: + * + * - `accepts` and `categories`: the PostgreSQL types, and type categories, + * of the columns it reads as such columns (integers below 64 bits for + * floats and narrower integers, as the in-memory engine; any type for + * `Null`, which holds only nulls). + * - `conditions`: the conditions every value satisfies, beyond its type, + * for its declared type to hold it exactly. + * + * `undefined` for column types it doesn't read yet. + */ +const tableTypes: { + readonly [K in Plan.TypeName]: + | { + readonly accepts?: ReadonlyArray + readonly categories?: ReadonlyArray + readonly conditions?: (x: string, type: S.DataType) => ReadonlyArray + } + | undefined +} = { + Int8: { accepts: smallIntegers, conditions: inRange }, + Int16: { accepts: smallIntegers, conditions: inRange }, + Int32: { accepts: smallIntegers }, + Int64: { accepts: ["bigint"] }, + UInt8: { accepts: smallIntegers, conditions: inRange }, + UInt16: { accepts: smallIntegers, conditions: inRange }, + UInt32: { accepts: smallIntegers, conditions: inRange }, + UInt64: { accepts: ["bigint"], conditions: inRange }, + Float16: undefined, + Float32: { accepts: [...smallIntegers, "real", "double precision"], conditions: float32 }, + Float64: { accepts: [...smallIntegers, "real", "double precision"] }, + // String types and enums. + Utf8: { categories: ["S", "E"] }, + Binary: { accepts: ["bytea"] }, + Bool: { accepts: ["boolean"] }, + Null: {}, + Date: { accepts: ["date"], conditions: (x, type) => [{ holds: `isfinite(${x})`, failure: exactly(type) }] }, + Time: undefined, + Timestamp: { + accepts: ["timestamp without time zone", "timestamp with time zone"], + conditions: (x, type) => [ + { + holds: `EXTRACT(EPOCH FROM ${x}) BETWEEN -8640000000000 AND 8640000000000.000999`, + failure: "has timestamps outside the range of DateTime" + }, + wholeUnits(x, type) + ] + }, + Duration: { + accepts: ["interval"], + conditions: (x, type) => [ + { + holds: `EXTRACT(YEAR FROM ${x}) = 0 AND EXTRACT(MONTH FROM ${x}) = 0`, + failure: "has durations in months, which have no fixed length" + }, + wholeUnits(x, type) + ] + }, + Decimal: { + accepts: ["numeric"], + conditions: (x, type) => { + const { precision } = type as S.Decimal + const scale = numericScale((type as S.Decimal).scale) + return [{ + holds: `ROUND(${x}, ${scale}) = ${x} AND ABS(${x}) < ${power(precision - scale)} AND ${x} * ${ + power(scale) + } BETWEEN ${-int64} AND ${int64 - BigInt(1)}`, + failure: exactly(type) + }] + } + }, + List: undefined, + Struct: undefined +} + +/** + * The type the SQL engine reads a declared table column as: the type the + * in-memory engine conforms it to. + */ +const tableType = (table: string, column: Plan.TableColumn): S.DataType => + tableTypes[column.type] === undefined + ? unsupported( + `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` + ) + : arrowType(column) + +/** The conditions of a declared column's values, `x` (see {@link tableTypes}). */ +const conditions = (x: string, column: Plan.TableColumn): ReadonlyArray => + tableTypes[column.type]!.conditions?.(x, arrowType(column)) ?? [] + +/** + * A check of a table: one SQL value, and a test of it that fails when the + * table breaks its declaration. + */ +interface Check { + readonly sql: string + readonly verify: (value: unknown) => void +} + +const utc = S.timestamp("microsecond") + +/** + * A declared column of a table's source as the engine reads it: in the + * PostgreSQL type of its declared type (strings and enums as text, + * timestamps as UTC timestamps without a time zone, whatever the session's + * time zone), and null where it breaks a condition (see {@link conditions}), + * so that reading such values doesn't fail before the checks of the rows do. + */ +const declared = (column: Plan.TableColumn): string => { + const x = quote(column.name) + const type = arrowType(column) + const value = type._tag === "Null" + ? `CAST(NULL AS ${sqlType(type)})` + : type._tag === "Timestamp" + ? fromInteger(toInteger(x, utc), utc) + : `CAST(${x} AS ${sqlType(type)})` + const holds = conditions(x, column) + return holds.length === 0 ? value : `(CASE WHEN ${holds.map((c) => c.holds).join(" AND ")} THEN ${value} END)` +} + +/** + * The checks of a table's declared column types that don't read its rows: + * each value is the PostgreSQL type of a column the engine doesn't read as + * declared, or null. Each column is qualified by its table: unqualified, a + * column the table lacks would be the `pg_type` column of its name. + */ +const typeChecks = (table: Plan.Table): Array => + (table.columns ?? []).flatMap((column): Array => { + if (column.type === "Null") return [] + const { accepts = [], categories = [] } = tableTypes[column.type]! + const accepted = [ + ...(accepts.length === 0 ? [] : [`format_type(oid, NULL) IN (${accepts.map(string).join(", ")})`]), + ...(categories.length === 0 ? [] : [`typcategory IN (${categories.map(string).join(", ")})`]) + ] + return [{ + sql: `(SELECT CASE WHEN ${ + accepted.join(" OR ") + } THEN NULL ELSE format_type(oid, NULL) END FROM pg_type WHERE oid = pg_typeof((SELECT t.${ + quote(column.name) + } FROM ${quote(table.name)} AS t LIMIT 0)))`, + verify: (actual) => { + if (actual !== null) V.fail(`column ${JSON.stringify(column.name)} is ${actual}, expected ${column.type}`) + } + }] + }) + +/** + * The tests of a table's checks, each of the values of one column, as a SQL + * engine runs them: a table with a test of whole rows fails. + */ +const columnChecks = (table: Plan.Table): Plan.TableChecks["columns"] => + table.checks === undefined + ? [] + : table.checks.rows === undefined + ? table.checks.columns + : V.fail("schemas with checks on whole rows aren't supported on SQL tables; check their fields instead") + +/** + * The checks of a table's rows beyond column types: nulls, the + * {@link conditions} of values, and the tests of the table's own checks + * (fields its schema constrains: literals, enums, refinements), run on + * distinct values. Each check aggregates the rows of the table's source. + */ +const rowChecks = (table: Plan.Table): Array => [ + ...(table.columns ?? []).flatMap((column) => { + const x = quote(column.name) + const any = (failed: string, failure: string): Check => ({ + sql: `BOOL_OR(${failed})`, + verify: (value) => { + if (value === true) V.fail(`column ${JSON.stringify(column.name)} ${failure}`) + } + }) + return [ + ...(column.type === "Null" + ? [any(`${x} IS NOT NULL`, "has values, expected Null")] + : column.nullable + ? [] + : [any(`${x} IS NULL`, "has missing values")]), + ...conditions(x, column).map((c) => any(`NOT (${c.holds})`, c.failure)) + ] + }), + ...columnChecks(table).map(([name, test]): Check => { + const column = table.columns!.find((c) => c.name === name)! + const type = arrowType(column) + return { + sql: `ARRAY_AGG(DISTINCT ${output(declared(column), type)})`, + verify: (values) => { + const rows = ((values ?? []) as ReadonlyArray).map((v) => [v]) + if (!F.toArray(decode(S.make([S.field(name, type, { nullable: true })]), rows), name).every(test)) { + V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) + } + } + } + }) +] + +/** + * The `MATERIALIZED` source of the tables PostgreSQL sees as table `name` + * (see {@link identifier}): their declared columns, read once per + * statement, so checks and result see the same rows. Columns are named as + * PostgreSQL keeps them, so declarations of one column that differ past its + * first 63 bytes all read it. + */ +const source = (name: string, tables: ReadonlyArray): string => { + const columns = [ + ...new Set(tables.flatMap((t) => identifier(t.name) === name ? t.columns!.map((c) => identifier(c.name)) : [])) + ] + return `MATERIALIZED (SELECT ${columns.length === 0 ? "1" : list(columns)} FROM ${quote(name)})` +} + +/** + * Quotes invented names with a prefix that no table or column name of the + * plan starts with (`__`, else the first free `__1_`, `__2_`, …), so a + * helper column can't hide a column and a source can't shadow a table it + * reads. Schemas are unknown, so tables can't be qualified instead. The + * prefix is short and truncation keeps a name's start, so invented names + * stay distinct after PostgreSQL truncates (see {@link identifier}). + */ +const inventor = (frames: ReadonlyMap): (name: string) => string => { + const names = [...frames].flatMap(([node, frame]) => [ + ...(node._tag === "Table" ? [node.name] : []), + ...frame.columns.map((col) => col.name) + ]) + let prefix = "__" + for (let k = 1; names.some((name) => name.startsWith(prefix)); k++) prefix = `__${k}_` + return (name) => quote(prefix + name) +} + +/** + * A name as PostgreSQL keeps it: the whole characters within its first 63 + * bytes in UTF-8 (`NAMEDATALEN - 1`). + */ +const identifier = (name: string): string => { + let bytes = 0 + let end = 0 + for (const char of name) { + const code = char.codePointAt(0)! + bytes += code < 0x80 ? 1 : code < 0x800 ? 2 : code < 0x10000 ? 3 : 4 + if (bytes > 63) break + end += char.length + } + return name.slice(0, end) +} + +/** + * Fails when two columns of a relation of the plan of `frames` are one + * identifier to PostgreSQL (see {@link identifier}): the in-memory engine + * tells them apart, the database can't. + */ +const distinguishable = (frames: ReadonlyMap): void => { + for (const frame of frames.values()) { + const seen = new Map() + for (const { name } of frame.columns) { + const id = identifier(name) + const other = seen.get(id) + if (other !== undefined) { + V.fail( + `columns ${JSON.stringify(other)} and ${JSON.stringify(name)} are one name to PostgreSQL (its first 63 bytes)` + ) + } + seen.set(id, name) + } + } +} + +/** One SQL query whose first row holds the values of checks. */ +interface Checked { + readonly sql: string + readonly checks: ReadonlyArray<(value: unknown) => void> +} + +/** @internal */ +export interface Compiled { + /** The query without the checks: the result's rows in the plan's row order. */ + readonly sql: string + /** + * The checks of the column types of the plan's tables, to run first (the + * query doesn't compile against tables of other column types), when the + * plan reads tables. + */ + readonly types: Checked | undefined + /** + * The query: a row of the checks of its tables' rows first, when there + * are some, then the result's rows in the plan's row order. + */ + readonly query: Checked + readonly schema: S.Schema +} + +/** + * A plan as SQL: the query, a query of its tables' column types, and the + * query with the checks of their rows. @internal + */ +export const compile = (plan: Plan.Node): Compiled => { + const frames = new Map() + const result = typed(plan, frames) + distinguishable(frames) + const hidden = inventor(frames) + const tables = [...frames.keys()].flatMap((node) => node._tag === "Table" ? [{ node, checks: rowChecks(node) }] : []) + const sources = new Map( + [...new Set(tables.map((t) => identifier(t.node.name)))].map((name, i) => [name, hidden(`table${i}`)]) + ) + const compiler: Compiler = { frames, hidden, row: hidden("row"), sources } + const sql = relation(plan, compiler) + const schema = S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) + const columns = result.columns.map((col) => `${output(quote(col.name), col.type)} AS ${quote(col.name)}`) + const nodes = tables.map((t) => t.node) + const ctes = [...sources].map(([name, cte]) => `${cte} AS ${source(name, nodes)}`) + const prelude = ctes.length === 0 ? "" : `WITH ${ctes.join(", ")} ` + const body = `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}` + const columnTypes = nodes.flatMap(typeChecks) + const types = columnTypes.length === 0 ? undefined : { + sql: `SELECT ${columnTypes.map((check) => check.sql).join(", ")}`, + checks: columnTypes.map((check) => check.verify) + } + const checked = tables.filter((t) => t.checks.length > 0) + const plain = prelude + body + if (checked.length === 0) return { sql: plain, types, query: { sql: plain, checks: [] }, schema } + // Check columns are read by position, so they need no names. + const header = checked.map((t, i) => + `(SELECT ${t.checks.map((check) => check.sql).join(", ")} FROM ${sources.get(identifier(t.node.name))}) AS ${ + hidden(`checks${i}`) + }` + ) + const rows = `(SELECT ${[...columns, compiler.row].join(", ")} FROM (${sql}) AS t) AS r` + const selected = [ + ...checked.map((_, i) => `${hidden(`checks${i}`)}.*`), + ...result.columns.map((col) => `r.${quote(col.name)}`) + ] + return { + sql: plain, + types, + query: { + sql: `${prelude}SELECT ${selected.join(", ")} FROM ${ + header.join(" CROSS JOIN ") + } FULL JOIN ${rows} ON FALSE ORDER BY r.${compiler.row} NULLS FIRST`, + checks: checked.flatMap((t) => t.checks.map((check) => check.verify)) + }, + schema + } +} + +/** + * Tests the checks in the first row of a checked query's result, failing + * when a table breaks its declaration, and returns the other rows without + * the checks. + * + * @internal + */ +export const verify = ( + checked: Checked, + rows: ReadonlyArray> +): ReadonlyArray> => { + const k = checked.checks.length + if (k === 0) return rows + checked.checks.forEach((test, i) => test(rows[0][i])) + return rows.slice(1).map((row) => row.slice(k)) +} + +/** The result rows of a compiled query, after its checks (see {@link verify}), as a frame. @internal */ +export const read = (compiled: Compiled, rows: ReadonlyArray>): F.Frame => + decode(compiled.schema, verify(compiled.query, rows)) + +/** + * The vector of a result column of decimals, or of lists of decimals, parsed + * from the strings of its values without a `BigDecimal` per value. + * `undefined` for other types and for values `Dec.fromStrings` doesn't read. + * `decode` then builds the column from what `reader` reads, which also + * reports invalid values. + */ +const decimals = (type: S.DataType, values: ReadonlyArray): V.Vector | undefined => { + if (type._tag === "Decimal") return Dec.fromStrings(type, values) + if (type._tag !== "List" || type.child.type._tag !== "Decimal") return undefined + const offsets = new Int32Array(values.length + 1) + const valid = new Uint8Array(values.length) + const items: Array = [] + let nulls = false + values.forEach((v, i) => { + if (v === null || v === undefined) nulls = true + else { + valid[i] = 1 + for (const item of v as ReadonlyArray) items.push(item) + } + offsets[i + 1] = items.length + }) + const child = Dec.fromStrings(type.child.type, items) + // A null item in a field declared without nulls is left for `decode` to report. + if (child === undefined || (child.valid !== undefined && !type.child.nullable)) return undefined + return V.list(V.storageType(type), offsets, child, nulls ? valid : undefined) +} + +/** The rows of a result, one array of values per row, as a frame of `schema`. */ +const decode = (schema: S.Schema, rows: ReadonlyArray>): F.Frame => + F.make( + schema.fields.map((f, i) => { + const cells = rows.map((row) => row[i]) + const vector = decimals(f.type, cells) + if (vector !== undefined) return new F.Column(f.name, vector, undefined, vector.valid !== undefined) + const read = reader(f.type) + let nullable = false + const values = cells.map((v) => { + if (v !== null && v !== undefined) return read(v) + nullable = true + return null + }) + const column = Result.getOrElse( + ArrowColumn.fromValues(S.field(f.name, f.type, { nullable }), values), + (error) => V.fail(`column ${JSON.stringify(f.name)}: ${error.message}`) + ) + return new F.Column(f.name, V.fromArrowData(column.chunks[0]), undefined, nullable) + }), + rows.length + ) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts new file mode 100644 index 00000000000..8170f048b35 --- /dev/null +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -0,0 +1,305 @@ +/** + * Running plans batch by batch over streamed tables. + * + * Row-wise operations (filters, projections and derived columns without + * windows, renames, `dropNulls`, `explode`, `unnest`) run on every batch as + * it arrives, a slice from the start stops pulling batches once it has its + * rows, joins stream their left side against the collected right side, and + * concatenations stream their inputs one after the other. Grouping and whole- + * table aggregation fold every batch into partial aggregates, so memory grows + * with the number of groups, not rows (medians, distinct counts and + * aggregates of windows collect instead). Every other operation collects its + * inputs first. + * + * Every batch of a node has the column types of the node's whole result: a + * table's batches (converted to its declared columns, if any) must share the + * columns of its first, and a concatenation converts its inputs to their + * common types. + * + * A node has at least one batch, possibly without rows, unless its columns + * are unknown: they depend on a table without declared columns of which no + * batch was read. Where no batch is read, a declared table's result is its + * declared columns without rows. Operations that need unknown columns + * (collecting, concatenating, joining, aggregating) fail, asking for them to + * be declared. + * + * @internal + */ +import * as Effect from "../../Effect.ts" +import * as Option from "../../Option.ts" +import * as Sink from "../../Sink.ts" +import * as Stream from "../../Stream.ts" +import type { Catalog } from "../Catalog.ts" +import type { DataFrameError } from "../DataFrame.ts" +import type * as Plan from "../Plan.ts" +import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" +import * as E from "./expr.ts" +import * as F from "./frame.ts" +import * as Partial from "./partial.ts" +import * as Q from "./query.ts" +import { arrowType, conform } from "./table.ts" +import * as V from "./vector.ts" + +type Batches = Stream.Stream + +/** Where a run reads the batches of its tables. */ +type Tables = Pick + +const reads = (node: Q.Node): boolean => node._tag === "Table" || Q.children(node).some(reads) + +/** + * What a node's columns depend on besides declared ones, so that they are + * unknown without reading its tables: tables without declared columns, and + * pivots without declared columns over tables (they take theirs from the + * values). + */ +const undeclared = (node: Q.Node): ReadonlyArray => [ + ...(node._tag === "Table" && node.columns === undefined ? [`table ${JSON.stringify(node.name)}`] : []), + ...(node._tag === "Pivot" && node.columns === undefined && reads(node.input) + ? [`the pivot on ${JSON.stringify(node.on)}`] + : []), + ...Q.children(node).flatMap(undeclared) +] + +/** + * The result of a node that reads no batch of its tables, as one batch: the + * node run over its tables' declared columns without rows. No batch when its + * columns are unknown. + */ +const unread = (node: Q.Node): Batches => { + if (undeclared(node).length > 0) return Stream.empty + const empty = (table: Plan.Table) => + Q.values(F.empty(table.columns!.map((c) => ({ name: c.name, type: arrowType(c), nullable: c.nullable })))) + return Stream.fromEffect(Effect.flatMap(Q.attempt(() => Q.mapTables(node, empty)), Q.execute)) +} + +/** Fails for a node without batches, whose columns are unknown. */ +const unknownColumns = (node: Q.Node): Effect.Effect => + Q.attempt(() => + V.fail( + `the columns of a result without rows are unknown, as no batch they depend on was read: ` + + `declare the columns of ${[...new Set(undeclared(node))].join(" and ")}` + ) + ) + +const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect => + Effect.flatMap(Stream.runCollect(batches), (frames) => + frames.length === 0 + ? unknownColumns(node) + : Q.attempt(() => frames.length === 1 ? frames[0] : F.concat(frames))) + +/** Each output row depends only on its input row, so the node runs per batch. */ +const perBatch = (node: Q.Node): boolean => { + switch (node._tag) { + case "Filter": + case "Select": + case "WithColumns": + case "Pick": + case "Omit": + case "Rename": + case "DropNulls": + case "Explode": + case "Unnest": + return Q.isRowWise(node) + default: + return false + } +} + +/** Runs a node by collecting its inputs (an operation that needs all rows). */ +const collected = (node: Q.Node, catalog: Tables): Batches => + Stream.fromEffect( + Effect.flatMap( + Effect.forEach(Q.children(node), (c) => collectFrames(batches(c, catalog), c)), + (inputs) => Q.attempt(() => Q.step(node, inputs)) + ) + ) + +const aggregate = ( + node: Extract, + catalog: Tables +): Batches => { + const keys = node._tag === "GroupBy" ? node.keys : [] + const specs = node.aggregates.map(({ expression, name }) => { + if (expression._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) + return { name, fn: Agg.kernel(expression.fn, expression.arg), arg: expression.arg } + }) + if (!Q.isRowWise(node) || !specs.every((s) => Partial.combinable(s.fn))) { + return collected(node, catalog) + } + const fns = specs.map((s) => s.fn) + return Stream.suspend(() => { + // Whether any batch's key column may hold nulls. + const nullable = keys.map(() => false) + return Stream.fromEffect( + Stream.runFoldEffect( + batches(node.input, catalog), + (): Partial.State | undefined => undefined, + (state, batch) => + Q.attempt(() => { + const ctx = F.context(batch) + keys.forEach((k, i) => nullable[i] ||= F.column(batch, k).nullable) + const part = Partial.group( + keys.map((k) => F.column(batch, k).values()), + specs.map((s) => ({ fn: s.fn, vector: s.arg === undefined ? undefined : E.evaluateColumn(s.arg, ctx) })), + batch.length + ) + return state === undefined ? part : Partial.merge(fns, [state, part]) + }) + ).pipe( + Effect.flatMap((state) => + state === undefined ? unknownColumns(node.input) : Q.attempt(() => { + const keyColumns = keys.map((k, i) => new F.Column(k, state.keys[i], undefined, nullable[i])) + const columns = specs.map((s, a) => + new F.Column(s.name, Partial.finish(s.fn, state.parts[a], state.types[a]), undefined) + ) + return F.make([...keyColumns, ...columns], state.count) + }) + ) + ) + ) + }) +} + +/** + * The inputs' batches one after the other, converted to the inputs' common + * types. An input's first batch gives its types, so every input starts before + * the first batch is emitted. + */ +const concat = (inputs: ReadonlyArray, catalog: Tables): Batches => + Stream.unwrap(Effect.gen(function*() { + const peeled = yield* Effect.forEach( + inputs, + (input) => + Effect.flatMap(Stream.peel(batches(input, catalog), Sink.head()), ([first, rest]) => + Option.isSome(first) + ? Effect.succeed({ first: first.value, rest: Stream.concat(Stream.succeed(first.value), rest) }) + : unknownColumns(input)) + ) + const types = yield* Q.attempt(() => F.concat(peeled.map(({ first }) => F.slice(first, 0, 0)))) + return peeled.map(({ rest }) => Stream.mapEffect(rest, (batch) => Q.attempt(() => F.coerce(batch, types)))) + .reduce((a, b) => Stream.concat(a, b)) + })) + +const describeColumns = (frame: F.Frame): string => + `[${frame.columns.map((c) => `${c.name}: ${D.describe(c.type)}`).join(", ")}]` + +/** + * Reads the batches of a table in turn: each converted to the declared + * columns, if any, and checked to have the columns of the first. + */ +const reader = (table: Plan.Table) => { + let first: F.Frame | undefined + return (frame: F.Frame): F.Frame => { + const batch = table.columns === undefined ? frame : conform(frame, table.columns, table.checks) + first ??= batch + const same = batch.columns.length === first.columns.length && + batch.columns.every((c, i) => c.name === first!.columns[i].name && D.sameType(c.type, first!.columns[i].type)) + return same ? batch : V.fail( + `a batch of table ${JSON.stringify(table.name)} has columns ${describeColumns(batch)}, ` + + `its first batch ${describeColumns(first)}: the batches of a streamed table share their columns` + ) + } +} + +const slice = (input: Batches, offset: number, length: number | undefined): Batches => + Stream.suspend(() => { + let skip = offset + let left = length ?? Infinity + return input.pipe( + Stream.map((batch) => { + const start = Math.min(skip, batch.length) + skip -= start + const end = Math.min(batch.length, start + left) + left -= end - start + return F.slice(batch, start, end) + }), + Stream.takeUntil(() => left === 0) + ) + }) + +/** + * The batches of a node's result: at least one unless its columns are + * unknown. Batches without rows stay, so a result whose rows are all dropped + * still has the types of its columns. + */ +const batches = (node: Q.Node, catalog: Tables): Batches => { + if (perBatch(node)) { + return Stream.mapEffect( + batches((node as { readonly input: Q.Node }).input, catalog), + (batch) => Q.attempt(() => Q.step(node, [batch])) + ) + } + switch (node._tag) { + case "Table": + return Stream.suspend(() => { + const read = reader(node) + return Stream.mapEffect( + catalog.stream(node.name), + (frame) => Q.attempt(() => read(frame as unknown as F.Frame)) + ) + }).pipe(Stream.orElseIfEmpty(() => unread(node))) + case "Values": + return Stream.succeed(node.frame as unknown as F.Frame) + case "Slice": + if (node.length === 0) return unread(node) + return node.offset >= 0 ? slice(batches(node.input, catalog), node.offset, node.length) : collected(node, catalog) + case "TopK": + return node.limit === 0 ? unread(node) : collected(node, catalog) + case "Join": { + if (node.how === "full") return collected(node, catalog) + const right = collectFrames(batches(node.right, catalog), node.right) + return Stream.unwrap(Effect.map(right, (r) => + Stream.mapEffect( + batches(node.left, catalog), + (batch) => Q.attempt(() => F.join(batch, r, node.on, node.how)) + ))) + } + case "Concat": + return node.inputs.length === 0 + ? collected(node, catalog) + : concat(node.inputs, catalog) + case "GroupBy": + case "Aggregate": + return aggregate(node, catalog) + default: + return collected(node, catalog) + } +} + +/** + * The optimised plan, and its tables with those it names more than once read + * into memory on first use: each run reads each table at most once, and only + * if a node needs its batches. + */ +const prepare = ( + plan: Plan.Node, + catalog: Catalog.Service +): Effect.Effect => + Effect.gen(function*() { + const optimized = yield* Q.attempt(() => Q.optimize(plan as Q.Node)) + const read = new Map>() + for (const [name, n] of Q.tables(optimized as Plan.Node)) { + if (n > 1) { + read.set( + name, + Stream.unwrap(Effect.map(yield* Effect.cached(Stream.runCollect(catalog.stream(name))), Stream.fromIterable)) + ) + } + } + return [optimized, { stream: (name: string) => read.get(name) ?? catalog.stream(name) }] as const + }) + +/** The non-empty batches of a plan's result, after optimisation. @internal */ +export const stream = (plan: Plan.Node, catalog: Catalog.Service): Batches => + Stream.unwrap(Effect.map(prepare(plan, catalog), ([optimized, read]) => batches(optimized, read))).pipe( + Stream.filter((batch) => batch.length > 0) + ) + +/** A plan's result as one frame, computed batch by batch. @internal */ +export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => + Effect.flatMap( + prepare(plan, catalog), + ([optimized, read]) => collectFrames(batches(optimized, read), optimized) + ) diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts new file mode 100644 index 00000000000..c25760d668d --- /dev/null +++ b/packages/effect/src/dataframe/internal/table.ts @@ -0,0 +1,541 @@ +/** + * Declared table columns: what a frame must hold to be read as a table, and + * the conversion of a frame to them. + * + * A declared column names its type once (`declareColumn`, `fromArrowSchema`). + * Validating a frame and reading a table convert to that declaration + * (`arrowType`, `conform`), so the same declaration always gives the same + * column types. The conversion works on the typed buffers of the columns and + * builds no value objects. + * + * A `Timestamp` column reads as `DateTime.Utc` values, so its timestamps + * must have no time zone (or a UTC one, which is dropped) and lie in the + * range `DateTime` represents. + * + * A declaration can carry tests of values and rows (`Plan.TableChecks`). + * They are compiled by whoever declares the table and run here as given, on + * the values of the columns they name and on the rows. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Layout from "../../arrow/internal/layout.ts" +import * as Values from "../../arrow/internal/values.ts" +import * as DateTime from "../../DateTime.ts" +import * as Duration from "../../Duration.ts" +import type * as Plan from "../Plan.ts" +import * as D from "./dtype.ts" +import * as F from "./frame.ts" +import * as K from "./kernels.ts" +import * as V from "./vector.ts" + +type Column = Omit + +/** The path of field `name` under `path`. @internal */ +export const field = (path: string, name: string) => (path === "" ? name : `${path}.${name}`) + +// ----------------------------------------------------------------------------- +// Declared columns +// ----------------------------------------------------------------------------- + +/** The plan type name of an Arrow type. */ +const typeName = (t: S.DataType): Plan.TypeName => { + switch (t._tag) { + case "Int": + case "Float": + return D.describe(t) as Plan.TypeName + case "Utf8": + case "LargeUtf8": + case "Utf8View": + return "Utf8" + case "Binary": + case "LargeBinary": + case "BinaryView": + case "FixedSizeBinary": + return "Binary" + case "LargeList": + return "List" + default: + return t._tag as Plan.TypeName + } +} + +const describe = (t: S.DataType): string => { + if (t._tag === "List" || t._tag === "LargeList") return `List<${describe(t.child.type)}>` + return t._tag === "Timestamp" || t._tag === "Duration" || t._tag === "Decimal" ? D.describe(t) : typeName(t) +} + +const describeColumn = (c: Column): string => c.type === "List" ? `List<${describeColumn(c.fields![0])}>` : c.type + +const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) + +/** + * Fails unless values of Arrow type `t` read as values of the column (not + * looking into lists and structs). Timestamps need no time zone or a UTC one. + */ +const checkType = (path: string, t: S.DataType, expected: Column): void => { + if (expected.type === "Timestamp" && t._tag === "Timestamp" && t.timezone !== undefined) { + if (utcZones.has(t.timezone)) return + return V.fail(`column ${JSON.stringify(path)} has time zone ${t.timezone}, expected UTC timestamps`) + } + if (!compatible(t, expected)) { + V.fail(`column ${JSON.stringify(path)} is ${describe(t)}, expected ${describeColumn(expected)}`) + } +} + +/** + * Integers below 64 bits read as floats and narrower integers, 64-bit + * integers (bigints) as 64-bit ones, any decimal as a decimal, and temporal + * types as the same kind in any unit. + */ +const compatible = (t: S.DataType, expected: Column): boolean => { + switch (expected.type) { + case "Float16": + case "Float32": + case "Float64": + return t._tag === "Float" || (t._tag === "Int" && t.bitWidth < 64) + case "Int8": + case "Int16": + case "Int32": + case "UInt8": + case "UInt16": + case "UInt32": + return t._tag === "Int" && t.bitWidth < 64 + case "Int64": + case "UInt64": + return t._tag === "Int" && t.bitWidth === 64 + default: + return typeName(t) === expected.type + } +} + +const missing = (path: string): never => V.fail(`missing column ${JSON.stringify(path)}`) + +/** + * The 128-bit decimal type of a declared precision and scale, failing unless + * Arrow allows them (`Layout.checkDecimal`). Declarations and plans give + * them unchecked, and converting values computes with them. + */ +const decimalType = (path: string, precision: number, scale: number): S.Decimal => { + const type = S.decimal(precision, scale) + Layout.checkDecimal(type, (message) => V.fail(`column ${JSON.stringify(path)}: ${message}`)) + return type +} + +/** + * The decimal type an Arrow decimal declares. Declared decimals are 128-bit, + * so another width fails instead of being declared as one. + */ +const declaredDecimal = (path: string, t: S.Decimal): S.Decimal => + t.bitWidth === 128 + ? decimalType(path, t.precision, t.scale) + : undeclarable(path, `${t.bitWidth}-bit decimals`, "declared decimals are 128-bit") + +/** The column declared for a field's `column` with Arrow type `t`, or its default type. @internal */ +export const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataType | undefined): Plan.TableColumn => { + if (t !== undefined) checkType(path, t, column) + switch (column.type) { + case "List": { + const child = t !== undefined && (t._tag === "List" || t._tag === "LargeList") ? t.child.type : undefined + return { ...column, fields: [declareColumn(`${path}[]`, column.fields![0], child)] } + } + case "Struct": + return { + ...column, + fields: column.fields!.map((f) => { + const p = field(path, f.name) + const g = t?._tag === "Struct" ? (t.fields.find((g) => g.name === f.name) ?? missing(p)).type : undefined + return declareColumn(p, f, g) + }) + } + case "Decimal": { + if (t?._tag !== "Decimal") return column + const { precision, scale } = declaredDecimal(path, t) + return { ...column, precision, scale } + } + case "Timestamp": + case "Duration": + return { ...column, unit: t?._tag === column.type ? t.unit : "microsecond" } + default: + return t === undefined || column.type === "Null" ? column : { ...column, type: typeName(t) } + } +} + +/** Arrow types frames hold as another type, which is the one to declare. */ +const heldAs: { readonly [tag: string]: string } = { + LargeUtf8: "Utf8", + Utf8View: "Utf8", + LargeBinary: "Binary", + BinaryView: "Binary", + LargeList: "List", + Dictionary: "their value type" +} + +const undeclarable = (path: string, what: string, why?: string): never => + V.fail(`column ${JSON.stringify(path)}: ${what} can't be declared${why === undefined ? "" : `: ${why}`}`) + +/** + * The declared column of an Arrow field. Only types that {@link arrowType} + * gives back are declared: a type frames can't hold, or hold as another + * type, fails. A UTC time zone is dropped, as {@link checkType} reads it. + */ +const fieldColumn = (path: string, f: S.Field): Plan.TableColumn => { + const t = f.type + const column = { name: f.name, nullable: f.nullable } + switch (t._tag) { + case "Null": + return f.nullable + ? { ...column, type: "Null" } + : V.fail(`column ${JSON.stringify(path)}: Null columns are nullable`) + case "Bool": + case "Utf8": + case "Binary": + case "Int": + return { ...column, type: typeName(t) } + case "Float": + return t.bitWidth === 16 ? undeclarable(path, "Float16 columns") : { ...column, type: typeName(t) } + case "Date": + return t.unit === "day" + ? { ...column, type: "Date" } + : undeclarable(path, `dates in ${t.unit}s`, "declared dates are in days") + case "Time": + return t.unit === "microsecond" + ? { ...column, type: "Time" } + : undeclarable(path, `times in ${t.unit}s`, "declared times are in microseconds") + case "Timestamp": + checkType(path, t, { type: "Timestamp", nullable: f.nullable }) + return { ...column, type: "Timestamp", unit: t.unit } + case "Duration": + return { ...column, type: "Duration", unit: t.unit } + case "Decimal": { + const { precision, scale } = declaredDecimal(path, t) + return { ...column, type: "Decimal", precision, scale } + } + case "List": + return { ...column, type: "List", fields: [fieldColumn(`${path}[]`, { ...t.child, name: "item" })] } + case "Struct": + return { ...column, type: "Struct", fields: t.fields.map((g) => fieldColumn(field(path, g.name), g)) } + default: + return undeclarable( + path, + `${t._tag} columns`, + heldAs[t._tag] === undefined ? undefined : `frames hold them as ${heldAs[t._tag]}` + ) + } +} + +/** + * The declared columns of an Arrow schema's fields: their types, + * nullability, nested fields, decimal precision and scale, and time units. + * + * @internal + */ +export const fromArrowSchema = (schema: S.Schema): Array => + schema.fields.map((f) => fieldColumn(f.name, f)) + +/** + * The Arrow type of a declared column. Timestamps and durations without a + * unit are microseconds, dates days and times microseconds; decimals need + * their precision and scale. + * + * @internal + */ +export const arrowType = (column: Plan.TableColumn, path: string = column.name): S.DataType => { + switch (column.type) { + case "List": { + const item = column.fields![0] + return S.list(S.field("item", arrowType(item, `${path}[]`), { nullable: item.nullable })) + } + case "Struct": + return S.struct( + column.fields!.map((f) => S.field(f.name, arrowType(f, field(path, f.name)), { nullable: f.nullable })) + ) + case "Utf8": + return S.utf8 + case "Binary": + return S.binary + case "Bool": + return S.bool + case "Null": + return S.nullType + case "Date": + return S.date("day") + case "Time": + return S.time("microsecond") + case "Timestamp": + return S.timestamp(column.unit ?? "microsecond") + case "Duration": + return S.duration(column.unit ?? "microsecond") + case "Decimal": + return column.precision === undefined || column.scale === undefined + ? V.fail( + `column ${ + JSON.stringify(path) + }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ) + : decimalType(path, column.precision, column.scale) + case "Float16": + return S.float16 + default: + // The other numeric types, named as cast targets. + return D.castTargets[column.type] + } +} + +/** + * The Arrow schema of declared columns. + * + * @internal + */ +export const toArrowSchema = (columns: ReadonlyArray): S.Schema => + S.make(columns.map((c) => S.field(c.name, arrowType(c), { nullable: c.nullable }))) + +// ----------------------------------------------------------------------------- +// Checking frames +// ----------------------------------------------------------------------------- + +const inRows = (rows: Uint8Array | undefined, i: number): boolean => rows === undefined || rows[i] === 1 + +const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { + for (let i = 0; i < v.length; i++) if (inRows(rows, i) && V.isValid(v, i)) return false + return true +} + +/** The stored values of a timestamp in each unit that `DateTime` represents (±8.64e15 ms). */ +const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { + second: [-BigInt("8640000000000"), BigInt("8640000000000")], + millisecond: [-BigInt("8640000000000000"), BigInt("8640000000000000")], + microsecond: [-BigInt("8640000000000000000"), BigInt("8640000000000000999")], + nanosecond: [-BigInt("9223372036854775808"), BigInt("9223372036854775807")] +} + +const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | undefined): void => { + const [lo, hi] = timestampRange[unit] + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { + V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) + } + } +} + +/** + * Fails for a decimal with more digits than its column's precision. A + * column can hold one (IPC decoding does not check stored values), and + * `convert` returns a column that has the target type as it is. + */ +const checkPrecision = (path: string, v: V.I64, type: S.Decimal, rows: Uint8Array | undefined): void => { + const limit = BigInt(10) ** BigInt(type.precision) + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && V.isValid(v, i) && (v.values[i] >= limit || v.values[i] <= -limit)) inexact(path, type) + } +} + +const sameValue = (a: V.Vector, b: V.Vector, i: number): boolean => { + switch (a.kind) { + case "f64": { + const x = a.values[i] + const y = (b as V.F64).values[i] + return Object.is(x, y) || (x !== x && y !== y) + } + case "i32": + case "i64": + case "bool": + return a.values[i] === (b as typeof a).values[i] + case "str": + return V.decodeString(a, i) === V.decodeString(b as V.Str, i) + default: + return true + } +} + +/** `v` converted to `target`, failing unless every present value converts exactly. */ +const convert = (path: string, v: V.Vector, target: S.DataType, rows: Uint8Array | undefined): V.Vector => { + if (D.sameType(v.type, target)) return v + const converted = K.cast(v, target) + const back = K.cast(converted, v.type) + for (let i = 0; i < v.length; i++) { + if ( + inRows(rows, i) && V.isValid(v, i) && + (!V.isValid(converted, i) || !sameValue(v, back, i)) + ) { + return inexact(path, target) + } + } + return converted +} + +const inexact = (path: string, target: S.DataType): never => + V.fail(`column ${JSON.stringify(path)} has values that ${D.describe(target)} can't hold exactly`) + +/** Microseconds in each unit of a time, as a fraction. */ +const microseconds: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { + second: [BigInt(1_000_000), BigInt(1)], + millisecond: [BigInt(1_000), BigInt(1)], + microsecond: [BigInt(1), BigInt(1)], + nanosecond: [BigInt(1), BigInt(1_000)] +} + +/** + * A date in days or a time in microseconds (`target`) from one in another + * unit, failing unless every present value converts exactly. `K.cast` + * reads the stored numbers of these types as they are, in any unit. + */ +const rescale = (path: string, v: V.I32 | V.I64, target: S.Date | S.Time, rows: Uint8Array | undefined): V.Vector => { + const [mul, div] = v.type._tag === "Time" ? microseconds[v.type.unit] : [BigInt(1), BigInt(86_400_000)] + const out = target._tag === "Date" ? new Int32Array(v.length) : new BigInt64Array(v.length) + for (let i = 0; i < v.length; i++) { + if (!inRows(rows, i) || !V.isValid(v, i)) continue + const x = BigInt(v.values[i]) * mul + const y = x / div + if (y * div !== x || BigInt.asIntN(out.BYTES_PER_ELEMENT * 8, y) !== y) return inexact(path, target) + out[i] = (target._tag === "Date" ? Number(y) : y) as never + } + return out instanceof Int32Array ? V.i32(target, out, v.valid) : V.i64(target, out, v.valid) +} + +/** + * A test of the values rows give a column, failing for a `DateTime` or + * `Duration` that is not a whole number of the column's unit: building the + * column rounds it down. `undefined` when the unit holds them all (a + * `DateTime` is in milliseconds, a `Duration` in nanoseconds). + */ +const exact = (path: string, column: Plan.TableColumn): ((value: unknown) => void) | undefined => { + switch (column.type) { + case "Timestamp": + return column.unit !== "second" ? undefined : (value) => { + if (DateTime.isDateTime(value) && DateTime.toEpochMillis(value) % 1000 !== 0) { + inexact(path, arrowType(column, path)) + } + } + case "Duration": { + const nanos = Values.nanosPerUnit[column.unit ?? "microsecond"] + return nanos === BigInt(1) ? undefined : (value) => { + if (Duration.isDuration(value) && Duration.toNanosUnsafe(value) % nanos !== BigInt(0)) { + inexact(path, arrowType(column, path)) + } + } + } + case "List": { + const item = exact(`${path}[]`, column.fields![0]) + return item && ((value) => { + if (value == null) return + // Any other iterable may be spent by now, with nothing left to test. + if (!Array.isArray(value)) return V.fail(`column ${JSON.stringify(path)} needs its lists as arrays`) + value.forEach(item) + }) + } + case "Struct": { + const fields = column.fields!.flatMap((f, k) => { + const test = exact(field(path, f.name), f) + return test === undefined ? [] : [[f.name, k, test] as const] + }) + return fields.length === 0 ? undefined : (value) => { + if (value == null) return + // An array gives a struct's values by position. + for (const [name, k, test] of fields) test(Array.isArray(value) ? value[k] : (value as any)[name]) + } + } + default: + return undefined + } +} + +/** + * Fails unless building the columns from `rows` keeps their `DateTime` + * and `Duration` values exactly: one finer than its column's unit fails, + * as it does when a frame is converted ({@link conform}). + * + * @internal + */ +export const checkExact = (columns: ReadonlyArray, rows: ReadonlyArray): void => { + for (const column of columns) { + const test = exact(column.name, column) + if (test !== undefined) { for (const row of rows) test(row[column.name]) } + } +} + +/** A column checked against its declaration and converted to its declared type. */ +const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint8Array | undefined): V.Vector => { + if (expected.type === "Null") { + if (!allNull(v, rows)) checkType(path, v.type, expected) + return V.nulls(v.length) + } + checkType(path, v.type, expected) + if (!expected.nullable) { + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && !V.isValid(v, i)) { + return V.fail(`column ${JSON.stringify(path)} has missing values`) + } + } + } + if (v.kind === "struct" && v.type._tag === "Struct") { + const present = new Uint8Array(v.length) + for (let i = 0; i < v.length; i++) present[i] = inRows(rows, i) && V.isValid(v, i) ? 1 : 0 + const fields = v.type.fields + const children = expected.fields!.map((f) => { + const k = fields.findIndex((g) => g.name === f.name) + return k === -1 ? missing(field(path, f.name)) : check(field(path, f.name), v.children[k], f, present) + }) + const type = S.struct(expected.fields!.map((f, k) => S.field(f.name, children[k].type, { nullable: f.nullable }))) + return V.struct(type, children, v.length, v.valid) + } + if (v.kind === "list") { + const present = new Uint8Array(v.child.length) + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) + } + const item = expected.fields![0] + const child = check(`${path}[]`, v.child, item, present) + return V.list(S.list(S.field("item", child.type, { nullable: item.nullable })), v.offsets, child, v.valid) + } + if (expected.type === "Timestamp" && v.kind === "i64" && v.type._tag === "Timestamp") { + checkRange(path, v, v.type.unit, rows) + } + const target = arrowType(expected, path) + if (target._tag === "Decimal" && v.kind === "i64" && D.sameType(v.type, target)) { + checkPrecision(path, v, target, rows) + } + return (target._tag === "Date" || target._tag === "Time") && !D.sameType(v.type, target) + ? rescale(path, v as V.I32 | V.I64, target, rows) + : convert(path, v, target, rows) +} + +/** + * Fails unless the values of the columns that `checks` names pass their + * test, and the rows the row test. + */ +const runChecks = (checks: Plan.TableChecks, frame: F.Frame): void => { + for (const [name, test] of checks.columns) { + if (!F.toArray(frame, name).every(test)) V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) + } + if (checks.rows !== undefined && !F.toRows(frame).every(checks.rows)) V.fail("rows have values their schema rejects") +} + +/** + * Checks that a frame has the declared columns and narrows it to them, in + * their order, converting each to its declared type (exactly: a value the + * declared type can't hold fails). With `checks`, the values of the columns + * they name, and the rows when they test rows, are then read and tested; + * other columns are never read as values. + * + * @internal + */ +export const conform = ( + frame: F.Frame, + columns: ReadonlyArray, + checks?: Plan.TableChecks | undefined +): F.Frame => { + const narrowed = F.make( + columns.map((expected) => { + const c = frame.columns.find((c) => c.name === expected.name) ?? missing(expected.name) + const values = c.values() + const checked = check(expected.name, values, expected, undefined) + return checked === values && c.nullable === expected.nullable + ? c + : new F.Column(c.name, checked, undefined, expected.nullable) + }), + frame.length + ) + if (checks !== undefined) runChecks(checks, narrowed) + return narrowed +} diff --git a/packages/effect/src/dataframe/internal/temporal.ts b/packages/effect/src/dataframe/internal/temporal.ts new file mode 100644 index 00000000000..40b9f14ea51 --- /dev/null +++ b/packages/effect/src/dataframe/internal/temporal.ts @@ -0,0 +1,337 @@ +/** + * Dates, timestamps and durations: unit conversions, calendar fields, + * truncation and casts, on their integer storage (days since the epoch for + * dates, units since the epoch for timestamps, units for durations). + * + * Calendar arithmetic uses Howard Hinnant's day-count algorithms, exact for + * negative days. Timestamps are read as UTC: their calendar fields are UTC + * fields (timestamps with another time zone are not supported yet). + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as V from "./vector.ts" + +/** @internal */ +export const isTemporal = (t: S.DataType): boolean => + t._tag === "Date" || t._tag === "Timestamp" || t._tag === "Duration" || t._tag === "Time" + +const perSecond: Record = { + second: 1, + millisecond: 1_000, + microsecond: 1_000_000, + nanosecond: 1_000_000_000 +} + +/** The coarser of two units (the common unit of mixed operands, as in Polars). @internal */ +export const coarser = (a: S.TimeUnit, b: S.TimeUnit): S.TimeUnit => (perSecond[a] <= perSecond[b] ? a : b) + +const zoned = (t: S.Timestamp): boolean => t.timezone !== undefined && t.timezone !== "UTC" && t.timezone !== "+00:00" + +/** Whether two timestamps' zones are compatible (no zone counts as UTC). */ +const sameZone = (a: S.Timestamp, b: S.Timestamp): boolean => (zoned(a) || zoned(b) ? a.timezone === b.timezone : true) + +/** The common type of two temporal types, if they have one. @internal */ +export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined => { + if (a._tag === "Timestamp" && b._tag === "Timestamp") { + return sameZone(a, b) ? S.timestamp(coarser(a.unit, b.unit), a.timezone ?? b.timezone) : undefined + } + if (a._tag === "Timestamp" && b._tag === "Date") return a + if (a._tag === "Date" && b._tag === "Timestamp") return b + if (a._tag === "Date" && b._tag === "Date") return S.date("day") + if (a._tag === "Duration" && b._tag === "Duration") return S.duration(coarser(a.unit, b.unit)) + return undefined +} + +// ----------------------------------------------------------------------------- +// Integer helpers +// ----------------------------------------------------------------------------- + +const I0 = BigInt(0) +const I1 = BigInt(1) + +/** Integer division rounding toward negative infinity. @internal */ +export const floorDiv = (a: bigint, b: bigint): bigint => { + const q = a / b + return a % b !== I0 && (a < I0) !== (b < I0) ? q - I1 : q +} + +/** `v`'s integers times `factor` as `target`; null where the product leaves 64 bits. */ +const multiply = (v: V.Vector, factor: bigint, target: S.DataType): V.Vector => { + const x = asI64(v) + const out = new BigInt64Array(x.length) + const valid = new Uint8Array(x.length) + for (let i = 0; i < x.length; i++) { + const y = x[i] * factor + out[i] = BigInt.asIntN(64, y) + valid[i] = V.isValid(v, i) && out[i] === y ? 1 : 0 + } + return V.i64(target, out, V.compactValid(valid)) +} + +/** + * Rescales integer counts of `from` units to `to` units as `target`, + * rounding down (toward zero when `truncate`, as Polars rescales + * durations); counts that leave 64 bits become null. + */ +const rescale = (v: V.Vector, from: S.TimeUnit, to: S.TimeUnit, target: S.DataType, truncate = false): V.Vector => { + if (perSecond[to] > perSecond[from]) return multiply(v, BigInt(perSecond[to] / perSecond[from]), target) + const f = BigInt(perSecond[from] / perSecond[to]) + return V.i64(target, BigInt64Array.from(asI64(v), (x) => (truncate ? x / f : floorDiv(x, f))), v.valid) +} + +const dayUnits = (unit: S.TimeUnit): bigint => BigInt(86_400 * perSecond[unit]) + +/** Units per day of a 64-bit date (milliseconds) or timestamp. */ +const unitsPerDay = (t: S.DataType): bigint => dayUnits(t._tag === "Timestamp" ? t.unit : "millisecond") + +/** Days since the epoch of every row (dates and timestamps). */ +const days = (v: V.Vector): Float64Array => { + const out = new Float64Array(v.length) + if (v.kind === "i32") { + for (let i = 0; i < v.length; i++) out[i] = v.values[i] + } else if (v.kind === "i64") { + const per = unitsPerDay(v.type) + for (let i = 0; i < v.length; i++) out[i] = Number(floorDiv(v.values[i], per)) + } + return out +} + +// ----------------------------------------------------------------------------- +// Civil dates +// ----------------------------------------------------------------------------- + +/** Year, month (1-12) and day (1-31) of a day count. @internal */ +export const civil = (z: number): readonly [number, number, number] => { + z += 719468 + const era = Math.floor(z / 146097) + const doe = z - era * 146097 + const yoe = Math.floor((doe - Math.floor(doe / 1460) + Math.floor(doe / 36524) - Math.floor(doe / 146096)) / 365) + const doy = doe - (365 * yoe + Math.floor(yoe / 4) - Math.floor(yoe / 100)) + const mp = Math.floor((5 * doy + 2) / 153) + const d = doy - Math.floor((153 * mp + 2) / 5) + 1 + const m = mp < 10 ? mp + 3 : mp - 9 + return [yoe + era * 400 + (m <= 2 ? 1 : 0), m, d] +} + +/** The day count of a date. */ +const daysFromCivil = (year: number, month: number, day: number): number => { + const y = month <= 2 ? year - 1 : year + const era = Math.floor(y / 400) + const yoe = y - era * 400 + const doy = Math.floor((153 * (month + (month > 2 ? -3 : 9)) + 2) / 5) + day - 1 + const doe = yoe * 365 + Math.floor(yoe / 4) - Math.floor(yoe / 100) + doy + return era * 146097 + doe - 719468 +} + +// ----------------------------------------------------------------------------- +// Fields +// ----------------------------------------------------------------------------- + +/** @internal */ +export type Field = + | "year" + | "month" + | "day" + | "hour" + | "minute" + | "second" + | "millisecond" + | "weekday" + | "ordinalDay" + +const fieldType: Record = { + year: S.int32, + month: S.int8, + day: S.int8, + hour: S.int8, + minute: S.int8, + second: S.int8, + millisecond: S.int32, + weekday: S.int8, + ordinalDay: S.int16 +} + +const checkTimestamp = (v: V.Vector, fn: string): void => { + if (v.type._tag !== "Date" && v.type._tag !== "Timestamp") { + V.fail(`${fn} is defined for dates and timestamps, not ${v.type._tag}`) + } + if (v.type._tag === "Timestamp" && zoned(v.type)) V.fail(`${fn} of timestamps with a time zone is not supported yet`) +} + +/** A calendar or clock field of every date or timestamp. @internal */ +export const field = (fn: Field, v: V.Vector): V.Vector => { + checkTimestamp(v, fn) + const n = v.length + const out = new Int32Array(n) + if (fn === "hour" || fn === "minute" || fn === "second" || fn === "millisecond") { + if (v.type._tag !== "Timestamp" || v.kind !== "i64") return V.i32(fieldType[fn], out, v.valid) + const unit = v.type.unit + const per = dayUnits(unit) + const second = BigInt(perSecond[unit]) + for (let i = 0; i < n; i++) { + const rem = v.values[i] - floorDiv(v.values[i], per) * per + const seconds = Number(rem / second) + out[i] = fn === "hour" + ? Math.floor(seconds / 3600) + : fn === "minute" + ? Math.floor(seconds / 60) % 60 + : fn === "second" + ? seconds % 60 + : Number((rem % second) * BigInt(1000) / second) + } + return V.i32(fieldType[fn], out, v.valid) + } + const d = days(v) + for (let i = 0; i < n; i++) { + const z = d[i] + switch (fn) { + case "weekday": + out[i] = (((z + 3) % 7) + 7) % 7 + 1 + break + case "ordinalDay": { + const [y] = civil(z) + out[i] = z - daysFromCivil(y, 1, 1) + 1 + break + } + default: { + const [y, m, day] = civil(z) + out[i] = fn === "year" ? y : fn === "month" ? m : day + } + } + } + return V.i32(fieldType[fn], out, v.valid) +} + +// ----------------------------------------------------------------------------- +// Truncation +// ----------------------------------------------------------------------------- + +const parseEvery = (every: string): { readonly n: number; readonly unit: string } => { + const match = /^(\d+)(ns|us|ms|s|m|h|d|w|mo|y)$/.exec(every) + if (match === null) return V.fail(`truncate expects an interval like "1h", "1d", "1w", "1mo" or "1y", got ${every}`) + const n = Number(match[1]) + if (n < 1) return V.fail(`truncate needs a positive interval, got ${every}`) + return { n, unit: match[2] } +} + +const seconds: Record = { ns: 1e-9, us: 1e-6, ms: 1e-3, s: 1, m: 60, h: 3600, d: 86_400, w: 604_800 } + +/** + * Every date or timestamp rounded down to a multiple of `every`: seconds, + * minutes, hours and days from the epoch, weeks from a Monday, months and + * years from the start of the calendar month or year. + * + * @internal + */ +export const truncate = (v: V.Vector, every: string): V.Vector => { + checkTimestamp(v, "truncate") + const { n: k, unit } = parseEvery(every) + const n = v.length + if (unit === "mo" || unit === "y") { + const months = unit === "y" ? 12 * k : k + const d = days(v) + const starts = new Float64Array(n) + for (let i = 0; i < n; i++) { + const [y, m] = civil(d[i]) + const index = Math.floor((y * 12 + (m - 1)) / months) * months + starts[i] = daysFromCivil(Math.floor(index / 12), (index % 12) + 1, 1) + } + return fromDays(v, starts) + } + if (v.kind === "i32") { + // dates: whole days + if (unit !== "d" && unit !== "w") return v + const size = k * (unit === "w" ? 7 : 1) + const offset = unit === "w" ? 3 : 0 // 1970-01-01 is a Thursday; weeks start on Monday + const out = new Int32Array(n) + for (let i = 0; i < n; i++) out[i] = Math.floor((v.values[i] + offset) / size) * size - offset + return V.i32(v.type, out, v.valid) + } + if (v.kind !== "i64" || v.type._tag !== "Timestamp") return V.fail(`truncate is not defined for ${v.type._tag}`) + const size = BigInt(Math.round(seconds[unit] * k * perSecond[v.type.unit])) + if (size <= I0) return v + const offset = unit === "w" ? BigInt(3) * dayUnits(v.type.unit) : I0 + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) out[i] = floorDiv(v.values[i] + offset, size) * size - offset + return V.i64(v.type, out, v.valid) +} + +/** Day counts as the input's type (dates stay dates). */ +const fromDays = (v: V.Vector, d: Float64Array): V.Vector => { + if (v.kind === "i32") return V.i32(v.type, Int32Array.from(d), v.valid) + const per = unitsPerDay(v.type) + return V.i64(v.type, BigInt64Array.from(d, (x) => BigInt(x) * per), v.valid) +} + +// ----------------------------------------------------------------------------- +// Durations +// ----------------------------------------------------------------------------- + +/** @internal */ +export type Total = "totalDays" | "totalHours" | "totalMinutes" | "totalSeconds" | "totalMilliseconds" + +const totalSeconds: Record = { + totalDays: 86_400, + totalHours: 3600, + totalMinutes: 60, + totalSeconds: 1, + totalMilliseconds: 0.001 +} + +/** A duration in whole units, rounded toward zero, as Int64. @internal */ +export const total = (fn: Total, v: V.Vector): V.Vector => { + if (v.type._tag !== "Duration" || v.kind !== "i64") { + return V.fail(`${fn} is defined for durations, not ${v.type._tag}`) + } + const per = BigInt(Math.round(totalSeconds[fn] * perSecond[v.type.unit])) + const out = new BigInt64Array(v.length) + for (let i = 0; i < v.length; i++) out[i] = v.values[i] / per + return V.i64(S.int64, out, v.valid) +} + +// ----------------------------------------------------------------------------- +// Casts +// ----------------------------------------------------------------------------- + +const asI64 = (v: V.Vector): BigInt64Array => + v.kind === "i64" ? v.values : v.kind === "i32" ? BigInt64Array.from(v.values, BigInt) : V.fail("expected integers") + +/** + * Casts between dates, timestamps, durations and their integers; `undefined` + * when neither side is temporal. Values the target can't hold become null. + * + * @internal + */ +export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { + const from = v.type + if (!isTemporal(from) && !isTemporal(target)) return undefined + if (from._tag === "Timestamp" && target._tag === "Timestamp") return rescale(v, from.unit, target.unit, target) + if (from._tag === "Duration" && target._tag === "Duration") return rescale(v, from.unit, target.unit, target, true) + if (from._tag === "Timestamp" && target._tag === "Date") { + return target.unit === "day" + ? V.i32(target, Int32Array.from(days(v)), v.valid) + : rescale(v, from.unit, "millisecond", target) + } + if (from._tag === "Date" && target._tag === "Timestamp") { + return from.unit === "day" + ? multiply(v, dayUnits(target.unit), target) + : rescale(v, "millisecond", target.unit, target) + } + // Integers and temporal values of the same width reinterpret each other. + if (target._tag === "Int" || isTemporal(target)) { + const kind = V.kindOf(target) + if (kind === "i64") return V.i64(target, asI64(v), v.valid) + if (kind === "i32" && v.kind === "i32") return V.i32(target, v.values, v.valid) + if (kind === "i32" && v.kind === "i64") { + const valid = new Uint8Array(v.length) + const out = Int32Array.from(v.values, (x, i) => { + const y = Number(BigInt.asIntN(32, x)) + valid[i] = V.isValid(v, i) && BigInt(y) === x ? 1 : 0 + return y + }) + return V.i32(target, out, V.compactValid(valid)) + } + } + return undefined +} diff --git a/packages/effect/src/dataframe/internal/vector.ts b/packages/effect/src/dataframe/internal/vector.ts new file mode 100644 index 00000000000..e3b9ec8fe09 --- /dev/null +++ b/packages/effect/src/dataframe/internal/vector.ts @@ -0,0 +1,881 @@ +/** + * Physical columns used by the DataFrame kernels. + * + * Every Arrow type the DataFrame supports maps onto one of five physical + * layouts, so kernels only need a loop per layout: + * + * - `f64`: `Float64Array` (Float32, Float64 and UInt32) + * - `i32`: `Int32Array` (Int8, Int16, Int32, UInt8, UInt16, Date32, Time32) + * - `i64`: `BigInt64Array` (Int64, UInt64 (bit pattern), Date64, Timestamp, + * Time64, Duration) + * - `bool`: `Uint8Array` with one byte per value (0 or 1) + * - `str`: UTF-8 bytes plus `Int32Array` offsets (Utf8, LargeUtf8, Utf8View + * and Binary variants) + * - `null`: no buffers + * + * Validity is one byte per slot (1 = valid), `undefined` when no slot is + * null. Values behind null slots are unspecified but always readable. + * + * The logical Arrow type rides along so results keep their type (an Int8 sum + * stays Int8, a timestamp stays a timestamp). + * + * @internal + */ +import type * as ArrowData from "../../arrow/ArrowData.ts" +import * as S from "../../arrow/ArrowSchema.ts" +import { typed, type TypedArrayConstructor } from "../../arrow/internal/buffers.ts" + +/** @internal */ +export type Vector = F64 | I32 | I64 | Bool | Str | Null | List | Struct + +/** @internal */ +export interface F64 { + readonly kind: "f64" + readonly type: S.DataType + readonly length: number + readonly values: Float64Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface I32 { + readonly kind: "i32" + readonly type: S.DataType + readonly length: number + readonly values: Int32Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface I64 { + readonly kind: "i64" + readonly type: S.DataType + readonly length: number + readonly values: BigInt64Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Bool { + readonly kind: "bool" + readonly type: S.DataType + readonly length: number + readonly values: Uint8Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Str { + readonly kind: "str" + readonly type: S.DataType + readonly length: number + /** `offsets[i]..offsets[i + 1]` is slot `i`'s byte range in `data`. */ + readonly offsets: Int32Array + readonly data: Uint8Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Null { + readonly kind: "null" + readonly type: S.DataType + readonly length: number + readonly valid: Uint8Array | undefined +} + +/** @internal */ +export interface List { + readonly kind: "list" + readonly type: S.DataType + readonly length: number + /** + * `offsets[i]..offsets[i + 1]` are slot `i`'s elements in `child`; + * `offsets[0]` is 0 and `child` holds exactly `offsets[length]` elements. + */ + readonly offsets: Int32Array + readonly child: Vector + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Struct { + readonly kind: "struct" + readonly type: S.DataType + readonly length: number + /** One vector per field, in the type's field order. */ + readonly children: ReadonlyArray + readonly valid: Uint8Array | undefined +} + +/** @internal */ +export class DataFrameDefect extends Error { + readonly _tag = "DataFrameDefect" +} + +/** @internal */ +export const fail = (message: string): never => { + throw new DataFrameDefect(message) +} + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** @internal */ +export const f64 = (type: S.DataType, values: Float64Array, valid?: Uint8Array): F64 => ({ + kind: "f64", + type, + length: values.length, + values, + valid +}) +/** @internal */ +export const i32 = (type: S.DataType, values: Int32Array, valid?: Uint8Array): I32 => ({ + kind: "i32", + type, + length: values.length, + values, + valid +}) +/** @internal */ +export const i64 = (type: S.DataType, values: BigInt64Array, valid?: Uint8Array): I64 => ({ + kind: "i64", + type, + length: values.length, + values, + valid +}) +/** @internal */ +export const bool = (values: Uint8Array, valid?: Uint8Array): Bool => ({ + kind: "bool", + type: S.bool, + length: values.length, + values, + valid +}) +/** @internal */ +export const str = (type: S.DataType, offsets: Int32Array, data: Uint8Array, valid?: Uint8Array): Str => ({ + kind: "str", + type, + length: offsets.length - 1, + offsets, + data, + valid +}) +/** @internal */ +export const list = (type: S.DataType, offsets: Int32Array, child: Vector, valid?: Uint8Array): List => ({ + kind: "list", + type, + length: offsets.length - 1, + offsets, + child, + valid +}) +/** @internal */ +export const struct = ( + type: S.DataType, + children: ReadonlyArray, + length: number, + valid?: Uint8Array +): Struct => ({ + kind: "struct", + type, + length, + children, + valid +}) +/** The element type of a list type. @internal */ +export const elementType = (type: S.DataType): S.DataType => + type._tag === "List" || type._tag === "LargeList" ? type.child.type : fail(`expected a list, got ${type._tag}`) +/** @internal */ +export const nulls = (length: number): Null => ({ kind: "null", type: S.nullType, length, valid: undefined }) + +/** @internal */ +export const withValid = (v: V, valid: Uint8Array | undefined): V => ({ ...v, valid }) + +/** @internal */ +export const withType = (v: V, type: S.DataType): V => ({ ...v, type }) + +/** Whether slot `i` holds a value. @internal */ +export const isValid = (v: Vector, i: number): boolean => + v.kind === "null" ? false : v.valid === undefined || v.valid[i] === 1 + +/** Number of null slots. */ +const nullCount = (v: Vector): number => { + if (v.kind === "null") return v.length + const valid = v.valid + if (valid === undefined) return 0 + let n = 0 + for (let i = 0; i < v.length; i++) n += valid[i] + return v.length - n +} + +/** Drops an all-ones validity mask. @internal */ +export const compactValid = (valid: Uint8Array | undefined): Uint8Array | undefined => { + if (valid === undefined) return undefined + for (let i = 0; i < valid.length; i++) if (valid[i] === 0) return valid + return undefined +} + +/** Validity of a slot-wise AND of two masks. @internal */ +export const andValid = ( + a: Uint8Array | undefined, + b: Uint8Array | undefined, + length: number +): Uint8Array | undefined => { + if (a === undefined) return b + if (b === undefined) return a + const out = new Uint8Array(length) + for (let i = 0; i < length; i++) out[i] = a[i] & b[i] + return out +} + +// ----------------------------------------------------------------------------- +// Physical classification of Arrow types +// ----------------------------------------------------------------------------- + +/** @internal */ +export type Kind = Vector["kind"] + +/** The physical layout of a supported logical type, or `undefined`. @internal */ +export const kindOf = (type: S.DataType): Kind | undefined => { + switch (type._tag) { + case "Null": + return "null" + case "Bool": + return "bool" + case "Int": + return type.bitWidth === 64 ? "i64" : type.bitWidth === 32 && !type.signed ? "f64" : "i32" + case "Float": + return type.bitWidth === 16 ? undefined : "f64" + case "Utf8": + case "LargeUtf8": + case "Utf8View": + case "Binary": + case "LargeBinary": + case "BinaryView": + return "str" + case "Date": + return type.unit === "day" ? "i32" : "i64" + case "Time": + return type.bitWidth === 32 ? "i32" : "i64" + case "Timestamp": + case "Duration": + return "i64" + case "Dictionary": + return kindOf(type.valueType) + case "Decimal": + return "i64" + case "List": + case "LargeList": + return kindOf(type.child.type) === undefined ? undefined : "list" + case "Struct": + return type.fields.every((f) => kindOf(f.type) !== undefined) ? "struct" : undefined + default: + return undefined + } +} + +/** The type a column of `type` has inside a DataFrame (string variants collapse). @internal */ +export const storageType = (type: S.DataType): S.DataType => { + switch (type._tag) { + case "LargeUtf8": + case "Utf8View": + return S.utf8 + case "LargeBinary": + case "BinaryView": + return S.binary + case "Dictionary": + return storageType(type.valueType) + case "List": + case "LargeList": + return S.list(S.field(type.child.name, storageType(type.child.type), { nullable: type.child.nullable })) + case "Struct": + return S.struct(type.fields.map((f) => S.field(f.name, storageType(f.type), { nullable: f.nullable }))) + default: + return type + } +} + +/** Wraps every value of an integer column to its logical type, in place. @internal */ +export const wrapInPlace = (type: S.DataType, values: Int32Array | Float64Array): void => { + if (type._tag !== "Int") return + const n = values.length + if (type.bitWidth === 8) { + if (type.signed) { for (let i = 0; i < n; i++) values[i] = (values[i] << 24) >> 24 } + else for (let i = 0; i < n; i++) values[i] = values[i] & 0xff + } else if (type.bitWidth === 16) { + if (type.signed) { for (let i = 0; i < n; i++) values[i] = (values[i] << 16) >> 16 } + else for (let i = 0; i < n; i++) values[i] = values[i] & 0xffff + } else if (type.bitWidth === 32 && !type.signed) { + for (let i = 0; i < n; i++) values[i] = values[i] >>> 0 + } +} + +/** Rounds every value of a Float32 column, in place. @internal */ +export const roundFloat32InPlace = (type: S.DataType, values: Float64Array): void => { + if (type._tag === "Float" && type.bitWidth === 32) { + for (let i = 0; i < values.length; i++) values[i] = Math.fround(values[i]) + } +} + +/** Whether `type` is UInt64 (stored as its Int64 bit pattern). @internal */ +export const isUnsigned64 = (type: S.DataType): boolean => type._tag === "Int" && type.bitWidth === 64 && !type.signed + +/** The bounds of an integer type's values; unbounded for other types. @internal */ +export const intRange = (t: S.DataType): [number, number] => { + if (t._tag !== "Int") return [-Infinity, Infinity] + if (t.signed) { + const max = 2 ** (t.bitWidth - 1) + return [-max, max - 1] + } + return [0, 2 ** t.bitWidth - 1] +} + +// ----------------------------------------------------------------------------- +// Arrow interop +// ----------------------------------------------------------------------------- + +const unpackBits = (bits: Uint8Array, offset: number, length: number): Uint8Array => { + const out = new Uint8Array(length) + if ((offset & 7) === 0) { + const base = offset >> 3 + const full = length >> 3 + for (let b = 0; b < full; b++) { + const byte = bits[base + b] + const o = b << 3 + out[o] = byte & 1 + out[o + 1] = (byte >> 1) & 1 + out[o + 2] = (byte >> 2) & 1 + out[o + 3] = (byte >> 3) & 1 + out[o + 4] = (byte >> 4) & 1 + out[o + 5] = (byte >> 5) & 1 + out[o + 6] = (byte >> 6) & 1 + out[o + 7] = (byte >> 7) & 1 + } + for (let i = full << 3; i < length; i++) out[i] = (bits[base + (i >> 3)] >> (i & 7)) & 1 + return out + } + for (let i = 0; i < length; i++) { + const j = offset + i + out[i] = (bits[j >> 3] >> (j & 7)) & 1 + } + return out +} + +/** Packs a byte-per-slot mask into an Arrow bitmap. */ +const packBits = (bytes: Uint8Array): Uint8Array => { + const n = bytes.length + const out = new Uint8Array((n + 7) >> 3) + const full = n >> 3 + for (let b = 0; b < full; b++) { + const o = b << 3 + out[b] = bytes[o] | (bytes[o + 1] << 1) | (bytes[o + 2] << 2) | (bytes[o + 3] << 3) | + (bytes[o + 4] << 4) | (bytes[o + 5] << 5) | (bytes[o + 6] << 6) | (bytes[o + 7] << 7) + } + for (let i = full << 3; i < n; i++) out[i >> 3] |= bytes[i] << (i & 7) + return out +} + +const I0 = BigInt(0) +const IM1 = BigInt(-1) + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +/** Decodes slot `i` of a string vector. @internal */ +export const decodeString = (v: Str, i: number): string => { + const start = v.offsets[i] + const end = v.offsets[i + 1] + if (end === start) return "" + // Short ASCII strings are cheaper to build by hand than through TextDecoder. + if (end - start <= 16) { + let s = "" + const data = v.data + for (let k = start; k < end; k++) { + const c = data[k] + if (c >= 0x80) return decoder.decode(data.subarray(start, end)) + s += String.fromCharCode(c) + } + return s + } + return decoder.decode(v.data.subarray(start, end)) +} + +/** Encodes a string to UTF-8. @internal */ +export const encodeString = (s: string): Uint8Array => encoder.encode(s) + +/** Builds a string vector from JS values (`null` for missing slots). @internal */ +export const stringsFrom = (type: S.DataType, values: ReadonlyArray): Str => { + const n = values.length + const offsets = new Int32Array(n + 1) + let data: Uint8Array = new Uint8Array(Math.max(16, n * 8)) + let valid: Uint8Array | undefined + let at = 0 + for (let i = 0; i < n; i++) { + const v = values[i] + if (v === null || v === undefined) { + if (valid === undefined) valid = new Uint8Array(n).fill(1) + valid[i] = 0 + } else if (typeof v === "string") { + // At most 3 bytes per UTF-16 code unit. + const need = at + v.length * 3 + if (need > data.length) data = grow(data, need) + at += encoder.encodeInto(v, data.subarray(at)).written + } else { + const need = at + v.length + if (need > data.length) data = grow(data, need) + data.set(v, at) + at += v.length + } + offsets[i + 1] = at + } + return str(type, offsets, data.subarray(0, at), valid) +} + +const grow = (data: Uint8Array, need: number): Uint8Array => { + const out = new Uint8Array(Math.max(need, data.length * 2)) + out.set(data) + return out +} + +const viewOffsets = (data: ArrowData.ArrowData, large: boolean): { offsets: Int32Array; bytes: Uint8Array } => { + const n = data.length + if (!large) { + const all = typed(Int32Array, data.buffers[0]) + const first = all[data.offset] + const bytes = data.buffers[1] + if (first === 0) return { offsets: all.subarray(data.offset, data.offset + n + 1), bytes } + const offsets = new Int32Array(n + 1) + for (let i = 0; i <= n; i++) offsets[i] = all[data.offset + i] - first + return { offsets, bytes: bytes.subarray(first) } + } + const all = typed(BigInt64Array, data.buffers[0]) + const first = Number(all[data.offset]) + const last = Number(all[data.offset + n]) + if (last - first > 0x7fffffff) fail("string column larger than 2 GiB") + const offsets = new Int32Array(n + 1) + for (let i = 0; i <= n; i++) offsets[i] = Number(all[data.offset + i]) - first + return { offsets, bytes: data.buffers[1].subarray(first) } +} + +const fromViews = (type: S.DataType, data: ArrowData.ArrowData, valid: Uint8Array | undefined): Str => { + const n = data.length + const views = data.buffers[0] + const dv = new DataView(views.buffer, views.byteOffset, views.byteLength) + let total = 0 + for (let i = 0; i < n; i++) total += dv.getInt32((data.offset + i) * 16, true) + const out = new Uint8Array(total) + const offsets = new Int32Array(n + 1) + let at = 0 + for (let i = 0; i < n; i++) { + const base = (data.offset + i) * 16 + const len = dv.getInt32(base, true) + if (len <= 12) out.set(views.subarray(base + 4, base + 4 + len), at) + else { + const buffer = data.buffers[1 + dv.getInt32(base + 8, true)] + const off = dv.getInt32(base + 12, true) + out.set(buffer.subarray(off, off + len), at) + } + at += len + offsets[i + 1] = at + } + return str(type, offsets, out, valid) +} + +/** Converts array data to a vector. Copies only where the layouts differ. @internal */ +export const fromArrowData = (data: ArrowData.ArrowData): Vector => { + const type = data.type + if (type._tag === "Dictionary") { + const dictionary = data.dictionary === undefined + ? fail("dictionary-encoded column without a dictionary") + : fromArrowData(data.dictionary) + const indexData: ArrowData.ArrowData = { ...data, type: type.indexType, dictionary: undefined } + const indices = fromArrowData(indexData) + const idx = indices.kind === "i64" + ? Int32Array.from(indices.values, Number) + : indices.kind === "f64" + ? Int32Array.from(indices.values) + : (indices as I32).values + return take(dictionary, idx, indices.valid) + } + const n = data.length + const valid = data.validity === undefined || data.nullCount === 0 + ? undefined + : unpackBits(data.validity, data.offset, n) + const storage = storageType(type) + /** Slots of the first buffer as `Ctor` (`extra` more for offsets). */ + const view = (Ctor: TypedArrayConstructor, extra = 0) => + typed(Ctor, data.buffers[0]).subarray(data.offset, data.offset + n + extra) + switch (type._tag) { + case "Null": + return nulls(n) + case "Bool": + return bool(unpackBits(data.buffers[0], data.offset, n), valid) + case "Int": + switch (type.bitWidth) { + case 8: + return i32(type, Int32Array.from(type.signed ? view(Int8Array) : view(Uint8Array)), valid) + case 16: + return i32(type, Int32Array.from(type.signed ? view(Int16Array) : view(Uint16Array)), valid) + case 32: + return type.signed + ? i32(type, view(Int32Array), valid) + : f64(type, Float64Array.from(view(Uint32Array)), valid) + default: + return i64(type, view(BigInt64Array), valid) + } + case "Float": + if (type.bitWidth === 64) return f64(type, view(Float64Array), valid) + if (type.bitWidth === 32) return f64(type, Float64Array.from(view(Float32Array)), valid) + return fail("Float16 columns are not supported") + case "Date": + case "Time": + case "Timestamp": + case "Duration": + return kindOf(type) === "i32" ? i32(type, view(Int32Array), valid) : i64(type, view(BigInt64Array), valid) + case "Utf8": + case "Binary": + case "LargeUtf8": + case "LargeBinary": { + const { bytes, offsets } = viewOffsets(data, type._tag === "LargeUtf8" || type._tag === "LargeBinary") + return str(storage, offsets, bytes, valid) + } + case "Utf8View": + case "BinaryView": + return fromViews(storage, data, valid) + case "Decimal": { + if (type.bitWidth === 32) return i64(type, BigInt64Array.from(view(Int32Array), BigInt), valid) + if (type.bitWidth === 64) return i64(type, view(BigInt64Array), valid) + // 128 or 256 bits: keep the low word when the rest only extends its sign. + const words = type.bitWidth / 64 + const all = typed(BigInt64Array, data.buffers[0]) + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) { + const base = (data.offset + i) * words + const lo = all[base] + const sign = lo < I0 ? IM1 : I0 + for (let w = 1; w < words; w++) { + if (all[base + w] !== sign && (valid === undefined || valid[i] === 1)) { + return fail(`decimal values beyond 18 digits are not supported (column of ${type.bitWidth}-bit decimals)`) + } + } + out[i] = lo + } + return i64(type, out, valid) + } + case "List": + case "LargeList": { + const raw = type._tag === "LargeList" ? Int32Array.from(view(BigInt64Array, 1), Number) : view(Int32Array, 1) + const child = fromArrowData(data.children[0]) + const first = raw[0] + if (first === 0 && raw[n] === child.length) return list(storage, raw, child, valid) + const offsets = first === 0 ? raw : Int32Array.from(raw, (o) => o - first) + return list(storage, offsets, slice(child, first, raw[n]), valid) + } + case "Struct": { + const children = data.children.map((c) => { + const v = fromArrowData(c) + return v.length === n && data.offset === 0 ? v : slice(v, data.offset, data.offset + n) + }) + return struct(storage, children, n, valid) + } + default: + return fail(`columns of type ${type._tag} are not supported`) + } +} + +/** Converts a vector to Arrow array data (offset 0). @internal */ +export const toArrowData = (v: Vector): ArrowData.ArrowData => { + const n = v.length + const nullCount_ = nullCount(v) + const validity = v.kind !== "null" && v.valid !== undefined && nullCount_ > 0 ? packBits(v.valid) : undefined + const base = { + type: v.type, + length: n, + offset: 0, + nullCount: nullCount_, + validity, + children: [], + dictionary: undefined + } + const bytes = (a: ArrayBufferView) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength) + switch (v.kind) { + case "null": + return { ...base, validity: undefined, buffers: [] } + case "list": + return { ...base, buffers: [bytes(v.offsets)], children: [toArrowData(v.child)] } + case "struct": + return { ...base, buffers: [], children: v.children.map(toArrowData) } + case "bool": + return { ...base, buffers: [packBits(v.values)] } + case "str": + return { ...base, buffers: [bytes(v.offsets), v.data] } + case "i64": { + const t = v.type + if (t._tag === "Decimal" && t.bitWidth !== 64) { + if (t.bitWidth === 32) return { ...base, buffers: [bytes(Int32Array.from(v.values, Number))] } + const words = t.bitWidth / 64 + const out = new BigInt64Array(n * words) + for (let i = 0; i < n; i++) { + out[i * words] = v.values[i] + const sign = v.values[i] < I0 ? IM1 : I0 + for (let w = 1; w < words; w++) out[i * words + w] = sign + } + return { ...base, buffers: [bytes(out)] } + } + return { ...base, buffers: [bytes(v.values)] } + } + case "i32": { + const t = v.type + if (t._tag === "Int" && t.bitWidth === 8) { + return { ...base, buffers: [bytes(t.signed ? Int8Array.from(v.values) : Uint8Array.from(v.values))] } + } + if (t._tag === "Int" && t.bitWidth === 16) { + return { ...base, buffers: [bytes(t.signed ? Int16Array.from(v.values) : Uint16Array.from(v.values))] } + } + return { ...base, buffers: [bytes(v.values)] } + } + case "f64": { + const t = v.type + if (t._tag === "Int") return { ...base, buffers: [bytes(Uint32Array.from(v.values))] } + if (t._tag === "Float" && t.bitWidth === 32) return { ...base, buffers: [bytes(Float32Array.from(v.values))] } + return { ...base, buffers: [bytes(v.values)] } + } + } +} + +// ----------------------------------------------------------------------------- +// Gather, slice, concat +// ----------------------------------------------------------------------------- + +/** + * Gathers `indices` from `v`. A negative index produces a null slot. + * `extraValid` (optional) marks additional null output slots. + * + * @internal + */ +export const take = (v: Vector, indices: Int32Array, extraValid?: Uint8Array): Vector => { + const n = indices.length + let hasMissing = false + for (let i = 0; i < n; i++) { + if (indices[i] < 0) { + hasMissing = true + break + } + } + if (v.kind === "null") return nulls(n) + let valid: Uint8Array | undefined + const src = v.valid + if (src !== undefined || hasMissing || extraValid !== undefined) { + valid = new Uint8Array(n) + if (src === undefined && !hasMissing) valid.set(extraValid!) + else { + for (let i = 0; i < n; i++) { + const j = indices[i] + valid[i] = j < 0 ? 0 : src === undefined ? 1 : src[j] + } + if (extraValid !== undefined) { for (let i = 0; i < n; i++) valid[i] &= extraValid[i] } + } + } + switch (v.kind) { + case "list": { + const so = v.offsets + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j >= 0) total += so[j + 1] - so[j] + offsets[i + 1] = total + } + const elements = new Int32Array(total) + let k = 0 + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j < 0) continue + for (let e = so[j]; e < so[j + 1]; e++) elements[k++] = e + } + return list(v.type, offsets, take(v.child, elements), valid) + } + case "struct": + return struct(v.type, v.children.map((c) => take(c, indices)), n, valid) + case "f64": { + const a = v.values + const o = new Float64Array(n) + if (hasMissing) { for (let i = 0; i < n; i++) o[i] = indices[i] < 0 ? 0 : a[indices[i]] } + else for (let i = 0; i < n; i++) o[i] = a[indices[i]] + return f64(v.type, o, valid) + } + case "i32": { + const a = v.values + const o = new Int32Array(n) + if (hasMissing) { for (let i = 0; i < n; i++) o[i] = indices[i] < 0 ? 0 : a[indices[i]] } + else for (let i = 0; i < n; i++) o[i] = a[indices[i]] + return i32(v.type, o, valid) + } + case "i64": { + // Gather 64-bit values as pairs of 32-bit halves: much faster than bigint reads. + const a = new Int32Array(v.values.buffer, v.values.byteOffset, v.values.length * 2) + const o = new Int32Array(n * 2) + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j >= 0) { + o[2 * i] = a[2 * j] + o[2 * i + 1] = a[2 * j + 1] + } + } + return i64(v.type, new BigInt64Array(o.buffer), valid) + } + case "bool": { + const a = v.values + const o = new Uint8Array(n) + if (hasMissing) { for (let i = 0; i < n; i++) o[i] = indices[i] < 0 ? 0 : a[indices[i]] } + else for (let i = 0; i < n; i++) o[i] = a[indices[i]] + return bool(o, valid) + } + case "str": { + const offsets = new Int32Array(n + 1) + const so = v.offsets + let total = 0 + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j >= 0) total += so[j + 1] - so[j] + offsets[i + 1] = total + } + const data = new Uint8Array(total) + const sd = v.data + let i = 0 + while (i < n) { + const j = indices[i] + if (j < 0) { + i++ + continue + } + // Copy runs of consecutive source rows in one go. + let e = i + 1 + while (e < n && indices[e] === indices[e - 1] + 1) e++ + const s = so[j] + const end = so[indices[e - 1] + 1] + const at = offsets[i] + const len = end - s + if (len < 16) { for (let k = 0; k < len; k++) data[at + k] = sd[s + k] } + else data.set(sd.subarray(s, end), at) + i = e + } + return str(v.type, offsets, data, valid) + } + } +} + +/** Slots `[start, end)`, sharing buffers. @internal */ +export const slice = (v: Vector, start: number, end: number): Vector => { + const valid = v.valid?.subarray(start, end) + switch (v.kind) { + case "list": { + const first = v.offsets[start] + const offsets = first === 0 + ? v.offsets.subarray(start, end + 1) + : Int32Array.from(v.offsets.subarray(start, end + 1), (o) => o - first) + return list(v.type, offsets, slice(v.child, first, v.offsets[end]), valid) + } + case "struct": + return struct(v.type, v.children.map((c) => slice(c, start, end)), end - start, valid) + case "null": + return nulls(end - start) + case "f64": + return f64(v.type, v.values.subarray(start, end), valid) + case "i32": + return i32(v.type, v.values.subarray(start, end), valid) + case "i64": + return i64(v.type, v.values.subarray(start, end), valid) + case "bool": + return bool(v.values.subarray(start, end), valid) + case "str": { + const first = v.offsets[start] + if (first === 0) return str(v.type, v.offsets.subarray(start, end + 1), v.data, valid) + const offsets = new Int32Array(end - start + 1) + for (let i = 0; i <= end - start; i++) offsets[i] = v.offsets[start + i] - first + return str(v.type, offsets, v.data.subarray(first, v.offsets[end]), valid) + } + } +} + +/** Concatenates vectors of the same kind and type. @internal */ +export const concat = (parts: ReadonlyArray): Vector => { + if (parts.length === 1) return parts[0] + const n = parts.reduce((s, p) => s + p.length, 0) + let valid: Uint8Array | undefined + if (parts.some((p) => p.kind === "null" || p.valid !== undefined)) { + valid = new Uint8Array(n) + let at = 0 + for (const p of parts) { + if (p.kind !== "null") { + if (p.valid === undefined) valid.fill(1, at, at + p.length) + else valid.set(p.valid, at) + } + at += p.length + } + } + const head = parts.find((p) => p.kind !== "null") + if (head === undefined) return nulls(n) + const type = head.type + const fill = (out: T): T => { + let at = 0 + for (const p of parts) { + if (p.kind !== "null") out.set((p as F64).values as any, at) + at += p.length + } + return out + } + switch (head.kind) { + case "f64": + return f64(type, fill(new Float64Array(n)), valid) + case "i32": + return i32(type, fill(new Int32Array(n)), valid) + case "i64": + return i64(type, fill(new BigInt64Array(n)), valid) + case "bool": + return bool(fill(new Uint8Array(n)), valid) + case "list": { + const offsets = new Int32Array(n + 1) + const children: Array = [] + let row = 0 + let at = 0 + for (const p of parts) { + if (p.kind === "list") { + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + p.offsets[i] + at += p.offsets[p.length] + children.push(p.child) + } else for (let i = 1; i <= p.length; i++) offsets[row + i] = at + row += p.length + } + return list(type, offsets, concat(children.length === 0 ? [nulls(0)] : children), valid) + } + case "struct": { + const children = head.children.map((_, f) => + concat(parts.map((p) => (p.kind === "struct" ? p.children[f] : nulls(p.length)))) + ) + return struct(type, children, n, valid) + } + case "str": { + const total = parts.reduce((s, p) => s + (p.kind === "str" ? p.offsets[p.length] - p.offsets[0] : 0), 0) + const data = new Uint8Array(total) + const offsets = new Int32Array(n + 1) + let at = 0 + let row = 0 + for (const p of parts) { + if (p.kind === "str") { + const base = p.offsets[0] + data.set(p.data.subarray(base, p.offsets[p.length]), at) + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + p.offsets[i] - base + at += p.offsets[p.length] - base + } else { + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + } + row += p.length + } + return str(type, offsets, data, valid) + } + } +} + +/** A vector of `length` copies of one value. @internal */ +export const broadcast = (v: Vector, length: number): Vector => { + if (v.length === length) return v + if (v.length !== 1) return fail(`cannot broadcast a column of length ${v.length} to ${length}`) + return take(v, new Int32Array(length)) +} diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts new file mode 100644 index 00000000000..8bb0ecb9238 --- /dev/null +++ b/packages/effect/src/dataframe/internal/window.ts @@ -0,0 +1,276 @@ +/** + * Window functions: values computed from the other rows of a row's + * partition, in row order (Polars semantics). Without a partition, the whole + * column is one partition. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import type * as Plan from "../Plan.ts" +import type * as Agg from "./aggregate.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as Hash from "./hash.ts" +import * as Sort from "./sort.ts" +import * as V from "./vector.ts" + +/** Rows' partition ids (dense, in order of first appearance). @internal */ +export type Partition = Agg.Grouping + +/** One partition holding every row. @internal */ +export const whole = (n: number): Partition => ({ ids: new Int32Array(n), count: n > 0 ? 1 : 0 }) + +const validOf = (v: V.Vector): Uint8Array | undefined => + v.kind === "null" ? new Uint8Array(v.length) : v.valid === undefined ? undefined : v.valid.slice() + +// ----------------------------------------------------------------------------- +// Cumulative +// ----------------------------------------------------------------------------- + +/** Running sum per partition; nulls stay null and are skipped. @internal */ +export const cumSum = (v: V.Vector, p: Partition): V.Vector => { + const n = v.length + const ids = p.ids + if (D.isPointInTime(v.type)) return V.fail(`cumSum is not defined for ${D.describe(v.type)}`) + const type = D.sumType(v.type) + const valid = validOf(v) + switch (v.kind) { + case "null": + return V.nulls(n) + case "f64": + case "bool": { + const x = v.values + const acc = new Float64Array(p.count) + const out = new Float64Array(n) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] += x[i] + V.wrapInPlace(type, out) + V.roundFloat32InPlace(type, out) + return V.f64(type, out, valid) + } + case "i32": { + const x = v.values + const acc = new Int32Array(p.count) + const out = new Int32Array(n) + for (let i = 0; i < n; i++) { + if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = (acc[ids[i]] + x[i]) | 0 + } + return V.i32(type, out, valid) + } + case "i64": { + const x = v.values + const acc = new BigInt64Array(p.count) + const out = new BigInt64Array(n) + const fit = Dec.fit(type) + for (let i = 0; i < n; i++) { + if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = fit(acc[ids[i]] + x[i]) + } + return V.i64(type, out, valid) + } + default: + return V.fail(`cumSum is not defined for ${D.describe(v.type)}`) + } +} + +/** Number of non-null values so far per partition (no nulls). @internal */ +export const cumCount = (v: V.Vector, p: Partition): V.Vector => { + const n = v.length + const acc = new Float64Array(p.count) + const out = new Float64Array(n) + for (let i = 0; i < n; i++) { + if (V.isValid(v, i)) acc[p.ids[i]]++ + out[i] = acc[p.ids[i]] + } + return V.f64(S.uint32, out) +} + +const isNaNAt = (v: V.Vector, i: number): boolean => v.kind === "f64" && v.values[i] !== v.values[i] + +/** Running minimum or maximum per partition; NaNs are skipped, nulls stay null. @internal */ +export const cumMinMax = (fn: "cumMin" | "cumMax", v: V.Vector, p: Partition): V.Vector => { + const n = v.length + const order = Sort.rowOrder(v) + const sign = fn === "cumMin" ? -1 : 1 + const best = new Int32Array(p.count).fill(-1) + const rows = new Int32Array(n) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) { + rows[i] = -1 + continue + } + const g = p.ids[i] + if (!isNaNAt(v, i) && (best[g] < 0 || order(i, best[g]) * sign > 0)) best[g] = i + rows[i] = best[g] < 0 ? i : best[g] + } + return V.take(v, rows) +} + +// ----------------------------------------------------------------------------- +// Shifts +// ----------------------------------------------------------------------------- + +/** The value `n` rows earlier in the partition (later for negative `n`). @internal */ +export const shift = (v: V.Vector, p: Partition, n: number): V.Vector => { + const { rows, starts } = Hash.rowsByGroup(p.ids, p.count) + const idx = new Int32Array(v.length).fill(-1) + for (let g = 0; g < p.count; g++) { + const s = starts[g] + const e = starts[g + 1] + for (let j = s; j < e; j++) { + const k = j - n + if (k >= s && k < e) idx[rows[j]] = rows[k] + } + } + return V.take(v, idx) +} + +// ----------------------------------------------------------------------------- +// Rank +// ----------------------------------------------------------------------------- + +/** Rank within the partition; nulls get no rank. @internal */ +export const rank = (v: V.Vector, p: Partition, method: Plan.RankMethod, descending: boolean): V.Vector => { + const n = v.length + const perm = Sort.permutation( + [{ vector: V.i32(S.int32, p.ids), descending: false }, { vector: v, descending }], + true, + n + ) + const words = Sort.words({ vector: v, descending }) + const same = (a: number, b: number) => + words.lo[a] === words.lo[b] && (words.hi === undefined || words.hi[a] === words.hi[b]) + const out = new Float64Array(n) + const valid = validOf(v) + let i = 0 + while (i < n) { + // One partition: perm[i..end). + const g = p.ids[perm[i]] + let end = i + while (end < n && p.ids[perm[end]] === g) end++ + let position = 0 + let dense = 0 + let s = i + while (s < end && V.isValid(v, perm[s])) { + let e = s + 1 + while (e < end && V.isValid(v, perm[e]) && same(perm[s], perm[e])) e++ + dense++ + const first = position + 1 + const last = position + (e - s) + for (let k = s; k < e; k++) { + const row = perm[k] + out[row] = method === "average" + ? (first + last) / 2 + : method === "min" + ? first + : method === "max" + ? last + : method === "dense" + ? dense + : position + (k - s) + 1 + } + position = last + s = e + } + i = end + } + return V.f64(method === "average" ? S.float64 : S.uint32, out, valid) +} + +// ----------------------------------------------------------------------------- +// Rolling +// ----------------------------------------------------------------------------- + +/** + * An aggregate over the last `window` rows of the partition (the current row + * included). Null when fewer than `minPeriods` of them are non-null; NaN + * propagates. + * + * @internal + */ +export const rolling = ( + fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax", + v: V.Vector, + p: Partition, + window: number, + minPeriods: number +): V.Vector => { + const n = v.length + const { rows, starts } = Hash.rowsByGroup(p.ids, p.count) + /** Calls `f` with each row and the window `rows[first..last]` ending at it. */ + const windows = (f: (row: number, first: number, last: number) => void) => { + for (let g = 0; g < p.count; g++) { + for (let j = starts[g]; j < starts[g + 1]; j++) f(rows[j], Math.max(starts[g], j - window + 1), j) + } + } + const ok = new Uint8Array(n) + if (fn === "rollingMin" || fn === "rollingMax") { + const order = Sort.rowOrder(v) + const sign = fn === "rollingMin" ? -1 : 1 + const idx = new Int32Array(n).fill(-1) + windows((row, first, last) => { + let best = -1 + let count = 0 + for (let k = first; k <= last; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + count++ + if (best >= 0 && isNaNAt(v, best)) continue + if (isNaNAt(v, r) || best < 0 || order(r, best) * sign > 0) best = r + } + if (count >= minPeriods) idx[row] = best + }) + return V.take(v, idx) + } + if ( + v.kind === "str" || v.kind === "null" || v.kind === "list" || v.kind === "struct" || D.isPointInTime(v.type) || + (fn === "rollingMean" && v.type._tag === "Duration") + ) { + return V.fail(`${fn} is not defined for ${D.describe(v.type)}`) + } + const type = fn === "rollingMean" ? D.meanType(v.type) : D.sumType(v.type) + if (v.kind === "i64") { + // Exact sums: 64-bit integers wrap and decimals fail outside 64 bits. + const fit = Dec.fit(type) + const scale = Dec.isDecimal(v.type) ? v.type.scale : 0 + const sums = new BigInt64Array(n) + const means = new Float64Array(n) + windows((row, first, last) => { + let sum = BigInt(0) + let count = 0 + for (let k = first; k <= last; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + sum += v.values[r] + count++ + } + if (count < minPeriods) return + ok[row] = 1 + if (fn === "rollingMean") means[row] = Dec.toNumber(sum, scale) / count + else sums[row] = fit(sum) + }) + return fn === "rollingMean" ? V.f64(type, means, ok) : V.i64(type, sums, ok) + } + const x = v.kind === "f64" ? v.values : Float64Array.from(v.values) + const out = new Float64Array(n) + windows((row, first, last) => { + let sum = 0 + let count = 0 + for (let k = first; k <= last; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + sum += x[r] + count++ + } + if (count >= minPeriods) { + ok[row] = 1 + out[row] = fn === "rollingMean" ? sum / count : sum + } + }) + if (fn === "rollingMean") return V.f64(type, out, ok) + switch (v.kind) { + case "i32": + return V.i32(type, Int32Array.from(out, (a) => a | 0), ok) + default: + V.wrapInPlace(type, out) + return V.f64(type, out, ok) + } +} diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts new file mode 100644 index 00000000000..4f1acf2f2b9 --- /dev/null +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -0,0 +1,866 @@ +import { assert, describe, it } from "@effect/vitest" +import { BigDecimal, DateTime, Duration, Effect, Equal, Option, Result, Schema, Stream } from "effect" +import { ArrowColumn, ArrowData, ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" +import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" +import * as InternalDateTime from "effect/internal/dateTime" +import { vi } from "vitest" + +const Reading = Schema.Struct({ + city: Schema.String, + temp: Schema.Number, + rain: Schema.NullOr(Schema.Number), + station: Schema.optional(Schema.String) +}) + +const readings = DataFrame.fromRows(Reading, [ + { city: "Vienna", temp: 21.5, rain: 0, station: "s1" }, + { city: "Berlin", temp: 26.5, rain: null }, + { city: "Vienna", temp: 19, rain: 2 } +]) + +const types = (df: DataFrame.DataFrame) => + df.schema.fields.map((f) => `${f.name}:${f.type._tag}:${f.nullable}`) + +/** + * What `fromRows`, `validate` and `Query.table` (in memory) say to rows + * held in the schema's columns: `"ok"`, or the message they fail with. + */ +const outcomes = ( + schema: Schema.Struct, + rows: ReadonlyArray> +): ReadonlyArray => { + const stored = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(DataFrame.empty(schema).schema, rows))) + const table = Query.table("t", schema).pipe(Effect.provide(Catalog.layer({ t: stored }))) + return [ + Result.try({ try: () => DataFrame.fromRows(schema, rows), catch: (e) => e as DataFrame.DataFrameError }), + DataFrame.validate(stored, schema), + Effect.runSync(Effect.result(table)) + ].map((result) => Result.isFailure(result) ? result.failure.message : "ok") +} + +describe("DataFrame", () => { + describe("constructors", () => { + // DF-C001 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows maps schema fields to Arrow types", () => { + assert.deepStrictEqual(types(readings), [ + "city:Utf8:false", + "temp:Float:false", + "rain:Float:true", + "station:Utf8:true" + ]) + assert.deepStrictEqual(DataFrame.toArray(readings, "station") as Array, ["s1", null, null]) + }) + + // DF-C002 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows takes per-column type overrides", () => { + const df = DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [{ n: 1 }, { n: 2 }], { + types: { n: ArrowSchema.int32 } + }) + assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.int32) + }) + + // DF-C003 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows builds decimal columns from options.types", () => { + const Price = Schema.Struct({ n: Schema.BigDecimal }) + const df = DataFrame.fromRows(Price, [{ n: BigDecimal.make(123n, 2) }], { + types: { n: ArrowSchema.decimal(18, 2) } + }) + assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.decimal(18, 2)) + assert.isTrue(BigDecimal.equals(DataFrame.toRows(df)[0].n, BigDecimal.make(123n, 2))) + assert.strictEqual(DataFrame.empty(Price, { types: { n: ArrowSchema.decimal(9, 0) } }).length, 0) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, Price, { types: { n: ArrowSchema.decimal(18, 2) } }))) + }) + + // DF-C004 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows rejects overrides that can't hold the field's values", () => { + assert.throws( + () => DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [], { types: { n: ArrowSchema.int64 } }), + `column "n" is Int64, expected Float64` + ) + assert.throws( + () => + DataFrame.fromRows(Schema.Struct({ t: Schema.DateTimeUtc }), [], { + types: { t: ArrowSchema.timestamp("second", "Europe/Berlin") } + }), + `column "t" has time zone Europe/Berlin, expected UTC timestamps` + ) + }) + + // DF-C033 (API contract): DataFrame.fromRows; pinned source in README.md. + it("rejects decimal overrides that aren't 128-bit", () => { + const Price = Schema.Struct({ n: Schema.BigDecimal }) + const df = DataFrame.fromRows(Price, [{ n: BigDecimal.make(7n, 0) }], { types: { n: ArrowSchema.decimal(9, 0) } }) + for (const width of [32, 64, 256] as const) { + const types = { n: ArrowSchema.decimal(9, 0, width) } + const message = `column "n": ${width}-bit decimals can't be declared: declared decimals are 128-bit` + assert.throws(() => DataFrame.fromRows(Price, [], { types }), DataFrame.DataFrameError, message) + assert.throws(() => DataFrame.empty(Price, { types }), DataFrame.DataFrameError, message) + assert.throws(() => Query.table("t", Price, { types }), DataFrame.DataFrameError, message) + const result = DataFrame.validate(df, Price, { types }) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + }) + + // DF-C005 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows rejects schemas without a column type", () => { + interface Tree { + readonly children: ReadonlyArray + } + const Tree: Schema.Codec = Schema.Struct({ children: Schema.Array(Schema.suspend(() => Tree)) }) + for (const xs of [Schema.Record(Schema.String, Schema.Number), Tree]) { + assert.throws(() => DataFrame.fromRows(Schema.Struct({ xs }), []), DataFrame.DataFrameError, `"xs`) + } + }) + + // DF-C006 (API contract): DataFrame.fromColumns; pinned source in README.md. + it("fromColumns keeps typed array element types", () => { + const df = DataFrame.fromColumns({ i: Int32Array.of(1, 2), b: BigInt64Array.of(1n, 2n), s: ["a", null] }) + assert.deepStrictEqual(types(df), ["i:Int:false", "b:Int:false", "s:Utf8:true"]) + assert.deepStrictEqual(DataFrame.toRows(df), [{ i: 1, b: 1n, s: "a" }, { i: 2, b: 2n, s: null }]) + }) + + // DF-C007 (API contract): DataFrame.fromColumns; pinned source in README.md. + it("fromColumns rejects columns of different lengths", () => { + assert.throws(() => DataFrame.fromColumns({ a: [1, 2], b: [1] }), DataFrame.DataFrameError) + }) + + // DF-C008 (API contract): DataFrame.empty; pinned source in README.md. + it("empty has the schema's columns and no rows", () => { + const df = DataFrame.empty(Reading) + assert.strictEqual(df.length, 0) + assert.deepStrictEqual(DataFrame.columnNames(df), ["city", "temp", "rain", "station"]) + }) + + // DF-C022 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows checks rows against the schema's refinements, literals and row checks", () => { + const Positive = Schema.Number.check(Schema.isGreaterThan(0)) + const Order = Schema.Struct({ + n: Positive, + kind: Schema.Literals(["x", "y"]), + note: Schema.optional(Schema.NonEmptyString), + lines: Schema.Array( + Schema.Struct({ qty: Schema.NullOr(Positive), unit: Schema.optional(Schema.Literal("kg")) }) + ) + }).check(Schema.makeFilter((row) => row.lines.length <= row.n || "more lines than n")) + const valid = { n: 2, kind: "x", lines: [{ qty: 1 }, { qty: null, unit: undefined }] } as const + const df = DataFrame.fromRows(Order, [valid, { ...valid, note: undefined }, { ...valid, note: "late" }]) + assert.deepStrictEqual(DataFrame.toArray(df, "note"), [null, null, "late"]) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, Order))) + const cases: ReadonlyArray = [ + [{ n: -1 }, `column "n" has values its schema rejects`], + [{ kind: "z" }, `column "kind" has values its schema rejects`], + [{ note: "" }, `column "note" has values its schema rejects`], + [{ lines: [{ qty: 1 }, { qty: 0 }] }, `column "lines" has values its schema rejects`], + [{ lines: [{ qty: 1, unit: "lb" }] }, `column "lines" has values its schema rejects`], + [{ n: 1 }, "rows have values their schema rejects"] + ] + for (const [fields, message] of cases) { + const rows = [valid, { ...valid, ...fields }] as unknown as ReadonlyArray + assert.throws(() => DataFrame.fromRows(Order, rows), DataFrame.DataFrameError, message) + } + }) + + // DF-C029 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows returns a frame that validates against the same declaration", () => { + const Row = Schema.Struct({ + n: Schema.Number.check(Schema.isGreaterThan(0)), + at: Schema.NullOr(Schema.DateTimeUtc), + waits: Schema.Array(Schema.Struct({ d: Schema.optional(Schema.Duration) })), + m: Schema.BigDecimal + }) + const types = { n: ArrowSchema.float32, at: ArrowSchema.timestamp("second"), m: ArrowSchema.decimal(4, 2) } + const row = { + n: 0.1, + at: DateTime.makeUnsafe(2000), + waits: [{ d: Duration.nanos(1_000n) }, {}], + m: BigDecimal.make(15n, 1) + } + const df = DataFrame.fromRows(Row, [row, { ...row, at: null }], { types }) + // A Float32 column rounds its numbers, and the frame holds the rounded ones. + assert.strictEqual(DataFrame.toArray(df, "n")[0], Math.fround(0.1)) + for (const built of [df, DataFrame.empty(Row, { types })]) { + const validated = Result.getOrThrow(DataFrame.validate(built, Row, { types })) + assert.deepStrictEqual(validated.schema, built.schema) + assert.deepStrictEqual(DataFrame.toRows(validated), DataFrame.toRows(built)) + } + + const beyond = DateTime.fromEpochSeconds(8_640_000_000_001) + const cases: ReadonlyArray = [ + // The check sees what the column stores: Math.fround(1e-46) is 0. + [{ n: 1e-46 }, `column "n" has values its schema rejects`], + [{ at: DateTime.makeUnsafe(1500) }, `column "at" has values that Timestamp[s] can't hold exactly`], + [{ at: beyond }, `column "at" has timestamps outside the range of DateTime`], + [ + { waits: [{ d: Duration.nanos(1_001n) }] }, + `column "waits[].d" has values that Duration[us] can't hold exactly` + ], + [{ m: BigDecimal.make(1_234n, 3) }, "1.234 has more than 2 decimal places"], + [{ m: BigDecimal.make(1_234n, 1) }, "123.4 has more than 4 digits at scale 2"] + ] + for (const [fields, message] of cases) { + assert.throws( + () => DataFrame.fromRows(Row, [row, { ...row, ...fields }], { types }), + DataFrame.DataFrameError, + message + ) + } + // A frame that holds such a decimal under the declared type fails too. Arrow builders + // refuse to store one, so the integers of a wider decimal are put under the declared type. + const Amount = Schema.Struct({ m: Schema.BigDecimal }) + const schema = DataFrame.empty(Amount, { types }).schema + const wide = Result.getOrThrow(ArrowTable.fromRows( + ArrowSchema.make([ArrowSchema.field("m", ArrowSchema.decimal(5, 2), { nullable: false })]), + [{ m: BigDecimal.make(1_234n, 1) }] + )) + const held = ArrowTable.make(schema, [ + ArrowTable.recordBatch(schema, 1, [{ ...wide.batches[0].columns[0], type: types.m }]) + ]) + const validated = DataFrame.validate(DataFrame.fromArrowTable(held), Amount, { types }) + assert.include( + Result.isFailure(validated) && validated.failure.message, + `column "m" has values that Decimal[4, 2] can't hold exactly` + ) + }) + + // DF-C030 (API contract): DataFrame.fromRows; pinned source in README.md. + it("a null member of a union keeps the checks of the other members", () => { + const Row = Schema.Struct({ s: Schema.Union([Schema.suspend(() => Schema.Null), Schema.Literal("ok")]) }) + assert.deepStrictEqual(outcomes(Row, [{ s: "ok" }, { s: null }]), ["ok", "ok", "ok"]) + const message = `column "s" has values its schema rejects` + assert.deepStrictEqual(outcomes(Row, [{ s: "ok" }, { s: "bad" }]), [message, message, message]) + }) + + // DF-C031 (API contract): DataFrame.fromRows; pinned source in README.md. + it("checks see stored values as the schema writes them, missing ones included", () => { + const seen: Array = [] + const record = Schema.makeFilter((value: unknown) => { + seen.push(value) + return true + }) + const Item = Schema.Struct({ unit: Schema.optional(Schema.String), qty: Schema.UndefinedOr(Schema.Number) }) + const Row = Schema.Struct({ + note: Schema.optional(Schema.String), + rain: Schema.NullOr(Schema.Number), + items: Schema.Array(Item).check(record) + }).check(record) + const rows = [{ rain: null, items: [{ qty: undefined }] }, { + note: undefined, + rain: 1, + items: [{ unit: "kg", qty: 2 }] + }] + assert.deepStrictEqual(outcomes(Row, rows), ["ok", "ok", "ok"]) + // What each of the three hands to the checks: the items of both rows, then both rows. + const items = [[{ qty: undefined }], [{ unit: "kg", qty: 2 }]] + const checked = [...items, { rain: null, items: items[0] }, { rain: 1, items: items[1] }] + assert.deepStrictEqual(seen, [...checked, ...checked, ...checked]) + + // A frame can't tell null from undefined or an absent key, so a check that could is rejected. + const Either = Schema.optional(Schema.NullOr(Schema.String)) + const rejected: ReadonlyArray, string]> = [ + [Schema.Struct({ a: Either }).check(record), `field "a"`], + [ + Schema.Struct({ p: Schema.Struct({ xs: Schema.Array(Schema.Struct({ a: Either })) }).check(record) }), + `field "p.xs[].a"` + ], + [Schema.Struct({ a: Schema.Union([Schema.Null, Schema.Undefined, Schema.String]).check(record) }), `field "a"`] + ] + for (const [schema, field] of rejected) { + const message = `${field}: a check sees a missing value that the schema writes in several ways` + assert.throws(() => DataFrame.fromRows(schema, []), DataFrame.DataFrameError, message) + assert.throws(() => Query.table("t", schema), DataFrame.DataFrameError, message) + const validated = DataFrame.validate(readings, schema) + assert.include(Result.isFailure(validated) && validated.failure.message, message) + } + // No check sees it here: the literal is tested on its own column. + const Unseen = Schema.Struct({ kind: Schema.Literal("x"), a: Either }) + assert.deepStrictEqual(outcomes(Unseen, [{ kind: "x" }, { kind: "x", a: null }]), ["ok", "ok", "ok"]) + }) + + // DF-C032 (API contract): DataFrame.fromRows; pinned source in README.md. + it("accepts the stored values that Schema.is accepts", () => { + const rejects = `column "v" has values its schema rejects` + const NotNull = Schema.Struct({ v: Schema.NullOr(Schema.String).check(Schema.makeFilter((v) => v !== null)) }) + assert.deepStrictEqual(outcomes(NotNull, [{ v: "a" }, { v: null }]), [rejects, rejects, rejects]) + const OneOf = Schema.Struct({ v: Schema.Union([Schema.String, Schema.Literal("ok")], { mode: "oneOf" }) }) + assert.deepStrictEqual(outcomes(OneOf, [{ v: "no" }, { v: "ok" }]), [rejects, rejects, rejects]) + + const Tagged = Schema.Union([ + Schema.Struct({ kind: Schema.Literal("a"), n: Schema.NullOr(Schema.Int) }), + Schema.Struct({ kind: Schema.Literals(["a", "b"]), n: Schema.NullOr(Schema.Number) }) + ], { mode: "oneOf" }) + const cases: ReadonlyArray, ReadonlyArray>]> = [ + [NotNull, [{ v: "a" }, { v: null }]], + [OneOf, [{ v: "no" }, { v: "ok" }]], + [Schema.Struct({ v: Schema.Array(Tagged) }), [ + { v: [{ kind: "b", n: 1.5 }, { kind: "a", n: 1.5 }] }, + { v: [{ kind: "a", n: 1 }] }, + { v: [{ kind: "a", n: null }] }, + { v: [{ kind: "c", n: 1 }] } + ]], + [Schema.Struct({ v: Schema.optional(Schema.String.check(Schema.isMinLength(2))) }), [ + {}, + { v: undefined }, + { v: "ab" }, + { v: "a" } + ]], + [Schema.Struct({ v: Schema.UndefinedOr(Schema.Number).check(Schema.makeFilter((v) => v !== 0)) }), [ + { v: undefined }, + { v: 1 }, + { v: 0 } + ]], + [Schema.Struct({ v: Schema.Union([Schema.Null.check(Schema.makeFilter(() => false)), Schema.String]) }), [ + { v: "a" }, + { v: null } + ]], + [ + Schema.Struct({ v: Schema.optionalKey(Schema.Number), w: Schema.NullOr(Schema.Number) }).check( + Schema.makeFilter((row) => ("v" in row) === (row.w === null)) + ), + [{ v: 1, w: null }, { w: 1 }, { w: null }, { v: 1, w: 1 }] + ], + // An optional field makes the check see a rebuilt row, which must keep + // `__proto__` as an own property + [ + Schema.Struct({ ["__proto__"]: Schema.String, opt: Schema.optional(Schema.Number) }).check( + Schema.makeFilter((row) => Object.keys(row).length === 0) + ), + [{ ["__proto__"]: "good" }] + ], + [ + Schema.Struct({ ["__proto__"]: Schema.String, opt: Schema.optional(Schema.Number) }).check( + Schema.makeFilter((row) => Object.hasOwn(row, "__proto__") && row["__proto__"] === "good") + ), + [{ ["__proto__"]: "good" }, { ["__proto__"]: "bad", opt: 1 }] + ], + [ + Schema.Struct({ + s: Schema.Struct({ ["__proto__"]: Schema.String, opt: Schema.optional(Schema.Number) }).check( + Schema.makeFilter((s) => Object.hasOwn(s, "__proto__") && s["__proto__"] === "good") + ) + }), + [{ s: { ["__proto__"]: "good" } }, { s: { ["__proto__"]: "bad" } }] + ] + ] + const verdicts = new Set() + for (const [schema, rows] of cases) { + for (const row of rows) { + const accepted = Schema.is(schema)(row) + verdicts.add(accepted) + assert.deepStrictEqual( + outcomes(schema, [row]).map((outcome) => outcome === "ok"), + [accepted, accepted, accepted], + JSON.stringify(row) + ) + } + } + assert.strictEqual(verdicts.size, 2) + }) + + // DF-C023 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows, empty and validate reject schemas that transform their rows", () => { + const cases: ReadonlyArray = [ + [Schema.Struct({ a: Schema.NumberFromString }), `field "a"`], + [Schema.Struct({ t: Schema.optional(Schema.DateTimeUtcFromString) }), `field "t"`], + [ + Schema.Struct({ p: Schema.Struct({ xs: Schema.Array(Schema.NullOr(Schema.NumberFromString)) }) }), + `field "p.xs[]"` + ], + [Schema.Struct({ a: Schema.Number.pipe(Schema.withDecodingDefault(Effect.succeed(1))) }), `field "a"`], + [Schema.Struct({ a: Schema.suspend(() => Schema.NumberFromString) }), `field "a"`], + [Schema.Struct({ a: Schema.Number }).pipe(Schema.encodeKeys({ a: "b" })), "the schema"] + ] + for (const [schema, subject] of cases) { + const message = + `${subject} has a transformation: frames hold decoded values and transformations are not run; ` + + "Schema.toType(schema) describes the decoded rows" + assert.throws(() => DataFrame.fromRows(schema, []), DataFrame.DataFrameError, message) + assert.throws(() => DataFrame.empty(schema), DataFrame.DataFrameError, message) + const result = DataFrame.validate(DataFrame.fromColumns({ a: ["1"] }), schema) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + const Decoded = Schema.toType(Schema.Struct({ a: Schema.NumberFromString, t: Schema.DateTimeUtcFromString })) + const row = { a: 1, t: DateTime.makeUnsafe(0) } + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromRows(Decoded, [row])), [row]) + }) + }) + + // DF-C021 (API contract): compatible Arrow layouts; pinned source in README.md. + it("reuses compatible numeric buffers across Arrow conversion", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("i", ArrowSchema.int32), + ArrowSchema.field("n", ArrowSchema.float64) + ]) + const input = Result.getOrThrow(ArrowTable.fromColumns(schema, [[1, -2, 3], [1.5, -2.5, 0]])) + const frame = DataFrame.fromArrowTable(input) + const output = DataFrame.toArrowTable(frame) + assert.deepStrictEqual(DataFrame.toRows(frame), [{ i: 1, n: 1.5 }, { i: -2, n: -2.5 }, { i: 3, n: 0 }]) + assert.deepStrictEqual(ArrowTable.toRows(output), DataFrame.toRows(frame)) + for (let i = 0; i < 2; i++) { + const before = input.batches[0].columns[i].buffers[0] + const after = output.batches[0].columns[i].buffers[0] + assert.strictEqual(after.buffer, before.buffer) + assert.strictEqual(after.byteOffset, before.byteOffset) + assert.strictEqual(after.byteLength, before.byteLength) + } + }) + + // DF-C028 (API contract): DataFrame.column; pinned source in README.md. + it("column gives a column's buffers and values without building rows", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("at", ArrowSchema.timestamp("nanosecond", "Europe/Vienna")), + ArrowSchema.field("n", ArrowSchema.int32, { nullable: false }), + ArrowSchema.field("s", ArrowSchema.utf8) + ]) + const input = Result.getOrThrow( + ArrowTable.fromColumns(schema, [[1700000000123456789n, null, -1n], [1, -2, 3], ["a", null, "bc"]]) + ) + const frame = DataFrame.fromArrowTable(input) + const built = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(InternalDateTime, "makeZonedProto")] + try { + const at = DataFrame.column(frame, "at") + assert.strictEqual(at.length, 3) + assert.deepStrictEqual(at.field.type, ArrowSchema.timestamp("nanosecond", "Europe/Vienna")) + assert.strictEqual(at.chunks.length, 1) + const nanos = Option.getOrThrow(ArrowData.values(at.chunks[0])) + assert.isTrue(nanos instanceof BigInt64Array) + assert.deepStrictEqual([nanos[0], nanos[2]], [1700000000123456789n, -1n]) + assert.deepStrictEqual([0, 1, 2].map((i) => ArrowData.isValid(at.chunks[0], i)), [true, false, true]) + + const n = frame.pipe(DataFrame.column("n")) + assert.strictEqual(n.field.nullable, false) + const ints = Option.getOrThrow(ArrowData.values(n.chunks[0])) + assert.deepStrictEqual(ints, Int32Array.of(1, -2, 3)) + assert.strictEqual(ints.buffer, input.batches[0].columns[1].buffers[0].buffer) + + const s = DataFrame.column(frame, "s").chunks[0] + assert.deepStrictEqual(Array.from(Option.getOrThrow(ArrowData.offsets(s)), Number), [0, 1, 1, 3]) + assert.deepStrictEqual(new TextDecoder().decode(s.buffers[1]), "abc") + assert.strictEqual(built.reduce((sum, spy) => sum + spy.mock.calls.length, 0), 0) + + // A value is built when it is asked for, at the millisecond DateTime holds. + const first = ArrowColumn.getUnsafe(at, 0) as DateTime.Zoned + assert.strictEqual(DateTime.toEpochMillis(first), 1700000000123) + assert.strictEqual(DateTime.zoneToString(first.zone), "Europe/Vienna") + assert.strictEqual(ArrowColumn.getUnsafe(at, 1), null) + assert.strictEqual(built.reduce((sum, spy) => sum + spy.mock.calls.length, 0), 1) + } finally { + built.forEach((spy) => spy.mockRestore()) + } + assert.deepStrictEqual(ArrowColumn.toArray(DataFrame.column(readings, "rain")), [0, null, 2]) + assert.throws( + () => DataFrame.column(frame as DataFrame.DataFrame, "missing"), + DataFrame.DataFrameError, + `unknown column "missing"` + ) + }) + + describe("validate", () => { + const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(readings)) + const arrowFrame = (columns: Record]>) => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make(Object.entries(columns).map(([name, [type]]) => ArrowSchema.field(name, type))), + Object.values(columns).map(([, values]) => values) + ))) + const column = (type: ArrowSchema.DataType, values: ReadonlyArray) => arrowFrame({ t: [type, values] }) + const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) + + // DF-C009 (API contract): DataFrame.validate; pinned source in README.md. + it("returns the frame typed by the schema, in schema order", () => { + const result = DataFrame.validate(untyped, Schema.Struct({ temp: Schema.Number, city: Schema.String })) + assert.deepStrictEqual(DataFrame.columnNames(Result.getOrThrow(result)), ["temp", "city"]) + }) + + // DF-C010 (API contract): DataFrame.validate; pinned source in README.md. + it("fails on missing columns, wrong types, unexpected nulls and values the schema rejects", () => { + const nested = DataFrame.fromRows( + Schema.Struct({ p: Schema.Struct({ x: Schema.String }), xs: Schema.Array(Schema.NullOr(Schema.Number)) }), + [{ p: { x: "a" }, xs: [1, null] }] + ) + const cases: ReadonlyArray, Schema.Struct, string]> = [ + [untyped, Schema.Struct({ country: Schema.String }), `missing column "country"`], + [untyped, Schema.Struct({ city: Schema.Number }), `column "city" is Utf8, expected Float64`], + [untyped, Schema.Struct({ rain: Schema.Number }), `column "rain" has missing values`], + [ + DataFrame.fromColumns({ n: Int32Array.of(1) }), + Schema.Struct({ n: Schema.BigInt }), + `column "n" is Int32, expected Int64` + ], + [nested, Schema.Struct({ p: Schema.Struct({ x: Schema.Number }) }), `column "p.x" is Utf8, expected Float64`], + [nested, Schema.Struct({ xs: Schema.Array(Schema.Number) }), `column "xs[]" has missing values`], + [ + DataFrame.fromColumns({ s: ["allowed", "nope"] }), + Schema.Struct({ s: Schema.Literal("allowed") }), + `column "s" has values its schema rejects` + ], + [ + DataFrame.fromColumns({ n: [1, 1.5] }), + Schema.Struct({ n: Schema.Int }), + `column "n" has values its schema rejects` + ], + [ + DataFrame.fromColumns({ lo: [1, 2], hi: [2, 1] }), + Schema.Struct({ lo: Schema.Number, hi: Schema.Number }).check( + Schema.makeFilter((row) => row.lo < row.hi || "lo must be below hi") + ), + "rows have values their schema rejects" + ], + [ + DataFrame.fromRows(Schema.Struct({ s: Schema.Struct({ a: Schema.String }) }), [{ s: { a: "x" } }, { + s: { a: "z" } + }]), + Schema.Struct({ + s: Schema.Union([Schema.Struct({ a: Schema.Literal("x") }), Schema.Struct({ a: Schema.Literal("y") })]) + }), + `column "s" has values its schema rejects` + ], + [ + DataFrame.fromColumns({ s: ["x", "z"] }), + Schema.Struct({ + s: Schema.Union([Schema.suspend(() => Schema.Literal("x")), Schema.suspend(() => Schema.Literal("y"))]) + }), + `column "s" has values its schema rejects` + ], + [ + column(ArrowSchema.decimal(18, 3), [BigDecimal.fromStringUnsafe("1.234")]), + Schema.Struct({ t: Schema.BigDecimal }), + `column "t": decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ], + [ + column(ArrowSchema.timestamp("nanosecond"), [1n]), + Utc, + `column "t" has values that Timestamp[us] can't hold exactly` + ], + [ + column(ArrowSchema.timestamp("millisecond", "Europe/Berlin"), [0n]), + Utc, + `column "t" has time zone Europe/Berlin, expected UTC timestamps` + ], + [ + column(ArrowSchema.timestamp("millisecond"), [9223372036854775807n]), + Utc, + `column "t" has timestamps outside the range of DateTime` + ], + [ + column(ArrowSchema.list(ArrowSchema.field("item", ArrowSchema.timestamp("microsecond"))), [[ + 8640000000000000999n, + 8640000000000001000n + ]]), + Schema.Struct({ t: Schema.Array(Schema.DateTimeUtc) }), + `column "t[]" has timestamps outside the range of DateTime` + ] + ] + for (const [df, schema, message] of cases) { + const result = DataFrame.validate(df, schema) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + }) + + // DF-C011 (API contract): DataFrame.validate; pinned source in README.md. + it("reads timestamps with a UTC time zone as DateTime.Utc", () => { + const ts = ArrowSchema.timestamp("millisecond", "UTC") + const nested = column(ArrowSchema.list(ArrowSchema.field("item", ts)), [[1000n]]) + const df = Result.getOrThrow(DataFrame.validate(nested, Schema.Struct({ t: Schema.Array(Schema.DateTimeUtc) }))) + assert.deepStrictEqual(DataFrame.toRows(df), [{ t: [DateTime.makeUnsafe(1000)] }]) + const built = DataFrame.fromRows(Utc, [{ t: DateTime.makeUnsafe(0) }], { + types: { t: ArrowSchema.timestamp("second", "UTC") } + }) + assert.deepStrictEqual(built.schema.fields[0].type, ArrowSchema.timestamp("second")) + assert.deepStrictEqual(DataFrame.toRows(built), [{ t: DateTime.makeUnsafe(0) }]) + }) + + // DF-C012 (API contract): DataFrame.validate; pinned source in README.md. + it("converts columns to the types fromRows gives the schema, exactly", () => { + const Row = Schema.Struct({ n: Schema.Number, d: Schema.BigDecimal, t: Schema.DateTimeUtc }) + const df = arrowFrame({ + n: [ArrowSchema.int32, [1]], + d: [ArrowSchema.decimal(18, 1), [BigDecimal.fromStringUnsafe("1.2")]], + t: [ArrowSchema.timestamp("millisecond"), [1n]] + }) + const types = { d: ArrowSchema.decimal(18, 2) } + const validated = Result.getOrThrow(DataFrame.validate(df, Row, { types })) + assert.deepStrictEqual(validated.schema, DataFrame.empty(Row, { types }).schema) + assert.deepStrictEqual(df.pipe(DataFrame.validate(Row, { types }), Result.getOrThrow).schema, validated.schema) + assert.deepStrictEqual(DataFrame.toRows(validated), [{ + n: 1, + d: BigDecimal.make(120n, 2), + t: DateTime.makeUnsafe(1) + }]) + const lossy = DataFrame.validate(df, Row, { types: { d: ArrowSchema.decimal(18, 0) } }) + assert.strictEqual( + Result.isFailure(lossy) && lossy.failure.message, + `column "d" has values that Decimal[18, 0] can't hold exactly` + ) + }) + + // DF-C024 (local policy): DataFrame.validate; pinned source in README.md. + it("builds values only for the fields the schema constrains", () => { + const Row = Schema.Struct({ t: Schema.DateTimeUtc, d: Schema.Duration, m: Schema.BigDecimal }) + const types = { m: ArrowSchema.decimal(18, 2) } + const df = arrowFrame({ + t: [ArrowSchema.timestamp("millisecond"), [1n, 2n, 3n]], + d: [ArrowSchema.duration("second"), [1n, 2n, 3n]], + m: [ArrowSchema.decimal(9, 2), ["1.5", "2.5", "3.5"].map(BigDecimal.fromStringUnsafe)] + }) + const row = { t: DateTime.makeUnsafe(1), d: Duration.seconds(1), m: BigDecimal.fromStringUnsafe("1.5") } + const built = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(Duration, "nanos"), vi.spyOn(BigDecimal, "make")] + const counts = () => built.map((spy) => spy.mock.calls.length) + try { + assert.strictEqual(DataFrame.fromRows(Row, [row, row], { types }).length, 2) + const validated = Result.getOrThrow(DataFrame.validate(df, Row, { types })) + const table = Query.table("t", Row, { types }).pipe(Effect.provide(Catalog.layer({ t: Stream.make(df, df) }))) + assert.strictEqual(Effect.runSync(table).length, 6) + assert.deepStrictEqual(counts(), [0, 0, 0]) + const After = Schema.Struct({ + ...Row.fields, + t: Schema.DateTimeUtc.check(Schema.makeFilter((t) => DateTime.toEpochMillis(t) > 0)) + }) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, After, { types }))) + assert.deepStrictEqual(counts(), [3, 0, 0]) + // Reading rows builds the values, which shows that the spies see them. + const rows = DataFrame.toRows(validated) + assert.deepStrictEqual(counts().map((n, i) => n > [3, 0, 0][i]), [true, true, true]) + assert.deepStrictEqual(rows[0], { + t: DateTime.makeUnsafe(1), + d: Duration.nanos(1_000_000_000n), + m: BigDecimal.make(150n, 2) + }) + } finally { + for (const spy of built) spy.mockRestore() + } + }) + + // DF-C025 (API contract): DataFrame.validate; pinned source in README.md. + it("takes an Arrow schema, converting the frame to its columns", () => { + const item = ArrowSchema.field("element", ArrowSchema.decimal(18, 2), { nullable: false }) + const schema = ArrowSchema.make([ + ArrowSchema.field("t", ArrowSchema.timestamp("second", "UTC")), + ArrowSchema.field("n", ArrowSchema.int32, { nullable: false }), + ArrowSchema.field("xs", ArrowSchema.list(item)), + ArrowSchema.field("p", ArrowSchema.struct([ArrowSchema.field("s", ArrowSchema.utf8, { nullable: false })])) + ]) + const frame = (n: number | null, unit: ArrowSchema.TimeUnit = "nanosecond", t = 1_000_000_000n) => + arrowFrame({ + extra: [ArrowSchema.bool, [true]], + p: [ArrowSchema.struct([ArrowSchema.field("s", ArrowSchema.largeUtf8)]), [{ s: "a" }]], + xs: [ArrowSchema.list(ArrowSchema.field("item", ArrowSchema.decimal(9, 1))), [[BigDecimal.make(15n, 1)]]], + n: [ArrowSchema.int8, [n]], + t: [ArrowSchema.timestamp(unit), [t]] + }) + const validated = Result.getOrThrow(DataFrame.validate(frame(1), schema)) + assert.deepStrictEqual( + validated.schema, + ArrowSchema.make([ + ArrowSchema.field("t", ArrowSchema.timestamp("second")), + schema.fields[1], + ArrowSchema.field("xs", ArrowSchema.list({ ...item, name: "item" })), + schema.fields[3] + ]) + ) + assert.deepStrictEqual(frame(1).pipe(DataFrame.validate(schema), Result.getOrThrow), validated) + assert.deepStrictEqual(DataFrame.toRows(validated), [{ + t: DateTime.makeUnsafe(1000), + n: 1, + xs: [BigDecimal.make(150n, 2)], + p: { s: "a" } + }]) + for ( + const [df, message] of [ + [frame(null), `column "n" has missing values`], + // A finer unit converts only when no digit is lost (DF-C010 has the same for struct schemas). + [frame(1, "millisecond", 1_500n), `column "t" has values that Timestamp[s] can't hold exactly`] + ] as const + ) { + const result = DataFrame.validate(df, schema) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + }) + + // DF-C027 (API contract): DataFrame.validate; pinned source in README.md. + it("converts dates and times to the declared unit", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("d", ArrowSchema.date("day")), + ArrowSchema.field("at", ArrowSchema.time("microsecond")) + ]) + const frame = (d: bigint | null, at: number | null) => + arrowFrame({ + d: [ArrowSchema.date("millisecond"), [d, null]], + at: [ArrowSchema.time("millisecond"), [at, null]] + }) + const source = frame(172_800_000n, 1_500) + const validated = Result.getOrThrow(DataFrame.validate(source, schema)) + assert.deepStrictEqual(validated.schema, schema) + assert.deepStrictEqual(DataFrame.toArray(validated, "d"), DataFrame.toArray(source, "d")) + // A time reads as its count of the column's unit. + assert.deepStrictEqual(DataFrame.toArray(validated, "at"), [1_500_000n, null]) + const result = DataFrame.validate(frame(172_800_001n, 1), schema) + assert.strictEqual( + Result.isFailure(result) && result.failure.message, + `column "d" has values that Date can't hold exactly` + ) + }) + + // DF-C026 (API contract): DataFrame.validate; pinned source in README.md. + it("rejects Arrow schemas with types a table can't declare", () => { + const { field } = ArrowSchema + const cases: ReadonlyArray = [ + [ + ArrowSchema.timestamp("second", "Europe/Berlin"), + `column "c" has time zone Europe/Berlin, expected UTC timestamps` + ], + [ArrowSchema.interval("yearMonth"), `column "c": Interval columns can't be declared`], + [ArrowSchema.float16, `column "c": Float16 columns can't be declared`], + [ + ArrowSchema.map(field("k", ArrowSchema.utf8), field("v", ArrowSchema.int32)), + `column "c": Map columns can't be declared` + ], + [ + ArrowSchema.dictionary(ArrowSchema.utf8), + `column "c": Dictionary columns can't be declared: frames hold them as their value type` + ], + [ArrowSchema.largeUtf8, `column "c": LargeUtf8 columns can't be declared: frames hold them as Utf8`], + [ + ArrowSchema.largeList(field("item", ArrowSchema.int32)), + `column "c": LargeList columns can't be declared: frames hold them as List` + ], + [ + ArrowSchema.decimal(18, 2, 64), + `column "c": 64-bit decimals can't be declared: declared decimals are 128-bit` + ], + [ + ArrowSchema.struct([field("x", ArrowSchema.decimal(39, 0))]), + `column "c.x": invalid decimal128 precision 39, expected an integer in 1..38` + ], + [ + ArrowSchema.date("millisecond"), + `column "c": dates in milliseconds can't be declared: declared dates are in days` + ], + [ + ArrowSchema.time("second"), + `column "c": times in seconds can't be declared: declared times are in microseconds` + ], + [ArrowSchema.list(field("item", ArrowSchema.float16)), `column "c[]": Float16 columns can't be declared`], + [ + ArrowSchema.struct([field("x", ArrowSchema.utf8View)]), + `column "c.x": Utf8View columns can't be declared: frames hold them as Utf8` + ] + ] + for (const [type, message] of cases) { + const result = DataFrame.validate(readings, ArrowSchema.make([field("c", type)])) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + const notNull = DataFrame.validate( + readings, + ArrowSchema.make([field("c", ArrowSchema.nullType, { nullable: false })]) + ) + assert.strictEqual(Result.isFailure(notNull) && notNull.failure.message, `column "c": Null columns are nullable`) + }) + }) + + describe("Arrow interop", () => { + // DF-C013 (API contract): DataFrame.toArrowTable; pinned source in README.md. + it("round-trips through the IPC stream format", () => { + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(readings))) + const back = DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) + assert.isTrue(Equal.equals(back, readings)) + }) + + // DF-C014 (API contract): DataFrame.decodeArrowStream; pinned source in README.md. + it.effect("decodes IPC stream bytes into a frame per batch", () => + Effect.gen(function*() { + const table = DataFrame.toArrowTable(readings) + const bytes = Result.getOrThrow( + ArrowIpc.encodeStream(ArrowTable.make(table.schema, [table.batches[0], table.batches[0]])) + ) + // Split the bytes at arbitrary points, as a socket would deliver them. + const chunks = [bytes.subarray(0, 100), bytes.subarray(100, 300), bytes.subarray(300)] + const frames = yield* DataFrame.decodeArrowStream(Stream.fromIterable(chunks)).pipe(Stream.runCollect) + assert.strictEqual(frames.length, 2) + for (const frame of frames) assert.isTrue(Equal.equals(frame, readings)) + })) + + // DF-C015 (API contract): DataFrame.toArrowTable; pinned source in README.md. + it.effect("exports late-selected columns", () => + Effect.gen(function*() { + const df = yield* Query.from(readings).pipe( + Query.filter((c) => Expr.isNotNull(c.rain)), + Query.sort((c) => [c.temp]) + ) + const table = DataFrame.toArrowTable(df) + assert.strictEqual(table.batches[0].length, 2) + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) + })) + + // DF-C016 (API contract): DataFrame.fromArrowTable; pinned source in README.md. + it("imports a table without batches with the nullability of its fields", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("x", ArrowSchema.int8, { nullable: false }), + ArrowSchema.field("s", ArrowSchema.utf8) + ]) + assert.deepStrictEqual(types(DataFrame.fromArrowTable(ArrowTable.make(schema, []))), [ + "x:Int:false", + "s:Utf8:true" + ]) + }) + + // DF-C017 (API contract): DataFrame.fromArrowTable; pinned source in README.md. + it.effect("imports sliced list columns with only their own elements", () => + Effect.gen(function*() { + const Lists = Schema.Struct({ xs: Schema.Array(Schema.Number) }) + const df = DataFrame.fromRows(Lists, [{ xs: [1] }, { xs: [2] }, { xs: [3] }]) + const table = DataFrame.toArrowTable(df) + const sliced = DataFrame.fromArrowTable(ArrowTable.slice(table, 0, 1)) + const sums = yield* Query.from(sliced).pipe( + Query.select((c) => ({ s: Expr.listSum(c.xs as Expr.Expr>) })) + ) + assert.deepStrictEqual(DataFrame.toRows(sums), [{ s: 1 }]) + const two = ArrowTable.make(table.schema, [ + ...ArrowTable.slice(table, 0, 1).batches, + ...ArrowTable.slice(table, 2, 3).batches + ]) + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(two)), [{ xs: [1] }, { xs: [3] }]) + })) + }) + + describe("values", () => { + // DF-C018 (API contract): DataFrame.equals; pinned source in README.md. + it("compare structurally, NaN included", () => { + const a = DataFrame.fromColumns({ x: [1, NaN, null] }) + const b = DataFrame.fromColumns({ x: [1, NaN, null] }) + const c = DataFrame.fromColumns({ x: [1, NaN, 2] }) + assert.isTrue(Equal.equals(a, b)) + assert.isFalse(Equal.equals(a, c)) + }) + + // DF-C019 (API contract): DataFrame.DataFrame; pinned source in README.md. + it("iterate as rows", () => { + assert.deepStrictEqual(Array.from(DataFrame.fromColumns({ city: DataFrame.toArray(readings, "city") })), [ + { city: "Vienna" }, + { city: "Berlin" }, + { city: "Vienna" } + ]) + }) + + // DF-C020 (local policy): DataFrame.format; pinned source in README.md. + it("format as a text table", () => { + assert.strictEqual( + DataFrame.format( + DataFrame.fromColumns({ + city: DataFrame.toArray(readings, "city"), + rain: DataFrame.toArray(readings, "rain") + }) + ), + [ + "DataFrame (3 rows × 2 columns)", + "┌──────────┬─────────┐", + "│ city │ rain │", + "│ Utf8 │ Float64 │", + "├──────────┼─────────┤", + `│ "Vienna" │ 0 │`, + `│ "Berlin" │ null │`, + `│ "Vienna" │ 2 │`, + "└──────────┴─────────┘" + ].join("\n") + ) + }) + }) +}) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts new file mode 100644 index 00000000000..1077ca5cfd7 --- /dev/null +++ b/packages/effect/test/dataframe/Query.test.ts @@ -0,0 +1,2187 @@ +import { assert, describe, it } from "@effect/vitest" +import { BigDecimal, DateTime, Duration, Effect, Equal, Exit, Fiber, Result, Scheduler, Schema, Stream } from "effect" +import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" +import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" +import * as InternalDateTime from "effect/internal/dateTime" +import { vi } from "vitest" +import * as Compare from "./oracle/Compare.ts" +import * as Gold from "./oracle/Gold.ts" +import * as Oracle from "./oracle/Plan.ts" + +const Reading = Schema.Struct({ + city: Schema.String, + temp: Schema.Number, + rain: Schema.NullOr(Schema.Number) +}) + +const readings = DataFrame.fromRows(Reading, [ + { city: "Vienna", temp: 21.5, rain: 0 }, + { city: "Berlin", temp: 26.5, rain: null }, + { city: "Vienna", temp: 19, rain: 2 } +]) + +const City = Schema.Struct({ city: Schema.String, country: Schema.String }) +const cities = DataFrame.fromRows(City, [ + { city: "Vienna", country: "AT" }, + { city: "Berlin", country: "DE" } +]) + +const tables = Catalog.layer({ readings, cities }) + +const rows = (query: Query.Query) => Effect.map(query, DataFrame.toRows) + +/** A frame of Arrow-typed columns. */ +const arrowFrame = (columns: Record]>) => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make(Object.entries(columns).map(([name, [type]]) => ArrowSchema.field(name, type))), + Object.values(columns).map(([, values]) => values) + ))) as DataFrame.DataFrame + +const Price = Schema.Struct({ d: Schema.BigDecimal }) +const price = (d: string, precision: number, scale: number) => + DataFrame.fromRows(Price, [{ d: BigDecimal.fromStringUnsafe(d) }], { + types: { d: ArrowSchema.decimal(precision, scale) } + }) + +describe("Query", () => { + describe("running", () => { + // DF-Q001 (API contract): Query.Query; pinned source in README.md. + it.effect("is an Effect producing a frame", () => + Effect.gen(function*() { + const df = yield* Query.from(readings).pipe(Query.filter((c) => Expr.gt(c.temp, 20))) + assert.deepStrictEqual(DataFrame.toArray(df, "city"), ["Vienna", "Berlin"]) + })) + + // DF-Q042 (API contract): Query yields between operations; pinned source in README.md. + it.effect("yields a long query so another fiber can interrupt it", () => + Effect.gen(function*() { + // Control elapsed time, not machine speed; disable automatic runtime yields + // so only the query's cooperative yielding can let the parent resume. + let now = 0 + const clock = vi.spyOn(performance, "now").mockImplementation(() => (now += 1_000)) + const tasks: Array<() => void> = [] + const scheduler: Scheduler.Scheduler = { + executionMode: "async", + shouldYield: () => false, + makeDispatcher: () => ({ + scheduleTask: (task) => { + tasks.push(task) + }, + flush() {} + }) + } + let loaded = false + let completed = false + try { + const query = Query.table("readings", Reading).pipe( + Query.sort((c) => [Expr.asc(c.temp)]), + Query.select((c) => ({ temp: Expr.add(c.temp, 1) })) + ) + const child = yield* query.pipe( + Effect.provideService( + Catalog.Catalog, + Catalog.make({ + readings: Effect.sync(() => { + loaded = true + return readings + }) + }) + ), + Effect.tap(() => + Effect.sync(() => { + completed = true + }) + ), + Effect.provideService(Scheduler.Scheduler, scheduler), + Effect.forkChild({ startImmediately: true }) + ) + try { + assert.isTrue(loaded) + assert.isUndefined(child.pollUnsafe()) + } finally { + child.interruptUnsafe() + while (tasks.length > 0) tasks.shift()!() + } + assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(child))) + assert.isFalse(completed) + } finally { + clock.mockRestore() + } + })) + + // DF-Q002 (API contract): Query.select; pinned source in README.md. + it.effect("expression callbacks run once at construction, not on execution", () => + Effect.gen(function*() { + let calls = 0 + const query = Query.from(readings).pipe(Query.select((c) => { + calls++ + return { city: c.city } + })) + assert.strictEqual(calls, 1) + const expected = [{ city: "Vienna" }, { city: "Berlin" }, { city: "Vienna" }] + assert.deepStrictEqual(yield* rows(query), expected) + assert.deepStrictEqual(yield* rows(query), expected) + assert.strictEqual(calls, 1) + })) + + // DF-Q003 (API contract): Query.Query; pinned source in README.md. + it.effect("does no work until it runs", () => + Effect.gen(function*() { + const query = Query.table("missing", City).pipe(Query.filter((c) => Expr.eq(c.city, "Vienna"))) + const error = yield* Effect.flip(query) + assert.strictEqual(error.message, `unknown table "missing"`) + })) + + // DF-Q004 (API contract): Catalog.Catalog; pinned source in README.md. + it.effect("reads each table once per run, from effects too", () => + Effect.gen(function*() { + let loads = 0 + const catalog = Catalog.make({ cities: Effect.sync(() => (loads++, cities)) }) + const query = Query.table("cities", City).pipe( + Query.join(Query.table("cities", City).pipe(Query.rename({ country: "other" })), { on: "city" }) + ) + const df = yield* Effect.provideService(query, Catalog.Catalog, catalog) + assert.strictEqual(df.length, 2) + assert.strictEqual(loads, 1) + yield* Effect.provideService(query, Catalog.Catalog, catalog) + assert.strictEqual(loads, 2) + })) + + // DF-Q005 (API contract): Catalog.Catalog; pinned source in README.md. + it.effect("tables can be queries (views)", () => + Effect.gen(function*() { + const warm = Query.table("readings", Reading).pipe(Query.filter((c) => Expr.gt(c.temp, 20))) + const query = Query.table("warm", Reading).pipe(Query.pick("city")) + const result = yield* rows(query).pipe(Effect.provide(Catalog.layer({ readings, warm }))) + assert.deepStrictEqual(result, [{ city: "Vienna" }, { city: "Berlin" }]) + })) + + // DF-Q006 (API contract): Query.table; pinned source in README.md. + it.effect("checks tables against their declared columns", () => + Effect.gen(function*() { + const error = yield* Effect.flip(Query.table("readings", City).pipe(Effect.provide(tables))) + assert.strictEqual(error.message, `missing column "country"`) + const bad = Stream.make(DataFrame.fromColumns({ city: [1] })) + const streamed = yield* Effect.flip(Query.table("t", City).pipe(Effect.provide(Catalog.layer({ t: bad })))) + assert.strictEqual(streamed.message, `column "city" is Float64, expected Utf8`) + const Kind = Schema.Struct({ kind: Schema.Literals(["a", "b"]) }) + const Range = Schema.Struct({ lo: Schema.Number, hi: Schema.Number }).check( + Schema.makeFilter((row) => row.lo < row.hi || "lo must be below hi") + ) + const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) + const berlin = arrowFrame({ t: [ArrowSchema.timestamp("millisecond", "Europe/Berlin"), [0n]] }) + for ( + const [schema, frame, message] of [ + [Kind, DataFrame.fromColumns({ kind: ["a", "c"] }), `column "kind" has values its schema rejects`], + [Range, DataFrame.fromColumns({ lo: [1, 2], hi: [2, 1] }), "rows have values their schema rejects"], + [Utc, berlin, `column "t" has time zone Europe/Berlin, expected UTC timestamps`] + ] as const + ) { + for (const t of [frame, Stream.make(frame)]) { + const error = yield* Effect.flip(Query.table("t", schema).pipe(Effect.provide(Catalog.layer({ t })))) + assert.strictEqual(error.message, message) + } + } + const ranges = Query.table("t", Range).pipe( + Effect.provide(Catalog.layer({ t: DataFrame.fromColumns({ lo: [1], hi: [2] }) })) + ) + assert.deepStrictEqual(DataFrame.toRows(yield* ranges), [{ lo: 1, hi: 2 }]) + })) + + // DF-Q007 (API contract): Query.table; pinned source in README.md. + it.effect("narrows tables to their declared columns", () => + Effect.gen(function*() { + const Temps = Schema.Struct({ temp: Schema.Number, city: Schema.String }) + const df = yield* Query.table("readings", Temps).pipe(Effect.provide(tables)) + assert.deepStrictEqual(DataFrame.columnNames(df), ["temp", "city"]) + })) + + // DF-Q043 (API contract): Query.table; pinned source in README.md. + it.effect("compiles a schema's checks when the table is declared, and does not encode them to JSON", () => + Effect.gen(function*() { + const Kind = Schema.Struct({ kind: Schema.Literals(["a", "b"]) }) + let reads = 0 + const counted = { + Type: Kind.Type, + get ast() { + reads++ + return Kind.ast + } + } + const query = Query.table("t", counted) + const declared = reads + const good = DataFrame.fromColumns({ kind: ["a", "b"] }) + const bad = DataFrame.fromColumns({ kind: ["c"] }) + const read = (query: Query.Query, batches: ReadonlyArray>) => + query.pipe(Effect.provide(Catalog.layer({ t: Stream.fromIterable(batches) }))) + assert.strictEqual((yield* read(query, [good, good, good, good])).length, 8) + const error = yield* Effect.flip(read(query, [good, good, bad])) + assert.strictEqual(error.message, `column "kind" has values its schema rejects`) + assert.strictEqual(reads, declared) + // The checks are functions without a JSON form: a plan that carries them is not encoded. + const first = query.pipe(Query.head(3)) + const rejected = /table "t" has checks[^]*Plan\.withoutChecks/ + assert.throws(() => Schema.encodeSync(Plan.NodeFromJson)(first.plan), rejected) + assert.throws(() => Schema.encodeSync(Schema.toCodecJson(Schema.Struct({ plan: Plan.Node })))(first), rejected) + // Nor is one decoded from JSON written without the codec, where the functions are lost. + assert.throws( + () => Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(JSON.stringify(first.plan))), + /table "t" has checks/ + ) + // Validating a plan keeps its checks. + const validated = Query.fromPlan(Schema.decodeUnknownSync(Plan.Node)(first.plan)) + assert.strictEqual( + (yield* Effect.flip(read(validated, [good, bad]))).message, + `column "kind" has values its schema rejects` + ) + // Removing them is explicit: the plan then encodes, and checks column types only. + const json = JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(Plan.withoutChecks(first.plan))) + const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(json))) + assert.deepStrictEqual(decoded.plan, { + _tag: "Slice", + input: { _tag: "Table", name: "t", columns: [{ name: "kind", type: "Utf8", nullable: false }] }, + offset: 0, + length: 3 + }) + assert.strictEqual((yield* read(decoded, [good, bad])).length, 3) + })) + + // DF-Q044 (API contract): Query.table; pinned source in README.md. + it.effect("tables declared by an Arrow schema convert their input to its columns", () => + Effect.gen(function*() { + const schema = ArrowSchema.make([ + ArrowSchema.field("d", ArrowSchema.decimal(18, 2), { nullable: false }), + ArrowSchema.field("n", ArrowSchema.int32) + ]) + const t = Query.table("t", schema) + assert.deepStrictEqual(t.plan, { + _tag: "Table", + name: "t", + columns: [ + { name: "d", type: "Decimal", nullable: false, precision: 18, scale: 2 }, + { name: "n", type: "Int32", nullable: true } + ] + }) + const batch = (d: string | null, scale: number, n: number | null) => + arrowFrame({ + n: [ArrowSchema.int8, [n]], + extra: [ArrowSchema.utf8, ["x"]], + d: [ArrowSchema.decimal(9, scale), [d === null ? null : BigDecimal.fromStringUnsafe(d)]] + }) + const read = (source: Catalog.Source) => t.pipe(Effect.provide(Catalog.layer({ t: source }))) + const expected = [{ d: BigDecimal.make(120n, 2), n: 1 }] + for (const source of [batch("1.2", 1, 1), Stream.make(batch("1.2", 1, 1)), Stream.make(batch("1.20", 2, 1))]) { + const df = yield* read(source) + assert.deepStrictEqual(df.schema, schema) + assert.deepStrictEqual(DataFrame.toRows(df), expected) + } + const two = yield* read(Stream.make(batch("1.2", 1, 1), batch("1.20", 2, null))) + assert.deepStrictEqual(DataFrame.toRows(two), [...expected, { d: BigDecimal.make(120n, 2), n: null }]) + assert.deepStrictEqual((yield* read(Stream.empty)).schema, schema) + for ( + const [bad, message] of [ + [batch(null, 1, 1), `column "d" has missing values`], + [batch("1.234", 3, 1), `column "d" has values that Decimal[18, 2] can't hold exactly`] + ] as const + ) { + for (const source of [bad, Stream.make(batch("1.2", 1, 1), bad)]) { + assert.strictEqual((yield* Effect.flip(read(source))).message, message) + } + } + })) + + // DF-Q045 (API contract): Query.table; pinned source in README.md. + it("fails when a table is declared by a schema that has no columns for it", () => { + assert.throws( + () => Query.table("t", Schema.Struct({ a: Schema.NumberFromString })), + DataFrame.DataFrameError, + `field "a" has a transformation: frames hold decoded values and transformations are not run; ` + + "Schema.toType(schema) describes the decoded rows" + ) + const zoned = ArrowSchema.make([ArrowSchema.field("t", ArrowSchema.timestamp("second", "Europe/Berlin"))]) + assert.throws( + () => Query.table("t", zoned), + DataFrame.DataFrameError, + `column "t" has time zone Europe/Berlin, expected UTC timestamps` + ) + }) + + // DF-Q048 (API contract): Query.table; pinned source in README.md. + it.effect("rejects a decimal precision or scale that a 128-bit decimal can't have", () => + Effect.gen(function*() { + const precision = (p: number) => `column "d": invalid decimal128 precision ${p}, expected an integer in 1..38` + const scale = (s: number) => `column "d": invalid decimal scale ${s}, expected a 32-bit integer` + const source = Catalog.layer({ t: price("7", 2, 0) }) + const native = (type: ArrowSchema.DataType) => + Query.table("t", ArrowSchema.make([ArrowSchema.field("d", type, { nullable: false })])) + const override = (type: ArrowSchema.DataType) => Query.table("t", Price, { types: { d: type } }) + const plan = (precision: number, scale: number): Plan.Node => ({ + _tag: "Table", + name: "t", + columns: [{ name: "d", type: "Decimal", nullable: false, precision, scale }] + }) + // The plan schema takes any integer: a decoded plan carries those, a hand-made one the others. + const cases: ReadonlyArray = [ + [2.5, 0, precision(2.5), false], + [0, 0, precision(0), true], + [39, 0, precision(39), true], + [2, 0.5, scale(0.5), false], + [2, 2 ** 31, scale(2 ** 31), true] + ] + for (const [p, s, message, decodes] of cases) { + const type = ArrowSchema.decimal(p, s) + assert.throws(() => native(type), DataFrame.DataFrameError, message) + assert.throws(() => override(type), DataFrame.DataFrameError, message) + const decoded = Schema.decodeUnknownResult(Plan.Node)(plan(p, s)) + assert.strictEqual(Result.isSuccess(decoded), decodes) + const query = Query.fromPlan(Result.getOrElse(decoded, () => plan(p, s))) + const error = yield* Effect.flip(query.pipe(Effect.provide(source))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, message) + const streamed = Query.stream(query).pipe(Stream.runCollect, Effect.provide(source)) + assert.strictEqual((yield* Effect.flip(streamed)).message, message) + } + const valid = ArrowSchema.decimal(38, 0) + const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.Node)(plan(38, 0))) + for (const query of [native(valid), override(valid), decoded]) { + const df = yield* query.pipe(Effect.provide(source)) + assert.deepStrictEqual(DataFrame.toRows(df), [{ d: BigDecimal.make(7n, 0) }]) + } + })) + + // DF-Q049 (API contract): Query.table, Expr.castDecimal; pinned source in README.md. + it.effect("decides decimal scales far apart from the 64 bits of storage", () => + Effect.gen(function*() { + const source = Catalog.layer({ t: price("7", 2, 0), zero: price("0", 2, 0) }) + const declared = (name: string, scale: number) => + Query.table( + name, + ArrowSchema.make([ArrowSchema.field("d", ArrowSchema.decimal(38, scale), { nullable: false })]) + ) + const cast = (name: string, scale: number) => + declared(name, 0).pipe(Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, scale) })), rows) + for (const scale of [2147483647, -2147483648, 10_000_000, 20, -20]) { + // The exact conversion of a declaration fails, the explicit cast gives null or rounds. + const inexact = `column "d" has values that Decimal[38, ${scale}] can't hold exactly` + const error = yield* Effect.flip(declared("t", scale).pipe(Effect.provide(source))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, inexact) + const streamed = Query.stream(declared("t", scale)).pipe(Stream.runCollect, Effect.provide(source)) + assert.strictEqual((yield* Effect.flip(streamed)).message, inexact) + assert.deepStrictEqual(yield* cast("t", scale).pipe(Effect.provide(source)), [ + { d: scale > 0 ? null : BigDecimal.make(0n, scale) } + ]) + // Zero is zero at every scale. + const zero = [{ d: BigDecimal.make(0n, scale) }] + assert.deepStrictEqual(yield* rows(declared("zero", scale)).pipe(Effect.provide(source)), zero) + assert.deepStrictEqual(yield* cast("zero", scale).pipe(Effect.provide(source)), zero) + } + // Up to 19 digits apart, values are rescaled as before: exactly, or half to even. + assert.deepStrictEqual(yield* rows(declared("t", 18)).pipe(Effect.provide(source)), [ + { d: BigDecimal.make(7_000_000_000_000_000_000n, 18) } + ]) + const halves = Query.from(DataFrame.fromColumns({ s: ["0.5", "1.5", "-2.5", "0.5000000000000000001"] })).pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.s, 38, 19) })), + Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, 0) })) + ) + assert.deepStrictEqual(yield* rows(halves), [ + { d: BigDecimal.make(0n, 0) }, + { d: null }, + { d: null }, + { d: BigDecimal.make(1n, 0) } + ]) + // Values held at the far scales: 7 and 0 units of 10^-2147483647 and of 10^2147483648. + const held = (scale: number) => + Query.from(arrowFrame<{ d: BigDecimal.BigDecimal }>({ + d: [ArrowSchema.decimal(38, scale), [BigDecimal.make(7n, scale), BigDecimal.make(0n, scale)]] + })) + const read = (scale: number) => + held(scale).pipe( + Query.select((c) => ({ + round: Expr.round(c.d, 0), + unit: Expr.castDecimal(c.d, 38, 0), + int: Expr.cast(c.d, "Int64"), + float: Expr.cast(c.d, "Float64"), + mean: Expr.rollingMean(c.d, 1) + })), + rows + ) + const whole = (n: bigint) => Expr.lit(BigDecimal.make(n, 0)) + const none = { unit: BigDecimal.make(0n, 0), int: 0n, float: 0, mean: 0 } + assert.deepStrictEqual(yield* read(2147483647), [ + { round: BigDecimal.make(0n, 2147483647), ...none }, + { round: BigDecimal.make(0n, 2147483647), ...none } + ]) + assert.deepStrictEqual(yield* read(-2147483648), [ + { round: BigDecimal.make(7n, -2147483648), unit: null, int: null, float: Infinity, mean: Infinity }, + { round: BigDecimal.make(0n, -2147483648), ...none } + ]) + for (const scale of [2147483647, -2147483648]) { + const sum = yield* Effect.flip(held(scale).pipe(Query.select((c) => ({ d: Expr.add(c.d, whole(1n)) })))) + assert.strictEqual(sum.message, "decimal value overflows 64 bits") + const text = yield* Effect.flip(held(scale).pipe(Query.select((c) => ({ d: Expr.cast(c.d, "Utf8") })))) + assert.strictEqual(text.message, `Decimal[38, ${scale}] values are too long to write as text`) + assert.deepStrictEqual( + yield* held(scale).pipe(Query.aggregate((c) => ({ mean: Expr.mean(Expr.multiply(c.d, whole(0n))) })), rows), + [{ mean: 0 }] + ) + } + })) + + // DF-Q050 (API contract): Expr.cast, Expr.castDecimal; pinned source in README.md. + it.effect("casts to and from decimals with a negative scale", () => + Effect.gen(function*() { + const tens = (unscaled: bigint) => BigDecimal.make(unscaled, -1) + const values = Query.from(DataFrame.fromColumns({ + i: Int32Array.of(0, 7, -15, 25), + b: BigInt64Array.of(0n, 7n, -15n, 25n), + f: [0, 7.5, -15, 26], + s: ["0", "7", "-15", "25.01"] + })) + // Half to even at the tens: 7 is 10, -15 is -20, 25 is 20. + const to = values.pipe(Query.select((c) => ({ + i: Expr.castDecimal(c.i, 38, -1), + b: Expr.castDecimal(c.b, 38, -1), + f: Expr.castDecimal(c.f, 38, -1), + s: Expr.castDecimal(c.s, 38, -1) + }))) + assert.deepStrictEqual(yield* rows(to), [ + { i: tens(0n), b: tens(0n), f: tens(0n), s: tens(0n) }, + { i: tens(1n), b: tens(1n), f: tens(1n), s: tens(1n) }, + { i: tens(-2n), b: tens(-2n), f: tens(-2n), s: tens(-2n) }, + { i: tens(2n), b: tens(2n), f: tens(3n), s: tens(3n) } + ]) + const from = (scale: number) => + values.pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.b, 38, 0) })), + Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, scale) })), + Query.select((c) => ({ + int: Expr.cast(c.d, "Int64"), + small: Expr.cast(c.d, "Int8"), + float: Expr.cast(c.d, "Float64"), + text: Expr.cast(c.d, "Utf8"), + above: Expr.gt(c.d, Expr.lit(BigDecimal.make(5n, 0))) + })), + rows + ) + assert.deepStrictEqual(yield* from(-1), [ + { int: 0n, small: 0, float: 0, text: "0", above: false }, + { int: 10n, small: 10, float: 10, text: "10", above: true }, + { int: -20n, small: -20, float: -20, text: "-20", above: false }, + { int: 20n, small: 20, float: 20, text: "20", above: true } + ]) + })) + + // DF-Q051 (API contract): Expr.castDecimal, Expr.multiply; pinned source in README.md. + it.effect("rejects decimal types a cast or a product can't have", () => + Effect.gen(function*() { + const sevens = Query.from(DataFrame.fromColumns({ b: BigInt64Array.of(7n) })) + const cast = (precision: number, scale: number) => + sevens.pipe(Query.select((c) => ({ d: Expr.castDecimal(c.b, precision, scale) }))) + const precision = (p: number) => `cast: invalid decimal128 precision ${p}, expected an integer in 1..38` + const scale = (s: number) => `cast: invalid decimal scale ${s}, expected a 32-bit integer` + const cases: ReadonlyArray = [ + [2.5, 0, precision(2.5)], + [0, 0, precision(0)], + [39, 0, precision(39)], + [1e9, 0, precision(1e9)], + [2, 0.5, scale(0.5)], + [2, 2 ** 31, scale(2 ** 31)] + ] + for (const [p, s, message] of cases) { + const error = yield* Effect.flip(cast(p, s)) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, message) + } + assert.deepStrictEqual(yield* rows(cast(38, 2)), [{ d: BigDecimal.make(700n, 2) }]) + // A product adds the scales of its factors. + const square = (s: number) => + cast(38, 0).pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, s) })), + Query.select((c) => ({ d: Expr.multiply(c.d, c.d) })) + ) + assert.strictEqual( + (yield* Effect.flip(square(-2147483648))).message, + "the scale -4294967296 of a decimal product is not a 32-bit integer" + ) + assert.deepStrictEqual(yield* rows(square(-1073741824)), [{ d: BigDecimal.make(0n, -2147483648) }]) + assert.deepStrictEqual(yield* rows(square(1)), [{ d: BigDecimal.make(4900n, 2) }]) + })) + + // DF-Q052 (API contract): Expr.Expr; pinned source in README.md. + it.effect("integers keep their value next to a decimal with a negative scale", () => + Effect.gen(function*() { + const tens = (unscaled: bigint) => BigDecimal.make(unscaled, -1) + const whole = (n: bigint) => BigDecimal.make(n, 0) + // 70 in every row, next to 69, 70 and 71. The columns are untyped, as in a frame + // read from Arrow: typed columns don't compare a decimal with an integer. + const mixed = Query.from(arrowFrame({ + d: [ArrowSchema.decimal(18, -1), [tens(7n), tens(7n), tens(7n)]], + i: [ArrowSchema.int32, [69, 70, 71]], + b: [ArrowSchema.int64, [69n, 70n, 71n]], + f: [ArrowSchema.float64, [69, 70, 71]] + })) + const compared = mixed.pipe(Query.select((c) => ({ + i: Expr.eq(c.d, c.i), + b: Expr.gt(c.d, c.b), + literal: Expr.eq(c.d, 69), + f: Expr.eq(c.d, c.f) + }))) + assert.deepStrictEqual(yield* rows(compared), [ + { i: false, b: true, literal: false, f: false }, + { i: true, b: false, literal: false, f: true }, + { i: false, b: false, literal: false, f: false } + ]) + const chosen = mixed.pipe(Query.select((c) => ({ + when: Expr.when(Expr.lit(false), c.d, c.i), + first: Expr.coalesce(c.i, c.d) + }))) + assert.deepStrictEqual(yield* rows(chosen), [ + { when: whole(69n), first: whole(69n) }, + { when: whole(70n), first: whole(70n) }, + { when: whole(71n), first: whole(71n) } + ]) + const stacked = Query.concat([ + mixed.pipe(Query.select((c) => ({ x: c.d }))), + mixed.pipe(Query.select((c) => ({ x: c.i }))) + ]) + assert.deepStrictEqual( + yield* rows(stacked), + [70n, 70n, 70n, 69n, 70n, 71n].map((n) => ({ x: whole(n) })) + ) + // Only the explicit cast rounds. + assert.deepStrictEqual( + yield* rows(mixed.pipe(Query.select((c) => ({ d: Expr.castDecimal(c.i, 38, -1) })))), + [{ d: tens(7n) }, { d: tens(7n) }, { d: tens(7n) }] + ) + // A decimal that scale 0 can't hold fails, as with any common type. + const far = Query.from(arrowFrame({ + d: [ArrowSchema.decimal(38, -400), [BigDecimal.make(7n, -400)]], + i: [ArrowSchema.int32, [1]] + })) + const error = yield* Effect.flip(far.pipe(Query.select((c) => ({ x: Expr.gt(c.d, c.i) })))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, "Decimal[38, -400] values don't fit their common type Decimal[38, 0]") + })) + + // DF-Q053 (API contract): Expr.lit; pinned source in README.md. + it.effect("decimal literals have a type their value fits", () => + Effect.gen(function*() { + const one = Query.from(DataFrame.fromColumns({ x: [1] })) + const select = (d: Expr.Expr) => one.pipe(Query.select(() => ({ d }))) + const typed = (value: bigint, precision: number, scale: number) => + select(Expr.fromPlan({ _tag: "Literal", value, type: { _tag: "Decimal", precision, scale } })) + /** The frame's only type and value, after a round trip through Arrow IPC. */ + const read = (df: DataFrame.DataFrame) => { + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df))) + const table = Result.getOrThrow(ArrowIpc.decodeStream(bytes)) + return [table.schema.fields[0].type, DataFrame.toRows(DataFrame.fromArrowTable(table))] as const + } + // The precision is the larger of the digits and the scale, capped at the 38 digits of a 128-bit decimal. + const inferred: ReadonlyArray = [ + [BigDecimal.make(5n, 2), ArrowSchema.decimal(2, 2)], + [BigDecimal.make(12345n, 2), ArrowSchema.decimal(5, 2)], + [BigDecimal.make(7n, -5), ArrowSchema.decimal(1, -5)], + [BigDecimal.make(1n, 400), ArrowSchema.decimal(38, 400)], + [BigDecimal.make(0n, 400), ArrowSchema.decimal(38, 400)], + [BigDecimal.make(-1n, 2147483647), ArrowSchema.decimal(38, 2147483647)] + ] + for (const [value, type] of inferred) { + assert.deepStrictEqual(read(yield* select(Expr.lit(value))), [type, [{ d: value }]]) + } + assert.deepStrictEqual(read(yield* typed(1n, 1, 400)), [ + ArrowSchema.decimal(1, 400), + [{ d: BigDecimal.make(1n, 400) }] + ]) + assert.deepStrictEqual(read(yield* typed(-9n, 1, 0)), [ArrowSchema.decimal(1, 0), [{ + d: BigDecimal.make(-9n, 0) + }]]) + const precision = (p: number) => + `decimal literal: invalid decimal128 precision ${p}, expected an integer in 1..38` + const failures: ReadonlyArray, string]> = [ + [typed(1n, 39, 0), precision(39)], + [typed(1n, 0, 0), precision(0)], + [typed(1n, 2, 2 ** 31), "decimal literal: invalid decimal scale 2147483648, expected a 32-bit integer"], + [ + select(Expr.lit(BigDecimal.make(1n, 2 ** 31))), + "decimal literal: invalid decimal scale 2147483648, expected a 32-bit integer" + ], + [typed(10n, 1, 0), "decimal literal 10 has more digits than its precision 1"], + [typed(-10n, 1, 0), "decimal literal -10 has more digits than its precision 1"], + [typed(2n ** 63n, 38, 0), "decimal value overflows 64 bits"] + ] + for (const [query, message] of failures) { + const error = yield* Effect.flip(query) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, message) + } + })) + + // DF-Q054 (API contract): Expr.cast, Expr.castDecimal, Expr.mean; pinned source in README.md. + it.effect("converts between floats and decimals at scales past a float's largest power of ten", () => + Effect.gen(function*() { + const tiny = Query.from(DataFrame.fromColumns({ f: [1e-310, -1e-310, 0, 1] })).pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.f, 38, 310) })) + ) + assert.deepStrictEqual(yield* rows(tiny), [ + { d: BigDecimal.make(1n, 310) }, + { d: BigDecimal.make(-1n, 310) }, + { d: BigDecimal.make(0n, 310) }, + { d: null } + ]) + // 10^18 units of 10^-310 are 10^-292. + const held = (scale: number, unscaled: ReadonlyArray) => + Query.from(arrowFrame<{ d: BigDecimal.BigDecimal }>({ + d: [ArrowSchema.decimal(38, scale), unscaled.map((x) => BigDecimal.make(x, scale))] + })) + const read = (query: Query.Query<{ d: BigDecimal.BigDecimal }>) => + Effect.all([ + query.pipe( + Query.select((c) => ({ float: Expr.cast(c.d, "Float64"), rolling: Expr.rollingMean(c.d, 2) })), + rows + ), + query.pipe(Query.aggregate((c) => ({ mean: Expr.mean(c.d) })), rows) + ]) + const big = 10n ** 18n + assert.deepStrictEqual(yield* read(held(310, [big, big, -big])), [ + [ + { float: 1e-292, rolling: null }, + { float: 1e-292, rolling: 1e-292 }, + { float: -1e-292, rolling: 0 } + ], + [{ mean: 1e-292 / 3 }] + ]) + // Below the smallest float the value is zero. + assert.deepStrictEqual(yield* read(held(400, [big, big])), [ + [{ float: 0, rolling: null }, { float: 0, rolling: 0 }], + [{ mean: 0 }] + ]) + })) + + // DF-Q055 (API contract): Expr.isIn; pinned source in README.md. + it.effect("isIn compares decimals, points in time and durations by value", () => + Effect.gen(function*() { + const dec = (unscaled: bigint, scale: number) => BigDecimal.make(unscaled, scale) + // The columns are untyped, as in a frame read from Arrow: typed columns take candidates of their own type. + const frame = arrowFrame({ + // 7.00, 7.50 and -7.00 + d: [ArrowSchema.decimal(18, 2), [dec(700n, 2), dec(750n, 2), dec(-700n, 2), null]], + // 70, 0 and 700 + n: [ArrowSchema.decimal(18, -1), [dec(7n, -1), dec(0n, -1), dec(70n, -1), null]], + b: [ArrowSchema.int64, [7n, 700n, 70n, null]], + at: [ArrowSchema.timestamp("millisecond", "UTC"), [0n, 1000n, 5n, null]], + took: [ArrowSchema.duration("millisecond"), [0n, 1000n, 5n, null]] + }) + const cases: ReadonlyArray, ReadonlyArray]> = [ + ["d", [dec(700n, 2)], [true, false, false, null]], + // Equal at another scale. + ["d", [dec(7n, 0), dec(7500n, 3)], [true, true, false, null]], + ["d", [dec(7001n, 3)], [false, false, false, null]], + ["d", [7], [true, false, false, null]], + ["d", [7n], [true, false, false, null]], + ["d", [7.5], [false, true, false, null]], + // Not the unscaled units 7.00 is stored as. + ["d", [700, 700n], [false, false, false, null]], + ["d", [7, dec(-700n, 2)], [true, false, true, null]], + ["d", [-7, null], [false, false, true, null]], + ["d", [], [false, false, false, null]], + ["d", ["7.00", true, NaN, Infinity, 2n ** 70n], [false, false, false, null]], + ["n", [70], [true, false, false, null]], + ["n", [7, 75], [false, false, false, null]], + ["n", [0, dec(700n, 0)], [false, true, true, null]], + // Controls: integers hold no fraction, and take numbers and bigints. + ["b", [7.5, NaN], [false, false, false, null]], + ["b", [7, 70n], [true, false, true, null]], + ["at", [DateTime.makeUnsafe(1000)], [false, true, false, null]], + ["took", [Duration.seconds(1), Duration.millis(5)], [false, true, true, null]] + ] + for (const [column, candidates, expected] of cases) { + const query = Query.from(frame).pipe(Query.select((c) => ({ x: Expr.isIn(c[column], candidates) }))) + assert.deepStrictEqual(yield* rows(query), expected.map((x) => ({ x })), `${column} in ${candidates}`) + // The plan holds no value its JSON form can't. + const plan = Schema.decodeSync(Plan.NodeFromJson)(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + assert.deepStrictEqual(yield* rows(Query.fromPlan(plan)), expected.map((x) => ({ x }))) + } + })) + + // DF-Q056 (API contract): Expr.isIn; pinned source in README.md. + it.effect("isIn takes long lists of typed candidates", () => + Effect.gen(function*() { + const count = 5000 + const last = BigInt(count - 1) + const frame = arrowFrame<{ + d: BigDecimal.BigDecimal | null + at: DateTime.Utc | null + took: Duration.Duration | null + }>({ + d: [ArrowSchema.decimal(18, 0), [ + BigDecimal.make(0n, 0), + BigDecimal.make(last, 0), + BigDecimal.make(last + 1n, 0), + null + ]], + at: [ArrowSchema.timestamp("millisecond", "UTC"), [0n, last, last + 1n, null]], + took: [ArrowSchema.duration("millisecond"), [0n, last, last + 1n, null]] + }) + const candidates = (make: (i: number) => A) => Array.from({ length: count }, (_, i) => make(i)) + const query = Query.from(frame).pipe(Query.select((c) => ({ + d: Expr.isIn(c.d, candidates((i) => BigDecimal.make(BigInt(i), 0))), + at: Expr.isIn(c.at, candidates((i) => DateTime.makeUnsafe(i))), + took: Expr.isIn(c.took, candidates((i) => Duration.millis(i))) + }))) + const expected = [true, true, false, null].map((x) => ({ d: x, at: x, took: x })) + assert.deepStrictEqual(yield* rows(query), expected) + // The plan is as deep as the logarithm of the list, in its JSON form too. + const plan = Schema.decodeSync(Plan.NodeFromJson)(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + assert.deepStrictEqual(yield* rows(Query.fromPlan(plan)), expected) + })) + + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. + it.effect("fails with DataFrameError on invalid expressions", () => + Effect.gen(function*() { + const error = yield* Effect.flip(Query.from(readings).pipe(Query.filter((c) => c.city as any))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") + })) + + // DF-Q009 (API contract): Expr.Expr; pinned source in README.md. + it.effect("decimal results fail instead of overflowing 64 bits", () => + Effect.gen(function*() { + const nines = (rows: number) => + DataFrame.fromColumns({ s: Array.from({ length: rows }, () => "999999999999999999") }) + const d = (c: Query.Columns<{ readonly s: string }>) => Expr.castDecimal(c.s, 19, 0) + const ten = Query.from(nines(10)) + const min = Query.from(DataFrame.fromColumns({ s: ["-9223372036854775808"] })) + const streamed = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( + Query.aggregate((c) => ({ total: Expr.sum(Expr.castDecimal(c.s as Expr.Expr, 18, 0)) })), + Effect.provide(Catalog.layer({ t: Stream.make(nines(5), nines(5)) })) + ) + const queries: ReadonlyArray> = [ + Query.aggregate(ten, (c) => ({ total: Expr.sum(d(c)) })), + Query.select(ten, (c) => ({ running: Expr.cumSum(d(c)) })), + Query.select(ten, (c) => ({ rolling: Expr.rollingSum(d(c), 10) })), + Query.select(min, (c) => ({ abs: Expr.abs(d(c)) })), + Query.select(min, (c) => ({ negated: Expr.negate(d(c)) })), + Query.select(min, () => ({ literal: Expr.lit(BigDecimal.fromBigInt(10n ** 20n)) })), + streamed + ] + for (const query of queries) { + assert.strictEqual((yield* Effect.flip(query)).message, "decimal value overflows 64 bits") + } + })) + + // DF-Q010 (API contract): Expr.round; pinned source in README.md. + it.effect("round rejects negative or fractional places", () => + Effect.gen(function*() { + for (const places of [-1, 0.5]) { + const error = yield* Effect.flip( + Query.from(readings).pipe(Query.select((c) => ({ t: Expr.round(c.temp, places) }))) + ) + assert.strictEqual(error.message, `round needs a non-negative whole number of decimals, got ${places}`) + } + })) + }) + + describe("operations", () => { + // DF-Q011 (API contract): Query.slice; pinned source in README.md. + it.effect("slice takes an offset and a length; negative offsets count from the end", () => + Effect.gen(function*() { + const n = Query.from(DataFrame.fromColumns({ n: Int32Array.of(0, 1, 2, 3, 4) })) + const values = (q: Query.Query<{ readonly n: number }>) => Effect.map(q, (df) => DataFrame.toArray(df, "n")) + assert.deepStrictEqual(yield* values(Query.slice(n, 1, 2)), [1, 2]) + assert.deepStrictEqual(yield* values(Query.slice(n, -2)), [3, 4]) + assert.deepStrictEqual(yield* values(Query.slice(n, -2, 1)), [3]) + assert.deepStrictEqual(yield* values(Query.head(n, 2)), [0, 1]) + assert.deepStrictEqual(yield* values(Query.tail(n, 2)), [3, 4]) + assert.deepStrictEqual(yield* values(Query.tail(n, 0)), []) + assert.deepStrictEqual(yield* values(Query.slice(n, 9)), []) + })) + + // DF-Q012 (API contract): Query.append; pinned source in README.md. + it.effect("append stacks rows; dropNulls removes incomplete rows", () => + Effect.gen(function*() { + const df = yield* Query.from(readings).pipe(Query.append(Query.from(readings)), Query.dropNulls(["rain"])) + assert.deepStrictEqual(DataFrame.toArray(df, "rain"), [0, 2, 0, 2]) + })) + + // DF-Q013 (API contract): Query.join; pinned source in README.md. + it.effect("joins on several keys", () => + Effect.gen(function*() { + const right = DataFrame.fromColumns({ city: ["Vienna", "Vienna"], temp: [19, 21.5], note: ["cool", "warm"] }) + const result = yield* rows( + Query.from(readings).pipe(Query.pick("city", "temp"), Query.join(Query.from(right), { on: ["city", "temp"] })) + ) + assert.deepStrictEqual(result, [ + { city: "Vienna", temp: 21.5, note: "warm" }, + { city: "Vienna", temp: 19, note: "cool" } + ]) + })) + + // DF-Q014 (API contract): Expr.cast; pinned source in README.md. + it.effect("cast reports values that don't fit as null", () => + Effect.gen(function*() { + const df = DataFrame.fromColumns({ x: [1.9, -1.9, 3e10, NaN] }) + const result = yield* rows( + Query.from(df).pipe(Query.select((c) => ({ i: Expr.cast(c.x, "Int32"), s: Expr.cast(c.x, "Utf8") }))) + ) + assert.deepStrictEqual(result, [ + { i: 1, s: "1.9" }, + { i: -1, s: "-1.9" }, + { i: null, s: "30000000000" }, + { i: null, s: "NaN" } + ]) + const decimals = DataFrame.fromColumns({ f: [1e308], u: BigUint64Array.of(2n ** 64n - 1n), d: ["-1.5"] }) + const casts = yield* rows( + Query.from(decimals).pipe(Query.select((c) => ({ + fromFloat: Expr.castDecimal(c.f, 18, 18), + fromUInt64: Expr.castDecimal(c.u, 20, 0), + toUInt64: Expr.cast(Expr.castDecimal(c.d, 2, 1), "UInt64") + }))) + ) + assert.deepStrictEqual(casts, [{ fromFloat: null, fromUInt64: null, toUInt64: null }]) + // Expected values from Polars 2.1.0 non-strict casts. + const temporal = arrowFrame<{ t: DateTime.Utc; d: Duration.Duration; i: bigint }>({ + t: [ArrowSchema.timestamp("millisecond"), [10n ** 17n]], + d: [ArrowSchema.duration("millisecond"), [10n ** 17n]], + i: [ArrowSchema.int64, [2n ** 40n]] + }) + const overflows = yield* rows( + Query.from(temporal).pipe(Query.select((c) => ({ + t: Expr.cast(c.t, "Timestamp[ns]"), + d: Expr.cast(c.d, "Duration[ns]"), + i: Expr.cast(c.i, "Date") + }))) + ) + assert.deepStrictEqual(overflows, [{ t: null, d: null, i: null }]) + })) + + // DF-Q015 (API contract): Expr.Expr; pinned source in README.md. + it.effect("implicit conversions to a common type fail on values it can't hold", () => + Effect.gen(function*() { + const big = Query.from(price("6000000000000000000", 19, 0)) + const small = Query.from(price("1.20", 18, 2)) + const failures: ReadonlyArray> = [ + Query.concat([big, small]), + Query.join(big, small, { on: "d", how: "left" }), + Query.filter(big, (c) => Expr.eq(c.d, BigDecimal.fromStringUnsafe("1.20"))) + ] + for (const query of failures) { + const error = yield* Effect.flip(query) + assert.strictEqual(error.message, "Decimal[19, 0] values don't fit their common type Decimal[38, 2]") + } + // Integer literals a column's type can't hold widen it, wherever they + // appear among the operands. Expected values from Polars 2.1.0 unless noted. + const { bool, float64, int8, int16, int32 } = ArrowSchema + const w = (x: Expr.Expr) => Expr.when(Expr.isNull(x), 1, 300) + const literals: ReadonlyArray< + readonly [ + string, + (c: Query.Columns<{ readonly x: number | null; readonly u: number | null }>) => Expr.Expr, + ArrowSchema.DataType, + ReadonlyArray + ] + > = [ + ["x + 300", (c) => Expr.add(c.x, 300), int16, [null, 305]], + ["300 + x", (c) => Expr.add(Expr.lit(300), c.x), int16, [null, 305]], + ["x > 1000", (c) => Expr.gt(c.x, 1000), bool, [null, false]], + ["when(x > 0, x, 300)", (c) => Expr.when(Expr.gt(c.x, 0), c.x, 300), int16, [300, 5]], + ["when(x > 0, 300, x)", (c) => Expr.when(Expr.gt(c.x, 0), 300, c.x), int16, [null, 300]], + ["coalesce(x, 300)", (c) => Expr.coalesce(c.x, 300), int16, [300, 5]], + ["coalesce(300, x)", (c) => Expr.coalesce(300, c.x), int16, [300, 300]], + ["fillNull(x, 300)", (c) => Expr.fillNull(c.x, 300), int16, [300, 5]], + ["coalesce(null, 300, x)", (c) => Expr.coalesce(null, 300, c.x), int16, [300, 300]], + ["coalesce(x, null, 300)", (c) => Expr.coalesce(c.x, null, 300), int16, [300, 5]], + ["coalesce(x, 300, 70000)", (c) => Expr.coalesce(c.x, 300, 70000), int32, [300, 5]], + // Polars gives Int16, which can't hold 70000. + ["coalesce(300, 70000, x)", (c) => Expr.coalesce(300, 70000, c.x), int32, [300, 300]], + ["coalesce(1, x)", (c) => Expr.coalesce(1, c.x), int8, [1, 1]], + ["coalesce(300, 1.5, x)", (c) => Expr.coalesce(300, 1.5, c.x), float64, [300, 300]], + ["coalesce(u, -1)", (c) => Expr.coalesce(c.u, -1), int16, [-1, 5]], + ["coalesce(-1, u)", (c) => Expr.coalesce(-1, c.u), int16, [-1, -1]], + ["x + cast(5, Int32)", (c) => Expr.add(c.x, Expr.cast(Expr.lit(5), "Int32")), int32, [null, 10]], + // Literal-only operands are fitted as a whole (w is when(x is null, 1, 300)); + // Polars gives Int8, making 5 + 300 null and (100 + 1) * 2 wrap to -54. + ["x + w", (c) => Expr.add(c.x, w(c.x)), int16, [null, 305]], + [ + "(coalesce(x, 100) + w) * 2", + (c) => Expr.multiply(Expr.add(Expr.coalesce(c.x, 100), w(c.x)), 2), + int16, + [202, 610] + ], + // Other operands computed from literals count as Int32; Polars gives + // Int8 with nulls. + ["x + -w", (c) => Expr.add(c.x, Expr.negate(w(c.x))), int32, [null, -295]] + ] + // x is Int8 and u UInt8, both [null, 5]: whole, streamed a row at a + // time and sliced to the first row, results have the same types. + const ints = (xs: ReadonlyArray) => + DataFrame.fromColumns({ x: Int8Array.from(xs), u: Uint8Array.from(xs) }) + const nullable = (input: Query.Query<{ readonly x: number; readonly u: number }>) => + Query.withColumns(input, (c) => ({ + x: Expr.when(Expr.gt(c.x, 0), c.x, null), + u: Expr.when(Expr.gt(c.u, 0), c.u, null) + })) + const whole = nullable(Query.from(ints([0, 5]))) + const streamed = nullable( + Query.table("t", Schema.Struct({ x: Schema.Number, u: Schema.Number }), { + types: { x: ArrowSchema.int8, u: ArrowSchema.uint8 } + }) + ) + const batches = Catalog.layer({ t: Stream.make(ints([0]), ints([5])) }) + const result = (query: Query.Query<{ readonly r: unknown }>) => + Effect.map(query, (df) => [df.schema.fields[0].type, DataFrame.toArray(df, "r")]) + for (const [name, expr, type, values] of literals) { + const select = (input: typeof whole) => Query.select(input, (c) => ({ r: expr(c) })) + assert.deepStrictEqual(yield* result(select(whole)), [type, values], name) + assert.deepStrictEqual(yield* result(select(streamed)).pipe(Effect.provide(batches)), [type, values], name) + assert.deepStrictEqual(yield* result(Query.head(select(whole), 1)), [type, values.slice(0, 1)], name) + } + })) + + // DF-Q016 (API contract): Expr.Expr; pinned source in README.md. + it.effect("temporal operands of mixed units meet in the coarser unit, as in Polars", () => + Effect.gen(function*() { + // Expected values from Polars 2.1.0: timestamps round down, durations toward zero. + const df = arrowFrame<{ t: DateTime.Utc; s: DateTime.Utc; d: Duration.Duration; e: Duration.Duration }>({ + t: [ArrowSchema.timestamp("millisecond"), [1n, -1n, 2n]], + s: [ArrowSchema.timestamp("microsecond"), [1500n, 1500n, -1n]], + d: [ArrowSchema.duration("millisecond"), [1n, -1n, 2n]], + e: [ArrowSchema.duration("microsecond"), [1500n, 1500n, -1n]] + }) + const int = (x: Expr.Expr) => Expr.cast(x, "Int64") + const result = yield* rows( + Query.from(df).pipe(Query.select((c) => ({ + tMinusS: int(Expr.subtract(c.t, c.s)), + tPlusE: int(Expr.add(c.t, c.e)), + tMinusE: int(Expr.subtract(c.t, c.e)), + dPlusE: int(Expr.add(c.d, c.e)), + dMinusE: int(Expr.subtract(c.d, c.e)), + eq: Expr.eq(c.t, c.s), + lt: Expr.lt(c.t, c.s), + durationsEq: Expr.eq(c.d, c.e), + s: int(Expr.cast(c.s, "Timestamp[ms]")), + e: int(Expr.cast(c.e, "Duration[ms]")) + }))) + ) + assert.deepStrictEqual(result, [ + { + tMinusS: 0n, + tPlusE: 2n, + tMinusE: 0n, + dPlusE: 2n, + dMinusE: 0n, + eq: true, + lt: false, + durationsEq: true, + s: 1n, + e: 1n + }, + { + tMinusS: -2n, + tPlusE: 0n, + tMinusE: -2n, + dPlusE: 0n, + dMinusE: -2n, + eq: false, + lt: true, + durationsEq: false, + s: 1n, + e: 1n + }, + { + tMinusS: 3n, + tPlusE: 2n, + tMinusE: 2n, + dPlusE: 2n, + dMinusE: 2n, + eq: false, + lt: false, + durationsEq: false, + s: -1n, + e: 0n + } + ]) + })) + + // DF-Q046 (API contract): Expr.sum; pinned source in README.md. + it.effect("sums durations to durations in the column's unit", () => + Effect.gen(function*() { + type Row = { readonly g: string; readonly q: string; readonly d: Duration.Duration | null } + const unit = ArrowSchema.duration("nanosecond") + const frame = (g: ReadonlyArray, q: ReadonlyArray, d: ReadonlyArray) => + arrowFrame({ g: [ArrowSchema.utf8, g], q: [ArrowSchema.utf8, q], d: [unit, d] }) + const whole = Query.from(frame(["a", "a", "a", "b", "c"], ["x", "y", "x", "x", "y"], [7n, 9n, null, 4n, null])) + const streamed = Query.table("t", whole.pipe(Effect.runSync).schema) as unknown as Query.Query + const batches = Catalog.layer({ + t: Stream.make( + frame(["a", "a"], ["x", "y"], [7n, 9n]), + frame(["a", "b", "c"], ["x", "x", "y"], [null, 4n, null]) + ) + }) + // Every column but the keys: its type and its values in nanoseconds. + const nanos = (query: Query.Query) => + Effect.map(query, (df) => + Object.fromEntries( + df.schema.fields.filter((f) => f.name !== "g").map((f) => [f.name, [ + f.type, + (DataFrame.toArray(df, f.name) as Array).map((d) => + d === null ? null : Duration.toNanosUnsafe(d) + ) + ]]) + )) + // A group without values sums to the zero duration. + const grouped = (input: Query.Query) => + input.pipe(Query.groupBy("g"), Query.agg((c) => ({ total: Expr.sum(c.d) }))) + assert.deepStrictEqual(yield* nanos(grouped(whole)), { total: [unit, [16n, 4n, 0n]] }) + assert.deepStrictEqual(yield* nanos(grouped(streamed)).pipe(Effect.provide(batches)), { + total: [unit, [16n, 4n, 0n]] + }) + assert.deepStrictEqual( + yield* nanos(whole.pipe(Query.select((c) => ({ + over: Expr.over(Expr.sum(c.d), [c.g]), + running: Expr.cumSum(c.d), + last2: Expr.rollingSum(c.d, 2), + least2: Expr.rollingMin(c.d, 2), + step: Expr.diff(c.d) + })))), + { + over: [unit, [16n, 16n, 16n, 4n, 0n]], + running: [unit, [7n, 16n, null, 20n, null]], + last2: [unit, [null, 16n, null, null, null]], + least2: [unit, [null, 7n, null, null, null]], + step: [unit, [null, 2n, null, null, null]] + } + ) + assert.deepStrictEqual( + yield* nanos(whole.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ all: Expr.list(c.d) })), + Query.select((c) => ({ g: c.g, total: Expr.listSum(c.all) })) + )), + { total: [unit, [16n, 4n, 0n]] } + ) + assert.deepStrictEqual( + yield* nanos( + whole.pipe(Query.pivot({ on: "q", index: ["g"], values: "d", aggregate: "sum", columns: ["x", "y"] })) + ), + { x: [unit, [7n, 4n, 0n]], y: [unit, [9n, 0n, 0n]] } + ) + })) + + // DF-Q047 (API contract): Expr.mean; pinned source in README.md. + it.effect("rejects aggregates that durations, timestamps, dates and times don't have", () => + Effect.gen(function*() { + const frame = (type: ArrowSchema.DataType, values: ReadonlyArray) => + arrowFrame<{ readonly g: string; readonly v: number | null }>({ + g: [ArrowSchema.utf8, values.map(() => "a")], + v: [type, values] + }) + // The types reject these columns too: `v` is read as a number here. + const message = (query: Query.Query) => Effect.map(Effect.flip(query), (e) => e.message) + const aggregated = + (input: Query.Query<{ readonly g: string; readonly v: number | null }>) => + (f: (v: Expr.Expr) => Expr.Aggregate) => + message(input.pipe(Query.groupBy("g"), Query.agg((c) => ({ r: f(c.v) })))) + const selected = + (input: Query.Query<{ readonly g: string; readonly v: number | null }>) => + (f: (v: Expr.Expr) => Expr.Expr) => + message(input.pipe(Query.select((c) => ({ r: f(c.v) })))) + + const durations = frame(ArrowSchema.duration("microsecond"), [7n, 9n, null]) + const notNumeric = "expected a numeric column, got Duration[us]" + for (const f of [Expr.mean, Expr.median, Expr.std, Expr.variance]) { + assert.strictEqual(yield* aggregated(Query.from(durations))(f), notNumeric) + } + // A streamed table's means combine from partial aggregates. + const streamed = Query.table("t", durations.schema) as unknown as Query.Query<{ g: string; v: number | null }> + const batches = Catalog.layer({ t: Stream.make(durations, durations) }) + for (const f of [Expr.mean, Expr.std]) { + assert.strictEqual(yield* aggregated(streamed)(f).pipe(Effect.provide(batches)), notNumeric) + } + assert.strictEqual( + yield* selected(Query.from(durations))((v) => Expr.rollingMean(v, 2)), + "rollingMean is not defined for Duration[us]" + ) + assert.strictEqual( + yield* message( + Query.from(durations).pipe( + Query.groupBy("g"), + Query.agg((c) => ({ all: Expr.list(c.v) })), + Query.select((c) => ({ r: Expr.listMean(c.all) })) + ) + ), + notNumeric + ) + + // Points in time have neither a sum nor a mean. + const points: ReadonlyArray, string]> = [ + [ArrowSchema.timestamp("microsecond"), [7n, 9n, null], "Timestamp[us]"], + [ArrowSchema.date("day"), [7, 9, null], "Date"], + [ArrowSchema.time("microsecond"), [7n, 9n, null], "Time"] + ] + for (const [type, values, name] of points) { + const input = Query.from(frame(type, values)) + assert.strictEqual(yield* aggregated(input)(Expr.sum), `sum is not defined for ${name}`) + assert.strictEqual(yield* aggregated(input)(Expr.mean), `expected a numeric column, got ${name}`) + assert.strictEqual(yield* selected(input)(Expr.cumSum), `cumSum is not defined for ${name}`) + assert.strictEqual( + yield* selected(input)((v) => Expr.rollingSum(v, 2)), + `rollingSum is not defined for ${name}` + ) + } + })) + }) + + describe("explain", () => { + // DF-Q017 (local policy): Query.explain; pinned source in README.md. + it.effect("shows the rewritten plan", () => + Effect.gen(function*() { + const query = Query.table("readings", Reading).pipe( + Query.withColumns((c) => ({ fahrenheit: Expr.add(Expr.multiply(c.temp, 1.8), 32), loud: Expr.lit(1) })), + Query.join(Query.table("cities", City), { on: "city" }), + Query.filter((c) => Expr.and(Expr.eq(c.country, "AT"), Expr.gt(c.temp, 20))), + Query.sort((c) => [Expr.desc(c.fahrenheit)]), + Query.select((c) => ({ city: c.city, fahrenheit: c.fahrenheit })), + Query.head(1) + ) + const plan = yield* Query.explain(query).pipe(Effect.provide(tables)) + assert.strictEqual( + plan, + [ + `Select [city = col("city"), fahrenheit = col("fahrenheit")]`, + ` TopK 1 [desc(col("fahrenheit"))]`, + ` Join inner on [city]`, + ` WithColumns [fahrenheit = add(multiply(col("temp"), 1.8), 32)]`, + ` Filter gt(col("temp"), 20)`, + ` Pick [city, temp]`, + ` Values 3 rows [city, temp, rain]`, + ` Filter eq(col("country"), "AT")`, + ` Values 2 rows [city, country]` + ].join("\n") + ) + assert.deepStrictEqual(yield* rows(query).pipe(Effect.provide(tables)), [{ city: "Vienna", fahrenheit: 70.7 }]) + })) + + // DF-Q018 (local policy): Query.explain; pinned source in README.md. + it.effect("pushes key predicates below groups and through renames", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe( + Query.rename({ city: "town" }), + Query.groupBy("town"), + Query.agg((c) => ({ n: Expr.count(), mean: Expr.mean(c.temp) })), + Query.filter((c) => Expr.and(Expr.neq(c.town, "Berlin"), Expr.gt(c.n, 1))) + ) + assert.strictEqual( + yield* Query.explain(query), + [ + `Filter gt(col("n"), 1)`, + ` GroupBy [town] [n = count(), mean = mean(col("temp"))]`, + ` Rename [city -> town]`, + ` Filter neq(col("city"), "Berlin")`, + ` Pick [city, temp]`, + ` Values 3 rows [city, temp, rain]` + ].join("\n") + ) + assert.deepStrictEqual(yield* rows(query), [{ town: "Vienna", n: 2, mean: 20.25 }]) + })) + + // DF-Q019 (local policy): Query.explain; pinned source in README.md. + it.effect("keeps predicates on join keys off the right side, whose keys the join drops", () => + Effect.gen(function*() { + // The result has the left side's Int8 key, where 100 + 100 wraps to -56; + // in the right side's Int64 it is 200. + const query = Query.from(DataFrame.fromColumns({ x: Int8Array.of(100), a: [1] })).pipe( + Query.join(Query.from(DataFrame.fromColumns({ x: BigInt64Array.of(100n), b: [2] })), { on: "x" }), + Query.filter((c) => Expr.lt(Expr.add(c.x, c.x), 0)) + ) + assert.deepStrictEqual(yield* rows(query), [{ x: 100, a: 1, b: 2 }]) + })) + + // DF-Q020 (local policy): Query.explain; pinned source in README.md. + it.effect("keeps predicates above left joins when they read the right side", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe( + Query.join(Query.from(cities), { on: "city", how: "left" }), + Query.filter((c) => Expr.isNull(c.country)) + ) + assert.strictEqual( + yield* Query.explain(query), + [ + `Filter isNull(col("country"))`, + ` Join left on [city]`, + ` Values 3 rows [city, temp, rain]`, + ` Values 2 rows [city, country]` + ].join("\n") + ) + })) + }) + + describe("plans", () => { + // DF-Q021 (API contract): Plan.Node; pinned source in README.md. + it("are plain data", () => { + const query = Query.from(readings).pipe(Query.filter((c) => Expr.gt(c.temp, 20)), Query.pick("city")) + assert.deepStrictEqual(query.plan, { + _tag: "Pick", + input: { + _tag: "Filter", + input: { _tag: "Values", frame: readings }, + predicate: { + _tag: "Call", + fn: "gt", + args: [{ _tag: "Column", name: "temp" }, { _tag: "Literal", value: 20 }], + options: undefined + } + }, + columns: ["city"] + }) + }) + + // DF-Q022 (API contract): Plan.NodeFromJson; pinned source in README.md. + it("encode to JSON without losing literal types", () => { + const values: ReadonlyArray = ["NaN", NaN, -0, Infinity, 2n ** 70n, "12", 12, true, null] + const expression: Plan.Expression = { + _tag: "Call", + fn: "isIn", + args: [{ _tag: "Column", name: "x" }], + options: { values } + } + const json = Schema.encodeSync(Plan.ExpressionFromJson)(expression) + assert.deepStrictEqual(json, { + _tag: "Call", + fn: "isIn", + args: [{ _tag: "Column", name: "x" }], + options: { + values: [ + "NaN", + { float: "NaN" }, + { float: "-0" }, + { float: "Infinity" }, + { + bigint: "1180591620717411303424" + }, + "12", + 12, + true, + null + ] + } + }) + const back = Schema.decodeUnknownSync(Plan.ExpressionFromJson)(JSON.parse(JSON.stringify(json))) + assert.deepStrictEqual(back, expression) + }) + + // DF-Q023 (API contract): Plan.NodeFromJson; pinned source in README.md. + it.effect("round-trip through JSON with their inputs", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe(Query.groupBy("city"), Query.agg((c) => ({ max: Expr.max(c.temp) }))) + const json = JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(json))) + assert.deepStrictEqual(yield* rows(decoded), yield* rows(query)) + })) + + // DF-Q024 (API contract): Plan.NodeFromJson; pinned source in README.md. + it("reject malformed JSON when decoding", () => { + const json = { _tag: "Filter", input: { _tag: "Table", name: "t" }, predicate: { _tag: "Call", fn: "explode" } } + assert.throws(() => Schema.decodeUnknownSync(Plan.NodeFromJson)(json)) + }) + + // DF-Q025 (API contract): Plan.Expression; pinned source in README.md. + it.effect("fail at run time on the wrong number of arguments", () => + Effect.gen(function*() { + const plan: Plan.Node = { + _tag: "Filter", + input: { _tag: "Values", frame: readings }, + predicate: { _tag: "Call", fn: "gt", args: [{ _tag: "Column", name: "temp" }] } + } + const error = yield* Effect.flip(Query.fromPlan(plan)) + assert.strictEqual(error.message, "gt takes 2 argument(s), got 1") + })) + + // DF-Q026 (API contract): Expr.toPlan; pinned source in README.md. + it("expressions convert to and from plans", () => { + const e = Expr.add(Expr.lit(1n), 2n) + assert.deepStrictEqual(Expr.toPlan(e), { + _tag: "Call", + fn: "add", + args: [{ _tag: "Literal", value: 1n }, { _tag: "Literal", value: 2n }], + options: undefined + }) + assert.strictEqual(String(Expr.fromPlan(Expr.toPlan(e))), "add(1n, 2n)") + assert.isTrue(Expr.isAggregate(Expr.fromPlan({ _tag: "AggregateCall", fn: "count" }))) + }) + }) + + // Every case of the Polars gold corpus (test/fixtures/dataframe/gold), with + // its inputs inline, as catalog frames and as streams of small batches. + describe("Polars oracle", () => { + const cases = Gold.cases() + for (const source of ["values", "frames", "batches"] as const) { + describe(source, () => { + for (const c of cases) { + // DF-I-POLARS//: interoperability evidence; pinned oracle provenance in README.md. + it.effect(`${c.id}: ${c.description}`, () => + Effect.gen(function*() { + const df = yield* Oracle.run(c, source) + assert.strictEqual(yield* Compare.diff(c.id, df, c.expected), undefined) + })) + } + }) + } + }) + + describe("streamed tables", () => { + const Row = Schema.Struct({ x: Schema.Number }) + const batches = Array.from({ length: 8 }, (_, b) => DataFrame.fromRows(Row, [{ x: 2 * b }, { x: 2 * b + 1 }])) + const t = Query.table("t", Row) + const streamed = (query: Query.Query, source: Catalog.Source) => + rows(query).pipe(Effect.provide(Catalog.layer({ t: source }))) + + // DF-Q027 (API contract): Catalog.Catalog; pinned source in README.md. + it.effect("a slice that has its rows pulls no further batch", () => + Effect.gen(function*() { + const one = Stream.concat(Stream.succeed(DataFrame.fromColumns({ x: [1] })), Stream.die("pulled again")) + assert.deepStrictEqual(yield* streamed(Query.head(t, 1), one), [{ x: 1 }]) + for (const none of [Query.head(t, 0), Query.head(Query.sort(t, (c) => [c.x]), 0)]) { + assert.deepStrictEqual(yield* streamed(none, Stream.die("pulled")), []) + } + })) + + // DF-Q028 (API contract): Expr.sum; pinned source in README.md. + it.effect("decimal sums don't depend on batch boundaries", () => + Effect.gen(function*() { + const query = Query.table("t", Schema.Struct({ s: Schema.String })).pipe( + Query.aggregate((c) => ({ total: Expr.sum(Expr.castDecimal(c.s, 19, 0)) })) + ) + const big = DataFrame.fromColumns({ s: ["6000000000000000000", "6000000000000000000"] }) + const negative = DataFrame.fromColumns({ s: ["-6000000000000000000", "-6000000000000000000"] }) + const result = yield* streamed(query, Stream.make(big, negative)) + assert.deepStrictEqual(result, [{ total: BigDecimal.fromBigInt(0n) }]) + })) + + // DF-Q029 (API contract): Catalog.Catalog; pinned source in README.md. + it.effect("batches convert to the declared column types; undeclared, they must share their columns", () => + Effect.gen(function*() { + const source = Stream.make(price("1.2", 18, 1), price("1.23", 18, 2), price("1.2", 18, 2)) + const declared = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }) + const d = (s: string) => BigDecimal.fromStringUnsafe(s) + const totals = declared.pipe( + Query.aggregate((c) => ({ s: Expr.sum(c.d), lo: Expr.min(c.d), hi: Expr.max(c.d) })) + ) + assert.deepStrictEqual(yield* streamed(totals, source), [{ s: d("3.63"), lo: d("1.20"), hi: d("1.23") }]) + const groups = declared.pipe( + Query.groupBy("d"), + Query.agg(() => ({ n: Expr.count() })), + Query.sort((c) => [c.d]) + ) + assert.deepStrictEqual(yield* streamed(groups, source), [{ d: d("1.20"), n: 2 }, { d: d("1.23"), n: 1 }]) + // Undeclared, a batch of other types would make the column types of + // earlier results depend on later batches. + const untyped = Query.fromPlan({ _tag: "Table", name: "t" }) + for ( + const query of [ + untyped, + Query.concat([untyped, untyped]), + Query.aggregate(untyped, () => ({ n: Expr.count() })) + ] + ) { + assert.strictEqual( + (yield* Effect.flip(streamed(query, source))).message, + `a batch of table "t" has columns [d: Decimal[18, 2]], its first batch [d: Decimal[18, 1]]: ` + + `the batches of a streamed table share their columns` + ) + } + const lossy = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 1) } }) + const error = yield* Effect.flip(streamed(lossy, source)) + assert.strictEqual(error.message, `column "d" has values that Decimal[18, 1] can't hold exactly`) + })) + + // DF-Q030 (API contract): Catalog.Catalog; pinned source in README.md. + it.effect("results over inputs of different types are those over their concatenation, streamed or not", () => + Effect.gen(function*() { + type Frame = DataFrame.DataFrame<{ readonly x: number }> + const x = (type: ArrowSchema.DataType, values: ReadonlyArray) => + DataFrame.fromRows(Row, values.map((x) => ({ x })), { types: { x: type } }) + const wide = (values: BigUint64Array | BigInt64Array) => + DataFrame.fromColumns({ x: values }) as unknown as Frame + const decimal = (d: string, precision: number, scale: number) => + DataFrame.fromRows(Schema.Struct({ x: Schema.BigDecimal }), [{ x: BigDecimal.fromStringUnsafe(d) }], { + types: { x: ArrowSchema.decimal(precision, scale) } + }) as unknown as Frame + // A query over the frames concatenated: in memory, and streamed with + // each frame a streamed table of its own. + const results = (frames: ReadonlyArray, query: (table: typeof t) => Query.Query) => { + const names = frames.map((_, i) => `t${i}`) + const concat = (tables: ReadonlyArray) => Query.concat(tables as [typeof t, ...Array]) + return [ + query(concat(frames.map(Query.from))), + query(concat(names.map((name) => Query.fromPlan({ _tag: "Table", name }) as typeof t))).pipe( + Effect.provide(Catalog.layer(Object.fromEntries(names.map((name, i) => [name, Stream.make(frames[i])])))) + ) + ] + } + const aggregates = (table: typeof t) => + Query.aggregate(table, (c) => ({ + sum: Expr.sum(c.x), + mean: Expr.mean(c.x), + min: Expr.min(c.x), + max: Expr.max(c.x), + count: Expr.count(c.x), + first: Expr.first(c.x), + last: Expr.last(c.x), + std: Expr.std(c.x), + variance: Expr.variance(c.x) + })) + const groups = (table: typeof t) => table.pipe(Query.groupBy("x"), Query.agg(() => ({ n: Expr.count() }))) + // Expected values from Polars 2.1.0 over the frames concatenated + // (`how="vertical_relaxed"`), in order and reversed: first and last, + // and groups in order of appearance. Every result is Float64 but + // count. std and variance are left out where floats round them + // differently depending on how the rows are split. + const { float32, float64, int32 } = ArrowSchema + const cases: ReadonlyArray< + readonly [ + ReadonlyArray, + Record, + readonly [readonly [number, number], readonly [number, number]], + readonly [ReadonlyArray, ReadonlyArray] + ] + > = [ + [ + [x(int32, [2147483647]), x(int32, [1]), x(float64, [0.5])], + { sum: 2147483648.5, mean: 715827882.8333334, min: 0.5, max: 2147483647, count: 3 }, + [[2147483647, 0.5], [0.5, 2147483647]], + [[[2147483647, 1], [1, 1], [0.5, 1]], [[0.5, 1], [1, 1], [2147483647, 1]]] + ], + [ + [x(float32, [1]), x(float64, [1.0000001])], + { sum: 2.0000001000000003, mean: 1.0000000500000001, min: 1, max: 1.0000001, count: 2 }, + [[1, 1.0000001], [1.0000001, 1]], + [[[1, 1], [1.0000001, 1]], [[1.0000001, 1], [1, 1]]] + ], + [ + [x(float32, [1, 3]), x(float64, [2])], + { sum: 6, mean: 2, min: 1, max: 3, count: 3, std: 1, variance: 1 }, + [[1, 2], [2, 3]], + [[[1, 1], [3, 1], [2, 1]], [[2, 1], [1, 1], [3, 1]]] + ], + [ + [x(float64, []), x(int32, [1, 2])], + { sum: 3, mean: 1.5, min: 1, max: 2, count: 2, std: Math.SQRT1_2, variance: 0.5 }, + [[1, 2], [1, 2]], + [[[1, 1], [2, 1]], [[1, 1], [2, 1]]] + ], + [ + [wide(BigUint64Array.of(2n ** 63n)), wide(BigInt64Array.of(-1n)), x(float64, [0.5])], + { sum: 2 ** 63, mean: 3074457345618258400, min: -1, max: 2 ** 63, count: 3 }, + [[2 ** 63, 0.5], [0.5, 2 ** 63]], + [[[2 ** 63, 1], [-1, 1], [0.5, 1]], [[0.5, 1], [-1, 1], [2 ** 63, 1]]] + ], + [ + [decimal("6000000000000000000", 19, 0), decimal("1.20", 18, 2), x(float64, [0.5])], + { sum: 6e18, mean: 2e18, min: 0.5, max: 6e18, count: 3 }, + [[6e18, 0.5], [0.5, 6e18]], + [[[6e18, 1], [1.2, 1], [0.5, 1]], [[0.5, 1], [1.2, 1], [6e18, 1]]] + ], + [ + [x(int32, [1, 2]), x(float64, [2, 0.5])], + { sum: 5.5, mean: 1.375, min: 0.5, max: 2, count: 4 }, + [[1, 0.5], [2, 2]], + [[[1, 1], [2, 2], [0.5, 1]], [[2, 2], [0.5, 1], [1, 1]]] + ] + ] + for (const [frames, expected, ends, grouped] of cases) { + const orders = [[frames, ends[0], grouped[0]], [[...frames].reverse(), ends[1], grouped[1]]] as const + for (const [order, [first, last], keys] of orders) { + for (const query of results(order, aggregates)) { + const df = yield* query + const [row] = DataFrame.toRows(df) + assert.deepStrictEqual( + Object.fromEntries( + [...Object.keys(expected), "first", "last"].map((k) => [k, row[k as keyof typeof row]]) + ), + { ...expected, first, last } + ) + assert.deepStrictEqual( + df.schema.fields.map((f) => f.type), + df.schema.fields.map((f) => f.name === "count" ? ArrowSchema.uint32 : float64) + ) + } + for (const query of results(order, groups)) { + assert.deepStrictEqual(DataFrame.toRows(yield* query), keys.map(([x, n]) => ({ x, n }))) + } + } + } + // Row-wise operations see the common type too, as in Polars: Int8 + // 100 doubles to 200 in Int64, where doubling in Int8 wraps to -56. + const small = [DataFrame.fromColumns({ x: Int8Array.of(100) }), wide(BigInt64Array.of(1n))] + const rowWise: ReadonlyArray Query.Query, ReadonlyArray]> = [ + [(table) => Query.select(table, (c) => ({ y: Expr.multiply(c.x, 2) })), [{ y: 200n }, { y: 2n }]], + [(table) => Query.filter(table, (c) => Expr.gt(Expr.add(c.x, c.x), 150)), [{ x: 100n }]] + ] + for (const [query, expected] of rowWise) { + for (const result of results(small, query)) assert.deepStrictEqual(DataFrame.toRows(yield* result), expected) + } + // A value the common type can't hold fails, as concatenating fails, though no minimum holds it. + const failures: ReadonlyArray, string]> = [ + [ + [wide(BigUint64Array.of(2n ** 63n, 0n)), wide(BigInt64Array.of(-1n))], + "UInt64 values don't fit their common type Int64" + ], + [ + [wide(BigInt64Array.of(-1n)), wide(BigUint64Array.of(2n ** 63n))], + "UInt64 values don't fit their common type Int64" + ], + [ + [decimal("6000000000000000000", 19, 0), decimal("1.20", 18, 2)], + "Decimal[19, 0] values don't fit their common type Decimal[38, 2]" + ] + ] + const min = (table: typeof t) => Query.aggregate(table, (c) => ({ min: Expr.min(c.x) })) + for (const [frames, message] of failures) { + const queries: ReadonlyArray> = [ + ...results(frames, min), + ...results(frames, groups) + ] + for (const query of queries) { + assert.strictEqual((yield* Effect.flip(query)).message, message) + } + } + })) + + // DF-Q031 (API contract): Query.aggregate; pinned source in README.md. + it.effect("zero-row batches aggregate like an empty table", () => + Effect.gen(function*() { + const query = Query.aggregate(t, (c) => ({ n: Expr.count(), total: Expr.sum(c.x), mean: Expr.mean(c.x) })) + const empty = Stream.make(DataFrame.fromColumns({ x: Float64Array.of() })) + assert.deepStrictEqual(yield* streamed(query, empty), [{ n: 0, total: 0, mean: null }]) + })) + + // DF-Q032 (API contract): Query.stream; pinned source in README.md. + it.effect("results without rows keep the types of their columns", () => + Effect.gen(function*() { + const Nested = Schema.Struct({ xs: Schema.Array(Schema.Number), s: Schema.Struct({ a: Schema.Number }) }) + const nested = Query.table("t", Nested).pipe( + Query.filter((c) => Expr.gt(Expr.listLength(c.xs), 5)), + Query.select((c) => ({ sum: Expr.listSum(c.xs), a: Expr.field(c.s, "a") })) + ) + const untyped = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( + Query.filter((c) => Expr.gt(c.x as Expr.Expr, 5)) + ) + const prices = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }).pipe( + Query.aggregate((c) => ({ total: Expr.sum(c.d), r: Expr.max(Expr.round(c.d, 1)) })) + ) + const cases: ReadonlyArray, Catalog.Source, ReadonlyArray]> = [ + [nested, Stream.empty, ["sum:Float", "a:Float"]], + [prices, Stream.empty, ["total:Decimal", "r:Decimal"]], + [nested, Stream.make(DataFrame.fromRows(Nested, [{ xs: [1], s: { a: 1 } }])), ["sum:Float", "a:Float"]], + [untyped, Stream.make(DataFrame.fromColumns({ x: [1], y: ["a"] })), ["x:Float", "y:Utf8"]] + ] + for (const [query, source, expected] of cases) { + const df = yield* query.pipe(Effect.provide(Catalog.layer({ t: source }))) + assert.deepStrictEqual(df.schema.fields.map((f) => `${f.name}:${f.type._tag}`), expected) + } + })) + + // DF-Q033 (API contract): Query.stream; pinned source in README.md. + it.effect("results without rows are those in memory, or fail when their columns are undeclared and unread", () => + Effect.gen(function*() { + const x = (...values: Array) => DataFrame.fromColumns({ x: Int8Array.from(values) }) + const xy = (x: number, y: number) => DataFrame.fromColumns({ x: Int8Array.of(x), y: Int8Array.of(y) }) + const undeclared = (name: string) => Query.fromPlan({ _tag: "Table", name }) as typeof t + const [u, r, e] = [undeclared("u"), undeclared("r"), undeclared("e")] + const d = Query.table("d", Row, { types: { x: ArrowSchema.int8 } }) + const p = Query.table("p", Schema.Struct({ i: Schema.Number, k: Schema.String, v: Schema.Number })) + // Streamed, no batch of u is read and the declared d has none. + const memory = Catalog.layer({ + u: x(1), + r: xy(1, 2), + d: x(), + p: DataFrame.fromColumns({ i: [1], k: ["a"], v: [1] }) + }) + const streamed = Catalog.layer({ + u: Stream.die("read"), + r: Stream.make(xy(1, 2)), + d: Stream.empty, + p: Stream.die("read"), + e: Stream.empty + }) + const declare = (what: string) => + `the columns of a result without rows are unknown, as no batch they depend on was read: declare the columns of ${what}` + const sums = (query: typeof t) => Query.aggregate(query, (c) => ({ n: Expr.count(), s: Expr.sum(c.x) })) + const cases: ReadonlyArray, Array, string?]> = [ + [Query.concat([Query.head(u, 0), Query.from(x(9))]), [{ x: 9 }], declare(`table "u"`)], + [Query.head(u, 0), [], declare(`table "u"`)], + [Query.head(Query.concat([u, u]), 0), [], declare(`table "u"`)], + [Query.unique(Query.head(u, 0)), [], declare(`table "u"`)], + [sums(Query.head(u, 0)), [{ n: 0, s: 0 }], declare(`table "u"`)], + [Query.join(d, Query.head(r, 0), { on: "x" }), [], declare(`table "r"`)], + [ + Query.head(Query.pivot(p, { on: "k", index: ["i"], values: "v", aggregate: "sum" }), 0), + [], + declare(`the pivot on "k"`) + ], + [Query.concat([Query.join(d, r, { on: "x" }), Query.from(xy(9, 3))]), [{ x: 9, y: 3 }]], + [Query.concat([Query.head(d, 0), Query.from(x(9))]), [{ x: 9 }]], + [sums(d), [{ n: 0, s: 0 }]] + ] + for (const [query, expected, error] of cases) { + const df = yield* Effect.provide(query, memory) + assert.deepStrictEqual(DataFrame.toRows(df), expected) + if (error === undefined) { + const result = yield* Effect.provide(query, streamed) + assert.deepStrictEqual([DataFrame.toRows(result), result.schema], [expected, df.schema]) + } else { + assert.strictEqual((yield* Effect.flip(Effect.provide(query, streamed))).message, error) + } + } + // A table without declared columns or batches has no in-memory counterpart. + const unknown: ReadonlyArray> = [ + e, + Query.concat([e, Query.from(x(9))]), + Query.concat([e, e]), + Query.aggregate(e, () => ({ n: Expr.count() })) + ] + for (const query of unknown) { + assert.strictEqual((yield* Effect.flip(Effect.provide(query, streamed))).message, declare(`table "e"`)) + } + })) + + // DF-Q034 (API contract): Catalog.Catalog; pinned source in README.md. + it.effect("reads a table named twice once per run, only if its batches are needed", () => + Effect.gen(function*() { + let loads = 0 + const load = Effect.sync(() => DataFrame.fromColumns({ x: [++loads] })) + const cases: ReadonlyArray, Array, number]> = [ + [Query.concat([t, t]), [{ x: 1 }, { x: 1 }], 1], + [Query.join(t, t, { on: "x" }), [{ x: 1 }], 1], + [Query.concat([t, Query.head(t, 0)]), [{ x: 1 }], 1], + [Query.head(Query.concat([t, t]), 0), [], 0], + [Query.concat([Query.head(t, 0), Query.head(t, 0)]), [], 0], + [Query.head(Query.sort(Query.concat([t, t]), (c) => [c.x]), 0), [], 0] + ] + for (const [query, expected, reads] of cases) { + loads = 0 + assert.deepStrictEqual(yield* streamed(query, Stream.fromEffect(load)), expected) + assert.strictEqual(loads, reads) + } + loads = 0 + const frames = yield* Query.concat([t, t]).pipe( + Query.stream, + Stream.runCollect, + Effect.provide(Catalog.layer({ t: load })) + ) + assert.deepStrictEqual(frames.flatMap((frame) => DataFrame.toArray(frame, "x")), [1, 1]) + assert.strictEqual(loads, 1) + })) + + // DF-Q035 (API contract): Query.explain; pinned source in README.md. + it.effect("explain shows the plan without reading the stream", () => + Effect.gen(function*() { + const plan = yield* Query.explain(Query.head(t, 1)).pipe( + Effect.provide(Catalog.layer({ t: Stream.die("read") })) + ) + assert.strictEqual(plan, [`Slice offset 0 length 1`, ` Table t [x]`].join("\n")) + })) + + // DF-Q036 (API contract): Query.head; pinned source in README.md. + it.effect("a head stops reading the stream", () => + Effect.gen(function*() { + let pulled = 0 + const source = Stream.fromIterable(batches).pipe(Stream.tap(() => Effect.sync(() => pulled++))) + const df = yield* t.pipe( + Query.filter((c) => Expr.gt(c.x, 0)), + Query.head(2), + Effect.provide(Catalog.layer({ t: source })) + ) + assert.deepStrictEqual(DataFrame.toArray(df, "x"), [1, 2]) + assert.isBelow(pulled, batches.length) + })) + + // DF-Q037 (API contract): Query.stream; pinned source in README.md. + it.effect("Query.stream emits a frame per non-empty batch", () => + Effect.gen(function*() { + const sizes = yield* t.pipe( + Query.filter((c) => Expr.lt(c.x, 5)), + Query.stream, + Stream.map((frame) => frame.length), + Stream.runCollect, + Effect.provide(Catalog.layer({ t: Stream.fromIterable(batches) })) + ) + assert.deepStrictEqual(sizes, [2, 2, 1]) + })) + }) + + // Inputs large enough to reach radix sorting, hash table growth and joins + // with many keys, checked against row-by-row reference implementations. + describe("large inputs", () => { + type Row = { + readonly f: number | null + readonly i: number | null + readonly b: bigint | null + readonly s: string | null + readonly p: boolean | null + } + type Key = { readonly name: keyof Row; readonly descending: boolean } + + const strings = ["a", "b", "ab", "B", "é", "😀", "\uffff", "", "zz", "a\u0000"] + + const generate = (seed: number, n: number, cardinality: number): Array => { + let state = seed + const next = () => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return (state >>> 0) / 4294967296 + } + const pick = (values: ReadonlyArray) => values[Math.floor(next() * values.length)] + const maybe = (a: A): A | null => (next() < 0.1 ? null : a) + return Array.from({ length: n }, () => ({ + f: maybe( + pick([NaN, -0, 0, Infinity, -Infinity, Math.floor(next() * cardinality) - cardinality / 2, 0.5, -2.5]) + ), + i: maybe(Math.floor(next() * cardinality) - Math.floor(cardinality / 2)), + b: maybe(BigInt(Math.floor(next() * cardinality)) * 4294967311n - 2n ** 40n), + s: maybe(`${pick(strings)}${Math.floor(next() * cardinality)}`), + p: maybe(next() < 0.5) + })) + } + + const compareCodePoints = (a: string, b: string): number => { + const x = Array.from(a, (c) => c.codePointAt(0)!) + const y = Array.from(b, (c) => c.codePointAt(0)!) + for (let k = 0; k < Math.min(x.length, y.length); k++) if (x[k] !== y[k]) return x[k] - y[k] + return x.length - y.length + } + + const compareValues = (a: unknown, b: unknown): number => { + if (typeof a === "number" && typeof b === "number") { + if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1 + return a < b ? -1 : a > b ? 1 : 0 + } + if (typeof a === "string") return compareCodePoints(a, b as string) + if (typeof a === "boolean") return Number(a) - Number(b) + return (a as bigint) < (b as bigint) ? -1 : (a as bigint) > (b as bigint) ? 1 : 0 + } + + const referenceSort = (rows: ReadonlyArray, keys: ReadonlyArray, nullsLast: boolean): Array => + [...rows].sort((r1, r2) => { + for (const { descending, name } of keys) { + const a = r1[name] + const b = r2[name] + if (a === null || b === null) { + if (a === b) continue + return (a === null ? -1 : 1) * (nullsLast ? -1 : 1) + } + const c = compareValues(a, b) * (descending ? -1 : 1) + if (c !== 0) return c + } + return 0 + }) + + const referenceGroups = (rows: ReadonlyArray, keys: ReadonlyArray) => { + const keyOf = (values: ReadonlyArray) => + JSON.stringify( + values, + (_, v) => typeof v === "bigint" ? `${v}n` : typeof v === "number" && !Number.isFinite(v) ? String(v) : v + ) + const groups = new Map>() + for (const r of rows) { + const k = keyOf(keys.map((name) => r[name])) + const g = groups.get(k) + if (g === undefined) groups.set(k, [r]) + else g.push(r) + } + return [...groups.values()] + } + + const data = generate(0x2545f491, 6000, 3000) + const df = Query.from(arrowFrame({ + f: [ArrowSchema.float64, data.map((r) => r.f)], + i: [ArrowSchema.int32, data.map((r) => r.i)], + b: [ArrowSchema.int64, data.map((r) => r.b)], + s: [ArrowSchema.utf8, data.map((r) => r.s)], + p: [ArrowSchema.bool, data.map((r) => r.p)] + })) + const sortKey = (c: { readonly [K in keyof Row]: Expr.Expr }, key: Key) => + key.descending ? Expr.desc(c[key.name] as Expr.Expr) : Expr.asc(c[key.name] as Expr.Expr) + + // DF-Q038 (API contract): Query.sort; pinned source in README.md. + it.effect("sorts by every column type, in both directions, nulls first or last", () => + Effect.gen(function*() { + for (const name of ["f", "i", "b", "s", "p"] as const) { + for (const descending of [false, true]) { + for (const nullsLast of [false, true]) { + const sorted = Query.sort(df, (c) => [sortKey(c, { name, descending })], { nullsLast }) + assert.deepStrictEqual(yield* rows(sorted), referenceSort(data, [{ name, descending }], nullsLast)) + } + } + } + })) + + // DF-Q039 (API contract): Query.sort; pinned source in README.md. + it.effect("sorts stably by several keys, also when sliced (top-k)", () => + Effect.gen(function*() { + const keys: ReadonlyArray = [ + { name: "p", descending: true }, + { name: "s", descending: false }, + { name: "f", descending: true } + ] + const sorted = Query.sort(df, (c) => keys.map((key) => sortKey(c, key))) + assert.deepStrictEqual(yield* rows(sorted), referenceSort(data, keys, false)) + for ( + const [name, descending, nullsLast, offset, length] of [ + ["f", true, false, 0, 10], + ["i", false, true, 0, 1], + ["s", false, false, 5, 20], + ["b", true, true, 0, 700] + ] as const + ) { + const top = df.pipe( + Query.sort((c) => [sortKey(c, { name, descending }), Expr.asc(c.i)], { nullsLast }), + Query.slice(offset, length) + ) + const expected = referenceSort(data, [{ name, descending }, { name: "i", descending: false }], nullsLast) + assert.deepStrictEqual(yield* rows(top), expected.slice(offset, offset + length)) + } + })) + + // DF-Q040 (API contract): Query.groupBy; pinned source in README.md. + it.effect("groups by keys of every type", () => + Effect.gen(function*() { + for (const keys of [["f"], ["b"], ["p", "i"], ["s", "b", "p"]] as const) { + const result = df.pipe( + Query.groupBy(...keys), + Query.agg((c) => ({ n: Expr.count(), ni: Expr.count(c.i), max: Expr.max(c.s), first: Expr.first(c.b) })) + ) + const expected = referenceGroups(data, keys).map((g) => { + const out: Record = {} + for (const k of keys) out[k] = g[0][k] + const present = g.map((r) => r.s).filter((s): s is string => s !== null) + out.n = g.length + out.ni = g.filter((r) => r.i !== null).length + out.max = present.length === 0 ? null : present.reduce((a, b) => (compareCodePoints(a, b) >= 0 ? a : b)) + out.first = g[0].b + return out + }) + assert.deepStrictEqual(yield* rows(result), expected) + } + assert.deepStrictEqual( + yield* rows(Query.unique(df, ["i", "p"])), + referenceGroups(data, ["i", "p"]).map((g) => g[0]) + ) + })) + + // DF-Q041 (API contract): Query.join; pinned source in README.md. + it.effect("joins on a high-cardinality key", () => + Effect.gen(function*() { + const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i ?? 0, rs: r.s })) + const rightFrame = DataFrame.fromColumns({ i: Int32Array.from(right, (r) => r.i), rs: right.map((r) => r.rs) }) + const byKey = new Map>() + for (const r of right) byKey.set(r.i, [...(byKey.get(r.i) ?? []), r.rs]) + for (const how of ["inner", "left", "semi", "anti"] as const) { + const result = Query.join(df.pipe(Query.pick("i", "s")), Query.from(rightFrame), { on: "i", how }) + const expected = data.flatMap((r) => { + const matches = r.i === null ? [] : byKey.get(r.i) ?? [] + switch (how) { + case "inner": + return matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "left": + return matches.length === 0 + ? [{ i: r.i, s: r.s, rs: null }] + : matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "semi": + return matches.length > 0 ? [{ i: r.i, s: r.s }] : [] + case "anti": + return matches.length === 0 ? [{ i: r.i, s: r.s }] : [] + } + }) + assert.deepStrictEqual(yield* rows(result), expected) + } + })) + }) + + // Columns hold temporal and decimal values as integers. Only rows and + // arrays of values turn them into DateTime, Duration and BigDecimal. + describe("columnar path", () => { + type Event = { + readonly us: DateTime.Utc | null + readonly zoned: DateTime.Utc | null + readonly ns: DateTime.Utc | null + readonly nsUtc: DateTime.Utc | null + readonly day: DateTime.Utc | null + readonly took: Duration.Duration | null + readonly amount: BigDecimal.BigDecimal | null + readonly key: string | null + } + const ns = [1_700_000_000_123_456_789n, 1_700_000_100_000_000_001n, null, -1n, 1_600_000_000_999_999_999n, 5n] + const micros = ns.map((x) => (x === null ? null : x / 1000n)) + const input = Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make([ + ArrowSchema.field("us", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("zoned", ArrowSchema.timestamp("microsecond", "Europe/Berlin")), + ArrowSchema.field("ns", ArrowSchema.timestamp("nanosecond")), + ArrowSchema.field("nsUtc", ArrowSchema.timestamp("nanosecond", "UTC")), + ArrowSchema.field("day", ArrowSchema.date("day")), + ArrowSchema.field("took", ArrowSchema.duration("microsecond")), + ArrowSchema.field("amount", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("key", ArrowSchema.utf8) + ]), + [ + micros, + micros, + ns, + ns, + [19_000, 19_001, 19_000, null, -1, 0], + [1_500_000n, -250n, 90_000_000_000n, null, 7n, 0n], + ["12.34", "-0.05", null, "100.25", "0.35", "7.45"].map((s) => + s === null ? null : BigDecimal.fromStringUnsafe(s) + ), + ["a", "b", "a", "b", "a", null] + ] + )) + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(input)) + /** Non-null values per column of `input`. */ + const present = 5 + + /** + * The DateTime, Duration and BigDecimal values built while `f` runs, by + * the functions `effect/arrow` and `effect/dataframe` build them with. + * `f` is synchronous, so no other test of this file runs meanwhile. + */ + const built = (f: () => A) => { + const count = (spies: ReadonlyArray<{ readonly mock: { readonly calls: ReadonlyArray } }>) => + spies.reduce((n, spy) => n + spy.mock.calls.length, 0) + const dateTimes = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(InternalDateTime, "makeZonedProto")] + const durations = [vi.spyOn(Duration, "nanos")] + const decimals = [ + vi.spyOn(BigDecimal, "make"), + vi.spyOn(BigDecimal, "round"), + vi.spyOn(BigDecimal, "fromStringUnsafe") + ] + try { + const value = f() + return { value, dateTimes: count(dateTimes), durations: count(durations), decimals: count(decimals) } + } finally { + for (const spy of [...dateTimes, ...durations, ...decimals]) spy.mockRestore() + } + } + const none = { dateTimes: 0, durations: 0, decimals: 0 } + + // DF-N001 (local policy): columnar path; pinned source in README.md. + it("decoding, querying and encoding build no DateTime, Duration or BigDecimal", () => { + const since = DateTime.makeUnsafe(1_650_000_000_000) + const selected = (events: Query.Query) => + events.pipe( + Query.filter((c) => Expr.gt(c.us, since)), + Query.sort((c) => [Expr.desc(c.us), c.ns]), + Query.select((c) => ({ + us: c.us, + zoned: c.zoned, + ns: c.ns, + year: Expr.year(c.us), + month: Expr.month(c.ns), + weekday: Expr.weekday(c.day), + millisecond: Expr.millisecond(c.us), + hourly: Expr.truncate(c.ns, "1h"), + later: Expr.add(c.us, c.took), + elapsed: Expr.subtract(c.ns, c.us), + seconds: Expr.totalSeconds(c.took), + tenths: Expr.castDecimal(c.amount, 18, 1), + rounded: Expr.round(c.amount, 1) + })) + ) + const grouped = (events: Query.Query) => + events.pipe( + Query.groupBy("key"), + Query.agg((c) => ({ + total: Expr.sum(c.amount), + mean: Expr.mean(c.amount), + least: Expr.min(c.amount), + most: Expr.max(c.amount), + tookTotal: Expr.sum(Expr.totalMilliseconds(c.took)), + tookMean: Expr.mean(Expr.totalSeconds(c.took)), + tookLeast: Expr.min(c.took), + tookMost: Expr.max(c.took), + first: Expr.min(c.us), + last: Expr.max(c.ns) + })) + ) + const table = Query.fromPlan({ _tag: "Table", name: "events" }) as Query.Query as Query.Query + const run = built(() => { + const frame = DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) as DataFrame.DataFrame< + unknown + > as DataFrame.DataFrame + const batches = Effect.runSync(Stream.runCollect( + DataFrame.decodeArrowStream(Stream.make(bytes.subarray(0, 200), bytes.subarray(200))) + )) + const streamed = (query: Query.Query) => + Effect.runSync( + Stream.runCollect(Query.stream(query)).pipe( + Effect.provide(Catalog.layer({ events: Stream.fromIterable([...batches, ...batches]) })) + ) + ) + const results = { + selected: Effect.runSync(selected(Query.from(frame))), + grouped: Effect.runSync(grouped(Query.from(frame))), + streamed: [...streamed(selected(table)), ...streamed(grouped(table))] + } + const frames: ReadonlyArray> = [ + results.selected, + results.grouped, + ...results.streamed + ] + return { + ...results, + encoded: frames.map((df) => + [df, Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df)))] as const + ) + } + }) + assert.deepStrictEqual({ dateTimes: run.dateTimes, durations: run.durations, decimals: run.decimals }, none) + // The results, read as rows afterwards: nothing was built, though the work was done. + const { encoded, grouped: groups, selected: events, streamed } = run.value + assert.deepStrictEqual( + DataFrame.toRows(events).map((r) => [ + r.us && DateTime.toEpochMillis(r.us), + r.year, + r.tenths && BigDecimal.format(r.tenths), + r.rounded && BigDecimal.format(r.rounded), + r.seconds + ]), + [ + [1_700_000_100_000, 2023, "0", "0", 0n], + [1_700_000_000_123, 2023, "12.3", "12.3", 1n] + ] + ) + assert.deepStrictEqual( + DataFrame.toRows(groups).map((r) => [ + r.key, + r.total && BigDecimal.format(r.total), + r.tookMost && Duration.toMillis(r.tookMost) + ]), + [["a", "12.69", 90_000_000], ["b", "100.2", -0.25], [null, "7.45", 0]] + ) + // Over the two batches: the sorted rows of both, then one row per group. + assert.deepStrictEqual(streamed.map((df) => df.length), [4, 3]) + for (const [df, bytes] of encoded) { + assert.isTrue(Equal.equals(DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))), df)) + } + }) + + // DF-N002 (local policy): DataFrame.toRows and DataFrame.toArray; pinned source in README.md. + it("rows build one DateTime, Duration or BigDecimal per value; columns keep nanoseconds", () => { + const frame = DataFrame.fromArrowTable(input) + const rows = built(() => DataFrame.toRows(frame)) + // us, zoned, ns, nsUtc and day; took; amount. + assert.deepStrictEqual( + { dateTimes: rows.dateTimes, durations: rows.durations, decimals: rows.decimals }, + { dateTimes: 5 * present, durations: present, decimals: present } + ) + for (const name of ["us", "zoned", "ns", "nsUtc", "day", "took", "amount"]) { + const column = built(() => DataFrame.toArray(frame, name)) + assert.strictEqual(column.dateTimes + column.durations + column.decimals, present, name) + } + // Rows read nanoseconds as DateTime, which keeps milliseconds. + assert.deepStrictEqual( + rows.value.map((r) => r.ns === null ? null : DateTime.toEpochMillis(r.ns as DateTime.Utc)), + [1_700_000_000_123, 1_700_000_100_000, null, -1, 1_600_000_000_999, 0] + ) + const output = built(() => Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(frame)))) + assert.deepStrictEqual( + { dateTimes: output.dateTimes, durations: output.durations, decimals: output.decimals }, + none + ) + assert.deepStrictEqual(output.value, bytes) + const stored = Effect.runSync( + Query.from(DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(output.value)))).pipe( + Query.select((c) => ({ ns: Expr.cast(c.ns as Expr.Expr, "Int64") })) + ) + ) + assert.deepStrictEqual(DataFrame.toArray(stored, "ns"), ns) + }) + + // DF-N003 (API contract): Expr.castDecimal and Expr.round; pinned source in README.md. + it.effect("dropping decimal digits rounds half to even, exactly", () => + Effect.gen(function*() { + /** + * Texts with `k` fraction digits brought to scale 0: parsed from the + * text, cast from a decimal of scale `k`, and rounded at that scale, + * the last two when 64 bits hold the text at scale `k`. + */ + const whole = (texts: ReadonlyArray, k: number) => + Query.from(DataFrame.fromColumns({ s: texts })).pipe( + Query.select((c) => { + const exact = Expr.castDecimal(c.s, 38, k) + return { + parsed: Expr.cast(Expr.castDecimal(c.s, 38, 0), "Int64"), + ...(k <= 18 && { + cast: Expr.cast(Expr.castDecimal(exact, 38, 0), "Int64"), + rounded: Expr.cast(Expr.round(exact, 0), "Int64") + }) + } + }), + Effect.map((df) => DataFrame.columnNames(df).map((name) => DataFrame.toArray(df, name))) + ) + const zeros = "0".repeat(38) + const pinned: ReadonlyArray = [ + [0, "7", 7n], + [0, "-7", -7n], + [1, "0.5", 0n], + [1, "1.5", 2n], + [1, "2.5", 2n], + [1, "-0.5", 0n], + [1, "-1.5", -2n], + [1, "-2.5", -2n], + [2, "0.51", 1n], + [2, "0.49", 0n], + [2, "-0.51", -1n], + [2, "-0.49", 0n], + [2, "-2.50", -2n], + [2, "-3.50", -4n], + [18, "8.500000000000000000", 8n], + [18, "-7.500000000000000000", -8n], + [18, "8.500000000000000001", 9n], + [18, "-8.499999999999999999", -8n], + [39, `0.5${zeros}`, 0n], + [39, `1.5${zeros}`, 2n], + [39, `-2.5${zeros}`, -2n], + [40, `0.5${zeros}1`, 1n], + [40, `-2.5${zeros}1`, -3n], + [39, `0.4${"9".repeat(38)}`, 0n] + ] + for (const k of new Set(pinned.map(([k]) => k))) { + const cases = pinned.filter((c) => c[0] === k) + for (const column of yield* whole(cases.map(([, text]) => text), k)) { + assert.deepStrictEqual(column, cases.map(([, , expected]) => expected), `${k} digits`) + } + } + // Against BigDecimal.round, over random values around and away from ties. + let state = 0x9e3779b9 + const next = (bound: number) => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return Math.floor(((state >>> 0) / 4294967296) * bound) + } + const digits = (n: number) => Array.from({ length: n }, () => next(10)).join("") + for (const k of [0, 1, 2, 9, 17, 18, 25, 40]) { + const fractions = k === 0 ? [""] : [ + "0".repeat(k), + `5${"0".repeat(k - 1)}`, + `4${"9".repeat(k - 1)}`, + `5${"0".repeat(k - 1)}`.replace(/0$/, "1"), + digits(k) + ] + // Up to 18 digits in all (below 9 at 18 fraction digits), so 64 bits hold the texts at scale `k`, rounded up too. + const integer = () => k === 18 ? String(next(9)) : digits(1 + next(k < 18 ? 18 - k : 17)) + const texts = Array.from({ length: 60 }, integer).flatMap((integer) => + fractions.map((fraction) => `${next(2) === 0 ? "-" : ""}${integer}${k === 0 ? "" : "."}${fraction}`) + ) + const expected = texts.map((text) => + BigDecimal.round(BigDecimal.make(BigInt(text.replace(".", "")), k), { scale: 0, mode: "half-even" }).value + ) + for (const column of yield* whole(texts, k)) assert.deepStrictEqual(column, expected, `${k} digits`) + } + })) + }) +}) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md new file mode 100644 index 00000000000..205e8d54ab4 --- /dev/null +++ b/packages/effect/test/dataframe/README.md @@ -0,0 +1,289 @@ +# DataFrame contract assertions + +Reference for maintainers reviewing the DataFrame layer. This is an assertion-to-test inventory, not a claim of complete standards or API coverage. Existing regressions remain in their conventional suites. + +## Authority and classification + +The approved API baseline is commit `d7121ccbd69f9f437ac60b09a704f216c0ca44ee`. Links below pin that immutable revision of `packages/effect/src/dataframe`; API documentation, not an external dataframe standard, defines the contract. The source symbol locates the surrounding documentation; it is not an assertion that every implementation detail is normative. + +- **C**: approved public API contract; **P**: local regression/policy detail (not an external requirement). +- **I**: interoperability/differential evidence; an independent engine is an oracle, not the authority for this API. +- No row is an external normative requirement. Gaps and exclusions are explicit below. +- IDs are permanent: retain them when tests move or titles change; do not renumber. Test comments resolve to this inventory and its pinned source. + +## Runtime assertions + +The assertion column is the exact leaf test title. Combine it with the file and surrounding describe blocks to select the test; each listed title is unique within its file. A row covers the concrete inputs/assertions in that test, not all possible values of the API. + +### `DataFrame.test.ts` + +| ID | Kind | Assertion / exact test title | API source | +| ------- | ---- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-C001 | C | fromRows maps schema fields to Arrow types | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C002 | C | fromRows takes per-column type overrides | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C003 | C | fromRows builds decimal columns from options.types | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C004 | C | fromRows rejects overrides that can't hold the field's values | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C005 | C | fromRows rejects schemas without a column type | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C006 | C | fromColumns keeps typed array element types | [DataFrame.fromColumns](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L274) | +| DF-C007 | C | fromColumns rejects columns of different lengths | [DataFrame.fromColumns](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L274) | +| DF-C008 | C | empty has the schema's columns and no rows | [DataFrame.empty](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L360) | +| DF-C009 | C | returns the frame typed by the schema, in schema order | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C010 | C | fails on missing columns, wrong types, unexpected nulls and values the schema rejects | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C011 | C | reads timestamps with a UTC time zone as DateTime.Utc | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C012 | C | converts columns to the types fromRows gives the schema, exactly | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C013 | C | round-trips through the IPC stream format | [DataFrame.toArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L448) | +| DF-C014 | C | decodes IPC stream bytes into a frame per batch | [DataFrame.decodeArrowStream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L336) | +| DF-C015 | C | exports late-selected columns | [DataFrame.toArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L448) | +| DF-C016 | C | imports a table without batches with the nullability of its fields | [DataFrame.fromArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L314) | +| DF-C017 | C | imports sliced list columns with only their own elements | [DataFrame.fromArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L314) | +| DF-C018 | C | compare structurally, NaN included | [DataFrame.equals](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L478) | +| DF-C019 | C | iterate as rows | [DataFrame.DataFrame](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L81) | +| DF-C020 | P | format as a text table | [DataFrame.format](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L467) | +| DF-C021 | C | reuses compatible numeric buffers across Arrow conversion | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | +| DF-C022 | C | fromRows checks rows against the schema's refinements, literals and row checks | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C023 | C | fromRows, empty and validate reject schemas that transform their rows | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C024 | P | builds values only for the fields the schema constrains | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C025 | C | takes an Arrow schema, converting the frame to its columns | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C026 | C | rejects Arrow schemas with types a table can't declare | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C027 | C | converts dates and times to the declared unit | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C028 | C | column gives a column's buffers and values without building rows | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | +| DF-C029 | C | fromRows returns a frame that validates against the same declaration | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C030 | C | a null member of a union keeps the checks of the other members | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C031 | C | checks see stored values as the schema writes them, missing ones included | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C032 | C | accepts the stored values that Schema.is accepts | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C033 | C | rejects decimal overrides that aren't 128-bit | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | + +DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. + +### `Query.test.ts` + +| ID | Kind | Assertion / exact test title | API source | +| ------- | ---- | ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-Q001 | C | is an Effect producing a frame | [Query.Query](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L108) | +| DF-Q002 | C | expression callbacks run once at construction, not on execution | [Query.select](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L7-L18) | +| DF-Q003 | C | does no work until it runs | [Query.Query](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L108) | +| DF-Q004 | C | reads each table once per run, from effects too | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q005 | C | tables can be queries (views) | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q006 | C | checks tables against their declared columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q007 | C | narrows tables to their declared columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q008 | C | fails with DataFrameError on invalid expressions | [DataFrame.DataFrameError](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L137) | +| DF-Q009 | C | decimal results fail instead of overflowing 64 bits | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q010 | C | round rejects negative or fractional places | [Expr.round](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L455) | +| DF-Q011 | C | slice takes an offset and a length; negative offsets count from the end | [Query.slice](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L391) | +| DF-Q012 | C | append stacks rows; dropNulls removes incomplete rows | [Query.append](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L492) | +| DF-Q013 | C | joins on several keys | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | +| DF-Q014 | C | cast reports values that don't fit as null | [Expr.cast](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L685) | +| DF-Q015 | C | implicit conversions to a common type fail on values it can't hold | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q016 | C | temporal operands of mixed units meet in the coarser unit, as in Polars | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q017 | P | shows the rewritten plan | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q018 | P | pushes key predicates below groups and through renames | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q019 | P | keeps predicates on join keys off the right side, whose keys the join drops | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q020 | P | keeps predicates above left joins when they read the right side | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q021 | C | are plain data | [Plan.Node](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L588) | +| DF-Q022 | C | encode to JSON without losing literal types | [Plan.NodeFromJson](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L1103) | +| DF-Q023 | C | round-trip through JSON with their inputs | [Plan.NodeFromJson](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L1103) | +| DF-Q024 | C | reject malformed JSON when decoding | [Plan.NodeFromJson](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L1103) | +| DF-Q025 | C | fail at run time on the wrong number of arguments | [Plan.Expression](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L380) | +| DF-Q026 | C | expressions convert to and from plans | [Expr.toPlan](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L251) | +| DF-Q027 | C | a slice that has its rows pulls no further batch | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q028 | C | decimal sums don't depend on batch boundaries | [Expr.sum](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L827) | +| DF-Q029 | C | batches convert to the declared column types; undeclared, they must share their columns | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q030 | C | results over inputs of different types are those over their concatenation, streamed or not | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q031 | C | zero-row batches aggregate like an empty table | [Query.aggregate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L653) | +| DF-Q032 | C | results without rows keep the types of their columns | [Query.stream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L310) | +| DF-Q033 | C | results without rows are those in memory, or fail when their columns are undeclared and unread | [Query.stream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L310) | +| DF-Q034 | C | reads a table named twice once per run, only if its batches are needed | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q035 | C | explain shows the plan without reading the stream | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q036 | C | a head stops reading the stream | [Query.head](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L407) | +| DF-Q037 | C | Query.stream emits a frame per non-empty batch | [Query.stream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L310) | +| DF-Q038 | C | sorts by every column type, in both directions, nulls first or last | [Query.sort](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L352) | +| DF-Q039 | C | sorts stably by several keys, also when sliced (top-k) | [Query.sort](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L352) | +| DF-Q040 | C | groups by keys of every type | [Query.groupBy](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L610) | +| DF-Q041 | C | joins on a high-cardinality key | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | +| DF-Q042 | C | yields a long query so another fiber can interrupt it | [Query yielding](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L65-L66) | +| DF-Q043 | C | compiles a schema's checks when the table is declared, and does not encode them to JSON | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q046 | C | sums durations to durations in the column's unit | [Expr.sum](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L827) | +| DF-Q047 | C | rejects aggregates that durations, timestamps, dates and times don't have | [Expr.mean](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L838) | +| DF-Q048 | C | rejects a decimal precision or scale that a 128-bit decimal can't have | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q049 | C | decides decimal scales far apart from the 64 bits of storage | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | +| DF-Q050 | C | casts to and from decimals with a negative scale | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | +| DF-Q051 | C | rejects decimal types a cast or a product can't have | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | +| DF-Q052 | C | integers keep their value next to a decimal with a negative scale | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q053 | C | decimal literals have a type their value fits | [Expr.lit](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L281) | +| DF-Q054 | C | converts between floats and decimals at scales past a float's largest power of ten | [Expr.cast](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L685) | +| DF-Q055 | C | isIn compares decimals, points in time and durations by value | [Expr.isIn](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L551) | +| DF-Q056 | C | isIn takes long lists of typed candidates | [Expr.isIn](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L551) | + +DF-C022 to DF-C033 and DF-Q043 to DF-Q056 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-Q052 to DF-Q054 cover the rest of the decimal contract at those scales. DF-Q052 holds 70 at scale -1 next to the integers 69, 70 and 71: comparisons, `when`, `coalesce` and `Query.concat` meet at scale 0, so no integer is rounded; only the explicit cast rounds, and a decimal that scale 0 can't hold fails as with any common type. DF-Q053 reads the type of a literal back through Arrow IPC: its precision is the larger of its number of digits and its scale, as SQL has it, capped at 38, so a scale of 400 or 2147483647 is a valid 128-bit decimal; a literal of a plan with a precision outside 1 to 38, a scale outside 32 bits or more digits than its precision fails when it is evaluated. DF-Q054 takes scales of 310 and 400, where `10^scale` is infinite as a float: a float of `1e-310` casts to one unit of scale 310, and a cast, a mean and a rolling mean of `10^18` units give `1e-292`; below the smallest float the value is zero. DF-Q055 covers `Expr.isIn` on decimals: a candidate is a value and not a number of the unscaled units a decimal stores, so 7.00 at scale 2 is one of `7`, `7n` and a `BigDecimal` equal to it at any scale, and not one of `700`; a number with a fraction matches as a comparison with a float does. `BigDecimal`, `DateTime` and `Duration` candidates have a literal type that a plan value can't carry, so they are compared with `eq`, and every plan of the test survives its JSON form. Null rows stay null, null candidates and candidates of another kind match nothing, and a 64-bit integer column with a candidate that has a fraction is the control for integers. DF-Q056 gives `Expr.isIn` 5000 `BigDecimal`, `DateTime` and `Duration` candidates each: their `eq` tests are joined in halves, so the plan is as deep as the logarithm of the list and runs, also after its JSON form, where a chain of that length overflowed the stack. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. + +## Supported Schema subset + +`DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` read a struct schema once and take from it column types and checks. They never decode or encode with it. + +| Schema | Column type | +| ----------------------------------------- | ------------------------------------------------------ | +| `String`, template literals | Utf8 | +| `Number` | Float64, unless `options.types` names another type | +| `Boolean` | Bool | +| `BigInt` | Int64 | +| `Null`, `Undefined` | Null | +| literals and enums of one primitive type | the type of their values | +| `DateTimeUtc` | Timestamp in microseconds, without a time zone | +| `Duration` | Duration in microseconds | +| `BigDecimal` | Decimal; `options.types` must give precision and scale | +| `Array` of a supported schema | List | +| `Struct` of supported schemas | Struct | +| `NullOr`, `UndefinedOr`, optional fields | the member's type, nullable | +| unions whose members have one column type | that type | + +Rejected with a `DataFrameError` that names the field: transformations (any schema with an encoding, such as `NumberFromString`, `decodeTo`, decoding defaults and renamed keys; `Schema.toType` gives the decoded side), `DateTimeZoned`, tuples, records, classes, unions of different column types, and recursive schemas. + +Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. A field with a value check accepts the stored values that `Schema.is` accepts for it (DF-C032): a check on a nullable field runs on its nulls too, a check on a union member runs on that member's values, and a union in `oneOf` mode takes a value of exactly one member. Fields without checks are checked by column type only, on the typed columns. + +`DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). + +The checks are compiled when the schema is read (`Plan.TableChecks`). They are functions and have no JSON form: `Plan.NodeFromJson` fails to encode a plan over a table that has them, naming the table, and fails to decode JSON that carries a `checks` key. `Plan.withoutChecks` removes them explicitly; the plan then encodes and checks column types only. A decoded plan gets its checks back by declaring the table again with `Query.table`. `Plan.Node` validates a plan as it is and keeps its checks. A SQL engine tests value checks on distinct values and fails for a table with row checks. + +Checks are handed stored values as the schema writes them (DF-C031), in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` alike. A frame holds `null` for every missing value, and a check sees an absent key for an optional field, `undefined` for `UndefinedOr` and `null` for `NullOr`. The rows `{}` and `{ a: undefined }` are one stored row, and a check sees an absent key for both. + +A declaration is rejected with a `DataFrameError` that names the field when a check would see a missing value the schema writes in several ways, as the frame holds `null` for each of them: + +- `null` next to `undefined` or an optional key (`optional(NullOr(String))`, `UndefinedOr(NullOr(String))`), under any check that is handed the value: a check of the field itself, or of a struct, array or union around it, row checks included. +- a union whose struct or array members write the same missing value in different ways, when its values are checked. + +Such fields are accepted where no check is handed their value. + +A table declared by an Arrow schema (DF-C025, DF-C026, DF-Q044) involves no Effect Schema and has no checks. + +### Columnar path + +Columns hold timestamps, dates, durations and decimals as integers under their Arrow type. `DateTime`, `Duration` and `BigDecimal` values are built only for rows and arrays of values (`DataFrame.toRows`, `DataFrame.toArray`). The tests are in `Query.test.ts`, under `Query > columnar path`, and in `packages/sql/pglite/test/DataFrameSql.test.ts`. + +| ID | Kind | File | Assertion / exact test title | API source | +| ------- | ---- | ---------------------- | ----------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-N001 | P | `Query.test.ts` | decoding, querying and encoding build no DateTime, Duration or BigDecimal | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | +| DF-N002 | P | `Query.test.ts` | rows build one DateTime, Duration or BigDecimal per value; columns keep nanoseconds | [DataFrame.toRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L427) | +| DF-N003 | C | `Query.test.ts` | dropping decimal digits rounds half to even, exactly | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699), [Expr.round](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L455) | +| DF-N004 | P | `DataFrameSql.test.ts` | decimal results are read from their text, without BigDecimal values | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | + +DF-N001, DF-N002 and DF-N004 count calls of the functions `effect/arrow` and `effect/dataframe` build these values with: `makeUtc` and `makeZonedProto` of `effect/internal/dateTime`, `Duration.nanos`, and `BigDecimal.make`, `BigDecimal.round` and `BigDecimal.fromStringUnsafe`. DF-N002 shows the counters see the values of rows. A value built another way would not be counted, and the tests measure no memory. + +DF-N001 covers one pipeline over timestamps in microseconds and nanoseconds (with and without a time zone), dates, durations, decimals and strings: IPC stream bytes decoded as a table and as a stream of batches, a filter against a timestamp literal, a sort, date and time fields, `truncate`, timestamp and duration arithmetic, a decimal cast to a lower scale, `round` on a decimal, and sums, means and extremes per group, run over a frame and over streamed batches, then exported and encoded. It makes no claim about other operations. Tables declared with a schema (`Query.table`) and `DataFrame.validate` are not part of it. + +DF-N003 compares with `BigDecimal.round` in mode `half-even` over generated values, besides the listed ties. DF-N004 covers the plain notation PostgreSQL writes decimals in (`-12.34`), alone and in lists. A column with a value in another form, with more fraction digits than its scale or beyond 64 bits is still read through `BigDecimal`, which reports the invalid ones. DF-N004 also pins the message of a decimal result beyond 64 bits, a local detail. + +## Oracle provenance and differential assertions + +**DF-I-POLARS//** (I) resolves exactly to `Query.test.ts > Query > Polars oracle > > : `. Sources are `values`, `frames`, and `batches`; case IDs and descriptions are the existing entries in the pinned [spec.json](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/gold/spec.json) and [extended.json](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/gold/extended.json). This expansion rule identifies each assertion without duplicating the corpus or renaming tests. The three modes exercise inline frames, catalog frames and streamed batches against the same expected columns, types, values and row order. + +The API authority is [Query semantics](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L46-L57) and [Expr semantics](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L22-L48). Separately, expected results were generated with **Python Polars 2.1.0**, recorded in the corpus metadata and [generator invocation](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/generate_gold.py#L1-L32). No oracle was regenerated in this change. + +Comparison policy is pinned in [Compare.ts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/dataframe/oracle/Compare.ts#L1-L45): float relative tolerance `1e-9` plus `1e-300`, NaN equals NaN, and signed zeros compare equal. `group/small-ints` allows Int32 sums for i8/u8 instead of Polars Int64; values must still match. Temporal values compare via epoch integers. These are local comparison policies, not Arrow requirements. + +## PostgreSQL / PGlite assertions + +The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not `packages/sql/sql-pg/test`. Its public contract is [Sql.ts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45). Each title below is under `effect/dataframe on PostgreSQL (PGlite)`. This is PostgreSQL integration evidence, not SQL-standard conformance. The dependency and resolved version are reproducible from the pinned [package manifest](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/sql/pglite/package.json) and [pnpm lockfile](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/pnpm-lock.yaml). + +**DF-I-SQL/** (I) identifies the exact test named by each `cases` entry, executed by `it.effect(name, () => both(query))`. This includes generated temporal-cast and mixed-unit names: their literal templates and unit arrays in the [pinned suite](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/sql/pglite/test/DataFrameSql.test.ts#L169-L485) define the finite expansion. `both` compares in-memory results with PGlite, including types, length and ordered values (numeric tolerance `1e-9 * max(1, abs(actual))`). Shared compiler/API bugs can escape differential agreement; it is not an independent proof of the whole API. + +| ID | Kind | Exact test title | API authority | +| ------- | ----- | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-S001 | C / I | negative slices clamp both ends | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S002 | C / I | rounds decimals half to even, keeping the scale | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S003 | C / I | errors carry the database's message | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S004 | C / I | tables must satisfy their schema, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S005 | C / I | tables reject values their declared types can't hold exactly, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S006 | C / I | tables are checked on the rows the query reads, whatever the relation | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S007 | C / I | tables are read from their names, whatever the compiler names | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S008 | C / I | columns are one to PostgreSQL when their first 63 bytes are | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S009 | C / I | timestamps with time zones are UTC, whatever the session's time zone | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S010 | C / I | filters, derives and sorts | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S011 | C / I | groups in order of first appearance | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S012 | C / I | joins keep the left row order | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S013 | C / I | semi and anti joins, unique and slices | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S014 | C / I | window functions over partitions | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S015 | C / I | dates and times | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S016 | C / I | explain shows the SQL | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S017 | C / I | sums durations to durations, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S018 | C / I | rejects aggregates that durations and timestamps don't have, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S019 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S020 | C / I | decimals take the scales numeric does | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S021 | C / I | decimals meet integers, literals and floats as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S022 | C / I | isIn compares decimals by value as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S023 | C / I | isIn keeps exact candidates apart from fractions and takes long lists | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | + +DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. DF-S022 asserts the same for DF-Q055: `Expr.isIn` over decimals of scale 2 and -1 with `BigDecimal`, integer, float and mixed candidates, and over a timestamp and a duration with `DateTime` and `Duration` candidates, gives the same rows on PostgreSQL as in memory. Candidates of another kind than the column (a string against a decimal, a bigint against an Int32) are not asserted: the documentation of `Expr.isIn` and of `Sql` calls them not portable, and PostgreSQL converts or rejects them. Two comparisons of supported candidates are not asserted either, because they follow the database, as the same documentation says: a number that is not an integer next to a decimal of more digits than a float holds or outside a float's range, and a `DateTime` next to a zoned timestamp outside a UTC session. DF-S023 holds 9007199254740993, the first integer a float can't hold, in a decimal, an Int64 and a UInt64 column. PostgreSQL compares a list with one float in it as floats, so the compiler writes the candidates with a fraction in a list of their own, against the column cast to a float, and the integers as their digits: a fraction next to 9007199254740992 doesn't make that candidate match, a bigint outside 64 bits matches nothing instead of failing, and a UInt64 above the signed range is one of its value and not of the signed integer with the same bits. The last column runs 3000 `BigDecimal` candidates on both engines. + +## Compiler assertions + +`packages/effect/typetest/DataFrame.tst.ts` contains the following exact `describe` / `it` names. These are public API inference and rejection checks, not runtime or wire-format evidence. Each entry covers only its concrete expressions; it does not establish every overload or editor quick-info rendering. + +### `DataFrame` + +- `fromRows takes the row type from the schema` +- `fromRows, empty, validate and Query.table read missing values back as null` +- `validate and Query.table take Arrow types for the schema's fields` +- `validate and Query.table take an Arrow schema and give untyped rows` +- `column types its values by the frame's rows` +- `fromColumns infers column types` + +### `Query` + +- `from and table take the row type from the frame or schema` +- `is an Effect producing a frame of its rows` +- `filter keeps the row type` +- `select and withColumns compute the row type, nulls included` +- `pick, omit and rename` +- `groupBy and agg` +- `joins add the other side's columns; left joins make them nullable` +- `rejects mistakes` + +### `Expr` + +- `propagates nullability` +- `data-last forms` +- `durations sum to durations and have no mean` + +### `Windows and reshaping` + +- `over keeps the value type; window functions type their results` +- `pivot with declared columns is typed; without them rows are untyped` +- `unpivot names the variable and value columns` +- `explode and unnest` + +### `Types` + +- `dates, times and durations` +- `lists and structs` +- `decimals` + +### `Streaming` + +- `Query.stream emits frames of the row type` + +Rejection assertions include unknown Arrow override fields (`DataFrame`), unknown columns/group keys, incompatible comparisons/arithmetic, non-aggregate aggregation, aggregate predicates and duplicate non-key join columns (`Query` / `rejects mistakes`), exploding a non-list column (`Windows and reshaping` / `explode and unnest`), extracting a year from a number (`Types` / `dates, times and durations`), and the mean, median, deviation and rolling mean of a duration and the sum of a timestamp (`Expr` / `durations sum to durations and have no mean`). Other entries assert exact inferred types, including nullability, Effect errors and stream elements. + +## Exclusions and remaining gaps + +- This inventory does not assert Arrow IPC or FlatBuffers wire conformance; DataFrame Arrow tests cover the adapter API and same-library round trips only. Wire formats belong to their own layer inventories. +- Polars corpus exclusions remain as recorded in `generate_gold.py:DEVIATIONS`: integral float literals, float-to-string formatting, automatic join suffixes, and mixed bigint/number arithmetic. No claim of universal Polars equivalence. +- SQL intentionally excludes pivot/unpivot/explode/unnest and databases other than PostgreSQL. Overflow, division by zero, collation and sub-microsecond temporal behavior follow the documented SQL deviations, not unconditional parity. PGlite is not a live PostgreSQL server/version matrix. +- The callback timing contract now has a direct test (DF-Q002); it supplements, rather than replaces, lazy loading and per-run catalog tests. +- DF-Q042 controls the host monotonic clock and disables automatic scheduler yields, then verifies a started public query lets its parent resume and interrupt it before completion. It does not assert a yield interval, timeout behavior, fairness among many fibers, or interruption within an operation. +- DF-C021 covers buffer reuse only for the compatible numeric layouts described above. Sliced/multi-batch inputs, validity buffers, other layouts and mutation/ownership guarantees remain outside that assertion. +- DF-C022 and DF-Q043 cover the check kinds named in their tests (a refinement, literals, an optional field, an array of structs, a row check). The subset table above is documentation: its rows are not each asserted here. +- Dedicated coverage for public guards, every dual overload, and every malformed plan/IPC input is not established by this inventory. Those are gaps, not passing assertions. +- Compiler inference/rejection coverage is inventoried above and is not counted as runtime coverage. Exact diagnostic strings and explain/format layouts are local regression details even where a test also verifies a public failure or result contract. + +## Validation + +Run from the repository root: + +```sh +nix develop -c pnpm test --run packages/effect/test/dataframe/DataFrame.test.ts packages/effect/test/dataframe/Query.test.ts packages/sql/pglite/test/DataFrameSql.test.ts +nix develop -c pnpm test-types DataFrame +``` + +Keep existing regressions and oracle cases when updating the inventory. Do not turn an excluded case into a coverage claim without a focused assertion and an authoritative contract source. diff --git a/packages/effect/test/dataframe/oracle/Compare.ts b/packages/effect/test/dataframe/oracle/Compare.ts new file mode 100644 index 00000000000..fb31463e00f --- /dev/null +++ b/packages/effect/test/dataframe/oracle/Compare.ts @@ -0,0 +1,91 @@ +/** + * Compares a DataFrame with a gold (Polars) result: column names, column + * types and values, in row order. + * + * Floats compare with a relative tolerance of 1e-9 (summation order differs + * from Polars), NaN equals NaN, and -0 equals 0. + */ +import { BigDecimal, Effect } from "effect" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Gold from "./Gold.ts" +import * as Plan from "./Plan.ts" + +/** + * Deliberate differences from Polars, by case id: the column types + * effect/dataframe produces instead. Values still have to match. + */ +const deviations: { readonly [id: string]: { readonly [column: string]: string } } = { + // Sums of 8/16-bit integers are Int32 (Polars: Int64, which would be a bigint in TypeScript). + "group/small-ints": { i8: "i32", u8: "i32" } +} + +const close = (a: number, b: number): boolean => { + if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) && Number.isNaN(b) + if (a === b) return true + if (!Number.isFinite(a) || !Number.isFinite(b)) return false + return Math.abs(a - b) <= 1e-9 * Math.max(Math.abs(a), Math.abs(b)) + 1e-300 +} + +const same = (actual: unknown, expected: unknown): boolean => { + if (actual === null || expected === null) return actual === expected + if (BigDecimal.isBigDecimal(expected)) return BigDecimal.isBigDecimal(actual) && BigDecimal.equals(actual, expected) + if (Array.isArray(expected)) { + return Array.isArray(actual) && actual.length === expected.length && actual.every((a, i) => same(a, expected[i])) + } + if (typeof expected === "object" && typeof actual === "object") { + const keys = Object.keys(expected) + return keys.length === Object.keys(actual).length && + keys.every((k) => same((actual as any)[k] ?? null, (expected as any)[k])) + } + if (typeof actual === "number" && typeof expected === "number") return close(actual, expected) + if (typeof actual === "bigint" || typeof expected === "bigint") { + return typeof actual === typeof expected ? actual === expected : Number(actual) === Number(expected) + } + return actual === expected +} + +const show = (u: unknown): string => + BigDecimal.isBigDecimal(u) + ? `${BigDecimal.format(u)}d` + : typeof u === "bigint" + ? `${u}n` + : typeof u === "number" + ? (Object.is(u, -0) ? "-0" : String(u)) + : JSON.stringify(u) ?? String(u) + +/** `undefined` when equal, otherwise a description of the first difference. */ +export const diff = Effect.fnUntraced(function*(id: string, actual: DataFrame.DataFrame, expected: Gold.Table) { + const names = DataFrame.columnNames(actual) + const expectedNames = expected.fields.map((f) => f.name) + if (JSON.stringify(names) !== JSON.stringify(expectedNames)) { + return `columns: got ${JSON.stringify(names)}, expected ${JSON.stringify(expectedNames)}` + } + const deviation = deviations[id] + for (const [i, f] of expected.fields.entries()) { + const got = Plan.typeName(actual.schema.fields[i].type) + const want = deviation?.[f.name] ?? f.type + if (got !== want) return `column ${f.name}: type ${got}, expected ${want}` + } + const length = Gold.rowCount(expected) + if (actual.length !== length) return `length ${actual.length}, expected ${length}` + // Temporal columns compare through their integers (days or units since the epoch). + const rows = DataFrame.toRows( + yield* Query.from(actual).pipe(Query.select((c: any) => { + const out: Record> = {} + for (const f of expected.fields) { + const t = Plan.temporal[f.type] + out[f.name] = t === undefined ? c[f.name] : Expr.cast(c[f.name], t.storage) + } + return out + })) + ) + for (const f of expected.fields) { + const column = expected.columns[f.name] + for (let i = 0; i < length; i++) { + const want = Gold.decodeValue(column[i], f.type) + const got = rows[i][f.name] + if (!same(got, want)) return `column ${f.name}, row ${i}: got ${show(got)}, expected ${show(want)}` + } + } + return undefined +}) diff --git a/packages/effect/test/dataframe/oracle/Gold.ts b/packages/effect/test/dataframe/oracle/Gold.ts new file mode 100644 index 00000000000..0f1bee3c673 --- /dev/null +++ b/packages/effect/test/dataframe/oracle/Gold.ts @@ -0,0 +1,152 @@ +/** + * The gold corpus: plans and Polars results from + * test/fixtures/dataframe/generate_gold.py. + */ +import * as BigDecimal from "effect/BigDecimal" +import * as Fs from "node:fs" + +export type TypeName = + | "bool" + | "i8" + | "i16" + | "i32" + | "i64" + | "u8" + | "u16" + | "u32" + | "u64" + | "f32" + | "f64" + | "utf8" + | "date" + | "datetime[us]" + | "datetime[ms]" + | "duration[us]" + | "duration[ms]" + | `decimal[${string}]` + | `list<${string}>` + | `struct<${string}>` + +export interface Table { + readonly fields: ReadonlyArray<{ readonly name: string; readonly type: TypeName }> + readonly columns: { readonly [name: string]: ReadonlyArray } +} + +export const rowCount = (table: Table): number => Object.values(table.columns)[0]?.length ?? 0 + +export interface Call { + readonly fn: string + readonly args: ReadonlyArray + readonly [option: string]: unknown +} + +export type Expr = + | { readonly over: Expr; readonly partitionBy: ReadonlyArray } + | { readonly col: string } + | { readonly lit: unknown; readonly type?: TypeName } + | Call + | { readonly agg: string; readonly arg?: Expr } + +export type Op = + | { readonly op: "filter"; readonly predicate: Expr } + | { readonly op: "select" | "withColumns"; readonly exprs: { readonly [name: string]: Expr } } + | { readonly op: "pick" | "omit"; readonly columns: ReadonlyArray } + | { readonly op: "rename"; readonly mapping: { readonly [name: string]: string } } + | { + readonly op: "sort" + readonly keys: ReadonlyArray<{ readonly expr: Expr; readonly descending?: boolean }> + readonly nullsLast?: boolean + } + | { readonly op: "head" | "tail"; readonly n: number } + | { readonly op: "slice"; readonly offset: number; readonly length?: number } + | { readonly op: "groupBy"; readonly keys: ReadonlyArray; readonly aggs: { readonly [name: string]: Expr } } + | { readonly op: "agg"; readonly aggs: { readonly [name: string]: Expr } } + | { + readonly op: "join" + readonly right: string + readonly on: ReadonlyArray + readonly how: "inner" | "left" | "full" | "semi" | "anti" + } + | { readonly op: "unique" | "dropNulls"; readonly columns?: ReadonlyArray } + | { readonly op: "concat"; readonly with: string } + | { + readonly op: "pivot" + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: string + readonly columns?: ReadonlyArray + } + | { readonly op: "explode" | "unnest"; readonly column: string } + | { + readonly op: "unpivot" + readonly index: ReadonlyArray + readonly on: ReadonlyArray + readonly variable?: string + readonly value?: string + } + +export interface Case { + readonly id: string + readonly description: string + readonly inputs: { readonly [name: string]: Table } + readonly ops: ReadonlyArray + readonly expected: Table +} + +const load = (name: string): ReadonlyArray => + JSON.parse(Fs.readFileSync(new URL(`../../fixtures/dataframe/gold/${name}.json`, import.meta.url), "utf8")).cases + +/** The cases of gold/spec.json and gold/extended.json. */ +export const cases = (): ReadonlyArray => [...load("spec"), ...load("extended")] + +/** Splits `a:x,b:list` on top-level commas. */ +const splitTop = (s: string): Array => { + const parts: Array = [] + let depth = 0 + let current = "" + for (const ch of s) { + if (ch === "<") depth++ + if (ch === ">") depth-- + if (ch === "," && depth === 0) { + parts.push(current) + current = "" + } else current += ch + } + if (current !== "") parts.push(current) + return parts +} + +/** The fields of a `struct<...>` type name. */ +export const structFields = (type: string): Array => + splitTop(type.slice(7, -1)).map((f) => { + const at = f.indexOf(":") + return [f.slice(0, at), f.slice(at + 1) as TypeName] as const + }) + +/** Decodes a JSON-encoded gold value. */ +export const decodeValue = (value: unknown, type: TypeName): unknown => { + if (value === null) return null + if (type.startsWith("decimal[")) return BigDecimal.fromStringUnsafe(value as string) + if (type.startsWith("list<")) { + return (value as ReadonlyArray).map((v) => decodeValue(v, type.slice(5, -1) as TypeName)) + } + if (type.startsWith("struct<")) { + const out: Record = {} + for (const [name, t] of structFields(type)) out[name] = decodeValue((value as any)[name] ?? null, t) + return out + } + switch (type) { + case "f32": + case "f64": + if (typeof value === "string") { + return ({ NaN: NaN, Infinity: Infinity, "-Infinity": -Infinity, "-0": -0 } as Record)[value] + } + return value + case "i64": + case "u64": + return BigInt(value as string) + default: + return value + } +} diff --git a/packages/effect/test/dataframe/oracle/Plan.ts b/packages/effect/test/dataframe/oracle/Plan.ts new file mode 100644 index 00000000000..25f6f6c4e57 --- /dev/null +++ b/packages/effect/test/dataframe/oracle/Plan.ts @@ -0,0 +1,318 @@ +/** + * Gold plans as effect/dataframe plans, run through the public API. + */ +import { BigDecimal, Effect, Schema, Stream } from "effect" +import { ArrowSchema, ArrowTable } from "effect/arrow" +import { Catalog, DataFrame, Plan, Query } from "effect/dataframe" +import * as Result from "effect/Result" +import * as Gold from "./Gold.ts" + +const arrowTypes = { + bool: ArrowSchema.bool, + i8: ArrowSchema.int8, + i16: ArrowSchema.int16, + i32: ArrowSchema.int32, + i64: ArrowSchema.int64, + u8: ArrowSchema.uint8, + u16: ArrowSchema.uint16, + u32: ArrowSchema.uint32, + u64: ArrowSchema.uint64, + f32: ArrowSchema.float32, + f64: ArrowSchema.float64, + utf8: ArrowSchema.utf8, + date: ArrowSchema.date("day"), + "datetime[us]": ArrowSchema.timestamp("microsecond"), + "datetime[ms]": ArrowSchema.timestamp("millisecond"), + "duration[us]": ArrowSchema.duration("microsecond"), + "duration[ms]": ArrowSchema.duration("millisecond") +} + +/** Temporal gold types: their integer storage type, and the literal type. */ +export const temporal: Partial< + Record +> = { + date: { storage: "Int32", literal: { _tag: "Date" } }, + "datetime[us]": { storage: "Int64", literal: { _tag: "Timestamp", unit: "microsecond" } }, + "datetime[ms]": { storage: "Int64", literal: { _tag: "Timestamp", unit: "millisecond" } }, + "duration[us]": { storage: "Int64", literal: { _tag: "Duration", unit: "microsecond" } }, + "duration[ms]": { storage: "Int64", literal: { _tag: "Duration", unit: "millisecond" } } +} + +/** Precision and scale of a `decimal[p,s]` name. */ +const decimalOf = (name: string): readonly [number, number] => { + const [p, s] = name.slice(8, -1).split(",").map(Number) + return [p, s] +} + +const arrowType = (name: Gold.TypeName): ArrowSchema.DataType => { + if (name.startsWith("decimal[")) return ArrowSchema.decimal(...decimalOf(name)) + if (name.startsWith("list<")) { + return ArrowSchema.list(ArrowSchema.field("item", arrowType(name.slice(5, -1) as Gold.TypeName))) + } + if (name.startsWith("struct<")) { + return ArrowSchema.struct(Gold.structFields(name).map(([n, t]) => ArrowSchema.field(n, arrowType(t)))) + } + return (arrowTypes as Record)[name] +} + +/** The gold type name of an Arrow type. */ +export const typeName = (type: ArrowSchema.DataType): string => { + if (type._tag === "Decimal") return `decimal[${type.precision},${type.scale}]` + if (type._tag === "List" || type._tag === "LargeList") return `list<${typeName(type.child.type)}>` + if (type._tag === "Struct") return `struct<${type.fields.map((f) => `${f.name}:${typeName(f.type)}`).join(",")}>` + for (const [name, t] of Object.entries(arrowTypes)) { + if (JSON.stringify(t) === JSON.stringify(type)) return name + } + return type._tag +} + +/** The table column declaring a gold column of Arrow type `t`. */ +const tableColumn = (name: string, t: ArrowSchema.DataType): Plan.TableColumn => { + switch (t._tag) { + case "Int": + return { name, type: `${t.signed ? "Int" : "UInt"}${t.bitWidth}` as Plan.TypeName, nullable: true } + case "Float": + return { name, type: `Float${t.bitWidth}` as Plan.TypeName, nullable: true } + case "List": + return { name, type: "List", nullable: true, fields: [tableColumn("item", t.child.type)] } + case "Struct": + return { name, type: "Struct", nullable: true, fields: t.fields.map((f) => tableColumn(f.name, f.type)) } + case "Decimal": + return { name, type: "Decimal", nullable: true, precision: t.precision, scale: t.scale } + case "Timestamp": + case "Duration": + return { name, type: t._tag, nullable: true, unit: t.unit } + default: + return { name, type: t._tag as Plan.TypeName, nullable: true } + } +} + +/** A gold input table as a frame. */ +const frameOf = (table: Gold.Table): DataFrame.DataFrame => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make(table.fields.map((f) => ArrowSchema.field(f.name, arrowType(f.type)))), + table.fields.map((f) => table.columns[f.name].map((v) => Gold.decodeValue(v, f.type))) + ))) + +const castNames: Partial> = { + i8: "Int8", + i16: "Int16", + i32: "Int32", + i64: "Int64", + u8: "UInt8", + u16: "UInt16", + u32: "UInt32", + u64: "UInt64", + f32: "Float32", + f64: "Float64", + utf8: "Utf8", + bool: "Bool", + date: "Date", + "datetime[us]": "Timestamp[us]", + "datetime[ms]": "Timestamp[ms]", + "duration[us]": "Duration[us]", + "duration[ms]": "Duration[ms]" +} + +/** Gold function names that differ from the plan's. */ +const functionNames: Record = { + sub: "subtract", + mul: "multiply", + div: "divide", + floorDiv: "floorDivide", + neg: "negate", + strLength: "length", + strLower: "toLowerCase", + strUpper: "toUpperCase", + strContains: "includes", + strStartsWith: "startsWith", + strEndsWith: "endsWith", + strConcat: "concat" +} + +/** Gold option names that differ from the plan's. */ +const optionNames: Record = { prefix: "pattern", suffix: "pattern", field: "name" } + +const options = ({ args: _, fn, ...rest }: Gold.Call): Plan.CallOptions | undefined => { + if (fn === "cast") { + const to = rest.to as Gold.TypeName + if (!to.startsWith("decimal[")) return { to: castNames[to] } + const [precision, scale] = decimalOf(to) + return { to: "Decimal", precision, scale } + } + const entries = Object.entries(rest) + return entries.length === 0 ? undefined : Object.fromEntries(entries.map(([k, v]) => [optionNames[k] ?? k, v])) +} + +const aggregateNames: Record = { len: "count", nUnique: "countDistinct", var: "variance" } + +/** A gold expression as plan data. */ +const expression = (e: Gold.Expr): Plan.Expression => { + if ("over" in e) { + const o = e as { readonly over: Gold.Expr; readonly partitionBy: ReadonlyArray } + return { _tag: "Over", expression: expression(o.over), partitionBy: o.partitionBy.map(expression) } + } + if ("col" in e) return { _tag: "Column", name: e.col as string } + if ("lit" in e) { + const { lit, type } = e as { readonly lit: unknown; readonly type?: Gold.TypeName } + if (type === undefined) return { _tag: "Literal", value: lit as Plan.Value } + if (type.startsWith("decimal[")) { + const [precision, scale] = decimalOf(type) + const value = BigDecimal.scale(BigDecimal.fromStringUnsafe(lit as string), scale).value + return { _tag: "Literal", value, type: { _tag: "Decimal", precision, scale } } + } + const t = temporal[type] + return t === undefined + ? { _tag: "Literal", value: Gold.decodeValue(lit, type) as Plan.Value } + : { _tag: "Literal", value: lit as number, type: t.literal } + } + if ("agg" in e) { + const a = e as { readonly agg: string; readonly arg?: Gold.Expr } + return { + _tag: "AggregateCall", + fn: aggregateNames[a.agg] ?? a.agg as Plan.AggregateName, + arg: a.arg === undefined ? undefined : expression(a.arg) + } + } + const call = e as Gold.Call + return { + _tag: "Call", + fn: functionNames[call.fn] ?? call.fn as Plan.FunctionName, + args: call.args.map(expression), + options: options(call) + } +} + +const named = (record: { readonly [name: string]: Gold.Expr }): Array => + Object.entries(record).map(([name, e]) => ({ name, expression: expression(e) })) + +/** + * How a case's inputs reach the query: inline `Values` nodes, frames in the + * catalog, or streams of two-row batches in the catalog, declared with their + * columns. + */ +export type Source = "values" | "frames" | "batches" + +/** A case's operations as one plan over its inputs. */ +const plan = (gold: Gold.Case, source: Source): Plan.Node => { + const input = (name: string): Plan.Node => + source === "values" + ? { _tag: "Values", frame: frameOf(gold.inputs[name]) } + : source === "batches" + ? { _tag: "Table", name, columns: gold.inputs[name].fields.map((f) => tableColumn(f.name, arrowType(f.type))) } + : { _tag: "Table", name } + let node = input("left") + for (const op of gold.ops) { + switch (op.op) { + case "filter": + node = { _tag: "Filter", input: node, predicate: expression(op.predicate) } + break + case "select": + node = { _tag: "Select", input: node, columns: named(op.exprs) } + break + case "withColumns": + node = { _tag: "WithColumns", input: node, columns: named(op.exprs) } + break + case "pick": + node = { _tag: "Pick", input: node, columns: op.columns } + break + case "omit": + node = { _tag: "Omit", input: node, columns: op.columns } + break + case "rename": + node = { _tag: "Rename", input: node, mapping: op.mapping } + break + case "sort": + node = { + _tag: "Sort", + input: node, + keys: op.keys.map((k) => ({ expression: expression(k.expr), descending: k.descending ?? false })), + nullsLast: op.nullsLast ?? false + } + break + case "head": + node = { _tag: "Slice", input: node, offset: 0, length: op.n } + break + case "tail": + node = op.n === 0 + ? { _tag: "Slice", input: node, offset: 0, length: 0 } + : { _tag: "Slice", input: node, offset: -op.n } + break + case "slice": + node = { _tag: "Slice", input: node, offset: op.offset, length: op.length } + break + case "groupBy": + node = { _tag: "GroupBy", input: node, keys: op.keys, aggregates: named(op.aggs) } + break + case "agg": + node = { _tag: "Aggregate", input: node, aggregates: named(op.aggs) } + break + case "join": + node = { _tag: "Join", left: node, right: input(op.right), on: op.on, how: op.how } + break + case "unique": + node = { _tag: "Unique", input: node, columns: op.columns } + break + case "dropNulls": + node = { _tag: "DropNulls", input: node, columns: op.columns } + break + case "concat": + node = { _tag: "Concat", inputs: [node, input(op.with)] } + break + case "pivot": + node = { + _tag: "Pivot", + input: node, + on: op.on, + index: op.index, + values: op.values, + aggregate: op.aggregate as Plan.PivotAggregate, + columns: op.columns + } + break + case "explode": + node = { _tag: "Explode", input: node, column: op.column } + break + case "unnest": + node = { _tag: "Unnest", input: node, column: op.column } + break + case "unpivot": + node = { + _tag: "Unpivot", + input: node, + index: op.index, + on: op.on, + variable: op.variable ?? "variable", + value: op.value ?? "value" + } + break + } + } + return node +} + +const batchesOf = (table: Gold.Table): Stream.Stream> => + Stream.fromIterable(Array.from({ length: Math.ceil(Gold.rowCount(table) / 2) }, (_, b) => + frameOf({ + fields: table.fields, + columns: Object.fromEntries( + Object.entries(table.columns).map(([name, values]) => [name, values.slice(b * 2, b * 2 + 2)]) + ) + }))) + +/** + * Runs a case: its plan goes through its JSON form (so the codecs are part + * of what is checked) and then runs as a query. + */ +export const run = ( + gold: Gold.Case, + source: Source +): Effect.Effect, DataFrame.DataFrameError> => { + const node = plan(gold, source) + const json = JSON.parse(JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(node))) + const decoded = Schema.decodeUnknownSync(Plan.NodeFromJson)(json) + const tables = Object.fromEntries( + Object.entries(gold.inputs).map(([name, t]) => [name, source === "batches" ? batchesOf(t) : frameOf(t)]) + ) + return Query.fromPlan(decoded).pipe(Effect.provide(Catalog.layer(tables))) +} diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py new file mode 100644 index 00000000000..3c9ca7e0013 --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -0,0 +1,936 @@ +""" +Gold corpus for the effect/dataframe oracle. + +Every case is a small input table (or two), a pipeline written in a neutral +JSON plan language, and the result Python Polars produces for that pipeline. +The TypeScript oracle (test/dataframe/oracle) replays each plan through the +public effect/dataframe API and compares the result with the Polars output. + + uv run --with polars==2.1.0 python generate_gold.py # writes gold/spec.json and gold/extended.json + +Polars is the semantic reference. Cases avoid the differences listed in +DEVIATIONS; differences in result types alone are listed in oracle/Compare.ts. +""" + +import json +import math +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path + +import polars as pl + +HERE = Path(__file__).parent + +# Deliberate differences from Polars. Cases that would exercise them are not +# generated. +DEVIATIONS = [ + "JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.", + "Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').", + "Joins never add '_right' suffixes: overlapping non-key columns are a type error.", + "Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast.", +] + +# --------------------------------------------------------------------------- +# Types and value encoding +# --------------------------------------------------------------------------- + +DTYPES = { + "bool": pl.Boolean, + "i8": pl.Int8, + "i16": pl.Int16, + "i32": pl.Int32, + "i64": pl.Int64, + "u8": pl.UInt8, + "u16": pl.UInt16, + "u32": pl.UInt32, + "u64": pl.UInt64, + "f32": pl.Float32, + "f64": pl.Float64, + "utf8": pl.String, + "date": pl.Date, + "datetime[us]": pl.Datetime("us"), + "datetime[ms]": pl.Datetime("ms"), + "duration[us]": pl.Duration("us"), + "duration[ms]": pl.Duration("ms"), +} + +# Temporal columns travel as their epoch integers (days, or units since the +# epoch): exact, and independent of how either side represents them. +TEMPORAL = {"date": pl.Int32, "datetime[us]": pl.Int64, "datetime[ms]": pl.Int64, "duration[us]": pl.Int64, "duration[ms]": pl.Int64} + + +def split_top(s): + """Splits on commas outside angle brackets.""" + parts, depth, cur = [], 0, "" + for ch in s: + if ch == "<": + depth += 1 + elif ch == ">": + depth -= 1 + if ch == "," and depth == 0: + parts.append(cur) + cur = "" + else: + cur += ch + if cur: + parts.append(cur) + return parts + + +def struct_fields(t): + return [f.split(":", 1) for f in split_top(t[7:-1])] + + +def dtype_of(name): + if name.startswith("decimal["): + p, sc = name[8:-1].split(",") + return pl.Decimal(int(p), int(sc)) + if name.startswith("list<"): + return pl.List(dtype_of(name[5:-1])) + if name.startswith("struct<"): + return pl.Struct({n: dtype_of(ft) for n, ft in struct_fields(name)}) + return DTYPES[name] + + +def type_name(dtype): + if isinstance(dtype, pl.Decimal): + return f"decimal[{dtype.precision},{dtype.scale}]" + if isinstance(dtype, pl.List): + return f"list<{type_name(dtype.inner)}>" + if isinstance(dtype, pl.Struct): + return "struct<" + ",".join(f"{f.name}:{type_name(f.dtype)}" for f in dtype.fields) + ">" + for name, d in DTYPES.items(): + if dtype == d: + return name + raise ValueError(f"unsupported dtype {dtype}") + + +def encode_value(value, t): + if value is None: + return None + if t.startswith("decimal["): + return str(value) + if t.startswith("list<"): + return [encode_value(v, t[5:-1]) for v in value] + if t.startswith("struct<"): + return {n: encode_value(value.get(n), ft) for n, ft in struct_fields(t)} + if t in ("f32", "f64"): + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Infinity" if value > 0 else "-Infinity" + if value == 0 and math.copysign(1.0, value) < 0: + return "-0" + return value + if t in ("i64", "u64"): + return str(value) + return value + + +def decode_value(value, t): + if value is None: + return None + if t.startswith("decimal["): + return Decimal(value) + if t.startswith("list<"): + return [decode_value(v, t[5:-1]) for v in value] + if t.startswith("struct<"): + return {n: decode_value(value.get(n), ft) for n, ft in struct_fields(t)} + if t in ("f32", "f64"): + if isinstance(value, str): + return {"NaN": float("nan"), "Infinity": float("inf"), "-Infinity": float("-inf"), "-0": -0.0}[value] + return float(value) + if t in ("i64", "u64"): + return int(value) + return value + + +def table(fields, columns): + """fields: [(name, type)], columns: {name: [python values]}""" + return { + "fields": [{"name": n, "type": t} for n, t in fields], + "columns": {n: [encode_value(v, t) for v in columns[n]] for n, t in fields}, + } + + +def to_polars(t): + columns = [] + for f in t["fields"]: + values = [decode_value(v, f["type"]) for v in t["columns"][f["name"]]] + if f["type"] in TEMPORAL: + columns.append(pl.Series(f["name"], values, dtype=TEMPORAL[f["type"]]).cast(DTYPES[f["type"]])) + else: + columns.append(pl.Series(f["name"], values, dtype=dtype_of(f["type"]))) + return pl.DataFrame(columns) + + +def from_polars(df): + fields = [(name, type_name(dtype)) for name, dtype in df.schema.items()] + def values(name, t): + series = df.get_column(name) + return (series.cast(TEMPORAL[t]) if t in TEMPORAL else series).to_list() + return table(fields, {name: values(name, t) for name, t in fields}) + + +# --------------------------------------------------------------------------- +# Plan language -> Polars +# --------------------------------------------------------------------------- + + +def lit(value, t=None): + return {"lit": value} if t is None else {"lit": encode_value(value, t), "type": t} + + +def col(name): + return {"col": name} + + +def fn(name, *args, **params): + return {"fn": name, "args": list(args), **params} + + +def agg(name, arg=None): + return {"agg": name} if arg is None else {"agg": name, "arg": arg} + + +def p_lit(e): + t = e.get("type") + if t is None: + return pl.lit(e["lit"]) + value = decode_value(e["lit"], t) + if t in TEMPORAL: + return pl.lit(value, dtype=TEMPORAL[t]).cast(DTYPES[t]) + return pl.lit(value, dtype=dtype_of(t)) + + +def p_expr(e): + if "over" in e: + return p_expr(e["over"]).over([p_expr(k) for k in e["partitionBy"]]) + if "col" in e: + return pl.col(e["col"]) + if "lit" in e: + return p_lit(e) + if "agg" in e: + return p_agg(e) + f = e["fn"] + a = [p_expr(x) for x in e["args"]] + binary = { + "add": lambda: a[0] + a[1], + "sub": lambda: a[0] - a[1], + "mul": lambda: a[0] * a[1], + "div": lambda: a[0] / a[1], + "floorDiv": lambda: a[0] // a[1], + "mod": lambda: a[0] % a[1], + "eq": lambda: a[0] == a[1], + "neq": lambda: a[0] != a[1], + "lt": lambda: a[0] < a[1], + "lte": lambda: a[0] <= a[1], + "gt": lambda: a[0] > a[1], + "gte": lambda: a[0] >= a[1], + "and": lambda: a[0] & a[1], + "or": lambda: a[0] | a[1], + } + if f in binary: + return binary[f]() + if f == "neg": + return -a[0] + if f == "abs": + return a[0].abs() + if f == "not": + return ~a[0] + if f == "isNull": + return a[0].is_null() + if f == "isNotNull": + return a[0].is_not_null() + if f == "isNaN": + return a[0].is_nan() + if f == "fillNull": + return a[0].fill_null(a[1]) + if f == "coalesce": + return pl.coalesce(a) + if f == "when": + return pl.when(a[0]).then(a[1]).otherwise(a[2]) + if f == "cast": + return a[0].cast(dtype_of(e["to"]), strict=False) + if f == "isIn": + return a[0].is_in(e["values"]) + if f == "between": + return a[0].is_between(a[1], a[2]) + if f == "round": + return a[0].round(e.get("decimals", 0)) + if f == "sqrt": + return a[0].sqrt() + if f == "strLength": + return a[0].str.len_chars() + if f == "strLower": + return a[0].str.to_lowercase() + if f == "strUpper": + return a[0].str.to_uppercase() + if f == "strContains": + return a[0].str.contains(e["pattern"], literal=True) + if f == "strStartsWith": + return a[0].str.starts_with(e["prefix"]) + if f == "strEndsWith": + return a[0].str.ends_with(e["suffix"]) + if f == "strConcat": + return pl.concat_str(a) + window = { + "cumSum": lambda: a[0].cum_sum(), + "cumMin": lambda: a[0].cum_min(), + "cumMax": lambda: a[0].cum_max(), + "cumCount": lambda: a[0].cum_count(), + "shift": lambda: a[0].shift(e.get("n", 1)), + "diff": lambda: a[0].diff(e.get("n", 1)), + "rank": lambda: a[0].rank(method=e.get("method", "average"), descending=e.get("descending", False)), + "rollingSum": lambda: a[0].rolling_sum(e["window"], min_samples=e.get("minPeriods")), + "rollingMean": lambda: a[0].rolling_mean(e["window"], min_samples=e.get("minPeriods")), + "rollingMin": lambda: a[0].rolling_min(e["window"], min_samples=e.get("minPeriods")), + "rollingMax": lambda: a[0].rolling_max(e["window"], min_samples=e.get("minPeriods")), + } + if f in window: + return window[f]() + temporal = { + "year": lambda: a[0].dt.year(), + "month": lambda: a[0].dt.month(), + "day": lambda: a[0].dt.day(), + "hour": lambda: a[0].dt.hour(), + "minute": lambda: a[0].dt.minute(), + "second": lambda: a[0].dt.second(), + "millisecond": lambda: a[0].dt.millisecond(), + "weekday": lambda: a[0].dt.weekday(), + "ordinalDay": lambda: a[0].dt.ordinal_day(), + "truncate": lambda: a[0].dt.truncate(e["every"]), + "totalDays": lambda: a[0].dt.total_days(), + "totalHours": lambda: a[0].dt.total_hours(), + "totalMinutes": lambda: a[0].dt.total_minutes(), + "totalSeconds": lambda: a[0].dt.total_seconds(), + "totalMilliseconds": lambda: a[0].dt.total_milliseconds(), + } + if f in temporal: + return temporal[f]() + nested = { + "listLength": lambda: a[0].list.len(), + "listGet": lambda: a[0].list.get(e["index"], null_on_oob=True), + "listSum": lambda: a[0].list.sum(), + "listMean": lambda: a[0].list.mean(), + "listMin": lambda: a[0].list.min(), + "listMax": lambda: a[0].list.max(), + "listContains": lambda: a[0].list.contains(a[1]), + "field": lambda: a[0].struct.field(e["field"]), + "struct": lambda: pl.struct([x.alias(n) for x, n in zip(a, e["names"])]), + } + if f in nested: + return nested[f]() + raise ValueError(f"unknown function {f}") + + +def p_agg(e, grouped=False): + name = e["agg"] + if name == "len": + return pl.len() + x = p_expr(e["arg"]) + if name == "list": + return x if grouped else x.implode() + return { + "sum": x.sum, + "mean": x.mean, + "min": x.min, + "max": x.max, + "count": x.count, + "first": x.first, + "last": x.last, + "nUnique": x.n_unique, + "std": x.std, + "var": x.var, + "median": x.median, + }[name]() + + +def run(inputs, ops): + frames = {name: to_polars(t) for name, t in inputs.items()} + df = frames["left"] + for op in ops: + kind = op["op"] + if kind == "filter": + df = df.filter(p_expr(op["predicate"])) + elif kind == "select": + df = df.select([p_expr(e).alias(n) for n, e in op["exprs"].items()]) + elif kind == "withColumns": + df = df.with_columns([p_expr(e).alias(n) for n, e in op["exprs"].items()]) + elif kind == "pick": + df = df.select(op["columns"]) + elif kind == "omit": + df = df.drop(op["columns"]) + elif kind == "rename": + df = df.rename(op["mapping"]) + elif kind == "sort": + df = df.sort( + [p_expr(k["expr"]) for k in op["keys"]], + descending=[k.get("descending", False) for k in op["keys"]], + nulls_last=op.get("nullsLast", False), + maintain_order=True, + ) + elif kind == "head": + df = df.head(op["n"]) + elif kind == "tail": + df = df.tail(op["n"]) + elif kind == "slice": + df = df.slice(op["offset"], op.get("length")) + elif kind == "groupBy": + df = df.group_by(op["keys"], maintain_order=True).agg( + [p_agg(a, grouped=True).alias(n) for n, a in op["aggs"].items()] + ) + elif kind == "agg": + df = df.select([p_agg(a).alias(n) for n, a in op["aggs"].items()]) + elif kind == "join": + right = frames[op["right"]] + how = op["how"] + kw = {} + if how in ("inner", "left", "full"): + kw["maintain_order"] = "left_right" + if how == "full": + kw["coalesce"] = True + df = df.join(right, on=op["on"], how=how, **kw) + elif kind == "unique": + df = df.unique(subset=op.get("columns"), keep="first", maintain_order=True) + elif kind == "concat": + df = pl.concat([df, frames[op["with"]]], how="vertical") + elif kind == "dropNulls": + df = df.drop_nulls(subset=op.get("columns")) + elif kind == "pivot": + source = df + df = df.pivot(on=op["on"], index=op["index"], values=op["values"], aggregate_function=op["aggregate"], sort_columns=False) + if "columns" in op: + # Declared columns that never occur are cells without rows: + # the aggregate of an empty set (0 for sums and counts). + empty = source.clear().select( + p_agg(agg(op["aggregate"], col(op["values"])) if op["aggregate"] != "len" else agg("len")).alias("v") + ) + for c in op["columns"]: + if c not in df.columns: + df = df.with_columns(pl.lit(empty.item(0, 0), dtype=empty.schema["v"]).alias(c)) + df = df.select(op["index"] + op["columns"]) + elif kind == "explode": + df = df.explode(op["column"]) + elif kind == "unnest": + df = df.unnest(op["column"]) + elif kind == "unpivot": + df = df.unpivot(on=op["on"], index=op["index"], variable_name=op.get("variable", "variable"), value_name=op.get("value", "value")) + else: + raise ValueError(f"unknown op {kind}") + return df + + +def case(id, description, inputs, ops): + expected = run(inputs, ops) + return {"id": id, "description": description, "inputs": inputs, "ops": ops, "expected": from_polars(expected)} + + +# --------------------------------------------------------------------------- +# Hand-written specification cases +# --------------------------------------------------------------------------- + +NAN = float("nan") +INF = float("inf") + +readings = table( + [("station", "utf8"), ("city", "utf8"), ("temp", "f64"), ("rain", "f64"), ("count", "i32"), ("id", "i64")], + { + "station": ["s1", "s2", "s3", "s4", "s5", "s6", "s7"], + "city": ["Vienna", "Vienna", "Berlin", "Berlin", "Zürich", None, "Vienna"], + "temp": [21.5, 19.0, 24.0, 26.5, 17.0, NAN, None], + "rain": [0.0, None, 3.5, 1.0, 7.0, 2.0, 0.5], + "count": [3, 1, None, 7, 2, 5, 4], + "id": [10, 20, 30, 40, 9007199254740993, -1, 0], + }, +) +stations = table( + [("station", "utf8"), ("elevation", "f64"), ("active", "bool")], + {"station": ["s1", "s3", "s3", "s9", None], "elevation": [190.0, 34.0, 35.0, 500.0, 1.0], "active": [True, False, None, True, True]}, +) +mixed = table( + [("a", "f64"), ("b", "f64"), ("i", "i32"), ("j", "i32"), ("s", "utf8"), ("t", "utf8"), ("p", "bool"), ("q", "bool")], + { + "a": [1.0, -0.0, 0.0, NAN, INF, -INF, None, 2.5, -2.5, 1e300], + "b": [2.0, 0.0, -0.0, NAN, 1.0, None, 3.0, 0.0, 2.0, 1e300], + "i": [7, -7, 0, 2147483647, -2147483648, None, 3, -3, 100, 5], + "j": [2, 2, 3, 1, -1, 4, None, 2, -7, 0], + "s": ["b", "a", "", "é", "\U0001F600", "￿", None, "B", "ab", "a\u0000b"], + "t": ["b", "A", "x", "É", "😀", "z", "n", None, "AB", "a"], + "p": [True, True, True, False, False, False, None, None, None, True], + "q": [True, False, None, True, False, None, True, False, None, True], + }, +) +ints = table( + [("i8", "i8"), ("u8", "u8"), ("i16", "i16"), ("u16", "u16"), ("u32", "u32"), ("i64", "i64"), ("u64", "u64"), ("f32", "f32")], + { + "i8": [127, -128, 0, None, 5], + "u8": [255, 0, 1, None, 5], + "i16": [32767, -32768, 0, 12, None], + "u16": [65535, 0, 7, 12, None], + "u32": [4294967295, 0, 1, None, 3], + "i64": [9223372036854775807, -9223372036854775808, 0, None, 42], + "u64": [18446744073709551615, 0, 1, None, 42], + "f32": [1.5, -0.25, None, 3.0, 1e10], + }, +) +empty = table([("k", "utf8"), ("v", "f64")], {"k": [], "v": []}) +allnull = table([("k", "utf8"), ("v", "f64"), ("i", "i32")], {"k": ["x", "x", "y"], "v": [None, None, 1.0], "i": [None, None, None]}) + + +def spec_cases(): + R = {"left": readings} + RS = {"left": readings, "right": stations} + M = {"left": mixed} + out = [] + add = lambda *a: out.append(case(*a)) + + # filter + add("filter/gt", "numeric comparison with a literal", R, [{"op": "filter", "predicate": fn("gt", col("temp"), lit(20.0))}]) + add("filter/null-is-false", "rows where the predicate is null are dropped", R, [{"op": "filter", "predicate": fn("gt", col("rain"), lit(0.5))}]) + add("filter/nan-total-order", "NaN compares greater than every number and equal to itself", M, + [{"op": "filter", "predicate": fn("gte", col("a"), col("b"))}]) + add("filter/string-eq", "string equality", R, [{"op": "filter", "predicate": fn("eq", col("city"), lit("Vienna"))}]) + add("filter/string-order", "strings compare by code point (UTF-8 byte order)", M, + [{"op": "filter", "predicate": fn("lt", col("s"), col("t"))}]) + add("filter/kleene", "and/or use three-valued logic", M, + [{"op": "select", "exprs": {"and": fn("and", col("p"), col("q")), "or": fn("or", col("p"), col("q")), "not": fn("not", col("p"))}}]) + add("filter/is-null", "isNull and isNotNull", R, + [{"op": "filter", "predicate": fn("or", fn("isNull", col("city")), fn("isNull", col("temp")))}]) + add("filter/is-in", "membership in a literal list", R, + [{"op": "filter", "predicate": fn("isIn", col("city"), values=["Berlin", "Zürich"])}]) + add("filter/between", "inclusive range", R, + [{"op": "filter", "predicate": fn("between", col("temp"), lit(19.0), lit(24.0))}]) + add("filter/compound", "compound predicate", R, + [{"op": "filter", "predicate": fn("and", fn("gt", col("count"), lit(1)), fn("neq", col("city"), lit("Berlin")))}]) + add("filter/i64", "64-bit integer comparison beyond 2^53", R, + [{"op": "filter", "predicate": fn("gt", col("id"), lit(9007199254740992, "i64"))}]) + add("filter/empty-result", "a predicate that matches nothing", R, + [{"op": "filter", "predicate": fn("gt", col("temp"), lit(1000.0))}]) + add("filter/all-null-literal", "a null literal predicate drops everything", R, + [{"op": "filter", "predicate": lit(None, "bool")}]) + + # arithmetic + add("arith/float", "float arithmetic with nulls, NaN and infinities", M, + [{"op": "select", "exprs": {"add": fn("add", col("a"), col("b")), "sub": fn("sub", col("a"), col("b")), + "mul": fn("mul", col("a"), col("b")), "div": fn("div", col("a"), col("b"))}}]) + add("arith/int32-wrap", "Int32 arithmetic wraps on overflow", M, + [{"op": "select", "exprs": {"add": fn("add", col("i"), col("j")), "sub": fn("sub", col("i"), col("j")), + "mul": fn("mul", col("i"), col("j")), "neg": fn("neg", col("i"))}}]) + add("arith/int32-div", "integer true division gives Float64; division by zero gives infinity", M, + [{"op": "select", "exprs": {"div": fn("div", col("i"), col("j"))}}]) + add("arith/int32-floor", "floor division and floored modulo", M, + [{"op": "filter", "predicate": fn("neq", col("j"), lit(0))}, + {"op": "select", "exprs": {"floor": fn("floorDiv", col("i"), col("j")), "mod": fn("mod", col("i"), col("j"))}}]) + add("arith/float-floor", "floor division and floored modulo on floats", M, + [{"op": "select", "exprs": {"floor": fn("floorDiv", col("a"), col("b")), "mod": fn("mod", col("a"), col("b"))}}]) + add("arith/mixed", "Int32 with a float literal widens to Float64", M, + [{"op": "select", "exprs": {"x": fn("mul", col("i"), lit(1.5)), "y": fn("add", col("i"), col("a"))}}]) + add("arith/int-literal", "an integer literal adopts the integer column type", M, + [{"op": "select", "exprs": {"x": fn("add", col("i"), lit(1)), "y": fn("mul", col("j"), lit(3))}}]) + add("arith/abs-sqrt", "abs and sqrt", M, + [{"op": "select", "exprs": {"abs": fn("abs", col("a")), "iabs": fn("abs", col("j")), "sqrt": fn("sqrt", col("b"))}}]) + add("arith/i64", "Int64 arithmetic", R, + [{"op": "select", "exprs": {"x": fn("add", col("id"), lit(1, "i64")), "y": fn("mul", col("id"), lit(2, "i64")), + "z": fn("sub", col("id"), col("id"))}}]) + add("arith/round", "round half to even", {"left": table([("x", "f64")], {"x": [0.5, 1.5, 2.5, -0.5, -1.5, 1.25, 2.675, None, NAN]})}, + [{"op": "select", "exprs": {"r0": fn("round", col("x")), "r1": fn("round", col("x"), decimals=1)}}]) + add("arith/small-ints", "8/16-bit arithmetic wraps in the column type", {"left": ints}, + [{"op": "select", "exprs": {"i8": fn("add", col("i8"), lit(1)), "u8": fn("add", col("u8"), lit(1)), + "i16": fn("mul", col("i16"), lit(2)), "u16": fn("sub", col("u16"), lit(1)), + "u32": fn("add", col("u32"), lit(1)), "f32": fn("mul", col("f32"), lit(2.0))}}]) + + # null handling + add("null/fill", "fillNull with a literal", R, + [{"op": "select", "exprs": {"city": fn("fillNull", col("city"), lit("?")), "temp": fn("fillNull", col("temp"), lit(0.0))}}]) + add("null/coalesce", "first non-null", M, + [{"op": "select", "exprs": {"x": fn("coalesce", col("a"), col("b"), lit(-1.0))}}]) + add("null/when", "when/then/otherwise", R, + [{"op": "select", "exprs": {"label": fn("when", fn("gt", col("temp"), lit(20.0)), lit("warm"), lit("cold"))}}]) + add("null/drop", "drop rows with nulls", R, [{"op": "dropNulls"}]) + add("null/drop-subset", "drop rows with nulls in some columns", R, [{"op": "dropNulls", "columns": ["city"]}]) + add("null/is-nan", "isNaN is distinct from isNull", M, [{"op": "select", "exprs": {"nan": fn("isNaN", col("a"))}}]) + + # casts + add("cast/float-int", "float to Int32 truncates; out of range is null", M, + [{"op": "select", "exprs": {"i": fn("cast", col("a"), to="i32"), "f": fn("cast", col("i"), to="f64")}}]) + add("cast/int-string", "integers to strings", M, [{"op": "select", "exprs": {"s": fn("cast", col("i"), to="utf8")}}]) + add("cast/string-number", "strings to numbers; unparsable is null", + {"left": table([("s", "utf8")], {"s": ["1", "-2", "3.5", "x", "", None, " 4", "1e3"]})}, + [{"op": "select", "exprs": {"f": fn("cast", col("s"), to="f64"), "i": fn("cast", col("s"), to="i32")}}]) + add("cast/bool", "booleans to and from numbers", M, + [{"op": "select", "exprs": {"b": fn("cast", col("j"), to="bool"), "n": fn("cast", col("p"), to="i32")}}]) + add("cast/i64", "Int64 to and from other types", R, + [{"op": "select", "exprs": {"f": fn("cast", col("id"), to="f64"), "i": fn("cast", col("count"), to="i64"), + "s": fn("cast", col("id"), to="utf8")}}]) + + # strings + add("str/length", "length in characters", M, [{"op": "select", "exprs": {"n": fn("strLength", col("s"))}}]) + add("str/case", "upper and lower case", M, + [{"op": "select", "exprs": {"u": fn("strUpper", col("s")), "l": fn("strLower", col("t"))}}]) + add("str/match", "contains, startsWith, endsWith with literals", M, + [{"op": "select", "exprs": {"c": fn("strContains", col("s"), pattern="a"), "p": fn("strStartsWith", col("t"), prefix="A"), + "e": fn("strEndsWith", col("s"), suffix="b")}}]) + add("str/concat", "concatenation; null in any argument gives null", M, + [{"op": "select", "exprs": {"c": fn("strConcat", col("s"), lit("-"), col("t"))}}]) + + # projection + add("project/pick", "pick columns in the given order", R, [{"op": "pick", "columns": ["temp", "station"]}]) + add("project/omit", "omit columns", R, [{"op": "omit", "columns": ["rain", "id"]}]) + add("project/rename", "rename columns", R, [{"op": "rename", "mapping": {"temp": "celsius", "city": "town"}}]) + add("project/with-replace", "withColumns replaces an existing column in place", R, + [{"op": "withColumns", "exprs": {"temp": fn("mul", col("temp"), lit(2.0)), "f": fn("add", fn("mul", col("temp"), lit(1.8)), lit(32.0))}}]) + add("project/literal-column", "a literal broadcasts to every row", R, + [{"op": "select", "exprs": {"station": col("station"), "half": lit(0.5), "one": lit(1), "tag": lit("x")}}]) + + # sorting and slicing + add("sort/asc", "ascending, nulls first, NaN last among numbers", R, [{"op": "sort", "keys": [{"expr": col("temp")}]}]) + add("sort/desc-nulls-last", "descending with nulls last", R, + [{"op": "sort", "keys": [{"expr": col("temp"), "descending": True}], "nullsLast": True}]) + add("sort/multi", "multi-key sort is stable", R, + [{"op": "sort", "keys": [{"expr": col("city")}, {"expr": col("count"), "descending": True}]}]) + add("sort/strings", "string sort by code point", M, [{"op": "sort", "keys": [{"expr": col("s")}]}]) + add("sort/zeros", "-0 and 0 sort as equal (stable)", M, [{"op": "sort", "keys": [{"expr": col("a")}]}]) + add("sort/i64", "Int64 sort", R, [{"op": "sort", "keys": [{"expr": col("id")}]}]) + add("sort/expr", "sort by an expression", R, [{"op": "sort", "keys": [{"expr": fn("neg", col("count"))}]}]) + add("sort/bool", "boolean sort", M, [{"op": "sort", "keys": [{"expr": col("p")}, {"expr": col("q"), "descending": True}]}]) + add("slice/head", "first rows", R, [{"op": "head", "n": 3}]) + add("slice/tail", "last rows", R, [{"op": "tail", "n": 2}]) + add("slice/offset", "slice in the middle", R, [{"op": "slice", "offset": 2, "length": 3}]) + add("slice/after-filter", "slicing after a filter", R, + [{"op": "filter", "predicate": fn("isNotNull", col("city"))}, {"op": "slice", "offset": 1, "length": 2}]) + + # aggregation + add("agg/global", "aggregates over the whole frame", R, + [{"op": "agg", "aggs": {"n": agg("len"), "c": agg("count", col("temp")), "sum": agg("sum", col("rain")), + "mean": agg("mean", col("rain")), "min": agg("min", col("count")), "max": agg("max", col("count")), + "first": agg("first", col("city")), "last": agg("last", col("city"))}}]) + add("agg/nan", "sum includes NaN; min and max skip it", R, + [{"op": "agg", "aggs": {"sum": agg("sum", col("temp")), "min": agg("min", col("temp")), "max": agg("max", col("temp"))}}]) + add("agg/stats", "std, var and median", R, + [{"op": "agg", "aggs": {"std": agg("std", col("rain")), "var": agg("var", col("rain")), "median": agg("median", col("rain")), + "imed": agg("median", col("count"))}}]) + add("agg/strings", "min, max and nUnique on strings", R, + [{"op": "agg", "aggs": {"min": agg("min", col("city")), "max": agg("max", col("city")), "n": agg("nUnique", col("city"))}}]) + add("agg/empty", "aggregates over an empty frame", {"left": empty}, + [{"op": "agg", "aggs": {"n": agg("len"), "sum": agg("sum", col("v")), "mean": agg("mean", col("v")), "min": agg("min", col("v"))}}]) + add("group/basic", "group by a string key in first-appearance order", R, + [{"op": "groupBy", "keys": ["city"], "aggs": {"n": agg("len"), "temp": agg("mean", col("temp")), "rain": agg("sum", col("rain"))}}]) + add("group/null-key", "null is its own group", R, + [{"op": "groupBy", "keys": ["city"], "aggs": {"stations": agg("count", col("station"))}}]) + add("group/multi-key", "group by two keys", R, + [{"op": "withColumns", "exprs": {"wet": fn("gt", col("rain"), lit(1.0))}}, + {"op": "groupBy", "keys": ["city", "wet"], "aggs": {"n": agg("len"), "max": agg("max", col("temp"))}}]) + add("group/all-null", "aggregates over all-null groups", {"left": allnull}, + [{"op": "groupBy", "keys": ["k"], "aggs": {"sum": agg("sum", col("v")), "mean": agg("mean", col("v")), "count": agg("count", col("v")), + "n": agg("len"), "min": agg("min", col("v")), "std": agg("std", col("v")), + "nu": agg("nUnique", col("v")), "first": agg("first", col("v")), "isum": agg("sum", col("i")), + "imax": agg("max", col("i"))}}]) + add("group/float-key", "NaN, -0 and 0 as group keys", M, + [{"op": "groupBy", "keys": ["a"], "aggs": {"n": agg("len")}}]) + add("group/i64-key", "Int64 group keys and sums", R, + [{"op": "withColumns", "exprs": {"k": fn("mod", col("id"), lit(3, "i64"))}}, + {"op": "groupBy", "keys": ["k"], "aggs": {"n": agg("len"), "s": agg("sum", col("id")), "mx": agg("max", col("id"))}}]) + add("group/bool-key", "boolean group keys", M, [{"op": "groupBy", "keys": ["p"], "aggs": {"n": agg("len"), "s": agg("sum", col("i"))}}]) + add("group/int-sum", "Int32 sums stay Int32; means are Float64", M, + [{"op": "groupBy", "keys": ["p"], "aggs": {"s": agg("sum", col("j")), "m": agg("mean", col("j")), "mn": agg("min", col("j"))}}]) + add("group/after-filter", "group a filtered frame", R, + [{"op": "filter", "predicate": fn("gt", col("rain"), lit(0.0))}, + {"op": "groupBy", "keys": ["city"], "aggs": {"n": agg("len"), "last": agg("last", col("station"))}}]) + add("group/expression-agg", "aggregate of an expression", R, + [{"op": "groupBy", "keys": ["city"], "aggs": {"x": agg("sum", fn("mul", col("rain"), lit(2.0)))}}]) + add("group/empty", "grouping an empty frame", {"left": empty}, [{"op": "groupBy", "keys": ["k"], "aggs": {"n": agg("len")}}]) + add("group/small-ints", "sums of small integer types", {"left": ints}, + [{"op": "agg", "aggs": {"i8": agg("sum", col("i8")), "u8": agg("sum", col("u8")), "u32": agg("sum", col("u32")), + "i64": agg("sum", col("i64")), "u64": agg("max", col("u64")), "f32": agg("mean", col("f32"))}}]) + + # joins + for how in ("inner", "left", "full", "semi", "anti"): + add(f"join/{how}", f"{how} join with duplicate and null keys", RS, + [{"op": "pick", "columns": ["station", "temp"]}, {"op": "join", "right": "right", "on": ["station"], "how": how}]) + add("join/multi-key", "join on two keys", + {"left": table([("a", "i32"), ("b", "utf8"), ("x", "f64")], {"a": [1, 1, 2, 2, None], "b": ["x", "y", "x", None, "x"], "x": [1.0, 2.0, 3.0, 4.0, 5.0]}), + "right": table([("a", "i32"), ("b", "utf8"), ("y", "bool")], {"a": [1, 2, 2, 1], "b": ["y", "x", "x", "z"], "y": [True, False, True, None]})}, + [{"op": "join", "right": "right", "on": ["a", "b"], "how": "left"}]) + add("join/i64-key", "join on Int64 keys", + {"left": table([("k", "i64"), ("x", "f64")], {"k": [9007199254740993, 9007199254740992, 1], "x": [1.0, 2.0, 3.0]}), + "right": table([("k", "i64"), ("y", "utf8")], {"k": [9007199254740993, 1, 1], "y": ["a", "b", "c"]})}, + [{"op": "join", "right": "right", "on": ["k"], "how": "inner"}]) + add("join/f64-key", "join on Float64 keys: NaN matches NaN, -0 matches 0", + {"left": table([("k", "f64"), ("x", "i32")], {"k": [NAN, -0.0, 1.5, None], "x": [1, 2, 3, 4]}), + "right": table([("k", "f64"), ("y", "i32")], {"k": [0.0, NAN, None, 1.5], "y": [10, 20, 30, 40]})}, + [{"op": "join", "right": "right", "on": ["k"], "how": "inner"}]) + + # unique / concat + add("unique/all", "distinct rows, keeping the first", {"left": table([("a", "i32"), ("b", "utf8")], {"a": [1, 1, 2, None, None, 1], "b": ["x", "x", "y", None, None, "y"]})}, + [{"op": "unique"}]) + add("unique/subset", "distinct by a subset", R, [{"op": "unique", "columns": ["city"]}]) + add("concat/vertical", "stack frames with the same schema", {"left": readings, "right": readings}, + [{"op": "concat", "with": "right"}, {"op": "pick", "columns": ["station", "temp"]}]) + + # pipelines + add("pipeline/summary", "filter, derive, group, sort", R, + [{"op": "filter", "predicate": fn("gt", col("temp"), lit(18.0))}, + {"op": "withColumns", "exprs": {"tempF": fn("add", fn("mul", col("temp"), lit(1.8)), lit(32.0))}}, + {"op": "groupBy", "keys": ["city"], "aggs": {"stations": agg("len"), "avgF": agg("mean", col("tempF")), "maxRain": agg("max", col("rain"))}}, + {"op": "sort", "keys": [{"expr": col("avgF"), "descending": True}]}]) + add("pipeline/join-group", "join then aggregate the joined columns", RS, + [{"op": "join", "right": "right", "on": ["station"], "how": "left"}, + {"op": "groupBy", "keys": ["active"], "aggs": {"n": agg("len"), "elev": agg("mean", col("elevation"))}}, + {"op": "sort", "keys": [{"expr": col("active")}]}]) + return out + + +# --------------------------------------------------------------------------- +# Extended cases: window functions, pivots, more types +# --------------------------------------------------------------------------- + +series = table( + [("g", "utf8"), ("t", "i32"), ("x", "f64"), ("i", "i32"), ("b", "bool")], + { + "g": ["a", "b", "a", "a", "b", None, "a", "b", "b", "a"], + "t": [1, 1, 2, 3, 2, 1, 4, 3, 4, 5], + "x": [1.5, 2.0, None, 4.0, NAN, 3.0, -1.0, 2.0, 0.5, 2.0], + "i": [3, 1, 4, 1, 5, 9, None, 6, 5, 3], + "b": [True, False, None, True, True, False, True, None, False, True], + }, +) + +EPOCH = datetime(1970, 1, 1) + + +def _us(dt): + return int((dt - EPOCH).total_seconds() * 1_000_000) if dt is not None else None + + +def _days(d): + return (d - date(1970, 1, 1)).days if d is not None else None + + +times = table( + [("k", "utf8"), ("ts", "datetime[us]"), ("ms", "datetime[ms]"), ("d", "date"), ("dur", "duration[us]")], + { + "k": ["a", "b", "c", "d", "e", "f"], + "ts": [_us(datetime(2024, 2, 29, 13, 45, 7, 123456)), _us(datetime(2023, 12, 31, 23, 59, 59, 999000)), _us(datetime(2024, 3, 4, 0, 0, 0)), + None, _us(datetime(1969, 7, 20, 20, 17, 40)), _us(datetime(2024, 3, 10, 6, 30, 0, 500))], + "ms": [_us(datetime(2024, 2, 29, 13, 45, 7, 123000)) // 1000, _us(datetime(2024, 1, 1)) // 1000, None, + _us(datetime(2024, 3, 1)) // 1000, _us(datetime(1970, 1, 1)) // 1000, _us(datetime(2024, 3, 9)) // 1000], + "d": [_days(date(2024, 2, 29)), _days(date(2023, 12, 31)), _days(date(2024, 3, 4)), _days(date(2024, 2, 29)), None, _days(date(2024, 3, 10))], + "dur": [90 * 60 * 1_000_000, -1_500_000, None, 3 * 86_400_000_000 + 1, 0, 25 * 3_600_000_000], + }, +) +holidays = table( + [("d", "date"), ("name", "utf8")], + {"d": [_days(date(2024, 2, 29)), _days(date(2024, 3, 10)), _days(date(2025, 1, 1))], "name": ["leap", "sunday", "new year"]}, +) + +nested = table( + [("k", "utf8"), ("g", "i32"), ("xs", "list"), ("ws", "list"), ("s", "struct")], + { + "k": ["a", "b", "c", "d", "e"], + "g": [1, 2, 1, 2, 1], + "xs": [[1, 2, 3], [], None, [4, None, 2], [7]], + "ws": [["a", "b"], ["c"], [], None, ["z", "a"]], + "s": [{"a": 1, "b": "x"}, {"a": None, "b": "y"}, None, {"a": 4, "b": None}, {"a": 5, "b": "z"}], + }, +) +nested_right = table( + [("k", "utf8"), ("tags", "list")], + {"k": ["a", "d", "x"], "tags": [["t1"], ["t2", "t3"], []]}, +) + +money = table( + [("k", "utf8"), ("price", "decimal[10,2]"), ("fee", "decimal[6,3]"), ("qty", "decimal[4,0]"), ("x", "f64")], + { + "k": ["a", "b", "a", "c", "b", "a"], + "price": ["12.34", "0.10", "-5.00", None, "99999999.99", "10.01"], + "fee": ["0.125", "1.000", None, "2.500", "-0.001", "10.010"], + "qty": ["3", "10", "2", "1", None, "7"], + "x": [1.005, -2.5, 123.456, None, 1e6, 0.0], + }, +) + +sales = table( + [("region", "utf8"), ("quarter", "utf8"), ("units", "i32"), ("price", "f64")], + { + "region": ["north", "north", "south", "south", "north", "east", "south", "north"], + "quarter": ["q1", "q2", "q1", "q3", "q1", "q2", "q1", None], + "units": [10, 20, 5, None, 7, 3, 8, 1], + "price": [1.5, 2.0, 1.0, 3.0, None, 2.5, 1.0, 9.0], + }, +) + + +def extended_cases(): + S = {"left": series} + P = {"left": sales} + out = [] + add = lambda *a: out.append(case(*a)) + over = lambda e, *keys: {"over": e, "partitionBy": [col(k) for k in keys]} + + # window functions over the whole frame + add("window/cum", "cumulative sum, min, max and count; nulls stay null", S, + [{"op": "select", "exprs": {"sx": fn("cumSum", col("x")), "si": fn("cumSum", col("i")), "mn": fn("cumMin", col("i")), + "mx": fn("cumMax", col("x")), "n": fn("cumCount", col("x")), "sb": fn("cumSum", col("b"))}}]) + add("window/shift-diff", "shift (lag and lead) and diff", S, + [{"op": "select", "exprs": {"lag": fn("shift", col("i")), "lead": fn("shift", col("x"), n=-2), "d": fn("diff", col("i")), + "d2": fn("diff", col("x"), n=2), "ls": fn("shift", col("g"))}}]) + add("window/rank", "rank methods, nulls and NaN", S, + [{"op": "select", "exprs": {"avg": fn("rank", col("x")), "min": fn("rank", col("i"), method="min"), + "max": fn("rank", col("i"), method="max"), "dense": fn("rank", col("i"), method="dense", descending=True), + "ord": fn("rank", col("x"), method="ordinal"), "s": fn("rank", col("g"), method="dense")}}]) + add("window/rolling", "rolling sum, mean, min and max", S, + [{"op": "select", "exprs": {"s": fn("rollingSum", col("x"), window=3), "m": fn("rollingMean", col("i"), window=2), + "mn": fn("rollingMin", col("x"), window=3, minPeriods=1), "mx": fn("rollingMax", col("i"), window=4, minPeriods=2)}}]) + # window functions per partition + add("window/over-agg", "aggregates broadcast over partitions", S, + [{"op": "withColumns", "exprs": {"total": over(agg("sum", col("x")), "g"), "n": over(agg("len"), "g"), + "mean": over(agg("mean", col("i")), "g"), "first": over(agg("first", col("t")), "g", "b")}}]) + add("window/over-cum", "cumulative and shifted values per partition, in row order", S, + [{"op": "withColumns", "exprs": {"cs": over(fn("cumSum", col("i")), "g"), "prev": over(fn("shift", col("x")), "g"), + "d": over(fn("diff", col("t")), "g"), "r": over(fn("rank", col("x"), method="min"), "g"), + "roll": over(fn("rollingSum", col("i"), window=2, minPeriods=1), "g")}}]) + add("window/over-filter", "filters after windows see the windowed values", S, + [{"op": "withColumns", "exprs": {"cs": over(fn("cumSum", col("t")), "g")}}, + {"op": "filter", "predicate": fn("gt", col("t"), lit(2))}]) + add("window/filter-first", "windows after a filter only see the remaining rows", S, + [{"op": "filter", "predicate": fn("isNotNull", col("x"))}, + {"op": "withColumns", "exprs": {"cs": fn("cumSum", col("x")), "r": over(fn("rank", col("x")), "g")}}, + {"op": "head", "n": 4}]) + + # pivots + add("pivot/sum", "pivot with a sum per cell; missing cells are null", P, + [{"op": "pivot", "on": "quarter", "index": ["region"], "values": "units", "aggregate": "sum"}]) + add("pivot/first", "pivot with the first value per cell", P, + [{"op": "pivot", "on": "region", "index": ["quarter"], "values": "price", "aggregate": "first"}]) + add("pivot/len", "pivot counting rows per cell", P, + [{"op": "pivot", "on": "quarter", "index": ["region"], "values": "units", "aggregate": "len"}]) + add("pivot/columns", "pivot with declared output columns", P, + [{"op": "pivot", "on": "quarter", "index": ["region"], "values": "price", "aggregate": "max", "columns": ["q1", "q2", "q4"]}]) + add("pivot/numeric-on", "pivot on a numeric column names columns by value", S, + [{"op": "filter", "predicate": fn("isNotNull", col("g"))}, + {"op": "pivot", "on": "t", "index": ["g"], "values": "i", "aggregate": "min"}]) + add("unpivot/basic", "unpivot stacks columns, column by column", P, + [{"op": "unpivot", "index": ["region"], "on": ["units", "price"]}]) + add("unpivot/names", "unpivot with named variable and value columns", P, + [{"op": "pick", "columns": ["region", "quarter"]}, + {"op": "unpivot", "index": [], "on": ["region", "quarter"], "variable": "field", "value": "text"}]) + T = {"left": times, "right": holidays} + add("temporal/extract", "calendar fields of timestamps and dates", T, + [{"op": "select", "exprs": {"y": fn("year", col("ts")), "mo": fn("month", col("ts")), "d": fn("day", col("ts")), + "h": fn("hour", col("ts")), "mi": fn("minute", col("ts")), "s": fn("second", col("ts")), + "ms": fn("millisecond", col("ts")), "wd": fn("weekday", col("ts")), "od": fn("ordinalDay", col("ts")), + "dy": fn("year", col("d")), "dwd": fn("weekday", col("d")), "dod": fn("ordinalDay", col("d"))}}]) + add("temporal/arith", "timestamp differences are durations; durations shift timestamps", T, + [{"op": "select", "exprs": {"gap": fn("sub", col("ts"), col("ms")), "later": fn("add", col("ts"), col("dur")), + "earlier": fn("sub", col("ts"), col("dur")), "days": fn("sub", col("d"), lit(18000, "date")), + "twice": fn("add", col("dur"), col("dur"))}}]) + add("temporal/truncate", "truncate to a calendar unit", T, + [{"op": "select", "exprs": {"h": fn("truncate", col("ts"), every="1h"), "d": fn("truncate", col("ts"), every="1d"), + "w": fn("truncate", col("ts"), every="1w"), "mo": fn("truncate", col("ts"), every="1mo"), + "y": fn("truncate", col("ts"), every="1y"), "dmo": fn("truncate", col("d"), every="1mo")}}]) + add("temporal/duration", "durations in whole units", T, + [{"op": "select", "exprs": {"d": fn("totalDays", col("dur")), "h": fn("totalHours", col("dur")), "m": fn("totalMinutes", col("dur")), + "s": fn("totalSeconds", col("dur")), "ms": fn("totalMilliseconds", col("dur"))}}]) + add("temporal/cast", "casts between dates, timestamps and integers", T, + [{"op": "select", "exprs": {"date": fn("cast", col("ts"), to="date"), "ts": fn("cast", col("d"), to="datetime[us]"), + "us": fn("cast", col("ts"), to="i64"), "days": fn("cast", col("d"), to="i32"), + "ms": fn("cast", col("ts"), to="datetime[ms]")}}]) + add("temporal/compare", "comparisons with timestamp literals and between units", T, + [{"op": "filter", "predicate": fn("and", fn("gte", col("ts"), lit(_us(datetime(2024, 3, 1)), "datetime[us]")), fn("lt", col("ms"), col("ts")))}]) + add("temporal/group-sort", "group by date, aggregate timestamps, sort", T, + [{"op": "groupBy", "keys": ["d"], "aggs": {"first": agg("min", col("ts")), "last": agg("max", col("ts")), "n": agg("len"), + "total": agg("sum", col("dur"))}}, + {"op": "sort", "keys": [{"expr": col("last"), "descending": True}]}]) + add("temporal/join", "join on a date", T, [{"op": "join", "right": "right", "on": ["d"], "how": "left"}]) + N = {"left": nested, "right": nested_right} + add("nested/list-ops", "list length, element access and aggregates", N, + [{"op": "select", "exprs": {"n": fn("listLength", col("xs")), "first": fn("listGet", col("xs"), index=0), + "last": fn("listGet", col("xs"), index=-1), "oob": fn("listGet", col("xs"), index=5), + "sum": fn("listSum", col("xs")), "mean": fn("listMean", col("xs")), "min": fn("listMin", col("xs")), + "max": fn("listMax", col("ws")), "has2": fn("listContains", col("xs"), lit(2)), + "hasA": fn("listContains", col("ws"), lit("a"))}}]) + add("nested/struct", "struct fields and struct construction", N, + [{"op": "select", "exprs": {"a": fn("field", col("s"), field="a"), "b": fn("field", col("s"), field="b"), + "pair": fn("struct", col("k"), fn("listLength", col("xs")), names=["key", "size"])}}]) + add("nested/explode", "explode lists into rows; a null list gives a null row, an empty list no rows", N, + [{"op": "pick", "columns": ["k", "xs"]}, {"op": "explode", "column": "xs"}]) + add("nested/unnest", "unnest a struct into columns", N, [{"op": "pick", "columns": ["k", "s"]}, {"op": "unnest", "column": "s"}]) + add("nested/agg-list", "collect values into lists per group", N, + [{"op": "groupBy", "keys": ["g"], "aggs": {"ks": agg("list", col("k")), "firsts": agg("first", col("xs")), + "n": agg("len")}}]) + add("nested/carry", "lists and structs survive filters, sorts, joins and slices", N, + [{"op": "filter", "predicate": fn("neq", col("k"), lit("c"))}, + {"op": "sort", "keys": [{"expr": col("g"), "descending": True}, {"expr": col("k")}]}, + {"op": "join", "right": "right", "on": ["k"], "how": "left"}, + {"op": "head", "n": 4}]) + D = {"left": money} + add("decimal/arith", "decimal addition, subtraction and multiplication", D, + [{"op": "select", "exprs": {"sum": fn("add", col("price"), col("fee")), "diff": fn("sub", col("price"), col("fee")), + "prod": fn("mul", col("price"), col("qty")), "plus": fn("add", col("price"), lit("0.5", "decimal[3,1]"))}}]) + add("decimal/compare", "decimal comparisons, also across scales", D, + [{"op": "filter", "predicate": fn("and", fn("gt", col("price"), lit("10.00", "decimal[10,2]")), fn("gte", col("price"), col("fee")))}]) + add("decimal/aggregate", "sums, extremes and means of decimals per group", D, + [{"op": "groupBy", "keys": ["k"], "aggs": {"total": agg("sum", col("price")), "min": agg("min", col("price")), + "max": agg("max", col("fee")), "mean": agg("mean", col("price")), "n": agg("count", col("price"))}}]) + add("decimal/cast", "casts between decimals, floats, integers and strings", D, + [{"op": "select", "exprs": {"f": fn("cast", col("price"), to="f64"), "i": fn("cast", col("price"), to="i32"), + "s": fn("cast", col("fee"), to="utf8"), "d": fn("cast", col("x"), to="decimal[10,2]"), + "r": fn("cast", col("price"), to="decimal[12,1]")}}]) + add("decimal/sort", "sort by a decimal", D, [{"op": "sort", "keys": [{"expr": col("price"), "descending": True}], "nullsLast": True}]) + add("decimal/round-ties", "narrowing the scale rounds exact ties", {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249"]})}, + [{"op": "select", "exprs": {"one": fn("cast", col("t"), to="decimal[6,1]"), "zero": fn("cast", col("t"), to="decimal[6,0]")}}]) + add("decimal/cast-float-ties", "floats cast to decimals round exact ties half to even, also below zero", + {"left": table([("x", "f64")], {"x": [-1.5, -2.5, 1.5, 2.5, -0.75]})}, + [{"op": "select", "exprs": {"zero": fn("cast", col("x"), to="decimal[18,0]"), "one": fn("cast", col("x"), to="decimal[18,1]")}}]) + add("decimal/round", "rounding decimals keeps the scale and rounds half to even", + {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249", None]})}, + [{"op": "select", "exprs": {"r0": fn("round", col("t")), "r1": fn("round", col("t"), decimals=1), + "r2": fn("round", col("t"), decimals=2), "r3": fn("round", col("t"), decimals=3)}}]) + add("decimal/rolling-mean", "rolling means of decimals are floats of their values", D, + [{"op": "select", "exprs": {"m": fn("rollingMean", col("price"), window=2), "mf": fn("rollingMean", col("fee"), window=3, minPeriods=1)}}]) + add("arith/u32-divide-by-zero", "unsigned 32-bit floor division and modulo by zero give null", + {"left": table([("n", "u32"), ("d", "u32")], {"n": [7, 0, 5], "d": [0, 0, 2]})}, + [{"op": "select", "exprs": {"q": fn("floorDiv", col("n"), col("d")), "m": fn("mod", col("n"), col("d")), + "qs": fn("floorDiv", col("n"), lit(0, "u32")), "ms": fn("mod", col("n"), lit(0, "u32"))}}]) + for offset, length in [(-5, 1), (-5, 3), (-5, 4), (-2, 5), (-5, None), (1, 0)]: + add(f"slice/clip{offset}{'' if length is None else f'+{length}'}", "slices reaching before the first row clip from the unclamped start", + {"left": table([("x", "i32")], {"x": [1, 2, 3]})}, [{"op": "slice", "offset": offset, **({} if length is None else {"length": length})}]) + # windows see exactly the rows of their input, whatever runs before or after them + W = {"left": table([("g", "i32"), ("h", "i32"), ("x", "f64")], {"g": [1, 2, 1, 2], "h": [1, 1, 2, 2], "x": [3.0, 1.0, 2.0, 4.0]}), + "right": table([("g", "i32"), ("name", "utf8")], {"g": [1], "name": ["one"]})} + add("window/filter-after-sort", "a window predicate sees the sorted rows", W, + [{"op": "sort", "keys": [{"expr": col("x")}]}, {"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(3.0))}]) + add("window/filter-after-sort-by-window", "a filter after a sort by a window leaves the sort's order alone", + {"left": table([("x", "f64")], {"x": [-3.0, 5.0, -1.0]})}, + [{"op": "sort", "keys": [{"expr": fn("cumSum", col("x"))}]}, {"op": "filter", "predicate": fn("neq", col("x"), lit(5.0))}]) + add("window/filter-after-filter", "a window predicate sees only the rows an earlier filter kept", W, + [{"op": "filter", "predicate": fn("gt", col("x"), lit(1.0))}, {"op": "filter", "predicate": fn("gt", fn("shift", col("x")), lit(2.0))}]) + add("window/filter-after-concat", "a window predicate sees every input of a concat", W, + [{"op": "concat", "with": "left"}, {"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(15.0))}]) + add("window/filter-after-join", "a window predicate sees only the joined rows", W, + [{"op": "join", "right": "right", "on": ["g"], "how": "inner"}, {"op": "filter", "predicate": fn("gt", fn("shift", col("x")), lit(2.0))}]) + add("window/filter-whole", "a window predicate sees the whole input", W, + [{"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(5.0))}]) + add("window/agg-of-window", "an aggregate of a window sees the whole input", W, + [{"op": "agg", "aggs": {"s": agg("sum", fn("cumSum", col("x"))), "n": agg("len")}}]) + add("window/grouped-agg-of-window", "windows inside grouped aggregates run per group", W, + [{"op": "groupBy", "keys": ["g"], "aggs": {"s": agg("sum", fn("cumSum", col("x"))), "l": agg("list", fn("cumSum", col("x"))), + "o": agg("sum", over(agg("sum", col("x")), "h"))}}]) + return out + + +def main(): + gold = HERE / "gold" + gold.mkdir(exist_ok=True) + header = {"generator": "generate_gold.py", "polars": pl.__version__, "deviations": DEVIATIONS} + spec = spec_cases() + (gold / "spec.json").write_text(json.dumps({**header, "cases": spec}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") + extended = extended_cases() + (gold / "extended.json").write_text(json.dumps({**header, "cases": extended}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") + print(f"spec: {len(spec)} cases, extended: {len(extended)} cases") + + +if __name__ == "__main__": + main() diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json new file mode 100644 index 00000000000..e79e2bffec4 --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -0,0 +1 @@ +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"decimal/rolling-mean","description":"rolling means of decimals are floats of their values","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"m":{"fn":"rollingMean","args":[{"col":"price"}],"window":2},"mf":{"fn":"rollingMean","args":[{"col":"fee"}],"window":3,"minPeriods":1}}}],"expected":{"fields":[{"name":"m","type":"f64"},{"name":"mf","type":"f64"}],"columns":{"m":[null,6.22,-2.45,null,null,50000005.0],"mf":[0.125,0.5625,0.5625,1.75,1.2495,4.169666666666667]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"slice/clip-5+1","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":1}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"slice/clip-5+3","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":3}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1]}}},{"id":"slice/clip-5+4","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":4}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2]}}},{"id":"slice/clip-2+5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-2,"length":5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[2,3]}}},{"id":"slice/clip-5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},{"id":"slice/clip1+0","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":1,"length":0}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-sort-by-window","description":"a filter after a sort by a window leaves the sort's order alone","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,5.0,-1.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"fn":"cumSum","args":[{"col":"x"}]}}]},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"x"},{"lit":5.0}]}}],"expected":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,-1.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} diff --git a/packages/effect/test/fixtures/dataframe/gold/spec.json b/packages/effect/test/fixtures/dataframe/gold/spec.json new file mode 100644 index 00000000000..d7e735dc6de --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/gold/spec.json @@ -0,0 +1 @@ +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"filter/gt","description":"numeric comparison with a literal","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"temp"},{"lit":20.0}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s3","s4","s6"],"city":["Vienna","Berlin","Berlin",null],"temp":[21.5,24.0,26.5,"NaN"],"rain":[0.0,3.5,1.0,2.0],"count":[3,null,7,5],"id":["10","30","40","-1"]}}},{"id":"filter/null-is-false","description":"rows where the predicate is null are dropped","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"rain"},{"lit":0.5}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s5","s6"],"city":["Berlin","Berlin","Zürich",null],"temp":[24.0,26.5,17.0,"NaN"],"rain":[3.5,1.0,7.0,2.0],"count":[null,7,2,5],"id":["30","40","9007199254740993","-1"]}}},{"id":"filter/nan-total-order","description":"NaN compares greater than every number and equal to itself","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"a"},{"col":"b"}]}}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":["-0",0.0,"NaN","Infinity",2.5,1e+300],"b":[0.0,"-0","NaN",1.0,0.0,1e+300],"i":[-7,0,2147483647,-2147483648,-3,5],"j":[2,3,1,-1,2,0],"s":["a","","é","😀","B","a\u0000b"],"t":["A","x","É","😀",null,"a"],"p":[true,true,false,false,null,true],"q":[false,null,true,false,false,true]}}},{"id":"filter/string-eq","description":"string equality","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"city"},{"lit":"Vienna"}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s7"],"city":["Vienna","Vienna","Vienna"],"temp":[21.5,19.0,null],"rain":[0.0,null,0.5],"count":[3,1,4],"id":["10","20","0"]}}},{"id":"filter/string-order","description":"strings compare by code point (UTF-8 byte order)","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"col":"s"},{"col":"t"}]}}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[0.0],"b":["-0"],"i":[0],"j":[3],"s":[""],"t":["x"],"p":[true],"q":[null]}}},{"id":"filter/kleene","description":"and/or use three-valued logic","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"and":{"fn":"and","args":[{"col":"p"},{"col":"q"}]},"or":{"fn":"or","args":[{"col":"p"},{"col":"q"}]},"not":{"fn":"not","args":[{"col":"p"}]}}}],"expected":{"fields":[{"name":"and","type":"bool"},{"name":"or","type":"bool"},{"name":"not","type":"bool"}],"columns":{"and":[true,false,null,false,false,false,null,false,null,true],"or":[true,true,true,true,false,null,true,null,null,true],"not":[false,false,false,true,true,true,null,null,null,false]}}},{"id":"filter/is-null","description":"isNull and isNotNull","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"isNull","args":[{"col":"city"}]},{"fn":"isNull","args":[{"col":"temp"}]}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s7"],"city":[null,"Vienna"],"temp":["NaN",null],"rain":[2.0,0.5],"count":[5,4],"id":["-1","0"]}}},{"id":"filter/is-in","description":"membership in a literal list","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"city"}],"values":["Berlin","Zürich"]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s5"],"city":["Berlin","Berlin","Zürich"],"temp":[24.0,26.5,17.0],"rain":[3.5,1.0,7.0],"count":[null,7,2],"id":["30","40","9007199254740993"]}}},{"id":"filter/between","description":"inclusive range","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"between","args":[{"col":"temp"},{"lit":19.0},{"lit":24.0}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3"],"city":["Vienna","Vienna","Berlin"],"temp":[21.5,19.0,24.0],"rain":[0.0,null,3.5],"count":[3,1,null],"id":["10","20","30"]}}},{"id":"filter/compound","description":"compound predicate","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"count"},{"lit":1}]},{"fn":"neq","args":[{"col":"city"},{"lit":"Berlin"}]}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s5","s7"],"city":["Vienna","Zürich","Vienna"],"temp":[21.5,17.0,null],"rain":[0.0,7.0,0.5],"count":[3,2,4],"id":["10","9007199254740993","0"]}}},{"id":"filter/i64","description":"64-bit integer comparison beyond 2^53","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"id"},{"lit":"9007199254740992","type":"i64"}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s5"],"city":["Zürich"],"temp":[17.0],"rain":[7.0],"count":[2],"id":["9007199254740993"]}}},{"id":"filter/empty-result","description":"a predicate that matches nothing","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"temp"},{"lit":1000.0}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6"],"city":[null],"temp":["NaN"],"rain":[2.0],"count":[5],"id":["-1"]}}},{"id":"filter/all-null-literal","description":"a null literal predicate drops everything","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"lit":null,"type":"bool"}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":[],"city":[],"temp":[],"rain":[],"count":[],"id":[]}}},{"id":"arith/float","description":"float arithmetic with nulls, NaN and infinities","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"add":{"fn":"add","args":[{"col":"a"},{"col":"b"}]},"sub":{"fn":"sub","args":[{"col":"a"},{"col":"b"}]},"mul":{"fn":"mul","args":[{"col":"a"},{"col":"b"}]},"div":{"fn":"div","args":[{"col":"a"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"add","type":"f64"},{"name":"sub","type":"f64"},{"name":"mul","type":"f64"},{"name":"div","type":"f64"}],"columns":{"add":[3.0,0.0,0.0,"NaN","Infinity",null,null,2.5,-0.5,2e+300],"sub":[-1.0,"-0",0.0,"NaN","Infinity",null,null,2.5,-4.5,0.0],"mul":[2.0,"-0","-0","NaN","Infinity",null,null,0.0,-5.0,"Infinity"],"div":[0.5,"NaN","NaN","NaN","Infinity",null,null,"Infinity",-1.25,1.0]}}},{"id":"arith/int32-wrap","description":"Int32 arithmetic wraps on overflow","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"add":{"fn":"add","args":[{"col":"i"},{"col":"j"}]},"sub":{"fn":"sub","args":[{"col":"i"},{"col":"j"}]},"mul":{"fn":"mul","args":[{"col":"i"},{"col":"j"}]},"neg":{"fn":"neg","args":[{"col":"i"}]}}}],"expected":{"fields":[{"name":"add","type":"i32"},{"name":"sub","type":"i32"},{"name":"mul","type":"i32"},{"name":"neg","type":"i32"}],"columns":{"add":[9,-5,3,-2147483648,2147483647,null,null,-1,93,5],"sub":[5,-9,-3,2147483646,-2147483647,null,null,-5,107,5],"mul":[14,-14,0,2147483647,-2147483648,null,null,-6,-700,0],"neg":[-7,7,0,-2147483647,-2147483648,null,-3,3,-100,-5]}}},{"id":"arith/int32-div","description":"integer true division gives Float64; division by zero gives infinity","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"div":{"fn":"div","args":[{"col":"i"},{"col":"j"}]}}}],"expected":{"fields":[{"name":"div","type":"f64"}],"columns":{"div":[3.5,-3.5,0.0,2147483647.0,2147483648.0,null,null,-1.5,-14.285714285714286,"Infinity"]}}},{"id":"arith/int32-floor","description":"floor division and floored modulo","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"j"},{"lit":0}]}},{"op":"select","exprs":{"floor":{"fn":"floorDiv","args":[{"col":"i"},{"col":"j"}]},"mod":{"fn":"mod","args":[{"col":"i"},{"col":"j"}]}}}],"expected":{"fields":[{"name":"floor","type":"i32"},{"name":"mod","type":"i32"}],"columns":{"floor":[3,-4,0,2147483647,-2147483648,null,-2,-15],"mod":[1,1,0,0,0,null,1,-5]}}},{"id":"arith/float-floor","description":"floor division and floored modulo on floats","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"floor":{"fn":"floorDiv","args":[{"col":"a"},{"col":"b"}]},"mod":{"fn":"mod","args":[{"col":"a"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"floor","type":"f64"},{"name":"mod","type":"f64"}],"columns":{"floor":[0.0,"NaN","NaN","NaN","Infinity",null,null,"Infinity",-2.0,1.0],"mod":[1.0,"NaN","NaN","NaN","NaN",null,null,"NaN",1.5,0.0]}}},{"id":"arith/mixed","description":"Int32 with a float literal widens to Float64","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"mul","args":[{"col":"i"},{"lit":1.5}]},"y":{"fn":"add","args":[{"col":"i"},{"col":"a"}]}}}],"expected":{"fields":[{"name":"x","type":"f64"},{"name":"y","type":"f64"}],"columns":{"x":[10.5,-10.5,0.0,3221225470.5,-3221225472.0,null,4.5,-4.5,150.0,7.5],"y":[8.0,-7.0,0.0,"NaN","Infinity",null,null,-0.5,97.5,1e+300]}}},{"id":"arith/int-literal","description":"an integer literal adopts the integer column type","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"add","args":[{"col":"i"},{"lit":1}]},"y":{"fn":"mul","args":[{"col":"j"},{"lit":3}]}}}],"expected":{"fields":[{"name":"x","type":"i32"},{"name":"y","type":"i32"}],"columns":{"x":[8,-6,1,-2147483648,-2147483647,null,4,-2,101,6],"y":[6,6,9,3,-3,12,null,6,-21,0]}}},{"id":"arith/abs-sqrt","description":"abs and sqrt","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"abs":{"fn":"abs","args":[{"col":"a"}]},"iabs":{"fn":"abs","args":[{"col":"j"}]},"sqrt":{"fn":"sqrt","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"abs","type":"f64"},{"name":"iabs","type":"i32"},{"name":"sqrt","type":"f64"}],"columns":{"abs":[1.0,0.0,0.0,"NaN","Infinity","Infinity",null,2.5,2.5,1e+300],"iabs":[2,2,3,1,1,4,null,2,7,0],"sqrt":[1.4142135623730951,0.0,"-0","NaN",1.0,null,1.7320508075688772,0.0,1.4142135623730951,1e+150]}}},{"id":"arith/i64","description":"Int64 arithmetic","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"add","args":[{"col":"id"},{"lit":"1","type":"i64"}]},"y":{"fn":"mul","args":[{"col":"id"},{"lit":"2","type":"i64"}]},"z":{"fn":"sub","args":[{"col":"id"},{"col":"id"}]}}}],"expected":{"fields":[{"name":"x","type":"i64"},{"name":"y","type":"i64"},{"name":"z","type":"i64"}],"columns":{"x":["11","21","31","41","9007199254740994","0","1"],"y":["20","40","60","80","18014398509481986","-2","0"],"z":["0","0","0","0","0","0","0"]}}},{"id":"arith/round","description":"round half to even","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[0.5,1.5,2.5,-0.5,-1.5,1.25,2.675,null,"NaN"]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"x"}]},"r1":{"fn":"round","args":[{"col":"x"}],"decimals":1}}}],"expected":{"fields":[{"name":"r0","type":"f64"},{"name":"r1","type":"f64"}],"columns":{"r0":[0.0,2.0,2.0,"-0",-2.0,1.0,3.0,null,"NaN"],"r1":[0.5,1.5,2.5,-0.5,-1.5,1.2,2.7,null,"NaN"]}}},{"id":"arith/small-ints","description":"8/16-bit arithmetic wraps in the column type","inputs":{"left":{"fields":[{"name":"i8","type":"i8"},{"name":"u8","type":"u8"},{"name":"i16","type":"i16"},{"name":"u16","type":"u16"},{"name":"u32","type":"u32"},{"name":"i64","type":"i64"},{"name":"u64","type":"u64"},{"name":"f32","type":"f32"}],"columns":{"i8":[127,-128,0,null,5],"u8":[255,0,1,null,5],"i16":[32767,-32768,0,12,null],"u16":[65535,0,7,12,null],"u32":[4294967295,0,1,null,3],"i64":["9223372036854775807","-9223372036854775808","0",null,"42"],"u64":["18446744073709551615","0","1",null,"42"],"f32":[1.5,-0.25,null,3.0,10000000000.0]}}},"ops":[{"op":"select","exprs":{"i8":{"fn":"add","args":[{"col":"i8"},{"lit":1}]},"u8":{"fn":"add","args":[{"col":"u8"},{"lit":1}]},"i16":{"fn":"mul","args":[{"col":"i16"},{"lit":2}]},"u16":{"fn":"sub","args":[{"col":"u16"},{"lit":1}]},"u32":{"fn":"add","args":[{"col":"u32"},{"lit":1}]},"f32":{"fn":"mul","args":[{"col":"f32"},{"lit":2.0}]}}}],"expected":{"fields":[{"name":"i8","type":"i8"},{"name":"u8","type":"u8"},{"name":"i16","type":"i16"},{"name":"u16","type":"u16"},{"name":"u32","type":"u32"},{"name":"f32","type":"f32"}],"columns":{"i8":[-128,-127,1,null,6],"u8":[0,1,2,null,6],"i16":[-2,0,0,24,null],"u16":[65534,65535,6,11,null],"u32":[0,1,2,null,4],"f32":[3.0,-0.5,null,6.0,20000000000.0]}}},{"id":"null/fill","description":"fillNull with a literal","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"city":{"fn":"fillNull","args":[{"col":"city"},{"lit":"?"}]},"temp":{"fn":"fillNull","args":[{"col":"temp"},{"lit":0.0}]}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"city":["Vienna","Vienna","Berlin","Berlin","Zürich","?","Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",0.0]}}},{"id":"null/coalesce","description":"first non-null","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"coalesce","args":[{"col":"a"},{"col":"b"},{"lit":-1.0}]}}}],"expected":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",3.0,2.5,-2.5,1e+300]}}},{"id":"null/when","description":"when/then/otherwise","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"label":{"fn":"when","args":[{"fn":"gt","args":[{"col":"temp"},{"lit":20.0}]},{"lit":"warm"},{"lit":"cold"}]}}}],"expected":{"fields":[{"name":"label","type":"utf8"}],"columns":{"label":["warm","cold","warm","warm","cold","warm","cold"]}}},{"id":"null/drop","description":"drop rows with nulls","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"dropNulls"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s4","s5"],"city":["Vienna","Berlin","Zürich"],"temp":[21.5,26.5,17.0],"rain":[0.0,1.0,7.0],"count":[3,7,2],"id":["10","40","9007199254740993"]}}},{"id":"null/drop-subset","description":"drop rows with nulls in some columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"dropNulls","columns":["city"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich","Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,null],"rain":[0.0,null,3.5,1.0,7.0,0.5],"count":[3,1,null,7,2,4],"id":["10","20","30","40","9007199254740993","0"]}}},{"id":"null/is-nan","description":"isNaN is distinct from isNull","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"nan":{"fn":"isNaN","args":[{"col":"a"}]}}}],"expected":{"fields":[{"name":"nan","type":"bool"}],"columns":{"nan":[false,false,false,true,false,false,null,false,false,false]}}},{"id":"cast/float-int","description":"float to Int32 truncates; out of range is null","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"i":{"fn":"cast","args":[{"col":"a"}],"to":"i32"},"f":{"fn":"cast","args":[{"col":"i"}],"to":"f64"}}}],"expected":{"fields":[{"name":"i","type":"i32"},{"name":"f","type":"f64"}],"columns":{"i":[1,0,0,null,null,null,null,2,-2,null],"f":[7.0,-7.0,0.0,2147483647.0,-2147483648.0,null,3.0,-3.0,100.0,5.0]}}},{"id":"cast/int-string","description":"integers to strings","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"cast","args":[{"col":"i"}],"to":"utf8"}}}],"expected":{"fields":[{"name":"s","type":"utf8"}],"columns":{"s":["7","-7","0","2147483647","-2147483648",null,"3","-3","100","5"]}}},{"id":"cast/string-number","description":"strings to numbers; unparsable is null","inputs":{"left":{"fields":[{"name":"s","type":"utf8"}],"columns":{"s":["1","-2","3.5","x","",null," 4","1e3"]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"s"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"s"}],"to":"i32"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"}],"columns":{"f":[1.0,-2.0,3.5,null,null,null,null,1000.0],"i":[1,-2,null,null,null,null,null,null]}}},{"id":"cast/bool","description":"booleans to and from numbers","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"b":{"fn":"cast","args":[{"col":"j"}],"to":"bool"},"n":{"fn":"cast","args":[{"col":"p"}],"to":"i32"}}}],"expected":{"fields":[{"name":"b","type":"bool"},{"name":"n","type":"i32"}],"columns":{"b":[true,true,true,true,true,true,null,true,true,false],"n":[1,1,1,0,0,0,null,null,null,1]}}},{"id":"cast/i64","description":"Int64 to and from other types","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"id"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"count"}],"to":"i64"},"s":{"fn":"cast","args":[{"col":"id"}],"to":"utf8"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i64"},{"name":"s","type":"utf8"}],"columns":{"f":[10.0,20.0,30.0,40.0,9007199254740992.0,-1.0,0.0],"i":["3","1",null,"7","2","5","4"],"s":["10","20","30","40","9007199254740993","-1","0"]}}},{"id":"str/length","description":"length in characters","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"strLength","args":[{"col":"s"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"}],"columns":{"n":[1,1,0,1,1,1,null,1,2,3]}}},{"id":"str/case","description":"upper and lower case","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"u":{"fn":"strUpper","args":[{"col":"s"}]},"l":{"fn":"strLower","args":[{"col":"t"}]}}}],"expected":{"fields":[{"name":"u","type":"utf8"},{"name":"l","type":"utf8"}],"columns":{"u":["B","A","","É","😀","￿",null,"B","AB","A\u0000B"],"l":["b","a","x","é","😀","z","n",null,"ab","a"]}}},{"id":"str/match","description":"contains, startsWith, endsWith with literals","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"c":{"fn":"strContains","args":[{"col":"s"}],"pattern":"a"},"p":{"fn":"strStartsWith","args":[{"col":"t"}],"prefix":"A"},"e":{"fn":"strEndsWith","args":[{"col":"s"}],"suffix":"b"}}}],"expected":{"fields":[{"name":"c","type":"bool"},{"name":"p","type":"bool"},{"name":"e","type":"bool"}],"columns":{"c":[false,true,false,false,false,false,null,false,true,true],"p":[false,true,false,false,false,false,false,null,true,false],"e":[true,false,false,false,false,false,null,false,true,true]}}},{"id":"str/concat","description":"concatenation; null in any argument gives null","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"c":{"fn":"strConcat","args":[{"col":"s"},{"lit":"-"},{"col":"t"}]}}}],"expected":{"fields":[{"name":"c","type":"utf8"}],"columns":{"c":["b-b","a-A","-x","é-É","😀-😀","￿-z",null,null,"ab-AB","a\u0000b-a"]}}},{"id":"project/pick","description":"pick columns in the given order","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"pick","columns":["temp","station"]}],"expected":{"fields":[{"name":"temp","type":"f64"},{"name":"station","type":"utf8"}],"columns":{"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"station":["s1","s2","s3","s4","s5","s6","s7"]}}},{"id":"project/omit","description":"omit columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"omit","columns":["rain","id"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"count","type":"i32"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"count":[3,1,null,7,2,5,4]}}},{"id":"project/rename","description":"rename columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"rename","mapping":{"temp":"celsius","city":"town"}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"town","type":"utf8"},{"name":"celsius","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"town":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"celsius":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},{"id":"project/with-replace","description":"withColumns replaces an existing column in place","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"withColumns","exprs":{"temp":{"fn":"mul","args":[{"col":"temp"},{"lit":2.0}]},"f":{"fn":"add","args":[{"fn":"mul","args":[{"col":"temp"},{"lit":1.8}]},{"lit":32.0}]}}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"},{"name":"f","type":"f64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[43.0,38.0,48.0,53.0,34.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"],"f":[70.7,66.2,75.2,79.7,62.6,"NaN",null]}}},{"id":"project/literal-column","description":"a literal broadcasts to every row","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"station":{"col":"station"},"half":{"lit":0.5},"one":{"lit":1},"tag":{"lit":"x"}}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"half","type":"f64"},{"name":"one","type":"i32"},{"name":"tag","type":"utf8"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"half":[0.5,0.5,0.5,0.5,0.5,0.5,0.5],"one":[1,1,1,1,1,1,1],"tag":["x","x","x","x","x","x","x"]}}},{"id":"sort/asc","description":"ascending, nulls first, NaN last among numbers","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"temp"}}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s7","s5","s2","s1","s3","s4","s6"],"city":["Vienna","Zürich","Vienna","Vienna","Berlin","Berlin",null],"temp":[null,17.0,19.0,21.5,24.0,26.5,"NaN"],"rain":[0.5,7.0,null,0.0,3.5,1.0,2.0],"count":[4,2,1,3,null,7,5],"id":["0","9007199254740993","20","10","30","40","-1"]}}},{"id":"sort/desc-nulls-last","description":"descending with nulls last","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"temp"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s4","s3","s1","s2","s5","s7"],"city":[null,"Berlin","Berlin","Vienna","Vienna","Zürich","Vienna"],"temp":["NaN",26.5,24.0,21.5,19.0,17.0,null],"rain":[2.0,1.0,3.5,0.0,null,7.0,0.5],"count":[5,7,null,3,1,2,4],"id":["-1","40","30","10","20","9007199254740993","0"]}}},{"id":"sort/multi","description":"multi-key sort is stable","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"city"}},{"expr":{"col":"count"},"descending":true}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s3","s4","s7","s1","s2","s5"],"city":[null,"Berlin","Berlin","Vienna","Vienna","Vienna","Zürich"],"temp":["NaN",24.0,26.5,null,21.5,19.0,17.0],"rain":[2.0,3.5,1.0,0.5,0.0,null,7.0],"count":[5,null,7,4,3,1,2],"id":["-1","30","40","0","10","20","9007199254740993"]}}},{"id":"sort/strings","description":"string sort by code point","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"s"}}]}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[null,0.0,2.5,"-0",1e+300,-2.5,1.0,"NaN","-Infinity","Infinity"],"b":[3.0,"-0",0.0,0.0,1e+300,2.0,2.0,"NaN",null,1.0],"i":[3,0,-3,-7,5,100,7,2147483647,null,-2147483648],"j":[null,3,2,2,0,-7,2,1,4,-1],"s":[null,"","B","a","a\u0000b","ab","b","é","￿","😀"],"t":["n","x",null,"A","a","AB","b","É","z","😀"],"p":[null,true,null,true,true,null,true,false,false,false],"q":[true,null,false,false,true,null,true,true,null,false]}}},{"id":"sort/zeros","description":"-0 and 0 sort as equal (stable)","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"a"}}]}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[null,"-Infinity",-2.5,"-0",0.0,1.0,2.5,1e+300,"Infinity","NaN"],"b":[3.0,null,2.0,0.0,"-0",2.0,0.0,1e+300,1.0,"NaN"],"i":[3,null,100,-7,0,7,-3,5,-2147483648,2147483647],"j":[null,4,-7,2,3,2,2,0,-1,1],"s":[null,"￿","ab","a","","b","B","a\u0000b","😀","é"],"t":["n","z","AB","A","x","b",null,"a","😀","É"],"p":[null,false,null,true,true,true,null,true,false,false],"q":[true,null,null,false,null,true,false,true,false,true]}}},{"id":"sort/i64","description":"Int64 sort","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"id"}}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s7","s1","s2","s3","s4","s5"],"city":[null,"Vienna","Vienna","Vienna","Berlin","Berlin","Zürich"],"temp":["NaN",null,21.5,19.0,24.0,26.5,17.0],"rain":[2.0,0.5,0.0,null,3.5,1.0,7.0],"count":[5,4,3,1,null,7,2],"id":["-1","0","10","20","30","40","9007199254740993"]}}},{"id":"sort/expr","description":"sort by an expression","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"fn":"neg","args":[{"col":"count"}]}}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s6","s7","s1","s5","s2"],"city":["Berlin","Berlin",null,"Vienna","Vienna","Zürich","Vienna"],"temp":[24.0,26.5,"NaN",null,21.5,17.0,19.0],"rain":[3.5,1.0,2.0,0.5,0.0,7.0,null],"count":[null,7,5,4,3,2,1],"id":["30","40","-1","0","10","9007199254740993","20"]}}},{"id":"sort/bool","description":"boolean sort","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"p"}},{"expr":{"col":"q"},"descending":true}]}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[-2.5,null,2.5,"-Infinity","NaN","Infinity",0.0,1.0,1e+300,"-0"],"b":[2.0,3.0,0.0,null,"NaN",1.0,"-0",2.0,1e+300,0.0],"i":[100,3,-3,null,2147483647,-2147483648,0,7,5,-7],"j":[-7,null,2,4,1,-1,3,2,0,2],"s":["ab",null,"B","￿","é","😀","","b","a\u0000b","a"],"t":["AB","n",null,"z","É","😀","x","b","a","A"],"p":[null,null,null,false,false,false,true,true,true,true],"q":[null,true,false,null,true,false,null,true,true,false]}}},{"id":"slice/head","description":"first rows","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"head","n":3}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3"],"city":["Vienna","Vienna","Berlin"],"temp":[21.5,19.0,24.0],"rain":[0.0,null,3.5],"count":[3,1,null],"id":["10","20","30"]}}},{"id":"slice/tail","description":"last rows","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"tail","n":2}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s7"],"city":[null,"Vienna"],"temp":["NaN",null],"rain":[2.0,0.5],"count":[5,4],"id":["-1","0"]}}},{"id":"slice/offset","description":"slice in the middle","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"slice","offset":2,"length":3}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s5"],"city":["Berlin","Berlin","Zürich"],"temp":[24.0,26.5,17.0],"rain":[3.5,1.0,7.0],"count":[null,7,2],"id":["30","40","9007199254740993"]}}},{"id":"slice/after-filter","description":"slicing after a filter","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"city"}]}},{"op":"slice","offset":1,"length":2}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s2","s3"],"city":["Vienna","Berlin"],"temp":[19.0,24.0],"rain":[null,3.5],"count":[1,null],"id":["20","30"]}}},{"id":"agg/global","description":"aggregates over the whole frame","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"n":{"agg":"len"},"c":{"agg":"count","arg":{"col":"temp"}},"sum":{"agg":"sum","arg":{"col":"rain"}},"mean":{"agg":"mean","arg":{"col":"rain"}},"min":{"agg":"min","arg":{"col":"count"}},"max":{"agg":"max","arg":{"col":"count"}},"first":{"agg":"first","arg":{"col":"city"}},"last":{"agg":"last","arg":{"col":"city"}}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"c","type":"u32"},{"name":"sum","type":"f64"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"i32"},{"name":"first","type":"utf8"},{"name":"last","type":"utf8"}],"columns":{"n":[7],"c":[6],"sum":[14.0],"mean":[2.3333333333333335],"min":[1],"max":[7],"first":["Vienna"],"last":["Vienna"]}}},{"id":"agg/nan","description":"sum includes NaN; min and max skip it","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"sum":{"agg":"sum","arg":{"col":"temp"}},"min":{"agg":"min","arg":{"col":"temp"}},"max":{"agg":"max","arg":{"col":"temp"}}}}],"expected":{"fields":[{"name":"sum","type":"f64"},{"name":"min","type":"f64"},{"name":"max","type":"f64"}],"columns":{"sum":["NaN"],"min":[17.0],"max":[26.5]}}},{"id":"agg/stats","description":"std, var and median","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"std":{"agg":"std","arg":{"col":"rain"}},"var":{"agg":"var","arg":{"col":"rain"}},"median":{"agg":"median","arg":{"col":"rain"}},"imed":{"agg":"median","arg":{"col":"count"}}}}],"expected":{"fields":[{"name":"std","type":"f64"},{"name":"var","type":"f64"},{"name":"median","type":"f64"},{"name":"imed","type":"f64"}],"columns":{"std":[2.6012817353502227],"var":[6.766666666666666],"median":[1.5],"imed":[3.5]}}},{"id":"agg/strings","description":"min, max and nUnique on strings","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"min":{"agg":"min","arg":{"col":"city"}},"max":{"agg":"max","arg":{"col":"city"}},"n":{"agg":"nUnique","arg":{"col":"city"}}}}],"expected":{"fields":[{"name":"min","type":"utf8"},{"name":"max","type":"utf8"},{"name":"n","type":"u32"}],"columns":{"min":["Berlin"],"max":["Zürich"],"n":[4]}}},{"id":"agg/empty","description":"aggregates over an empty frame","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"v","type":"f64"}],"columns":{"k":[],"v":[]}}},"ops":[{"op":"agg","aggs":{"n":{"agg":"len"},"sum":{"agg":"sum","arg":{"col":"v"}},"mean":{"agg":"mean","arg":{"col":"v"}},"min":{"agg":"min","arg":{"col":"v"}}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"sum","type":"f64"},{"name":"mean","type":"f64"},{"name":"min","type":"f64"}],"columns":{"n":[0],"sum":[0.0],"mean":[null],"min":[null]}}},{"id":"group/basic","description":"group by a string key in first-appearance order","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"groupBy","keys":["city"],"aggs":{"n":{"agg":"len"},"temp":{"agg":"mean","arg":{"col":"temp"}},"rain":{"agg":"sum","arg":{"col":"rain"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"n","type":"u32"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"}],"columns":{"city":["Vienna","Berlin","Zürich",null],"n":[3,2,1,1],"temp":[20.25,25.25,17.0,"NaN"],"rain":[0.5,4.5,7.0,2.0]}}},{"id":"group/null-key","description":"null is its own group","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"groupBy","keys":["city"],"aggs":{"stations":{"agg":"count","arg":{"col":"station"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"stations","type":"u32"}],"columns":{"city":["Vienna","Berlin","Zürich",null],"stations":[3,2,1,1]}}},{"id":"group/multi-key","description":"group by two keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"withColumns","exprs":{"wet":{"fn":"gt","args":[{"col":"rain"},{"lit":1.0}]}}},{"op":"groupBy","keys":["city","wet"],"aggs":{"n":{"agg":"len"},"max":{"agg":"max","arg":{"col":"temp"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"wet","type":"bool"},{"name":"n","type":"u32"},{"name":"max","type":"f64"}],"columns":{"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null],"wet":[false,null,true,false,true,true],"n":[2,1,1,1,1,1],"max":[21.5,19.0,24.0,26.5,17.0,"NaN"]}}},{"id":"group/all-null","description":"aggregates over all-null groups","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"v","type":"f64"},{"name":"i","type":"i32"}],"columns":{"k":["x","x","y"],"v":[null,null,1.0],"i":[null,null,null]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"sum":{"agg":"sum","arg":{"col":"v"}},"mean":{"agg":"mean","arg":{"col":"v"}},"count":{"agg":"count","arg":{"col":"v"}},"n":{"agg":"len"},"min":{"agg":"min","arg":{"col":"v"}},"std":{"agg":"std","arg":{"col":"v"}},"nu":{"agg":"nUnique","arg":{"col":"v"}},"first":{"agg":"first","arg":{"col":"v"}},"isum":{"agg":"sum","arg":{"col":"i"}},"imax":{"agg":"max","arg":{"col":"i"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"sum","type":"f64"},{"name":"mean","type":"f64"},{"name":"count","type":"u32"},{"name":"n","type":"u32"},{"name":"min","type":"f64"},{"name":"std","type":"f64"},{"name":"nu","type":"u32"},{"name":"first","type":"f64"},{"name":"isum","type":"i32"},{"name":"imax","type":"i32"}],"columns":{"k":["x","y"],"sum":[0.0,1.0],"mean":[null,1.0],"count":[0,1],"n":[2,1],"min":[null,1.0],"std":[null,null],"nu":[1,1],"first":[null,1.0],"isum":[0,0],"imax":[null,null]}}},{"id":"group/float-key","description":"NaN, -0 and 0 as group keys","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"groupBy","keys":["a"],"aggs":{"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"n","type":"u32"}],"columns":{"a":[1.0,"-0","NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"n":[1,2,1,1,1,1,1,1,1]}}},{"id":"group/i64-key","description":"Int64 group keys and sums","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"withColumns","exprs":{"k":{"fn":"mod","args":[{"col":"id"},{"lit":"3","type":"i64"}]}}},{"op":"groupBy","keys":["k"],"aggs":{"n":{"agg":"len"},"s":{"agg":"sum","arg":{"col":"id"}},"mx":{"agg":"max","arg":{"col":"id"}}}}],"expected":{"fields":[{"name":"k","type":"i64"},{"name":"n","type":"u32"},{"name":"s","type":"i64"},{"name":"mx","type":"i64"}],"columns":{"k":["1","2","0"],"n":[2,2,3],"s":["50","19","9007199254741023"],"mx":["40","20","9007199254740993"]}}},{"id":"group/bool-key","description":"boolean group keys","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"groupBy","keys":["p"],"aggs":{"n":{"agg":"len"},"s":{"agg":"sum","arg":{"col":"i"}}}}],"expected":{"fields":[{"name":"p","type":"bool"},{"name":"n","type":"u32"},{"name":"s","type":"i32"}],"columns":{"p":[true,false,null],"n":[4,3,3],"s":[5,-1,100]}}},{"id":"group/int-sum","description":"Int32 sums stay Int32; means are Float64","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"groupBy","keys":["p"],"aggs":{"s":{"agg":"sum","arg":{"col":"j"}},"m":{"agg":"mean","arg":{"col":"j"}},"mn":{"agg":"min","arg":{"col":"j"}}}}],"expected":{"fields":[{"name":"p","type":"bool"},{"name":"s","type":"i32"},{"name":"m","type":"f64"},{"name":"mn","type":"i32"}],"columns":{"p":[true,false,null],"s":[7,4,-5],"m":[1.75,1.3333333333333333,-2.5],"mn":[0,-1,-7]}}},{"id":"group/after-filter","description":"group a filtered frame","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"rain"},{"lit":0.0}]}},{"op":"groupBy","keys":["city"],"aggs":{"n":{"agg":"len"},"last":{"agg":"last","arg":{"col":"station"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"n","type":"u32"},{"name":"last","type":"utf8"}],"columns":{"city":["Berlin","Zürich",null,"Vienna"],"n":[2,1,1,1],"last":["s4","s5","s6","s7"]}}},{"id":"group/expression-agg","description":"aggregate of an expression","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"groupBy","keys":["city"],"aggs":{"x":{"agg":"sum","arg":{"fn":"mul","args":[{"col":"rain"},{"lit":2.0}]}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"x","type":"f64"}],"columns":{"city":["Vienna","Berlin","Zürich",null],"x":[1.0,9.0,14.0,4.0]}}},{"id":"group/empty","description":"grouping an empty frame","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"v","type":"f64"}],"columns":{"k":[],"v":[]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"n","type":"u32"}],"columns":{"k":[],"n":[]}}},{"id":"group/small-ints","description":"sums of small integer types","inputs":{"left":{"fields":[{"name":"i8","type":"i8"},{"name":"u8","type":"u8"},{"name":"i16","type":"i16"},{"name":"u16","type":"u16"},{"name":"u32","type":"u32"},{"name":"i64","type":"i64"},{"name":"u64","type":"u64"},{"name":"f32","type":"f32"}],"columns":{"i8":[127,-128,0,null,5],"u8":[255,0,1,null,5],"i16":[32767,-32768,0,12,null],"u16":[65535,0,7,12,null],"u32":[4294967295,0,1,null,3],"i64":["9223372036854775807","-9223372036854775808","0",null,"42"],"u64":["18446744073709551615","0","1",null,"42"],"f32":[1.5,-0.25,null,3.0,10000000000.0]}}},"ops":[{"op":"agg","aggs":{"i8":{"agg":"sum","arg":{"col":"i8"}},"u8":{"agg":"sum","arg":{"col":"u8"}},"u32":{"agg":"sum","arg":{"col":"u32"}},"i64":{"agg":"sum","arg":{"col":"i64"}},"u64":{"agg":"max","arg":{"col":"u64"}},"f32":{"agg":"mean","arg":{"col":"f32"}}}}],"expected":{"fields":[{"name":"i8","type":"i64"},{"name":"u8","type":"i64"},{"name":"u32","type":"u32"},{"name":"i64","type":"i64"},{"name":"u64","type":"u64"},{"name":"f32","type":"f32"}],"columns":{"i8":["4"],"u8":["261"],"u32":[3],"i64":["41"],"u64":["18446744073709551615"],"f32":[2500000000.0]}}},{"id":"join/inner","description":"inner join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"inner"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3"],"temp":[21.5,24.0,24.0],"elevation":[190.0,34.0,35.0],"active":[true,false,null]}}},{"id":"join/left","description":"left join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"left"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s2","s3","s3","s4","s5","s6","s7"],"temp":[21.5,19.0,24.0,24.0,26.5,17.0,"NaN",null],"elevation":[190.0,null,34.0,35.0,null,null,null,null],"active":[true,null,false,null,null,null,null,null]}}},{"id":"join/full","description":"full join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"full"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s2","s3","s3","s4","s5","s6","s7","s9",null],"temp":[21.5,19.0,24.0,24.0,26.5,17.0,"NaN",null,null,null],"elevation":[190.0,null,34.0,35.0,null,null,null,null,500.0,1.0],"active":[true,null,false,null,null,null,null,null,true,true]}}},{"id":"join/semi","description":"semi join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"semi"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"station":["s1","s3"],"temp":[21.5,24.0]}}},{"id":"join/anti","description":"anti join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"anti"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"station":["s2","s4","s5","s6","s7"],"temp":[19.0,26.5,17.0,"NaN",null]}}},{"id":"join/multi-key","description":"join on two keys","inputs":{"left":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"x","type":"f64"}],"columns":{"a":[1,1,2,2,null],"b":["x","y","x",null,"x"],"x":[1.0,2.0,3.0,4.0,5.0]}},"right":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"y","type":"bool"}],"columns":{"a":[1,2,2,1],"b":["y","x","x","z"],"y":[true,false,true,null]}}},"ops":[{"op":"join","right":"right","on":["a","b"],"how":"left"}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"x","type":"f64"},{"name":"y","type":"bool"}],"columns":{"a":[1,1,2,2,2,null],"b":["x","y","x","x",null,"x"],"x":[1.0,2.0,3.0,3.0,4.0,5.0],"y":[null,true,false,true,null,null]}}},{"id":"join/i64-key","description":"join on Int64 keys","inputs":{"left":{"fields":[{"name":"k","type":"i64"},{"name":"x","type":"f64"}],"columns":{"k":["9007199254740993","9007199254740992","1"],"x":[1.0,2.0,3.0]}},"right":{"fields":[{"name":"k","type":"i64"},{"name":"y","type":"utf8"}],"columns":{"k":["9007199254740993","1","1"],"y":["a","b","c"]}}},"ops":[{"op":"join","right":"right","on":["k"],"how":"inner"}],"expected":{"fields":[{"name":"k","type":"i64"},{"name":"x","type":"f64"},{"name":"y","type":"utf8"}],"columns":{"k":["9007199254740993","1","1"],"x":[1.0,3.0,3.0],"y":["a","b","c"]}}},{"id":"join/f64-key","description":"join on Float64 keys: NaN matches NaN, -0 matches 0","inputs":{"left":{"fields":[{"name":"k","type":"f64"},{"name":"x","type":"i32"}],"columns":{"k":["NaN","-0",1.5,null],"x":[1,2,3,4]}},"right":{"fields":[{"name":"k","type":"f64"},{"name":"y","type":"i32"}],"columns":{"k":[0.0,"NaN",null,1.5],"y":[10,20,30,40]}}},"ops":[{"op":"join","right":"right","on":["k"],"how":"inner"}],"expected":{"fields":[{"name":"k","type":"f64"},{"name":"x","type":"i32"},{"name":"y","type":"i32"}],"columns":{"k":["NaN","-0",1.5],"x":[1,2,3],"y":[20,10,40]}}},{"id":"unique/all","description":"distinct rows, keeping the first","inputs":{"left":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"a":[1,1,2,null,null,1],"b":["x","x","y",null,null,"y"]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"a":[1,2,null,1],"b":["x","y",null,"y"]}}},{"id":"unique/subset","description":"distinct by a subset","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"unique","columns":["city"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s3","s5","s6"],"city":["Vienna","Berlin","Zürich",null],"temp":[21.5,24.0,17.0,"NaN"],"rain":[0.0,3.5,7.0,2.0],"count":[3,null,2,5],"id":["10","30","9007199254740993","-1"]}}},{"id":"concat/vertical","description":"stack frames with the same schema","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"concat","with":"right"},{"op":"pick","columns":["station","temp"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7","s1","s2","s3","s4","s5","s6","s7"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null,21.5,19.0,24.0,26.5,17.0,"NaN",null]}}},{"id":"pipeline/summary","description":"filter, derive, group, sort","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"temp"},{"lit":18.0}]}},{"op":"withColumns","exprs":{"tempF":{"fn":"add","args":[{"fn":"mul","args":[{"col":"temp"},{"lit":1.8}]},{"lit":32.0}]}}},{"op":"groupBy","keys":["city"],"aggs":{"stations":{"agg":"len"},"avgF":{"agg":"mean","arg":{"col":"tempF"}},"maxRain":{"agg":"max","arg":{"col":"rain"}}}},{"op":"sort","keys":[{"expr":{"col":"avgF"},"descending":true}]}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"stations","type":"u32"},{"name":"avgF","type":"f64"},{"name":"maxRain","type":"f64"}],"columns":{"city":[null,"Berlin","Vienna"],"stations":[1,2,2],"avgF":["NaN",77.45,68.45],"maxRain":[2.0,3.5,0.0]}}},{"id":"pipeline/join-group","description":"join then aggregate the joined columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"join","right":"right","on":["station"],"how":"left"},{"op":"groupBy","keys":["active"],"aggs":{"n":{"agg":"len"},"elev":{"agg":"mean","arg":{"col":"elevation"}}}},{"op":"sort","keys":[{"expr":{"col":"active"}}]}],"expected":{"fields":[{"name":"active","type":"bool"},{"name":"n","type":"u32"},{"name":"elev","type":"f64"}],"columns":{"active":[null,false,true],"n":[6,1,1],"elev":[35.0,34.0,190.0]}}}]} diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts new file mode 100644 index 00000000000..951da8c4e4c --- /dev/null +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -0,0 +1,369 @@ +import type { BigDecimal, Result, Stream } from "effect" +import { DateTime, Duration, Effect, Schema } from "effect" +import { ArrowColumn, ArrowSchema } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" +import { describe, expect, it } from "tstyche" + +const Reading = Schema.Struct({ + station: Schema.String, + city: Schema.String, + temp: Schema.Number, + rain: Schema.NullOr(Schema.Number), + id: Schema.BigInt +}) +declare const readings: Query.Query + +const Station = Schema.Struct({ station: Schema.String, elevation: Schema.Number }) +declare const stations: Query.Query + +describe("DataFrame", () => { + it("fromRows takes the row type from the schema", () => { + expect(DataFrame.fromRows(Station, [])).type.toBe< + DataFrame.DataFrame<{ readonly station: string; readonly elevation: number }> + >() + }) + + it("fromRows, empty, validate and Query.table read missing values back as null", () => { + const S = Schema.Struct({ + s: Schema.optional(Schema.String), + u: Schema.UndefinedOr(Schema.String), + p: Schema.Struct({ x: Schema.optional(Schema.Number), xs: Schema.Array(Schema.UndefinedOr(Schema.Number)) }) + }) + type Row = { + readonly s: string | null + readonly u: string | null + readonly p: { readonly x: number | null; readonly xs: ReadonlyArray } + } + expect(DataFrame.fromRows(S, [{ u: undefined, p: { xs: [undefined] } }])).type.toBe>() + expect(DataFrame.empty(S)).type.toBe>() + expect(DataFrame.validate(DataFrame.empty(S), S)).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(Query.table("t", S)).type.toBe>() + }) + + it("validate and Query.table take Arrow types for the schema's fields", () => { + const Price = Schema.Struct({ d: Schema.BigDecimal }) + const types = { d: ArrowSchema.decimal(18, 2) } + expect(DataFrame.validate(DataFrame.empty(Price, { types }), Price, { types })).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(DataFrame.empty(Price, { types }).pipe(DataFrame.validate(Price, { types }))).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(Query.table("t", Price, { types })).type.toBe>() + expect(Query.table).type.not.toBeCallableWith("t", Price, { types: { e: ArrowSchema.int32 } }) + }) + + it("validate and Query.table take an Arrow schema and give untyped rows", () => { + const schema = ArrowSchema.make([ArrowSchema.field("n", ArrowSchema.int32)]) + type Untyped = { readonly [name: string]: unknown } + expect(DataFrame.validate(DataFrame.empty(Station), schema)).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(DataFrame.empty(Station).pipe(DataFrame.validate(schema))).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(Query.table("t", schema)).type.toBe>() + expect(Query.table).type.not.toBeCallableWith("t", schema, { types: { n: ArrowSchema.int64 } }) + expect(DataFrame.validate).type.not.toBeCallableWith(DataFrame.empty(Station), schema, { + types: { n: ArrowSchema.int64 } + }) + }) + + it("column types its values by the frame's rows", () => { + const frame = DataFrame.empty(Reading) + expect(DataFrame.column(frame, "temp")).type.toBe>() + expect(frame.pipe(DataFrame.column("rain"))).type.toBe>() + expect(ArrowColumn.toArray(DataFrame.column(frame, "rain"))).type.toBe>() + expect(ArrowColumn.getUnsafe(DataFrame.column(frame, "id"), 0)).type.toBe() + expect(DataFrame.column).type.not.toBeCallableWith(frame, "missing") + const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(frame)) + expect(DataFrame.column(untyped, "anything")).type.toBe>() + }) + + it("fromColumns infers column types", () => { + expect(DataFrame.fromColumns({ a: new Float64Array(1), b: ["x", null], c: new BigInt64Array(1) })).type.toBe< + DataFrame.DataFrame<{ readonly a: number; readonly b: string | null; readonly c: bigint }> + >() + }) +}) + +describe("Query", () => { + it("from and table take the row type from the frame or schema", () => { + expect(Query.from(DataFrame.fromRows(Station, []))).type.toBe< + Query.Query<{ readonly station: string; readonly elevation: number }> + >() + expect(Query.table("stations", Station)).type.toBe< + Query.Query<{ readonly station: string; readonly elevation: number }> + >() + }) + + it("is an Effect producing a frame of its rows", () => { + expect(Effect.runSync(stations)).type.toBe>() + expect(Effect.gen(function*() { + return yield* stations + })).type.toBe, DataFrame.DataFrameError>>() + }) + + it("filter keeps the row type", () => { + expect(readings.pipe(Query.filter((c) => Expr.gt(c.temp, 20)))).type.toBe() + }) + + it("select and withColumns compute the row type, nulls included", () => { + expect( + readings.pipe( + Query.select((c) => ({ + city: c.city, + wet: Expr.gt(c.rain, 1), + f: c.temp.pipe(Expr.multiply(1.8), Expr.add(32)) + })) + ) + ).type.toBe>() + expect(readings.pipe(Query.withColumns((c) => ({ temp: Expr.cast(c.temp, "Int32") })), Query.pick("temp"))) + .type.toBe>() + }) + + it("pick, omit and rename", () => { + expect(readings.pipe(Query.pick("city", "temp"))).type.toBe< + Query.Query<{ readonly city: string; readonly temp: number }> + >() + expect(readings.pipe(Query.omit("station", "rain", "id"))).type.toBe< + Query.Query<{ readonly city: string; readonly temp: number }> + >() + expect(readings.pipe(Query.pick("temp", "city"), Query.rename({ temp: "celsius" }))).type.toBe< + Query.Query<{ readonly city: string; readonly celsius: number }> + >() + }) + + it("groupBy and agg", () => { + expect( + readings.pipe( + Query.groupBy("city"), + Query.agg((c) => ({ + n: Expr.count(), + mean: Expr.mean(c.temp), + total: Expr.sum(c.id), + last: Expr.last(c.station) + })) + ) + ).type.toBe< + Query.Query< + { + readonly city: string + readonly n: number + readonly mean: number | null + readonly total: bigint + readonly last: string | null + } + > + >() + }) + + it("joins add the other side's columns; left joins make them nullable", () => { + expect(readings.pipe(Query.pick("station", "temp"), Query.join(stations, { on: "station" }))).type.toBe< + Query.Query<{ readonly station: string; readonly temp: number; readonly elevation: number }> + >() + expect(readings.pipe(Query.pick("station", "temp"), Query.join(stations, { on: "station", how: "left" }))) + .type.toBe< + Query.Query<{ readonly station: string; readonly temp: number; readonly elevation: number | null }> + >() + expect(readings.pipe(Query.pick("station", "temp"), Query.join(stations, { on: "station", how: "anti" }))) + .type.toBe>() + }) + + it("rejects mistakes", () => { + // no column "tmp" + // @ts-expect-error: Property 'tmp' does not exist + readings.pipe(Query.filter((c) => Expr.gt(c.tmp, 18))) + // a string compared with a number + // @ts-expect-error: Argument of type '18' is not assignable + readings.pipe(Query.filter((c) => Expr.gt(c.city, 18))) + // arithmetic on strings + // @ts-expect-error: Argument of type 'Expr' is not assignable + readings.pipe(Query.select((c) => ({ x: Expr.add(c.city, 1) }))) + // bigint and number don't mix without a cast + // @ts-expect-error: Argument of type 'Expr' is not assignable + readings.pipe(Query.select((c) => ({ x: Expr.add(c.id, c.temp) }))) + // agg needs aggregates + // @ts-expect-error: is missing in type 'Expr' but required in type 'Aggregate' + readings.pipe(Query.groupBy("city"), Query.agg((c) => ({ t: c.temp }))) + // a predicate can't be an aggregate + // @ts-expect-error: is missing in type 'Aggregate' + readings.pipe(Query.filter((c) => Expr.mean(c.temp))) + // unknown group key + // @ts-expect-error: Argument of type '"country"' is not assignable + readings.pipe(Query.groupBy("country")) + // "temp" exists on both sides and is not the key + // @ts-expect-error: rename the columns that exist on both sides: temp + readings.pipe(Query.join(readings.pipe(Query.pick("station", "temp")), { on: "station" })) + }) +}) + +declare const n: Expr.Expr +declare const nn: Expr.Expr +declare const s: Expr.Expr + +describe("Expr", () => { + it("propagates nullability", () => { + expect(Expr.add(n, 1)).type.toBe>() + expect(Expr.add(n, nn)).type.toBe>() + expect(Expr.gt(nn, 1)).type.toBe>() + expect(Expr.fillNull(nn, 0)).type.toBe>() + expect(Expr.coalesce(nn, s.pipe(Expr.length), 0)).type.toBe>() + expect(Expr.coalesce(nn, nn)).type.toBe>() + expect(Expr.when(Expr.gt(n, 0), "pos", "neg")).type.toBe>() + expect(Expr.concat(s, "-")).type.toBe>() + }) + + it("data-last forms", () => { + expect(n.pipe(Expr.gt(1))).type.toBe>() + expect(s.pipe(Expr.startsWith("a"))).type.toBe>() + expect(n.pipe(Expr.round(2))).type.toBe>() + }) + + it("durations sum to durations and have no mean", () => { + expect(Expr.sum(took)).type.toBe>() + expect(Expr.cumSum(took)).type.toBe>() + expect(Expr.rollingSum(took, 2)).type.toBe>() + expect(took.pipe(Expr.rollingMin(2))).type.toBe>() + expect(Expr.diff(took)).type.toBe>() + expect(Expr.listSum(tookEach)).type.toBe>() + expect(events.pipe(Query.pivot({ on: "id", index: [], values: "took", aggregate: "sum", columns: ["a", "b"] }))) + .type.toBe>() + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.mean(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.median(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.std(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.variance(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.rollingMean(took, 2) + // @ts-expect-error: is not assignable to parameter of type + Expr.listMean(tookEach) + // @ts-expect-error: is not assignable to parameter of type + Expr.sum(at) + }) +}) + +const Event = Schema.Struct({ + id: Schema.String, + at: Schema.DateTimeUtc, + took: Schema.Duration, + tags: Schema.Array(Schema.String), + place: Schema.NullOr(Schema.Struct({ city: Schema.String, zip: Schema.Number })), + amount: Schema.Number +}) +declare const events: Query.Query +declare const price: Expr.Expr +declare const at: Expr.Expr +declare const took: Expr.Expr +declare const tookEach: Expr.Expr> + +describe("Windows and reshaping", () => { + it("over keeps the value type; window functions type their results", () => { + expect(events.pipe(Query.select((c) => ({ + total: Expr.over(Expr.sum(c.amount), [c.id]), + running: Expr.cumSum(c.amount), + previous: Expr.shift(c.amount), + rank: Expr.rank(c.amount, { method: "dense" }), + avg3: Expr.rollingMean(c.amount, 3) + })))).type.toBe< + Query.Query<{ + readonly total: number + readonly running: number + readonly previous: number | null + readonly rank: number + readonly avg3: number | null + }> + >() + }) + + it("pivot with declared columns is typed; without them rows are untyped", () => { + expect(events.pipe(Query.pivot({ on: "id", index: [], values: "amount", aggregate: "sum", columns: ["a", "b"] }))) + .type.toBe>() + expect(events.pipe(Query.pivot({ on: "id", index: [], values: "amount", aggregate: "max" }))) + .type.toBe>() + }) + + it("unpivot names the variable and value columns", () => { + expect(readings.pipe(Query.unpivot({ index: ["city"], on: ["temp", "rain"], variable: "measure" }))).type.toBe< + Query.Query<{ readonly city: string; readonly measure: "temp" | "rain"; readonly value: number | null }> + >() + }) + + it("explode and unnest", () => { + expect(events.pipe(Query.pick("id", "tags"), Query.explode("tags"))).type.toBe< + Query.Query<{ readonly id: string; readonly tags: string | null }> + >() + expect(events.pipe(Query.pick("id", "place"), Query.unnest("place"))).type.toBe< + Query.Query<{ readonly id: string; readonly city: string | null; readonly zip: number | null }> + >() + // @ts-expect-error: Argument of type '"amount"' is not assignable + events.pipe(Query.explode("amount")) + }) +}) + +describe("Types", () => { + it("dates, times and durations", () => { + expect(events.pipe(Query.select((c) => ({ + year: Expr.year(c.at), + day: Expr.truncate(c.at, "1d"), + later: Expr.add(c.at, c.took), + gap: Expr.subtract(c.at, c.at), + hours: Expr.totalHours(c.took), + recent: Expr.gt(c.at, DateTime.makeUnsafe(0)), + long: Expr.gt(c.took, Duration.seconds(5)) + })))).type.toBe< + Query.Query<{ + readonly year: number + readonly day: DateTime.Utc + readonly later: DateTime.Utc + readonly gap: Duration.Duration + readonly hours: bigint + readonly recent: boolean + readonly long: boolean + }> + >() + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.year(Expr.lit(1)) + }) + + it("lists and structs", () => { + expect(events.pipe(Query.select((c) => ({ + n: Expr.listLength(c.tags), + first: Expr.listGet(c.tags, 0), + hasA: Expr.listContains(c.tags, "a"), + city: Expr.field(c.place, "city"), + pair: Expr.struct({ id: c.id, n: Expr.listLength(c.tags) }) + })))).type.toBe< + Query.Query<{ + readonly n: number + readonly first: string | null + readonly hasA: boolean + readonly city: string | null + readonly pair: { readonly id: string; readonly n: number } + }> + >() + expect(events.pipe(Query.groupBy("id"), Query.agg((c) => ({ amounts: Expr.list(c.amount) })))).type.toBe< + Query.Query<{ readonly id: string; readonly amounts: ReadonlyArray }> + >() + }) + + it("decimals", () => { + expect(Expr.add(price, price)).type.toBe>() + expect(Expr.sum(price)).type.toBe>() + expect(Expr.round(price, 1)).type.toBe>() + expect(Expr.castDecimal(Expr.lit(1.5), 10, 2)).type.toBe>() + }) +}) + +describe("Streaming", () => { + it("Query.stream emits frames of the row type", () => { + expect(Query.stream(stations)).type.toBe< + Stream.Stream, DataFrame.DataFrameError> + >() + }) +}) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts new file mode 100644 index 00000000000..4d4ac65e91b --- /dev/null +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -0,0 +1,1155 @@ +/** + * effect/dataframe queries run on PostgreSQL (PGlite) through `Sql.layer` + * give the rows the in-memory engine gives for the same data. + */ +import { PgliteClient } from "@effect/sql-pglite" +import { assert, describe, layer } from "@effect/vitest" +import { BigDecimal, DateTime, Duration, Effect, Layer, Result, Schema } from "effect" +import { ArrowSchema, ArrowTable } from "effect/arrow" +import { Catalog, DataFrame, Expr, Query, Sql } from "effect/dataframe" +import { SqlClient } from "effect/sql/SqlClient" +import { vi } from "vitest" + +const Sale = Schema.Struct({ + id: Schema.Number, + region: Schema.NullOr(Schema.String), + units: Schema.NullOr(Schema.Number), + price: Schema.Number, + at: Schema.DateTimeUtc +}) +const Region = Schema.Struct({ region: Schema.String, manager: Schema.String }) +/** Table names starting like the compiler's names: 63 bytes, the most PostgreSQL keeps, and 57. */ +const long = "_".repeat(62) + "a" +const underscores = "_".repeat(57) + +const sales = Array.from({ length: 40 }, (_, i) => ({ + id: i, + region: i % 7 === 0 ? null : ["north", "south", "east", "west"][i % 4], + units: i % 5 === 0 ? null : (i * 37) % 23, + price: Math.round(((i * 53) % 97) * 1.25 * 100) / 100, + at: DateTime.makeUnsafe(Date.UTC(2024, i % 12, 1 + (i % 27), i % 24)) +})) +const regions = [ + { region: "north", manager: "Ada" }, + { region: "south", manager: "Grace" }, + { region: "east", manager: "Edsger" } +] + +const Setup = Layer.effectDiscard( + Effect.gen(function*() { + const sql = yield* SqlClient + yield* sql`CREATE TABLE sales (id DOUBLE PRECISION, region TEXT, units DOUBLE PRECISION, price DOUBLE PRECISION, at TIMESTAMP)` + yield* sql`CREATE TABLE regions (region TEXT, manager TEXT)` + for (const s of sales) { + yield* sql`INSERT INTO sales VALUES (${s.id}, ${s.region}, ${s.units}, ${s.price}, ${ + DateTime.formatIso(s.at).replace("T", " ").replace("Z", "") + })` + } + for (const r of regions) yield* sql`INSERT INTO regions VALUES (${r.region}, ${r.manager})` + yield* sql`CREATE TABLE contract (s TEXT, n DOUBLE PRECISION)` + yield* sql`INSERT INTO contract VALUES ('bad', NULL), ('good', 1)` + // Tables named like the sources of other tables. + yield* sql`CREATE TABLE second (s TEXT)` + yield* sql`INSERT INTO second VALUES ('good')` + yield* sql`CREATE TABLE "__table0" (s TEXT)` + yield* sql`INSERT INTO "__table0" VALUES ('bad')` + yield* sql`CREATE TABLE "__table1" (s TEXT)` + yield* sql`INSERT INTO "__table1" VALUES ('bad')` + for (const name of [long, underscores]) { + yield* sql.unsafe(`CREATE TABLE "${name}" (s TEXT)`) + yield* sql.unsafe(`INSERT INTO "${name}" VALUES ('${name.length}')`) + } + yield* sql.unsafe(`CREATE TABLE shared ("${"z".repeat(63)}" DOUBLE PRECISION)`) + yield* sql`INSERT INTO shared VALUES (1)` + yield* sql`CREATE TABLE "a.b" (s TEXT)` + yield* sql`INSERT INTO "a.b" VALUES ('good')` + yield* sql`CREATE SCHEMA a` + yield* sql`CREATE TABLE a.b (s TEXT)` + yield* sql`INSERT INTO a.b VALUES ('bad')` + yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL, forever TIMESTAMP, endless DATE)` + yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month', 'infinity', 'infinity')` + yield* sql`CREATE TABLE declared (t TIMESTAMP, d INTERVAL, n NUMERIC)` + yield* sql`INSERT INTO declared VALUES ('1970-01-01 00:00:01', '-2 seconds', 12.5), ('1969-12-31 23:59:58', '3 seconds', -0.05), (NULL, NULL, NULL)` + yield* sql`CREATE TABLE inexact (s TIMESTAMP, ms TIMESTAMP, ns TIMESTAMP, ds INTERVAL, dms INTERVAL, dns INTERVAL, scale NUMERIC, digits NUMERIC, wide NUMERIC, i8 INTEGER, u32 INTEGER, u64 BIGINT, f32 DOUBLE PRECISION, f32i INTEGER)` + yield* sql`INSERT INTO inexact VALUES ('1970-01-01 00:00:01.5', '1970-01-01 00:00:01.0005', '2263-01-01', '-1.5 seconds', '0.0005 seconds', '110000 days', 1.005, 1000, 1, 256, -1, -1, 0.1, 16777217)` + yield* sql`CREATE TABLE exact (i8 INTEGER, f32 DOUBLE PRECISION)` + yield* sql`INSERT INTO exact VALUES (-128, 0.5), (127, 'NaN'), (0, '-Infinity'), (1, ${ + 2 ** -149 + }), (2, '-0'), (3, ${(2 - 2 ** -23) * 2 ** 127})` + yield* sql`CREATE TABLE arrowed (d DATE, at TIMESTAMP, n INTEGER, s TEXT, extra TEXT)` + yield* sql`INSERT INTO arrowed VALUES ('2024-01-01', '1970-01-01 00:00:01.5', 7, 'a', 'x'), (NULL, NULL, -7, NULL, 'y')` + yield* sql`CREATE TYPE mood AS ENUM ('zzz', 'aaa')` + yield* sql`CREATE TABLE texts (m mood, c CHAR(3), v VARCHAR(5), sm SMALLINT, r REAL)` + yield* sql`INSERT INTO texts VALUES ('zzz', 'ab', 'xy', 20000, 16777216), ('aaa', 'abc', 'x', -20000, 0.5)` + }) +) + +/** A frame with exactly the given Arrow columns. */ +const frame = (fields: ReadonlyArray, columns: ReadonlyArray>) => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns(ArrowSchema.make(fields), columns))) +const decimal = (s: string) => BigDecimal.fromStringUnsafe(s) + +const memory = Catalog.layer({ + sales: DataFrame.fromRows(Sale, sales), + regions: DataFrame.fromRows(Region, regions), + // The rows of the PostgreSQL tables `declared` and `inexact`, as PostgreSQL keeps them. + declared: frame([ + ArrowSchema.field("t", ArrowSchema.timestamp("microsecond"), { nullable: true }), + ArrowSchema.field("d", ArrowSchema.duration("microsecond"), { nullable: true }), + ArrowSchema.field("n", ArrowSchema.decimal(20, 4), { nullable: true }) + ], [[1_000_000n, -2_000_000n, null], [-2_000_000n, 3_000_000n, null], [decimal("12.5"), decimal("-0.05"), null]]), + inexact: frame([ + ...["s", "ms", "ns"].map((name) => ArrowSchema.field(name, ArrowSchema.timestamp("microsecond"))), + ...["ds", "dms", "dns"].map((name) => ArrowSchema.field(name, ArrowSchema.duration("microsecond"))), + ...["scale", "digits", "wide"].map((name) => ArrowSchema.field(name, ArrowSchema.decimal(20, 4))), + ...["i8", "u32"].map((name) => ArrowSchema.field(name, ArrowSchema.int32)), + ArrowSchema.field("u64", ArrowSchema.int64), + ArrowSchema.field("f32", ArrowSchema.float64), + ArrowSchema.field("f32i", ArrowSchema.int32) + ], [ + [1_500_000n], + [1_000_500n], + [9_246_182_400_000_000n], + [-1_500_000n], + [500n], + [9_504_000_000_000_000n], + [decimal("1.005")], + [decimal("1000")], + [decimal("1")], + [256], + [-1], + [-1n], + [0.1], + [16777217] + ]), + exact: frame([ + ArrowSchema.field("i8", ArrowSchema.int32), + ArrowSchema.field("f32", ArrowSchema.float64) + ], [[-128, 127, 0, 1, 2, 3], [0.5, NaN, -Infinity, 2 ** -149, -0, (2 - 2 ** -23) * 2 ** 127]]), + // Dates in milliseconds and timestamps in seconds convert to the declared days and milliseconds. + arrowed: frame([ + ArrowSchema.field("extra", ArrowSchema.utf8), + ArrowSchema.field("d", ArrowSchema.date("millisecond")), + ArrowSchema.field("at", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("n", ArrowSchema.int32), + ArrowSchema.field("s", ArrowSchema.utf8) + ], [["x", "y"], [BigInt(19723 * 86_400_000), null], [1_500_000n, null], [7, -7], ["a", null]]), + // PostgreSQL converts char(n) values to text without their trailing spaces. + texts: frame([ + ...["m", "c", "v"].map((name) => ArrowSchema.field(name, ArrowSchema.utf8)), + ArrowSchema.field("sm", ArrowSchema.int16), + ArrowSchema.field("r", ArrowSchema.float32) + ], [["zzz", "aaa"], ["ab", "abc"], ["xy", "x"], [20000, -20000], [16777216, 0.5]]) +}) + +const table = Query.table("sales", Sale) + +const close = (a: unknown, b: unknown) => + typeof a === "number" && typeof b === "number" + ? Object.is(a, b) || Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a)) + : false + +const types = (df: DataFrame.DataFrame) => + DataFrame.toArrowTable(df).schema.fields.map((f) => ({ name: f.name, type: f.type })) + +const onSql = (query: Query.Query) => query.pipe(Effect.provide(Sql.layer)) +const rowsOnSql = (query: Query.Query) => Effect.map(onSql(query), DataFrame.toRows) +/** The message of the error `query` fails with on SQL. */ +const rejects = (query: Query.Query) => Effect.map(Effect.flip(onSql(query)), (e) => e.message) +/** Queries of the table `name` declared with a schema. */ +const tableOf = (name: string) => (schema: S) => + Query.table(name, schema) + +const xs = Query.from(DataFrame.fromColumns({ x: [1, 2, 3] })) +const ints = Query.from(DataFrame.fromColumns({ x: Int32Array.of(1, 2, 3) })) +const bools = Query.from(DataFrame.fromColumns({ b: [true, null, false, true] })) +const groups = Query.from( + DataFrame.fromColumns({ g: ["a", "b", "a", "b", "a"], h: [1, 1, 2, 2, 1], x: [1, 3, 2, 6, 4] }) +) +const none = ints.pipe(Query.filter((c) => Expr.gt(c.x, 9))) +/** A query over a frame with exactly the given Arrow columns. */ +const arrow = (fields: ReadonlyArray, columns: ReadonlyArray>) => + Query.from(frame(fields, columns)) as Query.Query as Query.Query +const ties = [0.5, 1.5, 2.5, -0.5, -1.5, -2.5, 2.675, 1.005, 0.125, 0.375, 1.25, 1.35] +const decimals = arrow<{ d: BigDecimal.BigDecimal | null }>( + [ArrowSchema.field("d", ArrowSchema.decimal(18, 3))], + [["0.5", "1.5", "2.5", "-0.5", "-1.5", "-2.5", "2.675", "1.005", "0.125", null].map((s) => + s === null ? null : decimal(s) + )] +) +/** Durations in microseconds, with a group that has none. */ +const spans = arrow<{ g: string; d: Duration.Duration | null }>( + [ArrowSchema.field("g", ArrowSchema.utf8), ArrowSchema.field("d", ArrowSchema.duration("microsecond"))], + [["a", "a", "a", "b", "c"], [7n, 9n, null, 4n, null]] +) +const units = { s: "second", ms: "millisecond", us: "microsecond", ns: "nanosecond" } as const +const unitNames = ["s", "ms", "us", "ns"] as const +/** One column `v` of `type`, its integers microsecond multiples in every unit (PostgreSQL keeps microseconds). */ +const temporal = (type: ArrowSchema.DataType, values: ReadonlyArray = [1_001_000n, -1_500_000n, null]) => + arrow<{ v: unknown }>([ArrowSchema.field("v", type)], [values]) +/** Casts in both directions between temporal types of every unit and integers. */ +const temporalCasts = unitNames.flatMap((u) => [ + [ + `casts of Timestamp[${u}]`, + temporal(ArrowSchema.timestamp(units[u])).pipe(Query.select((c) => ({ + int: Expr.cast(c.v, "Int64"), + date: Expr.cast(c.v, "Date"), + duration: Expr.cast(c.v, `Duration[${u}]`), + s: Expr.cast(c.v, "Timestamp[s]"), + ms: Expr.cast(c.v, "Timestamp[ms]"), + us: Expr.cast(c.v, "Timestamp[us]"), + ns: Expr.cast(c.v, "Timestamp[ns]") + }))) + ] as const, + [ + `casts of Duration[${u}]`, + temporal(ArrowSchema.duration(units[u])).pipe(Query.select((c) => ({ + int: Expr.cast(c.v, "Int64"), + timestamp: Expr.cast(c.v, `Timestamp[${u}]`), + s: Expr.cast(c.v, "Duration[s]"), + ms: Expr.cast(c.v, "Duration[ms]"), + us: Expr.cast(c.v, "Duration[us]"), + ns: Expr.cast(c.v, "Duration[ns]") + }))) + ] as const, + [ + `casts of integers to Timestamp[${u}] and Duration[${u}]`, + temporal(ArrowSchema.int64).pipe(Query.select((c) => ({ + timestamp: Expr.cast(c.v, `Timestamp[${u}]`), + duration: Expr.cast(c.v, `Duration[${u}]`) + }))) + ] as const +]) +/** Integers of a temporal column of `unit`, microsecond multiples (PostgreSQL keeps microseconds). */ +const exact = (unit: typeof unitNames[number], values: ReadonlyArray) => + values.map((v) => (v === null ? null : BigInt(v) * (unit === "ns" ? 1000n : 1n))) +/** Timestamps `t`, `s` and durations `d`, `e`: `t` and `d` of unit `u`, `s` and `e` of unit `v`. */ +const mixed = (u: typeof unitNames[number], v: typeof unitNames[number]) => + arrow<{ + t: DateTime.Utc | null + s: DateTime.Utc | null + d: Duration.Duration | null + e: Duration.Duration | null + }>([ + ArrowSchema.field("t", ArrowSchema.timestamp(units[u])), + ArrowSchema.field("s", ArrowSchema.timestamp(units[v])), + ArrowSchema.field("d", ArrowSchema.duration(units[u])), + ArrowSchema.field("e", ArrowSchema.duration(units[v])) + ], [ + exact(u, [1500, -1, 2, 1000, null]), + exact(v, [1, 1500, -1, 1000, 7]), + exact(u, [1500, -1, 2, 1000, null]), + exact(v, [1, 1500, -1, 1000, 7]) + ]) +/** Temporal operations on operands of two different units, which meet in the coarser one. */ +const mixedUnits = unitNames.flatMap((u) => + unitNames.filter((v) => v !== u).map((v) => + [ + `temporal operations mixing ${u} and ${v}`, + mixed(u, v).pipe(Query.select((c) => { + const int = (x: Expr.Expr) => Expr.cast(x, "Int64") + return { + tMinusS: int(Expr.subtract(c.t, c.s)), + tPlusE: int(Expr.add(c.t, c.e)), + tMinusE: int(Expr.subtract(c.t, c.e)), + sPlusD: int(Expr.add(c.s, c.d)), + dPlusE: int(Expr.add(c.d, c.e)), + dMinusE: int(Expr.subtract(c.d, c.e)), + eq: Expr.eq(c.t, c.s), + lt: Expr.lt(c.t, c.s), + gte: Expr.gte(c.s, c.t), + durationsEq: Expr.eq(c.d, c.e), + durationsGt: Expr.gt(c.d, c.e), + between: Expr.between(c.t, c.s, c.s), + coalesce: int(Expr.coalesce(c.t, c.s)), + when: int(Expr.when(Expr.isNull(c.d), c.e, c.d)) + } + })) + ] as const + ) +) + +/** Queries whose SQL result must equal the in-memory result, column types included. */ +const cases: ReadonlyArray]> = [ + ["int64 columns", Query.from(DataFrame.fromColumns({ d: BigInt64Array.of(1n, 9007199254740993n, -3n) }))], + [ + "aggregate types", + ints.pipe( + Query.aggregate((c) => ({ n: Expr.count(), s: Expr.sum(c.x), m: Expr.mean(c.x), d: Expr.countDistinct(c.x) })) + ) + ], + ["empty results", none.pipe(Query.withColumns((c) => ({ y: Expr.divide(c.x, 2), s: Expr.concat("a", "b") })))], + ["aggregates of empty input", none.pipe(Query.aggregate((c) => ({ n: Expr.countDistinct(c.x), s: Expr.sum(c.x) })))], + [ + "boolean sums", + Query.from(DataFrame.fromColumns({ b: [true, false, true] })).pipe( + Query.select((c) => ({ s: Expr.over(Expr.sum(c.b), []), r: Expr.cumSum(c.b) })) + ) + ], + [ + "boolean sum aggregate", + Query.from(DataFrame.fromColumns({ b: [true, false, true] })).pipe(Query.aggregate((c) => ({ s: Expr.sum(c.b) }))) + ], + [ + "boolean minimum and maximum", + bools.pipe(Query.aggregate((c) => ({ min: Expr.min(c.b), max: Expr.max(c.b) }))) + ], + [ + "boolean minimum and maximum over windows", + bools.pipe(Query.select((c) => ({ + min: Expr.over(Expr.min(c.b), []), + max: Expr.over(Expr.max(c.b), []), + cumMin: Expr.cumMin(c.b), + cumMax: Expr.cumMax(c.b) + }))) + ], + ["unique on no columns keeps the first row", xs.pipe(Query.unique([]))], + [ + "windows in aggregate arguments see only their group", + groups.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ + running: Expr.sum(Expr.cumSum(c.x)), + previous: Expr.sum(Expr.shift(c.x)), + rank: Expr.max(Expr.rank(c.x, { method: "min" })), + centered: Expr.max(Expr.subtract(c.x, Expr.over(Expr.mean(c.x), []))), + nested: Expr.sum(Expr.over(Expr.cumSum(c.x), [c.h])) + })) + ) + ], + [ + "windows in aggregate arguments over all rows", + groups.pipe(Query.aggregate((c) => ({ running: Expr.sum(Expr.cumSum(c.x)), first: Expr.first(Expr.shift(c.x)) }))) + ], + ["filters on a window function", xs.pipe(Query.filter((c) => Expr.gt(Expr.cumSum(c.x), 3)))], + ["filters on an aggregate over the input", xs.pipe(Query.filter((c) => Expr.gt(c.x, Expr.over(Expr.mean(c.x), []))))], + [ + "over an empty partition list", + xs.pipe(Query.select((c) => ({ n: Expr.over(Expr.sum(c.x), []), m: Expr.over(Expr.first(c.x), []) }))) + ], + [ + "temporal and decimal inputs", + arrow<{ d: Duration.Duration | null; t: DateTime.Utc | null }>([ + ArrowSchema.field("d", ArrowSchema.duration("microsecond")), + ArrowSchema.field("t", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("day", ArrowSchema.date("day")), + ArrowSchema.field("n", ArrowSchema.decimal(18, 2)) + ], [ + [1500n, -90_000_001n, null], + [1_700_000_000_123_456n, -1n, null], + [19_000, -1, null], + [decimal("12.34"), decimal("-0.05"), null] + ]).pipe(Query.withColumns((c) => ({ s: Expr.totalSeconds(c.d), year: Expr.year(c.t) }))) + ], + ...temporalCasts, + ...mixedUnits, + [ + "joins on timestamps of mixed units", + mixed("s", "ms").pipe( + Query.select((c) => ({ k: c.t, x: Expr.lit(1) })), + Query.join(mixed("s", "ms").pipe(Query.select((c) => ({ k: c.s, y: Expr.lit(2) }))), { on: "k", how: "full" }), + Query.select((c) => ({ k: Expr.cast(c.k, "Int64"), x: c.x, y: c.y })) + ) + ], + [ + "concatenations of durations of mixed units", + Query.concat([ + mixed("s", "ms").pipe(Query.select((c) => ({ d: c.e }))), + mixed("s", "ms").pipe(Query.select((c) => ({ d: c.d }))) + ]).pipe( + Query.select((c) => ({ d: Expr.cast(c.d, "Int64"), ms: Expr.cast(Expr.cast(c.d, "Duration[ms]"), "Int64") })) + ) + ], + [ + "casts of integers beyond double precision to Timestamp[us] and Duration[us]", + temporal(ArrowSchema.int64, [253_402_300_799_999_999n, -62_135_596_800_000_001n, 9_007_199_254_740_993n]).pipe( + Query.select((c) => ({ timestamp: Expr.cast(c.v, "Timestamp[us]"), duration: Expr.cast(c.v, "Duration[us]") })) + ) + ], + [ + "casts of zoned timestamps", + temporal(ArrowSchema.timestamp("microsecond", "UTC")).pipe( + Query.select((c) => ({ int: Expr.cast(c.v, "Int64"), s: Expr.cast(c.v, "Timestamp[s]") })) + ) + ], + [ + "casts of dates", + temporal(ArrowSchema.date("day"), [19_000, -1, null]).pipe(Query.select((c) => ({ + int: Expr.cast(c.v, "Int32"), + long: Expr.cast(c.v, "Int64"), + text: Expr.cast(c.v, "Utf8"), + s: Expr.cast(c.v, "Timestamp[s]"), + ns: Expr.cast(c.v, "Timestamp[ns]"), + date: Expr.cast(Expr.cast(c.v, "Int32"), "Date") + }))) + ], + [ + "durations", + Query.from(DataFrame.fromColumns({ d: [Duration.seconds(30), Duration.seconds(90)] })).pipe( + Query.filter((c) => Expr.gt(Expr.totalSeconds(c.d), 60n)) + ) + ], + [ + "columns named like helpers", + Query.from(DataFrame.fromColumns({ __row: [1, 2], __n: [3, 4], __source: ["a", "b"] })) + ], + [ + "tail of columns named __n and long", + Query.from(DataFrame.fromColumns({ __n: [1, 2], [long]: [3, 4] })).pipe(Query.tail(1)) + ], + [ + "columns of 63 bytes, multi-byte characters included", + Query.from(DataFrame.fromColumns({ ["é".repeat(31) + "a"]: [1], ["é".repeat(31) + "b"]: [2] })) + ], + [ + "concat of a column named __source", + Query.concat([ + Query.from(DataFrame.fromColumns({ __source: [1] })), + Query.from(DataFrame.fromColumns({ __source: [2] })) + ]) + ], + [ + "sqrt of negative numbers", + Query.from(DataFrame.fromColumns({ x: [-1, 4, null, -0] })).pipe(Query.select((c) => ({ r: Expr.sqrt(c.x) }))) + ], + [ + "rounding floats", + Query.from(DataFrame.fromColumns({ x: ties })).pipe( + Query.select((c) => ({ r0: Expr.round(c.x), r1: Expr.round(c.x, 1), r2: Expr.round(c.x, 2) })) + ) + ], + ...unitNames.map((u) => + [ + `tables declared in ${u} and with a decimal`, + Query.table( + "declared", + Schema.Struct({ + t: Schema.NullOr(Schema.DateTimeUtc), + d: Schema.NullOr(Schema.Duration), + n: Schema.NullOr(Schema.BigDecimal) + }), + { + types: { t: ArrowSchema.timestamp(units[u]), d: ArrowSchema.duration(units[u]), n: ArrowSchema.decimal(5, 2) } + } + ).pipe(Query.withColumns((c) => ({ + ti: Expr.cast(c.t, "Int64"), + di: Expr.cast(c.d, "Int64"), + text: Expr.cast(c.n, "Utf8"), + sum: Expr.add(c.n, c.n) + }))) + ] as const + ), + [ + "tables declared with narrower types, holding them exactly", + Query.table("exact", Schema.Struct({ i8: Schema.Number, f32: Schema.Number }), { + types: { i8: ArrowSchema.int8, f32: ArrowSchema.float32 } + }) + ], + [ + "tables declared by an Arrow schema", + Query.table( + "arrowed", + ArrowSchema.make([ + ArrowSchema.field("d", ArrowSchema.date("day")), + ArrowSchema.field("at", ArrowSchema.timestamp("millisecond")), + ArrowSchema.field("n", ArrowSchema.int32, { nullable: false }), + ArrowSchema.field("s", ArrowSchema.utf8) + ]) + ) + ], + [ + "enum, char, varchar, smallint and real columns, as their declared types", + Query.table( + "texts", + Schema.Struct({ m: Schema.String, c: Schema.String, v: Schema.String, sm: Schema.Number, r: Schema.Number }), + { types: { sm: ArrowSchema.int32 } } + ).pipe( + Query.select((c) => ({ + m: c.m, + lower: Expr.toLowerCase(c.m), + before: Expr.lt(c.m, "b"), + c: c.c, + padded: Expr.eq(c.c, "ab "), + length: Expr.length(c.c), + v: Expr.concat(c.v, "|"), + sm: Expr.add(c.sm, c.sm), + r: Expr.add(c.r, 1) + })), + Query.sort((c) => [c.m]) + ) + ], + [ + "rounding integers", + Query.from(DataFrame.fromColumns({ x: Int32Array.of(5, -15, 25) })).pipe( + Query.select((c) => ({ r0: Expr.round(c.x), r1: Expr.round(c.x, 1) })) + ) + ], + [ + "integer literals fitted whichever rows select them", + Query.from(DataFrame.fromColumns({ x: Int8Array.of(100, 0) })).pipe( + Query.select((c) => ({ r: Expr.multiply(Expr.add(c.x, Expr.when(Expr.eq(c.x, 0), 300, 1)), 2) })) + ) + ] +] + +describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => { + layer(Layer.provideMerge(Setup, PgliteClient.layer()), { timeout: "60 seconds" })((it) => { + const both = Effect.fnUntraced(function*(query: Query.Query) { + const expected = yield* query.pipe(Effect.provide(memory)) + const actual = yield* onSql(query) + assert.deepStrictEqual(types(actual), types(expected)) + const a = DataFrame.toRows(actual) as Array> + const b = DataFrame.toRows(expected) as Array> + assert.strictEqual(a.length, b.length) + for (let r = 0; r < b.length; r++) { + for (const k of Object.keys(b[r])) { + if (!close(a[r][k], b[r][k])) { + assert.deepStrictEqual({ row: r, column: k, value: a[r][k] }, { row: r, column: k, value: b[r][k] }) + } + } + } + }) + + // DF-I-SQL/: differential evidence; exact expansion and pins in effect/test/dataframe/README.md. + for (const [name, query] of cases) it.effect(name, () => both(query)) + + // DF-S001: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("negative slices clamp both ends", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* rowsOnSql(xs.pipe(Query.slice(-5, 1))), []) + assert.deepStrictEqual(yield* rowsOnSql(xs.pipe(Query.slice(-5, 3))), [{ x: 1 }]) + })) + + // DF-S002: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("rounds decimals half to even, keeping the scale", () => + Effect.gen(function*() { + const df = yield* onSql( + decimals.pipe(Query.select((c) => ({ r0: Expr.round(c.d, 0), r1: Expr.round(c.d, 1) }))) + ) + assert.deepStrictEqual(types(df).map((f) => f.type), [ArrowSchema.decimal(19, 3), ArrowSchema.decimal(19, 3)]) + assert.deepStrictEqual( + DataFrame.toRows(df).map((r: any) => [r.r0 && BigDecimal.format(r.r0), r.r1 && BigDecimal.format(r.r1)]), + [ + ["0", "0.5"], + ["2", "1.5"], + ["2", "2.5"], + ["0", "-0.5"], + ["-2", "-1.5"], + ["-2", "-2.5"], + ["3", "2.7"], + ["1", "1"], + ["0", "0.1"], + [null, null] + ] + ) + })) + + // DF-S003: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("errors carry the database's message", () => + Effect.gen(function*() { + assert.include( + yield* rejects(ints.pipe(Query.select((c) => ({ y: Expr.add(c.x, 2147483647) })))), + "integer out of range" + ) + })) + + // DF-S004: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("tables must satisfy their schema, as in memory", () => + Effect.gen(function*() { + const contract = tableOf("contract") + const odd = tableOf("odd") + assert.strictEqual( + yield* rejects(contract(Schema.Struct({ s: Schema.Literal("good") }))), + `column "s" has values its schema rejects` + ) + assert.strictEqual( + yield* rejects(contract(Schema.Struct({ n: Schema.Number }))), + `column "n" has missing values` + ) + // Columns the query doesn't read are checked too. + assert.strictEqual( + yield* rejects( + contract(Schema.Struct({ s: Schema.Literal("good"), n: Schema.NullOr(Schema.Number) })).pipe( + Query.pick("n") + ) + ), + `column "s" has values its schema rejects` + ) + assert.deepStrictEqual( + yield* rowsOnSql(contract(Schema.Struct({ s: Schema.Literals(["bad", "good"]) }))), + [{ s: "bad" }, { s: "good" }] + ) + for ( + const [query, message] of [ + [odd(Schema.Struct({ nil: Schema.Null })), `column "nil" has values, expected Null`], + [contract(Schema.Struct({ s: Schema.NullOr(Schema.Number) })), `column "s" is text, expected Float64`], + [ + contract(Schema.Struct({ n: Schema.NullOr(Schema.BigInt) })), + `column "n" is double precision, expected Int64` + ], + [ + odd(Schema.Struct({ far: Schema.NullOr(Schema.DateTimeUtc) })), + `column "far" has timestamps outside the range of DateTime` + ], + [ + odd(Schema.Struct({ months: Schema.Duration })), + `column "months" has durations in months, which have no fixed length` + ], + [ + contract(Schema.Struct({ s: Schema.String }).check(Schema.makeFilter((row) => row.s !== ""))), + "schemas with checks on whole rows aren't supported on SQL tables; check their fields instead" + ], + [ + odd(Schema.Struct({ forever: Schema.DateTimeUtc })), + `column "forever" has timestamps outside the range of DateTime` + ], + [ + Query.fromPlan({ + _tag: "Table", + name: "odd", + columns: [{ name: "endless", type: "Date", nullable: true }] + }), + `column "endless" has values that Date can't hold exactly` + ] + ] as const + ) { + assert.strictEqual(yield* rejects(query), message) + } + assert.deepStrictEqual(yield* rowsOnSql(odd(Schema.Struct({ nothing: Schema.Null }))), [{ nothing: null }]) + // A declared column the table doesn't have, named like a column of PostgreSQL's catalog. + for (const schema of [Schema.Struct({ oid: Schema.BigInt }), Schema.Struct({ typname: Schema.String })]) { + assert.include(yield* rejects(contract(schema)), `.${Object.keys(schema.fields)[0]} does not exist`) + } + })) + + // DF-S005: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("tables reject values their declared types can't hold exactly, as in memory", () => + Effect.gen(function*() { + for ( + const [column, schema, type] of [ + ["s", Schema.DateTimeUtc, ArrowSchema.timestamp("second")], + ["ms", Schema.DateTimeUtc, ArrowSchema.timestamp("millisecond")], + ["ns", Schema.DateTimeUtc, ArrowSchema.timestamp("nanosecond")], + ["ds", Schema.Duration, ArrowSchema.duration("second")], + ["dms", Schema.Duration, ArrowSchema.duration("millisecond")], + ["dns", Schema.Duration, ArrowSchema.duration("nanosecond")], + ["scale", Schema.BigDecimal, ArrowSchema.decimal(5, 2)], + ["digits", Schema.BigDecimal, ArrowSchema.decimal(5, 2)], + ["wide", Schema.BigDecimal, ArrowSchema.decimal(38, 20)], + ["i8", Schema.Number, ArrowSchema.int8], + ["u32", Schema.Number, ArrowSchema.uint32], + ["u64", Schema.BigInt, ArrowSchema.uint64], + ["f32", Schema.Number, ArrowSchema.float32], + ["f32i", Schema.Number, ArrowSchema.float32] + ] as const + ) { + const table = Query.table("inexact", Schema.Struct({ [column]: schema }), { types: { [column]: type } }) + // A count reads none of the table's columns. + for (const query of [table, table.pipe(Query.aggregate(() => ({ n: Expr.count() })))]) { + const expected = yield* Effect.flip(query.pipe(Effect.provide(memory))) + assert.strictEqual(yield* rejects(query), expected.message) + } + } + })) + + // DF-S006: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("tables are checked on the rows the query reads, whatever the relation", () => + Effect.gen(function*() { + const sql = yield* SqlClient + // A view whose rows differ in each scan, without volatile functions. + yield* sql.unsafe( + `CREATE FUNCTION tick() RETURNS INTEGER STABLE LANGUAGE plpgsql AS $$ + DECLARE n INTEGER := COALESCE(NULLIF(current_setting('df.tick', TRUE), ''), '0')::INTEGER + 1; + BEGIN PERFORM set_config('df.tick', n::TEXT, FALSE); RETURN n; END $$` + ) + // Named with 63 bytes: names longer than that are one with it to PostgreSQL. + const view = "ticking".padEnd(63, "_") + yield* sql.unsafe(`CREATE VIEW "${view}" AS SELECT CASE WHEN tick() <= 1 THEN 'good' ELSE 'bad' END AS s`) + const ticking = (name = view) => Query.table(name, Schema.Struct({ s: Schema.Literal("good") })) + const tick = (query: Query.Query) => + Effect.andThen(sql`SELECT set_config('df.tick', '0', FALSE)`, rowsOnSql(query)) + assert.deepStrictEqual(yield* tick(ticking()), [{ s: "good" }]) + assert.deepStrictEqual( + yield* tick(Query.concat([ticking(), ticking(), ticking(view + "s")])), + [{ s: "good" }, { s: "good" }, { s: "good" }] + ) + yield* sql`CREATE SEQUENCE seq START 1` + yield* sql`CREATE VIEW fluctuating AS SELECT CASE WHEN nextval('seq') <= 2 THEN 'good' ELSE 'bad' END AS s` + const query = Query.table("fluctuating", Schema.Struct({ s: Schema.Literal("good") })) + const read = sql.withTransaction( + Effect.andThen(sql`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ`, Effect.result(rowsOnSql(query))) + ) + const results = [yield* read, yield* read, yield* read] + assert.deepStrictEqual(results.map((r) => Result.isSuccess(r) ? r.success : r.failure.message), [ + [{ s: "good" }], + [{ s: "good" }], + `column "s" has values its schema rejects` + ]) + })) + + // DF-S007: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("tables are read from their names, whatever the compiler names", () => + Effect.gen(function*() { + const s = (name: string) => Query.table(name, Schema.Struct({ s: Schema.String })) + assert.deepStrictEqual( + yield* rowsOnSql(Query.concat([s("contract"), s("__table0"), s("second"), s("__table1")])), + [{ s: "bad" }, { s: "good" }, { s: "bad" }, { s: "good" }, { s: "bad" }] + ) + assert.strictEqual( + yield* rejects(Query.concat([ + s("contract").pipe(Query.filter((c) => Expr.eq(c.s, "good"))), + s("second"), + Query.table("__table1", Schema.Struct({ s: Schema.Literal("good") })) + ])), + `column "s" has values its schema rejects` + ) + assert.deepStrictEqual(yield* rowsOnSql(s("a.b")), [{ s: "good" }]) + assert.deepStrictEqual(yield* rowsOnSql(s(long)), [{ s: "63" }]) + const two = Query.concat([s(underscores), s("second")]) + assert.deepStrictEqual(yield* rowsOnSql(two), [{ s: "57" }, { s: "good" }]) + })) + + // DF-S008: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("columns are one to PostgreSQL when their first 63 bytes are", () => + Effect.gen(function*() { + // Declarations of one table read its one column `"z".repeat(63)`, each with its checks. + const shared = (name: string, schema: Schema.Top & { readonly Type: number } = Schema.Number) => + Query.table("shared", Schema.Struct({ [name]: schema })).pipe(Query.select((c) => ({ x: c[name] }))) + const [za, zb] = ["z".repeat(63) + "a", "z".repeat(63) + "b"] + assert.deepStrictEqual(yield* rowsOnSql(Query.concat([shared(za), shared(zb)])), [{ x: 1 }, { x: 1 }]) + assert.strictEqual( + yield* rejects(Query.concat([shared(za), shared(zb, Schema.Literal(2))])), + `column ${JSON.stringify(zb)} has values its schema rejects` + ) + // Columns of one relation are rejected: PostgreSQL keeps the whole characters within an identifier's first 63 bytes. + const pairs = [["x".repeat(63) + "a", "x".repeat(63) + "b"], ["x".repeat(62) + "é", "x".repeat(62) + "ж"]] + for (const [a, b] of pairs) { + const added = Query.from(DataFrame.fromColumns({ [a]: [1] })).pipe( + Query.withColumns(() => ({ [b]: Expr.lit(2) })) + ) + assert.include(yield* rejects(added), `columns ${JSON.stringify(a)} and ${JSON.stringify(b)} are one name`) + } + })) + + // DF-S009: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => + Effect.gen(function*() { + const sql = yield* SqlClient + const query = Query.table("odd", Schema.Struct({ at: Schema.DateTimeUtc })).pipe( + Query.select((c) => ({ + at: c.at, + hour: Expr.hour(c.at), + early: Expr.lt(c.at, Expr.lit(DateTime.makeUnsafe("2024-01-01T00:45:00Z"))) + })) + ) + const rows = yield* sql`SET TIME ZONE 'Europe/Berlin'`.pipe( + Effect.andThen(rowsOnSql(query)), + Effect.ensuring(Effect.orDie(sql`RESET TIME ZONE`)) + ) + assert.deepStrictEqual(rows, [{ at: DateTime.makeUnsafe("2024-01-01T00:30:00Z"), hour: 0, early: true }]) + })) + + // DF-S010: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("filters, derives and sorts", () => + both(table.pipe( + Query.filter((c) => Expr.and(Expr.isNotNull(c.units), Expr.gt(c.price, 20))), + Query.withColumns((c) => ({ revenue: Expr.multiply(c.units, c.price), tag: Expr.concat(c.region, "/", "x") })), + Query.sort((c) => [Expr.desc(c.revenue), c.id]), + Query.head(10) + ))) + + // DF-S011: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("groups in order of first appearance", () => + both(table.pipe( + Query.groupBy("region"), + Query.agg((c) => ({ + n: Expr.count(), + units: Expr.sum(c.units), + mean: Expr.mean(c.price), + min: Expr.min(c.price), + max: Expr.max(c.price), + first: Expr.first(c.id), + last: Expr.last(c.id), + median: Expr.median(c.price), + sd: Expr.std(c.price) + })) + ))) + + // DF-S012: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("joins keep the left row order", () => + both(table.pipe( + Query.pick("id", "region"), + Query.join(Query.table("regions", Region), { on: "region", how: "left" }), + Query.filter((c) => Expr.lt(c.id, 15)) + ))) + + // DF-S013: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("semi and anti joins, unique and slices", () => + Effect.gen(function*() { + yield* both(table.pipe(Query.join(Query.table("regions", Region), { on: "region", how: "anti" }))) + yield* both(table.pipe(Query.unique(["region"]), Query.pick("id", "region"))) + yield* both(table.pipe(Query.tail(5))) + yield* both(table.pipe(Query.slice(7, 4))) + })) + + // DF-S014: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("window functions over partitions", () => + both(table.pipe( + Query.select((c) => ({ + id: c.id, + region: c.region, + running: Expr.over(Expr.cumSum(c.units), [c.region]), + previous: Expr.over(Expr.shift(c.price), [c.region]), + rank: Expr.over(Expr.rank(c.price, { method: "min" }), [c.region]), + share: Expr.divide(c.price, Expr.over(Expr.sum(c.price), [c.region])) + })) + ))) + + // DF-S015: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("dates and times", () => + both(table.pipe( + Query.select((c) => ({ + id: c.id, + year: Expr.year(c.at), + month: Expr.month(c.at), + weekday: Expr.weekday(c.at), + hour: Expr.hour(c.at) + })) + ))) + + // DF-S016: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("explain shows the SQL", () => + Effect.gen(function*() { + const text = yield* Query.explain(table.pipe(Query.head(1))).pipe( + Effect.provide(Sql.layer) + ) + assert.include(text, `FROM "sales"`) + })) + + // DF-S017: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("sums durations to durations, as in memory", () => + Effect.gen(function*() { + const totals = spans.pipe(Query.groupBy("g"), Query.agg((c) => ({ total: Expr.sum(c.d) }))) + yield* both(totals) + yield* both(spans.pipe(Query.select((c) => ({ + over: Expr.over(Expr.sum(c.d), [c.g]), + running: Expr.cumSum(c.d), + last2: Expr.rollingSum(c.d, 2), + least2: Expr.rollingMin(c.d, 2), + step: Expr.diff(c.d) + })))) + // A group without values sums to the zero duration. + const result = yield* onSql(totals) + assert.deepStrictEqual(types(result)[1].type, ArrowSchema.duration("microsecond")) + assert.deepStrictEqual(DataFrame.toArray(result, "total").map(Duration.toNanosUnsafe), [16_000n, 4_000n, 0n]) + })) + + // DF-S018: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("rejects aggregates that durations and timestamps don't have, as in memory", () => + Effect.gen(function*() { + // The types reject these columns too: they are read as numbers here. + type Numbers = Query.Query<{ readonly g: string; readonly d: number | null }> + const inMemory = (query: Query.Query) => Effect.map(Effect.flip(query), (e) => e.message) + const rejected = Effect.fnUntraced(function*(query: Query.Query, message: string) { + assert.strictEqual(yield* rejects(query), message) + assert.strictEqual(yield* inMemory(query), message) + }) + const durations = spans as unknown as Numbers + for (const f of [Expr.mean, Expr.median, Expr.std, Expr.variance]) { + yield* rejected( + durations.pipe(Query.groupBy("g"), Query.agg((c) => ({ r: f(c.d) }))), + "expected a numeric column, got Duration[us]" + ) + } + yield* rejected( + durations.pipe(Query.select((c) => ({ r: Expr.over(Expr.mean(c.d), [c.g]) }))), + "expected a numeric column, got Duration[us]" + ) + yield* rejected( + durations.pipe(Query.select((c) => ({ r: Expr.rollingMean(c.d, 2) }))), + "rollingMean is not defined for Duration[us]" + ) + const timestamps = arrow<{ g: string; d: number | null }>( + [ArrowSchema.field("g", ArrowSchema.utf8), ArrowSchema.field("d", ArrowSchema.timestamp("microsecond"))], + [["a", "a"], [7n, null]] + ) + yield* rejected( + timestamps.pipe(Query.groupBy("g"), Query.agg((c) => ({ r: Expr.sum(c.d) }))), + "sum is not defined for Timestamp[us]" + ) + yield* rejected( + timestamps.pipe(Query.select((c) => ({ r: Expr.cumSum(c.d) }))), + "cumSum is not defined for Timestamp[us]" + ) + })) + + // DF-N004: local policy for SQL results; pins in effect/test/dataframe/README.md. + it.effect("decimal results are read from their text, without BigDecimal values", () => + Effect.gen(function*() { + const amounts = arrow<{ g: string; d: BigDecimal.BigDecimal | null }>( + [ArrowSchema.field("g", ArrowSchema.utf8), ArrowSchema.field("d", ArrowSchema.decimal(18, 2))], + [ + ["a", "a", "b", "b", "c"], + [decimal("-12.34"), decimal("0"), decimal("9999999999999999.99"), null, decimal("-9999999999999999.99")] + ] + ) + const query = amounts.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ total: Expr.sum(c.d), all: Expr.list(c.d) })) + ) + const { calls, grouped, plain } = yield* Effect.acquireUseRelease( + Effect.sync(() => [ + vi.spyOn(BigDecimal, "make"), + vi.spyOn(BigDecimal, "round"), + vi.spyOn(BigDecimal, "fromStringUnsafe") + ]), + (spies) => + Effect.gen(function*() { + const count = () => spies.reduce((n, spy) => n + spy.mock.calls.length, 0) + const plain = yield* onSql(amounts) + const grouped = yield* onSql(query) + const read = count() + DataFrame.toRows(plain) + DataFrame.toRows(grouped) + return { plain, grouped, calls: [read, count() - read] } + }), + (spies) => Effect.sync(() => spies.forEach((spy) => spy.mockRestore())) + ) + // None to read the results; one per decimal of their rows: 4 in `d`, 3 in `total`, 4 in `all`. + assert.deepStrictEqual(calls, [0, 11]) + const unscaled = (d: unknown) => BigDecimal.isBigDecimal(d) ? [d.value, d.scale] : d + assert.deepStrictEqual(types(plain).map((f) => f.type), [ArrowSchema.utf8, ArrowSchema.decimal(18, 2)]) + assert.deepStrictEqual(DataFrame.toArray(plain, "d").map(unscaled), [ + [-1234n, 2], + [0n, 2], + [999999999999999999n, 2], + null, + [-999999999999999999n, 2] + ]) + assert.deepStrictEqual(types(grouped).map((f) => f.type), [ + ArrowSchema.utf8, + ArrowSchema.decimal(38, 2), + ArrowSchema.list(ArrowSchema.field("item", ArrowSchema.decimal(18, 2))) + ]) + assert.deepStrictEqual( + DataFrame.toRows(grouped).map((r) => [r.g, unscaled(r.total), r.all.map(unscaled)]), + [ + ["a", [-1234n, 2], [[-1234n, 2], [0n, 2]]], + ["b", [999999999999999999n, 2], [[999999999999999999n, 2], null]], + ["c", [-999999999999999999n, 2], [[-999999999999999999n, 2]]] + ] + ) + // Results beyond the 64 bits of a decimal column fail, in lists too. + const beyond = "decimal values beyond 18 digits are not supported (column of 128-bit decimals)" + assert.strictEqual( + yield* rejects(amounts.pipe(Query.select((c) => ({ p: Expr.multiply(c.d, c.d) })))), + beyond + ) + assert.strictEqual( + yield* rejects(amounts.pipe(Query.aggregate((c) => ({ p: Expr.list(Expr.multiply(c.d, c.d)) })))), + beyond + ) + })) + + // DF-S020: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("decimals take the scales numeric does", () => + Effect.gen(function*() { + const tens = (unscaled: bigint) => BigDecimal.make(unscaled, -1) + const negative = arrow<{ d: BigDecimal.BigDecimal | null }>( + [ArrowSchema.field("d", ArrowSchema.decimal(18, -1))], + [[tens(7n), tens(-15n), tens(0n), null]] + ) + yield* both(negative.pipe(Query.select((c) => ({ + d: c.d, + lit: Expr.lit(tens(-7n)), + cast: Expr.castDecimal(c.d, 38, 2), + text: Expr.cast(c.d, "Utf8"), + above: Expr.gt(c.d, Expr.lit(tens(1n))) + })))) + assert.deepStrictEqual( + yield* rowsOnSql(negative.pipe(Query.select((c) => ({ d: c.d, lit: Expr.lit(tens(-7n)) })))), + [ + { d: tens(7n), lit: tens(-7n) }, + { d: tens(-15n), lit: tens(-7n) }, + { d: tens(0n), lit: tens(-7n) }, + { d: null, lit: tens(-7n) } + ] + ) + for (const scale of [1001, -1001, 2147483647, -2147483648]) { + const message = `a decimal scale of ${scale} is not supported by the SQL engine` + const declared = Query.table("t", ArrowSchema.make([ArrowSchema.field("d", ArrowSchema.decimal(38, scale))])) + const literal = ints.pipe(Query.select(() => ({ d: Expr.lit(BigDecimal.make(7n, scale)) }))) + for (const query of [declared, literal]) { + const compiled = Sql.compile(query.plan) + assert.strictEqual(Result.isFailure(compiled) && compiled.failure.message, message) + assert.strictEqual(yield* rejects(query), message) + } + } + })) + + // DF-S021: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("decimals meet integers, literals and floats as in memory", () => + Effect.gen(function*() { + // 70 in every row, next to 69, 70 and 71. The columns are untyped, as in a frame + // read from Arrow: typed columns don't compare a decimal with an integer. + const mixed = arrow( + [ArrowSchema.field("d", ArrowSchema.decimal(18, -1)), ArrowSchema.field("i", ArrowSchema.int32)], + [[7n, 7n, 7n].map((x) => BigDecimal.make(x, -1)), [69, 70, 71]] + ) + const chosen = mixed.pipe(Query.select((c) => ({ + equal: Expr.eq(c.d, c.i), + above: Expr.gt(c.d, c.i), + literal: Expr.eq(c.d, 69), + when: Expr.when(Expr.lit(false), c.d, c.i), + first: Expr.coalesce(c.i, c.d) + }))) + yield* both(chosen) + assert.deepStrictEqual(yield* rowsOnSql(chosen), [ + { equal: false, above: true, literal: false, when: BigDecimal.make(69n, 0), first: BigDecimal.make(69n, 0) }, + { equal: true, above: false, literal: false, when: BigDecimal.make(70n, 0), first: BigDecimal.make(70n, 0) }, + { equal: false, above: false, literal: false, when: BigDecimal.make(71n, 0), first: BigDecimal.make(71n, 0) } + ]) + // A literal's precision is at most 38, whatever its scale. + const small = BigDecimal.make(1n, 400) + const literal = ints.pipe(Query.head(1), Query.select(() => ({ d: Expr.lit(small) }))) + yield* both(literal) + assert.deepStrictEqual(yield* rowsOnSql(literal), [{ d: small }]) + // 10^18 units of 10^-310 are 10^-292, past the largest power of ten a float holds. + const held = arrow<{ d: BigDecimal.BigDecimal }>( + [ArrowSchema.field("d", ArrowSchema.decimal(38, 310))], + [[BigDecimal.make(10n ** 18n, 310), BigDecimal.make(-(10n ** 18n), 310)]] + ) + const floats = held.pipe(Query.select((c) => ({ f: Expr.cast(c.d, "Float64") }))) + yield* both(floats) + assert.deepStrictEqual(yield* rowsOnSql(floats), [{ f: 1e-292 }, { f: -1e-292 }]) + })) + + // DF-S022: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("isIn compares decimals by value as in memory", () => + Effect.gen(function*() { + const dec = (unscaled: bigint, scale: number) => BigDecimal.make(unscaled, scale) + // 7.00, 7.50 and -7.00 next to 70, 0 and 700. The columns are untyped, as in a frame + // read from Arrow: typed columns take candidates of their own type. + const held = arrow( + [ + ArrowSchema.field("d", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("n", ArrowSchema.decimal(18, -1)), + ArrowSchema.field("at", ArrowSchema.timestamp("microsecond", "UTC")), + ArrowSchema.field("took", ArrowSchema.duration("microsecond")) + ], + [ + [dec(700n, 2), dec(750n, 2), dec(-700n, 2), null], + [dec(7n, -1), dec(0n, -1), dec(70n, -1), null], + [0n, 1_000_000n, 5n, null], + [0n, 1_000_000n, 5n, null] + ] + ) + const chosen = held.pipe(Query.select((c) => ({ + same: Expr.isIn(c.d, [dec(700n, 2)]), + rescaled: Expr.isIn(c.d, [dec(7n, 0), dec(7500n, 3)]), + integer: Expr.isIn(c.d, [7, -7n]), + float: Expr.isIn(c.d, [7.5]), + units: Expr.isIn(c.d, [700, 750n]), + mixed: Expr.isIn(c.d, [7, dec(-700n, 2), null]), + none: Expr.not(Expr.isIn(c.d, [7, dec(750n, 2)])), + negative: Expr.isIn(c.n, [70, dec(700n, 0)]), + at: Expr.isIn(c.at, [DateTime.makeUnsafe(1000)]), + took: Expr.isIn(c.took, [Duration.seconds(1), Duration.micros(5n)]) + }))) + yield* both(chosen) + const row = (...values: ReadonlyArray) => ({ + same: values[0], + rescaled: values[1], + integer: values[2], + float: values[3], + units: values[4], + mixed: values[5], + none: values[6], + negative: values[7], + at: values[8], + took: values[9] + }) + assert.deepStrictEqual(yield* rowsOnSql(chosen), [ + row(true, true, true, false, false, true, false, true, false, false), + row(false, true, false, true, false, false, false, false, true, true), + row(false, false, true, false, false, true, true, true, false, true), + row(null, null, null, null, null, null, null, null, null, null) + ]) + })) + + // DF-S023: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("isIn keeps exact candidates apart from fractions and takes long lists", () => + Effect.gen(function*() { + const dec = (unscaled: bigint, scale: number) => BigDecimal.make(unscaled, scale) + // The first integer a float can't hold, next to the float below it. + const odd = 9007199254740993n + const below = 9007199254740992 + const held = arrow( + [ + ArrowSchema.field("d", ArrowSchema.decimal(18, 0)), + ArrowSchema.field("e", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("b", ArrowSchema.int64), + ArrowSchema.field("u", ArrowSchema.uint64) + ], + [ + [dec(odd, 0), dec(0n, 0), null], + [dec(odd * 100n + 50n, 2), dec(50n, 2), null], + [odd, 0n, null], + [2n ** 64n - 1n, odd, null] + ] + ) + const chosen = held.pipe(Query.select((c) => ({ + // A fraction in the list doesn't round the other candidates or the column. + rounded: Expr.isIn(c.d, [0.5, below]), + exact: Expr.isIn(c.d, [0.5, odd]), + zero: Expr.isIn(c.d, [-0, NaN, 2n ** 70n]), + fraction: Expr.isIn(c.e, [0.5, below]), + integer: Expr.isIn(c.b, [0.5, below]), + wide: Expr.isIn(c.b, [2n ** 70n, -0, odd]), + unsigned: Expr.isIn(c.u, [0.5, 2n ** 64n - 1n, below]), + signed: Expr.isIn(c.u, [-1n]), + long: Expr.isIn(c.d, Array.from({ length: 3000 }, (_, i) => dec(BigInt(i), 0))) + }))) + yield* both(chosen) + const row = (...values: ReadonlyArray) => ({ + rounded: values[0], + exact: values[1], + zero: values[2], + fraction: values[3], + integer: values[4], + wide: values[5], + unsigned: values[6], + signed: values[7], + long: values[8] + }) + assert.deepStrictEqual(yield* rowsOnSql(chosen), [ + row(false, true, false, false, false, true, true, false, false), + row(false, false, true, true, false, true, false, false, true), + row(null, null, null, null, null, null, null, null, null) + ]) + })) + + // DF-S019: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("unions and nullable fields are checked as in memory", () => + Effect.gen(function*() { + const contract = tableOf("contract") + const orNull = (member: S) => Schema.Union([Schema.suspend(() => Schema.Null), member]) + assert.deepStrictEqual( + yield* rowsOnSql(contract(Schema.Struct({ s: orNull(Schema.Literals(["bad", "good"])) }))), + [{ s: "bad" }, { s: "good" }] + ) + const notNull = Schema.makeFilter((n: number | null) => n !== null) + for ( + const [schema, column] of [ + [Schema.Struct({ s: orNull(Schema.Literal("good")) }), "s"], + // A check of a nullable field is run on its nulls. + [Schema.Struct({ n: Schema.NullOr(Schema.Number).check(notNull) }), "n"], + // "good" is a value of both members. + [Schema.Struct({ s: Schema.Union([Schema.String, Schema.Literal("good")], { mode: "oneOf" }) }), "s"] + ] as const + ) { + assert.strictEqual(yield* rejects(contract(schema)), `column "${column}" has values its schema rejects`) + } + })) + }) +})